Files
Homelab-manager/server/test/monthlyReport.test.ts
bobbanandClaude Sonnet 5.5 be6c5552bd Patch plans, a monthly report and new-port alerts
Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an
ordinary update run, stopping at the first failure by default, silencing what depends on a
server while it is updated, with one message at the end.

Monthly report (Operations -> Monthly report, and optionally a message on a day of the month):
availability, incidents, updates, restore tests, backups and renewals for a calendar month,
printable. Backed by daily endpoint figures and a record of renewals.

New ports: the first report with ports is the baseline; a TCP port that begins listening later and
is not noted on the server's Ports card is a warning, cleared by Mark as expected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-19 23:16:14 +02:00

290 lines
19 KiB
TypeScript

/**
* The monthly report: which month is which, what renewals and daily figures are kept, what the report gathers for a month, how it reads as a message, and the
* routes and schedule around it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-monthly-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, alertIncidents, updateRuns, patchPlanRuns, patchPlans, restoreTests, renewals, endpointChecks, endpointDaily, endpointResults } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { isMonthId, lastFullMonth, recentMonths, monthBounds, monthLabel, buildMonthlyReport, monthlyReportText } = await import("../src/services/monthlyReport.js");
const { isRenewal, noteRenewal } = await import("../src/services/renewals.js");
const { runCheck, pruneResults } = await import("../src/services/endpointChecks.js");
const { cronForMonthlyReport } = await import("../src/services/monthlyReportScheduler.js");
const { reportsRouter } = await import("../src/routes/reports.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { sql, eq } = await import("drizzle-orm");
await runMigrations();
describe("which month is which", () => {
test("the last whole month, and the twelve to choose from, on the clock of the time zone", () => {
assert.equal(lastFullMonth(new Date("2026-10-19T12:00:00Z"), "UTC"), "2026-09");
assert.equal(lastFullMonth(new Date("2027-01-02T12:00:00Z"), "UTC"), "2026-12", "across the new year");
assert.equal(lastFullMonth(new Date("2026-10-31T23:30:00Z"), "Europe/Stockholm"), "2026-10", "it is already November in Stockholm");
const months = recentMonths(new Date("2026-10-19T12:00:00Z"), "UTC");
assert.equal(months.length, 12);
assert.deepEqual([months[0], months[1], months[11]], [{ id: "2026-10", label: "October 2026" }, { id: "2026-09", label: "September 2026" }, { id: "2025-11", label: "November 2025" }]);
assert.equal(monthLabel("2026-02"), "February 2026");
});
test("a month begins and ends where the clock of the time zone says, summer or winter", () => {
const sept = monthBounds("2026-09", "Europe/Stockholm");
assert.deepEqual([sept.start.toISOString(), sept.end.toISOString()], ["2026-08-31T22:00:00.000Z", "2026-09-30T22:00:00.000Z"]);
const oct = monthBounds("2026-10", "Europe/Stockholm");
assert.equal(oct.end.toISOString(), "2026-10-31T23:00:00.000Z", "the clocks went back on the last Sunday");
assert.equal(monthBounds("2026-12", "UTC").end.toISOString(), "2027-01-01T00:00:00.000Z");
});
test("only a real month is a month", () => {
for (const ok of ["2026-09", "1999-12"]) assert.equal(isMonthId(ok), true, ok);
for (const bad of ["2026-13", "2026-00", "2026-9", "26-09", "2026-09-01", "", undefined, 202609]) assert.equal(isMonthId(bad), false, String(bad));
});
test("the schedule is a day of the month, 1 to 28 so every month has it", () => {
assert.equal(cronForMonthlyReport(5, "07:30"), "30 7 5 * *");
assert.equal(cronForMonthlyReport(31, "07:30"), "30 7 1 * *");
assert.equal(cronForMonthlyReport(0, "bad"), "0 8 1 * *");
});
});
describe("renewals and the daily figures", () => {
test("an expiry that moves a clear step later is a renewal; a day's rounding, a move earlier and a gap in the data are not", () => {
assert.equal(isRenewal("2026-10-01", "2026-12-30"), true);
assert.equal(isRenewal("2026-10-01T08:00:00.000Z", "2026-11-20"), true, "a full time or a day, only the day counts");
assert.equal(isRenewal("2026-10-01", "2026-10-02"), false);
assert.equal(isRenewal("2026-10-01", "2026-10-20"), false, "nineteen days is not enough");
assert.equal(isRenewal("2026-10-21", "2026-10-01"), false);
assert.equal(isRenewal(null, "2026-12-01"), false, "there was nothing to compare with");
assert.equal(isRenewal("2026-10-01", "not a date"), false);
});
test("a renewal is written down once", async () => {
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2027-10-01"), true);
assert.equal(await noteRenewal("domain", "example.se", "2026-10-02", "2027-10-01"), false, "the same one seen again");
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2026-10-03"), false);
assert.equal((await db.select().from(renewals).where(eq(renewals.subject, "example.se"))).length, 1);
await db.delete(renewals);
});
test("each answer adds to its day; a certificate that is replaced is a renewal; old days go", async () => {
const [check] = await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning();
const answer = (ok: boolean, ms: number | null, cert?: string) => async () => ({ ok, ms, message: "m", certExpiresAt: cert });
const t = (h: number) => new Date(`2026-09-10T0${h}:00:00Z`);
let row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(1), answer(true, 100, "2026-10-05T00:00:00.000Z"));
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(2), answer(true, 300, "2026-10-05T00:00:00.000Z"));
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(3), answer(false, null, "2026-10-05T00:00:00.000Z"));
const [day] = await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id));
assert.deepEqual([day.day, day.total, day.ok, day.msSum, day.msCount], ["2026-09-10", 3, 2, 400, 2], "a failure counts as a check but adds no answer time");
assert.equal((await db.select().from(renewals)).length, 0, "the same certificate three times is not a renewal");
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(4), answer(true, 100, "2026-12-04T00:00:00.000Z"));
const [renewed] = await db.select().from(renewals);
assert.deepEqual([renewed.kind, renewed.subject, renewed.fromDate, renewed.toDate], ["certificate", "Gitea", "2026-10-05", "2026-12-04"]);
await db.insert(endpointDaily).values({ checkId: check.id, day: "2020-01-01", total: 1, ok: 1 });
await pruneResults(new Date("2026-10-19T00:00:00Z"));
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2020-01-01"))).length, 0, "kept for years, not for ever");
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2026-09-10"))).length, 1);
await db.delete(renewals);
});
test("the migration started the daily figures from the results that already existed", async () => {
const [check] = await db.insert(endpointChecks).values({ name: "Backfilled", kind: "tcp", target: "h", port: 22 }).returning();
await db.insert(endpointResults).values([
{ checkId: check.id, at: "2026-10-01T01:00:00.000Z", ok: true, ms: 10 },
{ checkId: check.id, at: "2026-10-01T02:00:00.000Z", ok: true, ms: 30 },
{ checkId: check.id, at: "2026-10-01T03:00:00.000Z", ok: false, ms: null },
{ checkId: check.id, at: "2026-10-02T01:00:00.000Z", ok: true, ms: 50 },
]);
const migration = readFileSync(new URL("../drizzle/0043_puzzling_prism.sql", import.meta.url), "utf8");
const statement = migration.split("--> statement-breakpoint").pop()!;
assert.match(statement, /INSERT INTO `endpoint_daily`/);
await db.run(sql.raw(statement));
const rows = (await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id))).sort((a, b) => a.day.localeCompare(b.day));
assert.deepEqual(rows.map((r) => [r.day, r.total, r.ok, r.msSum, r.msCount]), [["2026-10-01", 3, 2, 40, 2], ["2026-10-02", 1, 1, 50, 1]]);
});
});
describe("a month's report", () => {
let nasId = 0;
let checkId = 0;
const audit = (action: string, id: number, at: string) => ({ category: "server", action, targetType: "server", targetId: String(id), createdAt: at });
const incident = (extra: Partial<typeof alertIncidents.$inferInsert>) => ({ alertKey: "k", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "a disk", openedAt: "2026-09-02T00:00:00.000Z", lastSeenAt: "2026-09-02T00:00:00.000Z", ...extra });
before(async () => {
await updateSettings({ notifications: { timezone: "UTC" } });
await db.delete(endpointChecks); // the earlier tests left checks of their own, and their daily figures go with them
nasId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
const elsaId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "e", apiTokenPrefix: "e", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
void elsaId;
// nas was offline 13 hours in September, and a night in August that does not count
await db.insert(auditLog).values([audit("went_offline", nasId, "2026-09-10 08:00:00"), audit("back_online", nasId, "2026-09-10 21:00:00"), audit("went_offline", nasId, "2026-08-02 01:00:00"), audit("back_online", nasId, "2026-08-02 05:00:00")]);
await db.insert(alertIncidents).values([
incident({ message: "nas: / is 96% full", category: "disk", openedAt: "2026-09-03T10:00:00.000Z", resolvedAt: "2026-09-03T12:00:00.000Z", peakSeverity: "critical" }),
incident({ message: "elsa stopped reporting", category: "offline", openedAt: "2026-09-12T10:00:00.000Z", resolvedAt: "2026-09-12T10:30:00.000Z", targetType: "server", targetId: 2, targetName: "elsa" }),
incident({ message: "Backup of anna failed", category: "backup", openedAt: "2026-09-20T00:00:00.000Z" }),
incident({ message: "in August", openedAt: "2026-08-20T00:00:00.000Z", resolvedAt: "2026-08-20T01:00:00.000Z" }),
incident({ message: "began in August, ended in September", openedAt: "2026-08-31T20:00:00.000Z", resolvedAt: "2026-09-01T02:00:00.000Z" }),
]);
await db.insert(updateRuns).values([
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-09-05T02:00:00.000Z", updatesBefore: 12, updatesAfter: 2 },
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "failed", startedAt: "2026-09-12T02:00:00.000Z", updatesBefore: 3, updatesAfter: null },
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-10-05T02:00:00.000Z", updatesBefore: 9, updatesAfter: 0 },
]);
const [plan] = await db.insert(patchPlans).values({ name: "Lab", days: "[0]", startTime: "02:00", timezone: "UTC", integrationId: 1, projectId: 1, templateId: 1, templateName: "t" }).returning();
await db.insert(patchPlanRuns).values([
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "done", startedAt: "2026-09-06T02:00:00.000Z" },
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "failed", startedAt: "2026-09-13T02:00:00.000Z" },
]);
await db.insert(restoreTests).values([{ subjectKey: "custom:photos", label: "Photos share", testedAt: "2026-09-14", result: "ok" }, { subjectKey: "custom:old", label: "Old", testedAt: "2026-08-14", result: "ok" }]);
await db.insert(renewals).values([{ kind: "certificate", subject: "git.lab", at: "2026-09-12T03:00:00.000Z", fromDate: "2026-09-14", toDate: "2026-12-13" }, { kind: "domain", subject: "example.se", at: "2026-10-03T03:00:00.000Z", fromDate: "2026-10-04", toDate: "2027-10-04" }]);
checkId = (await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning({ id: endpointChecks.id }))[0].id;
await db.insert(endpointDaily).values([
{ checkId, day: "2026-09-01", total: 100, ok: 100, msSum: 10_000, msCount: 100 },
{ checkId, day: "2026-09-30", total: 100, ok: 90, msSum: 9_000, msCount: 90 },
{ checkId, day: "2026-08-31", total: 100, ok: 0 },
{ checkId, day: "2026-10-01", total: 100, ok: 0 },
]);
});
const NOW = new Date("2026-10-19T12:00:00Z");
test("availability: what was down and for how long, from what the audit log recorded; the neighbouring months are not in it", async () => {
const r = await buildMonthlyReport("2026-09", NOW);
assert.deepEqual([r.month, r.label, r.partial, r.timezone], ["2026-09", "September 2026", false, "UTC"]);
const nas = r.servers.find((s) => s.name === "nas")!;
assert.equal(nas.downMinutes, 13 * 60);
assert.equal(nas.percent, 98.2, "13 of 720 hours down");
assert.equal(r.servers[0].name, "nas", "the worst first");
assert.deepEqual(r.endpoints.map((e) => [e.name, e.percent, e.avgMs, e.checks]), [["Gitea", 95, 100, 200]], "the first and last day of the month, not the days either side");
});
test("incidents: how many began and ended, the time they took, the longest, by kind", async () => {
const i = (await buildMonthlyReport("2026-09", NOW)).incidents;
assert.deepEqual([i.began, i.resolved, i.stillOpen], [3, 3, 1], "the open one is still open; the August one that ended in September ended in it");
assert.equal(i.totalMinutes, 150, "2 h and 30 min, of those that began and ended in the month");
assert.equal(i.averageMinutes, 75);
assert.equal(i.longest!.message, "Backup of anna failed", "still open, so it has been going the longest");
assert.deepEqual(i.byCategory, [{ category: "backup", count: 1 }, { category: "disk", count: 1 }, { category: "offline", count: 1 }]);
assert.deepEqual(i.list.map((x) => x.message), ["Backup of anna failed", "nas: / is 96% full", "elsa stopped reporting"]);
assert.equal(i.list[2].where, "elsa");
});
test("updates, patch plans, restore tests and renewals for that month only", async () => {
const r = await buildMonthlyReport("2026-09", NOW);
assert.deepEqual([r.updates.runs, r.updates.done, r.updates.failed, r.updates.installed], [2, 1, 1, 10]);
assert.deepEqual(r.updates.byServer, [{ name: "nas", runs: 2, failed: 1, installed: 10 }]);
assert.deepEqual(r.updates.planRuns, { runs: 2, done: 1, failed: 1 });
assert.deepEqual(r.restoreTests.map((t) => t.label), ["Photos share"]);
assert.deepEqual(r.renewals.map((x) => [x.kind, x.subject]), [["certificate", "git.lab"]]);
assert.equal(r.backups, null, "no backup coverage has been read here");
const oct = await buildMonthlyReport("2026-10", NOW);
assert.equal(oct.partial, true, "October is not over");
assert.deepEqual(oct.renewals.map((x) => x.subject), ["example.se"]);
assert.equal(oct.updates.runs, 1);
});
test("as a message: a section only where there is something to say, with the numbers that matter", async () => {
const { title, message } = monthlyReportText(await buildMonthlyReport("2026-09", NOW));
assert.equal(title, "Homelab monthly report — September 2026");
assert.match(message, /^AVAILABILITY\nServers: 9\d\.\d% on average \(2 servers\)\.\n- nas: 9\d\.\d% \(13 h down\)/);
assert.match(message, /Endpoints: 95% on average \(1 check\)\.\n- Gitea: 95%, 100 ms on average/);
assert.match(message, /INCIDENTS\n3 incidents began, 3 ended, 1 still open\.\nA resolved incident took 1 h 15 min on average\./);
assert.match(message, /- Backup of anna failed — 11 d and still open/);
assert.match(message, /UPDATES\n2 update runs: 1 done, 1 failed — 10 updates installed\.\nPatch plans ran 2 times, 1 not cleanly\./);
assert.match(message, /BACKUPS\n- Restore test of Photos share \(2026-09-14\) passed/);
assert.match(message, /RENEWED\n- Certificate git\.lab: 2026-09-14 → 2026-12-13/);
const partial = monthlyReportText(await buildMonthlyReport("2026-10", NOW));
assert.equal(partial.title, "Homelab monthly report — October 2026 (so far)");
const empty = monthlyReportText(await buildMonthlyReport("2025-01", NOW));
assert.equal(empty.message.includes("Nothing was recorded") || !/INCIDENTS\n[1-9]/.test(empty.message), true);
});
});
describe("through the routes", () => {
let app: Server;
let hook: Server;
let base = "";
let who = "viewer";
const messages: string[] = [];
before(async () => {
await db.insert(users).values([
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
{ oidcSub: "admin", name: "Ada", role: "admin" },
]);
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/reports", reportsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/reports`;
});
after(() => {
app.close();
hook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("anyone signed in can read it: the last whole month by default, the twelve to choose from, or one of them; nothing else is a month", async () => {
const body = await (await fetch(`${base}/monthly`)).json();
assert.equal(body.months.length, 12);
assert.equal(body.report.month, body.months[1].id, "last month, not this one");
assert.equal((await (await fetch(`${base}/monthly?month=${body.months[0].id}`)).json()).report.partial, true);
for (const bad of ["2026-13", "2019-01", "abc", "2026-9"]) assert.equal((await fetch(`${base}/monthly?month=${bad}`)).status, 400, bad);
const text = await (await fetch(`${base}/monthly?format=text`)).json();
assert.match(text.title, /^Homelab monthly report — /);
assert.equal(typeof text.message, "string");
});
test("an administrator can send it now to the channels; nobody else can", async () => {
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 403);
who = "admin";
const res = await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
assert.equal(res.status, 200);
assert.equal(messages.length, 1);
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ month: "nope" }) })).status, 400);
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "send_monthly"))).length, 1);
});
});