Patch plans, a monthly report and new-port alerts
Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an ordinary update run, stopping at the first failure by default, silencing what depends on a server while it is updated, with one message at the end. Monthly report (Operations -> Monthly report, and optionally a message on a day of the month): availability, incidents, updates, restore tests, backups and renewals for a calendar month, printable. Backed by daily endpoint figures and a record of renewals. New ports: the first report with ports is the baseline; a TCP port that begins listening later and is not noted on the server's Ports card is a warning, cleared by Mark as expected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ddd0aa622a
commit
be6c5552bd
46 files changed
+7836
-26
No files matched your search
@@ -33,7 +33,7 @@ await runMigrations();
|
||||
|
||||
type Data = Parameters<typeof agendaFromData>[0];
|
||||
const NOW = new Date("2026-10-19T12:00:00Z");
|
||||
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
|
||||
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], patchPlans: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
|
||||
const titles = (data: Data, days?: number, tz = "UTC") => agendaFromData(data, NOW, tz, days).items.map((i) => [i.daysLeft, i.kind, i.title]);
|
||||
|
||||
describe("what is gathered, and how it is dated", () => {
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
/**
|
||||
* The monthly report: which month is which, what renewals and daily figures are kept, what the report gathers for a month, how it reads as a message, and the
|
||||
* routes and schedule around it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-monthly-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog, alertIncidents, updateRuns, patchPlanRuns, patchPlans, restoreTests, renewals, endpointChecks, endpointDaily, endpointResults } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { isMonthId, lastFullMonth, recentMonths, monthBounds, monthLabel, buildMonthlyReport, monthlyReportText } = await import("../src/services/monthlyReport.js");
|
||||
const { isRenewal, noteRenewal } = await import("../src/services/renewals.js");
|
||||
const { runCheck, pruneResults } = await import("../src/services/endpointChecks.js");
|
||||
const { cronForMonthlyReport } = await import("../src/services/monthlyReportScheduler.js");
|
||||
const { reportsRouter } = await import("../src/routes/reports.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { sql, eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("which month is which", () => {
|
||||
test("the last whole month, and the twelve to choose from, on the clock of the time zone", () => {
|
||||
assert.equal(lastFullMonth(new Date("2026-10-19T12:00:00Z"), "UTC"), "2026-09");
|
||||
assert.equal(lastFullMonth(new Date("2027-01-02T12:00:00Z"), "UTC"), "2026-12", "across the new year");
|
||||
assert.equal(lastFullMonth(new Date("2026-10-31T23:30:00Z"), "Europe/Stockholm"), "2026-10", "it is already November in Stockholm");
|
||||
const months = recentMonths(new Date("2026-10-19T12:00:00Z"), "UTC");
|
||||
assert.equal(months.length, 12);
|
||||
assert.deepEqual([months[0], months[1], months[11]], [{ id: "2026-10", label: "October 2026" }, { id: "2026-09", label: "September 2026" }, { id: "2025-11", label: "November 2025" }]);
|
||||
assert.equal(monthLabel("2026-02"), "February 2026");
|
||||
});
|
||||
|
||||
test("a month begins and ends where the clock of the time zone says, summer or winter", () => {
|
||||
const sept = monthBounds("2026-09", "Europe/Stockholm");
|
||||
assert.deepEqual([sept.start.toISOString(), sept.end.toISOString()], ["2026-08-31T22:00:00.000Z", "2026-09-30T22:00:00.000Z"]);
|
||||
const oct = monthBounds("2026-10", "Europe/Stockholm");
|
||||
assert.equal(oct.end.toISOString(), "2026-10-31T23:00:00.000Z", "the clocks went back on the last Sunday");
|
||||
assert.equal(monthBounds("2026-12", "UTC").end.toISOString(), "2027-01-01T00:00:00.000Z");
|
||||
});
|
||||
|
||||
test("only a real month is a month", () => {
|
||||
for (const ok of ["2026-09", "1999-12"]) assert.equal(isMonthId(ok), true, ok);
|
||||
for (const bad of ["2026-13", "2026-00", "2026-9", "26-09", "2026-09-01", "", undefined, 202609]) assert.equal(isMonthId(bad), false, String(bad));
|
||||
});
|
||||
|
||||
test("the schedule is a day of the month, 1 to 28 so every month has it", () => {
|
||||
assert.equal(cronForMonthlyReport(5, "07:30"), "30 7 5 * *");
|
||||
assert.equal(cronForMonthlyReport(31, "07:30"), "30 7 1 * *");
|
||||
assert.equal(cronForMonthlyReport(0, "bad"), "0 8 1 * *");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renewals and the daily figures", () => {
|
||||
test("an expiry that moves a clear step later is a renewal; a day's rounding, a move earlier and a gap in the data are not", () => {
|
||||
assert.equal(isRenewal("2026-10-01", "2026-12-30"), true);
|
||||
assert.equal(isRenewal("2026-10-01T08:00:00.000Z", "2026-11-20"), true, "a full time or a day, only the day counts");
|
||||
assert.equal(isRenewal("2026-10-01", "2026-10-02"), false);
|
||||
assert.equal(isRenewal("2026-10-01", "2026-10-20"), false, "nineteen days is not enough");
|
||||
assert.equal(isRenewal("2026-10-21", "2026-10-01"), false);
|
||||
assert.equal(isRenewal(null, "2026-12-01"), false, "there was nothing to compare with");
|
||||
assert.equal(isRenewal("2026-10-01", "not a date"), false);
|
||||
});
|
||||
|
||||
test("a renewal is written down once", async () => {
|
||||
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2027-10-01"), true);
|
||||
assert.equal(await noteRenewal("domain", "example.se", "2026-10-02", "2027-10-01"), false, "the same one seen again");
|
||||
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2026-10-03"), false);
|
||||
assert.equal((await db.select().from(renewals).where(eq(renewals.subject, "example.se"))).length, 1);
|
||||
await db.delete(renewals);
|
||||
});
|
||||
|
||||
test("each answer adds to its day; a certificate that is replaced is a renewal; old days go", async () => {
|
||||
const [check] = await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning();
|
||||
const answer = (ok: boolean, ms: number | null, cert?: string) => async () => ({ ok, ms, message: "m", certExpiresAt: cert });
|
||||
const t = (h: number) => new Date(`2026-09-10T0${h}:00:00Z`);
|
||||
let row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(1), answer(true, 100, "2026-10-05T00:00:00.000Z"));
|
||||
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(2), answer(true, 300, "2026-10-05T00:00:00.000Z"));
|
||||
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(3), answer(false, null, "2026-10-05T00:00:00.000Z"));
|
||||
const [day] = await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id));
|
||||
assert.deepEqual([day.day, day.total, day.ok, day.msSum, day.msCount], ["2026-09-10", 3, 2, 400, 2], "a failure counts as a check but adds no answer time");
|
||||
assert.equal((await db.select().from(renewals)).length, 0, "the same certificate three times is not a renewal");
|
||||
|
||||
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(4), answer(true, 100, "2026-12-04T00:00:00.000Z"));
|
||||
const [renewed] = await db.select().from(renewals);
|
||||
assert.deepEqual([renewed.kind, renewed.subject, renewed.fromDate, renewed.toDate], ["certificate", "Gitea", "2026-10-05", "2026-12-04"]);
|
||||
|
||||
await db.insert(endpointDaily).values({ checkId: check.id, day: "2020-01-01", total: 1, ok: 1 });
|
||||
await pruneResults(new Date("2026-10-19T00:00:00Z"));
|
||||
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2020-01-01"))).length, 0, "kept for years, not for ever");
|
||||
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2026-09-10"))).length, 1);
|
||||
await db.delete(renewals);
|
||||
});
|
||||
|
||||
test("the migration started the daily figures from the results that already existed", async () => {
|
||||
const [check] = await db.insert(endpointChecks).values({ name: "Backfilled", kind: "tcp", target: "h", port: 22 }).returning();
|
||||
await db.insert(endpointResults).values([
|
||||
{ checkId: check.id, at: "2026-10-01T01:00:00.000Z", ok: true, ms: 10 },
|
||||
{ checkId: check.id, at: "2026-10-01T02:00:00.000Z", ok: true, ms: 30 },
|
||||
{ checkId: check.id, at: "2026-10-01T03:00:00.000Z", ok: false, ms: null },
|
||||
{ checkId: check.id, at: "2026-10-02T01:00:00.000Z", ok: true, ms: 50 },
|
||||
]);
|
||||
const migration = readFileSync(new URL("../drizzle/0043_puzzling_prism.sql", import.meta.url), "utf8");
|
||||
const statement = migration.split("--> statement-breakpoint").pop()!;
|
||||
assert.match(statement, /INSERT INTO `endpoint_daily`/);
|
||||
await db.run(sql.raw(statement));
|
||||
const rows = (await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id))).sort((a, b) => a.day.localeCompare(b.day));
|
||||
assert.deepEqual(rows.map((r) => [r.day, r.total, r.ok, r.msSum, r.msCount]), [["2026-10-01", 3, 2, 40, 2], ["2026-10-02", 1, 1, 50, 1]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a month's report", () => {
|
||||
let nasId = 0;
|
||||
let checkId = 0;
|
||||
const audit = (action: string, id: number, at: string) => ({ category: "server", action, targetType: "server", targetId: String(id), createdAt: at });
|
||||
const incident = (extra: Partial<typeof alertIncidents.$inferInsert>) => ({ alertKey: "k", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "a disk", openedAt: "2026-09-02T00:00:00.000Z", lastSeenAt: "2026-09-02T00:00:00.000Z", ...extra });
|
||||
|
||||
before(async () => {
|
||||
await updateSettings({ notifications: { timezone: "UTC" } });
|
||||
await db.delete(endpointChecks); // the earlier tests left checks of their own, and their daily figures go with them
|
||||
nasId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
|
||||
const elsaId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "e", apiTokenPrefix: "e", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
|
||||
void elsaId;
|
||||
// nas was offline 13 hours in September, and a night in August that does not count
|
||||
await db.insert(auditLog).values([audit("went_offline", nasId, "2026-09-10 08:00:00"), audit("back_online", nasId, "2026-09-10 21:00:00"), audit("went_offline", nasId, "2026-08-02 01:00:00"), audit("back_online", nasId, "2026-08-02 05:00:00")]);
|
||||
await db.insert(alertIncidents).values([
|
||||
incident({ message: "nas: / is 96% full", category: "disk", openedAt: "2026-09-03T10:00:00.000Z", resolvedAt: "2026-09-03T12:00:00.000Z", peakSeverity: "critical" }),
|
||||
incident({ message: "elsa stopped reporting", category: "offline", openedAt: "2026-09-12T10:00:00.000Z", resolvedAt: "2026-09-12T10:30:00.000Z", targetType: "server", targetId: 2, targetName: "elsa" }),
|
||||
incident({ message: "Backup of anna failed", category: "backup", openedAt: "2026-09-20T00:00:00.000Z" }),
|
||||
incident({ message: "in August", openedAt: "2026-08-20T00:00:00.000Z", resolvedAt: "2026-08-20T01:00:00.000Z" }),
|
||||
incident({ message: "began in August, ended in September", openedAt: "2026-08-31T20:00:00.000Z", resolvedAt: "2026-09-01T02:00:00.000Z" }),
|
||||
]);
|
||||
await db.insert(updateRuns).values([
|
||||
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-09-05T02:00:00.000Z", updatesBefore: 12, updatesAfter: 2 },
|
||||
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "failed", startedAt: "2026-09-12T02:00:00.000Z", updatesBefore: 3, updatesAfter: null },
|
||||
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-10-05T02:00:00.000Z", updatesBefore: 9, updatesAfter: 0 },
|
||||
]);
|
||||
const [plan] = await db.insert(patchPlans).values({ name: "Lab", days: "[0]", startTime: "02:00", timezone: "UTC", integrationId: 1, projectId: 1, templateId: 1, templateName: "t" }).returning();
|
||||
await db.insert(patchPlanRuns).values([
|
||||
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "done", startedAt: "2026-09-06T02:00:00.000Z" },
|
||||
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "failed", startedAt: "2026-09-13T02:00:00.000Z" },
|
||||
]);
|
||||
await db.insert(restoreTests).values([{ subjectKey: "custom:photos", label: "Photos share", testedAt: "2026-09-14", result: "ok" }, { subjectKey: "custom:old", label: "Old", testedAt: "2026-08-14", result: "ok" }]);
|
||||
await db.insert(renewals).values([{ kind: "certificate", subject: "git.lab", at: "2026-09-12T03:00:00.000Z", fromDate: "2026-09-14", toDate: "2026-12-13" }, { kind: "domain", subject: "example.se", at: "2026-10-03T03:00:00.000Z", fromDate: "2026-10-04", toDate: "2027-10-04" }]);
|
||||
checkId = (await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning({ id: endpointChecks.id }))[0].id;
|
||||
await db.insert(endpointDaily).values([
|
||||
{ checkId, day: "2026-09-01", total: 100, ok: 100, msSum: 10_000, msCount: 100 },
|
||||
{ checkId, day: "2026-09-30", total: 100, ok: 90, msSum: 9_000, msCount: 90 },
|
||||
{ checkId, day: "2026-08-31", total: 100, ok: 0 },
|
||||
{ checkId, day: "2026-10-01", total: 100, ok: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
const NOW = new Date("2026-10-19T12:00:00Z");
|
||||
|
||||
test("availability: what was down and for how long, from what the audit log recorded; the neighbouring months are not in it", async () => {
|
||||
const r = await buildMonthlyReport("2026-09", NOW);
|
||||
assert.deepEqual([r.month, r.label, r.partial, r.timezone], ["2026-09", "September 2026", false, "UTC"]);
|
||||
const nas = r.servers.find((s) => s.name === "nas")!;
|
||||
assert.equal(nas.downMinutes, 13 * 60);
|
||||
assert.equal(nas.percent, 98.2, "13 of 720 hours down");
|
||||
assert.equal(r.servers[0].name, "nas", "the worst first");
|
||||
assert.deepEqual(r.endpoints.map((e) => [e.name, e.percent, e.avgMs, e.checks]), [["Gitea", 95, 100, 200]], "the first and last day of the month, not the days either side");
|
||||
});
|
||||
|
||||
test("incidents: how many began and ended, the time they took, the longest, by kind", async () => {
|
||||
const i = (await buildMonthlyReport("2026-09", NOW)).incidents;
|
||||
assert.deepEqual([i.began, i.resolved, i.stillOpen], [3, 3, 1], "the open one is still open; the August one that ended in September ended in it");
|
||||
assert.equal(i.totalMinutes, 150, "2 h and 30 min, of those that began and ended in the month");
|
||||
assert.equal(i.averageMinutes, 75);
|
||||
assert.equal(i.longest!.message, "Backup of anna failed", "still open, so it has been going the longest");
|
||||
assert.deepEqual(i.byCategory, [{ category: "backup", count: 1 }, { category: "disk", count: 1 }, { category: "offline", count: 1 }]);
|
||||
assert.deepEqual(i.list.map((x) => x.message), ["Backup of anna failed", "nas: / is 96% full", "elsa stopped reporting"]);
|
||||
assert.equal(i.list[2].where, "elsa");
|
||||
});
|
||||
|
||||
test("updates, patch plans, restore tests and renewals for that month only", async () => {
|
||||
const r = await buildMonthlyReport("2026-09", NOW);
|
||||
assert.deepEqual([r.updates.runs, r.updates.done, r.updates.failed, r.updates.installed], [2, 1, 1, 10]);
|
||||
assert.deepEqual(r.updates.byServer, [{ name: "nas", runs: 2, failed: 1, installed: 10 }]);
|
||||
assert.deepEqual(r.updates.planRuns, { runs: 2, done: 1, failed: 1 });
|
||||
assert.deepEqual(r.restoreTests.map((t) => t.label), ["Photos share"]);
|
||||
assert.deepEqual(r.renewals.map((x) => [x.kind, x.subject]), [["certificate", "git.lab"]]);
|
||||
assert.equal(r.backups, null, "no backup coverage has been read here");
|
||||
const oct = await buildMonthlyReport("2026-10", NOW);
|
||||
assert.equal(oct.partial, true, "October is not over");
|
||||
assert.deepEqual(oct.renewals.map((x) => x.subject), ["example.se"]);
|
||||
assert.equal(oct.updates.runs, 1);
|
||||
});
|
||||
|
||||
test("as a message: a section only where there is something to say, with the numbers that matter", async () => {
|
||||
const { title, message } = monthlyReportText(await buildMonthlyReport("2026-09", NOW));
|
||||
assert.equal(title, "Homelab monthly report — September 2026");
|
||||
assert.match(message, /^AVAILABILITY\nServers: 9\d\.\d% on average \(2 servers\)\.\n- nas: 9\d\.\d% \(13 h down\)/);
|
||||
assert.match(message, /Endpoints: 95% on average \(1 check\)\.\n- Gitea: 95%, 100 ms on average/);
|
||||
assert.match(message, /INCIDENTS\n3 incidents began, 3 ended, 1 still open\.\nA resolved incident took 1 h 15 min on average\./);
|
||||
assert.match(message, /- Backup of anna failed — 11 d and still open/);
|
||||
assert.match(message, /UPDATES\n2 update runs: 1 done, 1 failed — 10 updates installed\.\nPatch plans ran 2 times, 1 not cleanly\./);
|
||||
assert.match(message, /BACKUPS\n- Restore test of Photos share \(2026-09-14\) passed/);
|
||||
assert.match(message, /RENEWED\n- Certificate git\.lab: 2026-09-14 → 2026-12-13/);
|
||||
const partial = monthlyReportText(await buildMonthlyReport("2026-10", NOW));
|
||||
assert.equal(partial.title, "Homelab monthly report — October 2026 (so far)");
|
||||
const empty = monthlyReportText(await buildMonthlyReport("2025-01", NOW));
|
||||
assert.equal(empty.message.includes("Nothing was recorded") || !/INCIDENTS\n[1-9]/.test(empty.message), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let hook: Server;
|
||||
let base = "";
|
||||
let who = "viewer";
|
||||
const messages: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
]);
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/reports", reportsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/reports`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("anyone signed in can read it: the last whole month by default, the twelve to choose from, or one of them; nothing else is a month", async () => {
|
||||
const body = await (await fetch(`${base}/monthly`)).json();
|
||||
assert.equal(body.months.length, 12);
|
||||
assert.equal(body.report.month, body.months[1].id, "last month, not this one");
|
||||
assert.equal((await (await fetch(`${base}/monthly?month=${body.months[0].id}`)).json()).report.partial, true);
|
||||
for (const bad of ["2026-13", "2019-01", "abc", "2026-9"]) assert.equal((await fetch(`${base}/monthly?month=${bad}`)).status, 400, bad);
|
||||
const text = await (await fetch(`${base}/monthly?format=text`)).json();
|
||||
assert.match(text.title, /^Homelab monthly report — /);
|
||||
assert.equal(typeof text.message, "string");
|
||||
});
|
||||
|
||||
test("an administrator can send it now to the channels; nobody else can", async () => {
|
||||
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 403);
|
||||
who = "admin";
|
||||
const res = await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ month: "nope" }) })).status, 400);
|
||||
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "send_monthly"))).length, 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* New ports: what counts as one, that the first report is the baseline, and what the health check, the Alerts page, maintenance, routing and
|
||||
* the server's Ports card do with them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-newports-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverPorts } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { newListeningPorts, baselineFrom, parseBaseline, newPortMessage } = await import("../src/services/newPorts.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { syncServerTasks } = await import("../src/services/taskSync.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
|
||||
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { serverPortsRouter } = await import("../src/routes/serverPorts.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const sock = (port: number, extra: Partial<{ protocol: "tcp" | "udp"; address: string; process: string }> = {}) => ({ protocol: "tcp" as const, port, address: "0.0.0.0", ...extra });
|
||||
const none = new Set<string>();
|
||||
|
||||
describe("what counts as a new port", () => {
|
||||
const baseline = parseBaseline(baselineFrom([sock(22), sock(443), sock(53, { protocol: "udp" })]));
|
||||
|
||||
test("the baseline is everything the first report listened on, in any protocol", () => {
|
||||
assert.deepEqual([...baseline!].sort(), ["tcp:22", "tcp:443", "udp:53"]);
|
||||
assert.equal(parseBaseline(null), null);
|
||||
assert.equal(parseBaseline("not json"), null);
|
||||
assert.equal(parseBaseline(JSON.stringify({ not: "a list" })), null);
|
||||
});
|
||||
|
||||
test("with no baseline yet, nothing is new", () => {
|
||||
assert.deepEqual(newListeningPorts([sock(8080)], null, none), []);
|
||||
assert.deepEqual(newListeningPorts(null, baseline, none), []);
|
||||
});
|
||||
|
||||
test("a TCP port that is listening and was not in the baseline is new, with what is listening on it", () => {
|
||||
const found = newListeningPorts([sock(22), sock(8080, { process: "node" }), sock(8080, { address: "::", process: "" }), sock(3000)], baseline, none);
|
||||
assert.deepEqual(found, [{ protocol: "tcp", port: 3000, process: null }, { protocol: "tcp", port: 8080, process: "node" }]);
|
||||
});
|
||||
|
||||
test("not UDP, not the dynamic range, not a port bound to this machine alone", () => {
|
||||
const found = newListeningPorts([sock(5353, { protocol: "udp" }), sock(49152), sock(60000), sock(9000, { address: "127.0.0.1" }), sock(9001, { address: "::1" }), sock(9002, { address: "[::1]" }), sock(9003, { address: "127.0.0.1" }), sock(9003, { address: "10.0.0.5" })], baseline, none);
|
||||
assert.deepEqual(found.map((p) => p.port), [9003], "one socket on a real address is enough");
|
||||
assert.deepEqual(newListeningPorts([sock(49151)], baseline, none).map((p) => p.port), [49151], "the last port before the dynamic range is still a service");
|
||||
});
|
||||
|
||||
test("a port somebody noted as expected is not new", () => {
|
||||
assert.deepEqual(newListeningPorts([sock(8080), sock(9090)], baseline, new Set(["tcp:8080"])).map((p) => p.port), [9090]);
|
||||
});
|
||||
|
||||
test("the message says what it is and what to do", () => {
|
||||
assert.equal(newPortMessage("nas", { protocol: "tcp", port: 8080, process: "node" }), "nas: a new port is listening — 8080/tcp (node). If it should be there, note it on the server's Ports card.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health check, the alert and the routing", () => {
|
||||
const snap = (extra: object) => ({ servers: [{ id: 3, name: "nas", lastSeenAt: new Date().toISOString(), disks: [], ...extra }], proxmox: [], synology: [] });
|
||||
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
||||
|
||||
test("each new port is a warning keyed by server and port; an offline server's are not judged", () => {
|
||||
const list = evaluateHealth(snap({ newPorts: [{ protocol: "tcp", port: 8080, process: "node" }, { protocol: "tcp", port: 9090, process: null }] }), limits, Date.now());
|
||||
assert.deepEqual(list.map((c) => [c.key, c.severity, c.source]), [["new-port:server:3:tcp:8080", "warning", "server"], ["new-port:server:3:tcp:9090", "warning", "server"]]);
|
||||
assert.match(list[0].message, /nas: a new port is listening — 8080\/tcp \(node\)/);
|
||||
const offline = evaluateHealth(snap({ lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString(), newPorts: [{ protocol: "tcp", port: 8080, process: null }] }), limits, Date.now());
|
||||
assert.equal(offline.some((c) => c.key.startsWith("new-port")), false);
|
||||
});
|
||||
|
||||
test("maintenance on the server holds it back, and routing treats it as monitoring with that server's tags", () => {
|
||||
assert.equal(subjectOfConditionKey("new-port:server:3:tcp:8080"), "server:3");
|
||||
assert.deepEqual(metaOfConditionKey("new-port:server:3:tcp:8080", "warning"), { category: "monitoring", severity: "warning", serverId: 3 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("from a report to the Ports card", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let serverId = 0;
|
||||
let who = "operator";
|
||||
const report = (ports: ReturnType<typeof sock>[]) => ({ tasks: [], system: { listening_ports: ports } });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown; currentUser: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
// The real servers router puts a signed-in user on the request before this; here the role is read the same way the middleware does.
|
||||
const { requireAuth } = await import("../src/auth/middleware.js");
|
||||
a.use("/api/servers/:id/ports", requireAuth, serverPortsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers/${serverId}/ports`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const newKeys = async () => (await collectSnapshot()).snapshot.servers.find((s) => s.id === serverId)!.newPorts!.map((p) => p.port);
|
||||
|
||||
test("the first report that carries ports is the baseline: nothing in it is new, and it is never changed by a later one", async () => {
|
||||
await syncServerTasks(serverId, report([sock(22), sock(443)]));
|
||||
let [row] = await db.select().from(servers).where(eq(servers.id, serverId));
|
||||
assert.deepEqual(JSON.parse(row.portBaseline!), ["tcp:22", "tcp:443"]);
|
||||
assert.deepEqual(await newKeys(), []);
|
||||
await syncServerTasks(serverId, report([sock(22), sock(443), sock(8080, { process: "node" })]));
|
||||
[row] = await db.select().from(servers).where(eq(servers.id, serverId));
|
||||
assert.deepEqual(JSON.parse(row.portBaseline!), ["tcp:22", "tcp:443"], "8080 is not folded into what is normal");
|
||||
assert.deepEqual(await newKeys(), [8080]);
|
||||
});
|
||||
|
||||
test("a report with no ports at all does not start a baseline", async () => {
|
||||
const [other] = await db.insert(servers).values({ name: "bare", apiTokenHash: "b", apiTokenPrefix: "b" }).returning({ id: servers.id });
|
||||
await syncServerTasks(other.id, { tasks: [], system: {} });
|
||||
const [row] = await db.select().from(servers).where(eq(servers.id, other.id));
|
||||
assert.equal(row.portBaseline, null);
|
||||
});
|
||||
|
||||
test("it is an alert on the Alerts page, in monitoring, linked to the server — and silenced by maintenance on it", async () => {
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((a) => a.message.includes("a new port is listening"))!;
|
||||
assert.ok(alert, "the alert is there");
|
||||
assert.deepEqual([alert.category, alert.severity, alert.silenced], ["monitoring", "warning", false]);
|
||||
assert.deepEqual(alert.target && [alert.target.type, alert.target.id], ["server", serverId]);
|
||||
await startOrExtendWindow("server", serverId, 30, "work", "Ada");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((a) => a.message.includes("a new port is listening"))!.silenced, true);
|
||||
});
|
||||
|
||||
test("the Ports card marks it new, and noting it as expected ends the alert; a scan-only row does not", async () => {
|
||||
who = "viewer";
|
||||
let list = (await (await fetch(base)).json()).ports as { port: number; isNew: boolean }[];
|
||||
assert.deepEqual(list.filter((p) => p.isNew).map((p) => p.port), [8080]);
|
||||
await db.insert(serverPorts).values({ serverId, port: 8080, protocol: "tcp", open: true });
|
||||
assert.deepEqual(await newKeys(), [8080], "a row that only records an open scan says nothing");
|
||||
await db.delete(serverPorts).where(eq(serverPorts.port, 8080));
|
||||
|
||||
who = "operator";
|
||||
const res = await fetch(base, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ port: 8080, protocol: "tcp", label: "Expected" }) });
|
||||
assert.equal(res.status, 200);
|
||||
list = (await res.json()).ports;
|
||||
assert.equal(list.find((p) => p.port === 8080)!.isNew, false);
|
||||
assert.deepEqual(await newKeys(), []);
|
||||
});
|
||||
|
||||
test("it can be switched off, and then nothing is new", async () => {
|
||||
await syncServerTasks(serverId, report([sock(22), sock(443), sock(8080), sock(9999)]));
|
||||
assert.deepEqual(await newKeys(), [9999]);
|
||||
await updateSettings({ healthChecks: { newPortAlerts: false } });
|
||||
assert.deepEqual(await newKeys(), []);
|
||||
who = "viewer";
|
||||
assert.equal(((await (await fetch(base)).json()).ports as { isNew: boolean }[]).some((p) => p.isNew), false);
|
||||
await updateSettings({ healthChecks: { newPortAlerts: true } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,407 @@
|
||||
/**
|
||||
* Patch plans: who a plan covers, when it is due, that it goes one server at a time and waits for each, what it does when one fails, what it silences,
|
||||
* the message at the end, and who may do what.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-patchplans-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, patchPlans, patchPlanRuns, updateRuns, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { planServers, dueOccurrence, summariseItems, startPlanRun, advancePlanRuns, startDuePlans, cancelPlanRun, parseItems, PlanError } = await import("../src/services/patchPlans.js");
|
||||
const { advanceUpdateRuns, setSemaphoreForTests } = await import("../src/services/updateRuns.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { agendaFromData } = await import("../src/services/agenda.js");
|
||||
const { patchPlansRouter } = await import("../src/routes/patchPlans.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
// ── a Semaphore that answers as told ──
|
||||
const fake = { runs: [] as { limit?: string }[], task: { id: 41, status: "waiting", created: null as string | null, start: null as string | null, end: null as string | null }, refuseFor: new Set<string>() };
|
||||
setSemaphoreForTests(async (integrationId) =>
|
||||
integrationId === 9
|
||||
? {
|
||||
ping: async () => ({ ok: true }),
|
||||
listTemplatesWithStatus: async () => [],
|
||||
checkTemplates: async () => ({ templates: [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "u.yml", lastTask: null }], failedProjectIds: [] }),
|
||||
runTemplate: async (_p: number, _t: number, options?: { limit?: string }) => {
|
||||
if (options?.limit && fake.refuseFor.has(options.limit)) throw new Error("Semaphore is having a moment");
|
||||
fake.runs.push({ limit: options?.limit });
|
||||
return { ...fake.task } as never;
|
||||
},
|
||||
getTask: async () => ({ ...fake.task }) as never,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const T0 = new Date("2026-10-18T02:05:00.000Z"); // a Sunday
|
||||
const at = (m: number) => new Date(T0.getTime() + m * 60_000);
|
||||
|
||||
describe("who a plan covers, and when it is due", () => {
|
||||
const all = [
|
||||
{ id: 1, name: "web", tags: JSON.stringify(["lab"]) },
|
||||
{ id: 2, name: "api", tags: JSON.stringify(["prod"]) },
|
||||
{ id: 3, name: "ångström", tags: JSON.stringify(["lab", "prod"]) },
|
||||
{ id: 4, name: "db", tags: null },
|
||||
];
|
||||
|
||||
test("servers with any of the tags, and those named, by name; each only once", () => {
|
||||
assert.deepEqual(planServers({ tags: JSON.stringify(["lab"]), serverIds: "[]" }, all).map((s) => s.name), ["ångström", "web"].sort((x, y) => x.localeCompare(y)));
|
||||
assert.deepEqual(planServers({ tags: JSON.stringify(["lab"]), serverIds: JSON.stringify([4, 3]) }, all).map((s) => s.name), ["db", "ångström", "web"].sort((x, y) => x.localeCompare(y)), "in the order the app sorts names");
|
||||
assert.deepEqual(planServers({ tags: "[]", serverIds: "[]" }, all), [], "a plan that names no one covers no one");
|
||||
});
|
||||
|
||||
const plan = { enabled: true, days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", lastRunDate: null as string | null };
|
||||
|
||||
test("due from its time to half an hour after, once a day, only on its days", () => {
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T01:59:00Z")), null, "not yet");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:00:00Z"))?.date, "2026-10-18");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:29:00Z"))?.date, "2026-10-18", "the app may have been down for a while");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:31:00Z")), null, "too late: that day is skipped");
|
||||
assert.equal(dueOccurrence({ ...plan, lastRunDate: "2026-10-18" }, new Date("2026-10-18T02:10:00Z")), null, "already started today");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-19T02:10:00Z")), null, "Monday is not a day it runs");
|
||||
assert.equal(dueOccurrence({ ...plan, enabled: false }, new Date("2026-10-18T02:10:00Z")), null);
|
||||
});
|
||||
|
||||
test("its time is on the clock of its time zone, winter and summer", () => {
|
||||
const stockholm = { ...plan, timezone: "Europe/Stockholm" };
|
||||
assert.equal(dueOccurrence(stockholm, new Date("2026-10-18T00:05:00Z"))?.date, "2026-10-18", "02:05 in Stockholm is 00:05 UTC while summer time lasts");
|
||||
assert.equal(dueOccurrence(stockholm, new Date("2026-11-01T01:05:00Z"))?.date, "2026-11-01", "and 01:05 UTC after the clocks go back");
|
||||
assert.equal(dueOccurrence(stockholm, new Date("2026-11-01T00:05:00Z")), null);
|
||||
});
|
||||
|
||||
test("a finished run says how each went in one line", () => {
|
||||
const item = (status: string) => ({ serverId: 1, name: "x", status, updateRunId: null, message: null, windowIds: [] }) as never;
|
||||
assert.equal(summariseItems([item("done"), item("done"), item("failed"), item("skipped")]), "2 updated, 1 failed, 1 left alone of 4 servers.");
|
||||
assert.equal(summariseItems([item("done")]), "1 updated of 1 server.");
|
||||
});
|
||||
|
||||
test("a plan is in the agenda as maintenance, with how many servers it updates", () => {
|
||||
const data = { secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], patchPlans: [{ id: 5, name: "Lab nights", days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", servers: 3 }] };
|
||||
const items = agendaFromData(data, new Date("2026-10-19T12:00:00Z"), "UTC", 20).items.filter((i) => i.id.startsWith("plan:"));
|
||||
assert.deepEqual(items.map((i) => [i.title, i.detail, i.at.slice(0, 16), i.kind]), [["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-10-25T02:00", "maintenance"], ["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-11-01T02:00", "maintenance"], ["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-11-08T02:00", "maintenance"]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a run, from start to end", () => {
|
||||
let hook: Server;
|
||||
const messages: string[] = [];
|
||||
const ids: Record<string, number> = {};
|
||||
let planId = 0;
|
||||
|
||||
const plan = (extra: Partial<typeof patchPlans.$inferInsert> = {}) => ({
|
||||
name: "Lab nights", days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", tags: JSON.stringify(["lab"]), serverIds: "[]", integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", limitFrom: "name", stopOnFailure: true, onlyWithUpdates: true, silenceDependents: true, windowMinutes: 90, ...extra,
|
||||
});
|
||||
const updates = (count: number, reportedAt = T0.toISOString()) => JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: false, reportedAt });
|
||||
const runOf = async (id: number) => (await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)))[0];
|
||||
const itemsOf = async (id: number) => parseItems((await runOf(id)).items);
|
||||
|
||||
/** The playbook for the server being updated finishes and the server reports again — as the update run follows. */
|
||||
async function finishCurrent(server: string, minute: number) {
|
||||
fake.task = { id: 41, status: "success", created: null, start: null, end: at(minute).toISOString() };
|
||||
await advanceUpdateRuns(at(minute + 1));
|
||||
await db.update(servers).set({ lastSeenAt: at(minute + 2).toISOString(), pendingUpdates: updates(0, at(minute + 2).toISOString()) }).where(eq(servers.id, ids[server]));
|
||||
await advanceUpdateRuns(at(minute + 3));
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
}
|
||||
|
||||
async function reset(extra: Partial<typeof patchPlans.$inferInsert> = {}) {
|
||||
await db.delete(patchPlans);
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows); // the tests share one clock, so a window left by the last would still be open
|
||||
fake.runs = [];
|
||||
fake.refuseFor.clear();
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
for (const name of ["web", "api"]) await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: updates(4) }).where(eq(servers.id, ids[name]));
|
||||
await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: updates(0) }).where(eq(servers.id, ids.idle));
|
||||
messages.length = 0;
|
||||
planId = (await db.insert(patchPlans).values(plan(extra)).returning({ id: patchPlans.id }))[0].id;
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
for (const name of ["web", "api", "idle"]) {
|
||||
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: JSON.stringify(["lab"]), lastSeenAt: T0.toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
}
|
||||
ids.prod = (await db.insert(servers).values({ name: "prod", apiTokenHash: "p", apiTokenPrefix: "p", tags: JSON.stringify(["prod"]), lastSeenAt: T0.toISOString(), pendingUpdates: updates(9) }).returning({ id: servers.id }))[0].id;
|
||||
// a guest that runs on web: what depends on web
|
||||
ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: T0.toISOString(), parentServerId: ids.web }).returning({ id: servers.id }))[0].id;
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
});
|
||||
|
||||
after(() => {
|
||||
hook.close();
|
||||
setSemaphoreForTests(null);
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("one server at a time, each waited for, those with nothing waiting left alone, and a message at the end", async () => {
|
||||
await reset();
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => [i.name, i.status]), [["api", "pending"], ["idle", "pending"], ["web", "pending"]], "by name");
|
||||
|
||||
await advancePlanRuns(T0);
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["running", "pending", "pending"]);
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["api"], "only the first has been started");
|
||||
assert.equal((await listActiveWindows(T0)).some((w) => w.targetType === "server" && w.targetId === ids.api), true, "and it is under maintenance");
|
||||
|
||||
await advancePlanRuns(at(1));
|
||||
assert.equal(fake.runs.length, 1, "while it is still running, nothing else starts");
|
||||
|
||||
await finishCurrent("api", 5);
|
||||
await advancePlanRuns(at(9));
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["done", "skipped", "running"], "idle has nothing waiting, so it is passed over, and web is next");
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["api", "web"]);
|
||||
assert.equal((await itemsOf(run.id))[1].message, "No updates were waiting.");
|
||||
assert.equal((await listActiveWindows(at(9))).some((w) => w.targetId === ids.api), false, "api's maintenance ended with its update");
|
||||
|
||||
await finishCurrent("web", 15);
|
||||
await advancePlanRuns(at(19));
|
||||
const done = await runOf(run.id);
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.message, "2 updated, 1 left alone of 3 servers.");
|
||||
assert.equal(messages.length >= 3, true, "each update run says how it went, and so does the plan");
|
||||
const last = messages[messages.length - 1];
|
||||
assert.match(last, /2 updated, 1 left alone of 3 servers\./);
|
||||
assert.match(last, /✓ api/);
|
||||
assert.match(last, /– idle — No updates were waiting\./);
|
||||
assert.match(last, /✓ web/);
|
||||
});
|
||||
|
||||
test("what depends on a server is silenced while it is updated, and let go when it is done", async () => {
|
||||
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
const during = await listActiveWindows(T0);
|
||||
assert.equal(during.some((w) => w.targetId === ids.guest), true, "the guest that runs on web");
|
||||
assert.match(during.find((w) => w.targetId === ids.guest)!.reason ?? "", /Depends on web — patch plan "Lab nights"/);
|
||||
assert.equal((await itemsOf(run.id))[0].windowIds.length, 1);
|
||||
await finishCurrent("web", 5);
|
||||
await advancePlanRuns(at(9));
|
||||
assert.equal((await listActiveWindows(at(9))).some((w) => w.targetId === ids.guest), false);
|
||||
assert.equal((await runOf(run.id)).status, "done");
|
||||
});
|
||||
|
||||
test("it can be told not to silence what depends on a server", async () => {
|
||||
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]), silenceDependents: false });
|
||||
await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
assert.equal((await listActiveWindows(T0)).some((w) => w.targetId === ids.guest), false);
|
||||
});
|
||||
|
||||
test("a failure stops the plan and leaves the rest alone, saying where it stopped", async () => {
|
||||
await reset();
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
fake.task = { id: 41, status: "error", created: null, start: null, end: null };
|
||||
await advanceUpdateRuns(at(2));
|
||||
await advancePlanRuns(at(3));
|
||||
const after = await runOf(run.id);
|
||||
assert.equal(after.status, "failed");
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => [i.name, i.status]), [["api", "failed"], ["idle", "skipped"], ["web", "skipped"]]);
|
||||
assert.equal((await itemsOf(run.id))[2].message, "Not started, because an earlier server failed.");
|
||||
assert.match(after.message ?? "", /1 failed, 2 left alone of 3 servers\. Stopped after api\./);
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["api"], "web was never touched");
|
||||
assert.match(messages[messages.length - 1], /✕ api/);
|
||||
assert.equal(messages.some((m) => m.includes("didn't finish cleanly")), true);
|
||||
});
|
||||
|
||||
test("told to carry on, it does: the failed one is reported and the rest are updated", async () => {
|
||||
await reset({ stopOnFailure: false });
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
fake.task = { id: 41, status: "error", created: null, start: null, end: null };
|
||||
await advanceUpdateRuns(at(2));
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
await advancePlanRuns(at(3));
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["failed", "skipped", "running"], "web is next");
|
||||
await finishCurrent("web", 10);
|
||||
await advancePlanRuns(at(14));
|
||||
const done = await runOf(run.id);
|
||||
assert.equal(done.status, "failed", "something failed, so the run says so");
|
||||
assert.equal(done.message, "1 updated, 1 failed, 1 left alone of 3 servers.");
|
||||
});
|
||||
|
||||
test("a server that can't be started counts as a failure; a server with nothing to update can be updated all the same if asked", async () => {
|
||||
await reset({ onlyWithUpdates: false, tags: "[]", serverIds: JSON.stringify([ids.idle]) });
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["idle"], "it had nothing waiting but the plan was told not to mind");
|
||||
await updateRunsCancelAll();
|
||||
await cancelPlanRun(run.id, "Olle", at(1));
|
||||
|
||||
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
|
||||
fake.refuseFor.add("web");
|
||||
const refused = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
const items = await itemsOf(refused.id);
|
||||
assert.equal(items[0].status, "failed");
|
||||
assert.match(items[0].message ?? "", /Semaphore wouldn't start it: Semaphore is having a moment/);
|
||||
assert.equal((await runOf(refused.id)).status, "failed");
|
||||
});
|
||||
|
||||
async function updateRunsCancelAll() {
|
||||
await db.update(updateRuns).set({ status: "cancelled" });
|
||||
}
|
||||
|
||||
test("a plan does not overlap itself, and one with no servers says so rather than doing nothing silently", async () => {
|
||||
await reset();
|
||||
await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await assert.rejects(startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 }), (e: unknown) => e instanceof PlanError && e.code === "already_running");
|
||||
await db.update(patchPlanRuns).set({ status: "cancelled" });
|
||||
|
||||
await db.update(patchPlans).set({ tags: JSON.stringify(["nothing-has-this"]) }).where(eq(patchPlans.id, planId));
|
||||
await assert.rejects(startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 }), (e: unknown) => e instanceof PlanError && e.code === "no_servers");
|
||||
});
|
||||
|
||||
test("stopping a run that is going leaves the rest alone, ends the maintenance it made and stops the update run being followed", async () => {
|
||||
await reset();
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
const stopped = await cancelPlanRun(run.id, "Olga", at(2));
|
||||
assert.equal(stopped!.status, "cancelled");
|
||||
assert.deepEqual(parseItems(stopped!.items).map((i) => i.status), ["failed", "skipped", "skipped"]);
|
||||
assert.equal((await listActiveWindows(at(2))).some((w) => w.targetId === ids.api), false);
|
||||
assert.equal((await db.select().from(updateRuns))[0].status, "cancelled");
|
||||
await assert.rejects(cancelPlanRun(run.id, "Olga", at(3)), (e: unknown) => e instanceof PlanError && e.code === "not_active");
|
||||
assert.equal(await cancelPlanRun(99999, "Olga"), null);
|
||||
});
|
||||
|
||||
test("the schedule starts a plan once on its day, and a restart of the app does not start it again", async () => {
|
||||
await reset();
|
||||
assert.equal(await startDuePlans(T0), 1);
|
||||
const [row] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId));
|
||||
assert.equal(row.lastRunDate, "2026-10-18");
|
||||
assert.equal((await db.select().from(patchPlanRuns))[0].trigger, "schedule");
|
||||
await db.update(patchPlanRuns).set({ status: "done" });
|
||||
assert.equal(await startDuePlans(at(10)), 0, "not twice in a day");
|
||||
assert.equal(await startDuePlans(new Date("2026-10-25T02:03:00Z")), 1, "but the next Sunday it does");
|
||||
});
|
||||
|
||||
test("a plan that is due but has no servers leaves a failed run to say so", async () => {
|
||||
await reset({ tags: JSON.stringify(["nothing-has-this"]) });
|
||||
await db.delete(patchPlanRuns);
|
||||
assert.equal(await startDuePlans(T0), 0);
|
||||
const [run] = await db.select().from(patchPlanRuns);
|
||||
assert.deepEqual([run.status, run.trigger], ["failed", "schedule"]);
|
||||
assert.match(run.message ?? "", /^Didn't start: No server has this plan's tags/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
const body = (extra: object = {}) => ({ name: "Weekend patching", days: [6, 0], startTime: "03:30", timezone: "Europe/Stockholm", tags: ["Lab"], serverIds: [], integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", ...extra });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/patch-plans", patchPlansRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/patch-plans`;
|
||||
await db.delete(patchPlans);
|
||||
await db.delete(patchPlanRuns);
|
||||
});
|
||||
|
||||
after(() => app.close());
|
||||
|
||||
const send = (method: string, path: string, b?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: b === undefined ? undefined : JSON.stringify(b) });
|
||||
let id = 0;
|
||||
|
||||
test("an administrator makes a plan, with its tags written the way tags are; a wrong one is refused with a reason; nobody else can", async () => {
|
||||
const res = await send("POST", "/", body());
|
||||
assert.equal(res.status, 201);
|
||||
const plan = (await res.json()).plan;
|
||||
id = plan.id;
|
||||
assert.deepEqual([plan.tags, plan.days, plan.limitFrom, plan.stopOnFailure, plan.onlyWithUpdates, plan.windowMinutes], [["lab"], [0, 6], "name", true, true, 90]);
|
||||
assert.deepEqual(plan.servers.map((s: { name: string }) => s.name), ["api", "idle", "web"], "the three tagged lab");
|
||||
assert.equal(plan.next.length, 3, "its next times, so nobody has to work out when");
|
||||
for (const bad of [{ days: [] }, { startTime: "3:30" }, { timezone: "Mars/Olympus" }, { tags: [], serverIds: [] }, { tags: ["bad tag!"] }, { windowMinutes: 5 }, { name: " " }, { integrationId: 0 }]) {
|
||||
const r = await send("POST", "/", body(bad));
|
||||
assert.equal(r.status, 400, JSON.stringify(bad));
|
||||
assert.ok((await r.json()).message.length > 0);
|
||||
}
|
||||
who = "operator";
|
||||
assert.equal((await send("POST", "/", body())).status, 403);
|
||||
assert.equal((await send("PUT", `/${id}`, body())).status, 403);
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 403);
|
||||
who = "viewer";
|
||||
const list = await (await send("GET", "/")).json();
|
||||
assert.equal(list.plans.length, 1, "everyone can see what is planned");
|
||||
assert.ok(list.tags.includes("lab") && list.servers.length >= 4);
|
||||
});
|
||||
|
||||
test("changing when it happens forgets which day it last started on; changing anything else does not", async () => {
|
||||
who = "admin";
|
||||
await db.update(patchPlans).set({ lastRunDate: "2026-10-18" }).where(eq(patchPlans.id, id));
|
||||
await send("PUT", `/${id}`, body({ name: "Renamed" }));
|
||||
assert.equal((await db.select().from(patchPlans).where(eq(patchPlans.id, id)))[0].lastRunDate, "2026-10-18");
|
||||
await send("PUT", `/${id}`, body({ name: "Renamed", startTime: "04:00" }));
|
||||
assert.equal((await db.select().from(patchPlans).where(eq(patchPlans.id, id)))[0].lastRunDate, null);
|
||||
assert.equal((await send("PUT", "/99999", body())).status, 404);
|
||||
});
|
||||
|
||||
test("an operator can run it now and stop it; a viewer cannot; and it cannot be removed while it is running", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", `/${id}/run`)).status, 403);
|
||||
who = "operator";
|
||||
const res = await send("POST", `/${id}/run`);
|
||||
assert.equal(res.status, 201);
|
||||
const run = (await res.json()).run;
|
||||
assert.equal(run.status, "running");
|
||||
assert.equal(run.trigger, "manual");
|
||||
assert.ok(run.items.every((i: { windowIds?: unknown }) => i.windowIds === undefined), "the windows it made are its own business");
|
||||
assert.equal((await send("POST", `/${id}/run`)).status, 409, "not twice at once");
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 409);
|
||||
who = "operator";
|
||||
const stopped = await send("POST", `/runs/${run.id}/cancel`);
|
||||
assert.equal(stopped.status, 200);
|
||||
assert.equal((await stopped.json()).run.status, "cancelled");
|
||||
assert.equal((await send("POST", `/runs/${run.id}/cancel`)).status, 409);
|
||||
assert.equal((await send("POST", "/runs/99999/cancel")).status, 404);
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 204);
|
||||
assert.equal((await db.select().from(patchPlanRuns)).length, 0, "its history goes with it");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user