Patch plans, a monthly report and new-port alerts

Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an
ordinary update run, stopping at the first failure by default, silencing what depends on a
server while it is updated, with one message at the end.

Monthly report (Operations -> Monthly report, and optionally a message on a day of the month):
availability, incidents, updates, restore tests, backups and renewals for a calendar month,
printable. Backed by daily endpoint figures and a record of renewals.

New ports: the first report with ports is the baseline; a TCP port that begins listening later and
is not noted on the server's Ports card is a warning, cleared by Mark as expected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:16:14 +02:00
1 parent ddd0aa622a
commit be6c5552bd
46 files changed
+7836 -26

No files matched your search

+66
View File
@@ -0,0 +1,66 @@
CREATE TABLE `endpoint_daily` (
`check_id` integer NOT NULL,
`day` text NOT NULL,
`total` integer DEFAULT 0 NOT NULL,
`ok` integer DEFAULT 0 NOT NULL,
`ms_sum` integer DEFAULT 0 NOT NULL,
`ms_count` integer DEFAULT 0 NOT NULL,
PRIMARY KEY(`check_id`, `day`),
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE TABLE `patch_plan_runs` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`plan_id` integer NOT NULL,
`plan_name` text NOT NULL,
`trigger` text NOT NULL,
`started_by` text,
`status` text NOT NULL,
`message` text,
`items` text DEFAULT '[]' NOT NULL,
`started_at` text NOT NULL,
`finished_at` text,
FOREIGN KEY (`plan_id`) REFERENCES `patch_plans`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `patch_plan_runs_plan_idx` ON `patch_plan_runs` (`plan_id`,`started_at`);--> statement-breakpoint
CREATE INDEX `patch_plan_runs_status_idx` ON `patch_plan_runs` (`status`);--> statement-breakpoint
CREATE TABLE `patch_plans` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`enabled` integer DEFAULT true NOT NULL,
`days` text NOT NULL,
`start_time` text NOT NULL,
`timezone` text NOT NULL,
`tags` text DEFAULT '[]' NOT NULL,
`server_ids` text DEFAULT '[]' NOT NULL,
`integration_id` integer NOT NULL,
`project_id` integer NOT NULL,
`template_id` integer NOT NULL,
`template_name` text NOT NULL,
`limit_from` text DEFAULT 'name' NOT NULL,
`stop_on_failure` integer DEFAULT true NOT NULL,
`only_with_updates` integer DEFAULT true NOT NULL,
`silence_dependents` integer DEFAULT true NOT NULL,
`window_minutes` integer DEFAULT 90 NOT NULL,
`last_run_date` text,
`created_by` text,
`created_at` text DEFAULT (current_timestamp) NOT NULL
);
--> statement-breakpoint
CREATE TABLE `renewals` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`kind` text NOT NULL,
`subject` text NOT NULL,
`at` text NOT NULL,
`from_date` text NOT NULL,
`to_date` text NOT NULL
);
--> statement-breakpoint
CREATE INDEX `renewals_at_idx` ON `renewals` (`at`);--> statement-breakpoint
ALTER TABLE `servers` ADD `port_baseline` text;
--> statement-breakpoint
-- The daily rollup starts with what the endpoint results already hold (they are kept two weeks), so the first report is not blank.
INSERT INTO `endpoint_daily` (`check_id`, `day`, `total`, `ok`, `ms_sum`, `ms_count`)
SELECT `check_id`, substr(`at`, 1, 10), count(*), sum(`ok`), sum(CASE WHEN `ok` = 1 AND `ms` IS NOT NULL THEN `ms` ELSE 0 END), sum(CASE WHEN `ok` = 1 AND `ms` IS NOT NULL THEN 1 ELSE 0 END)
FROM `endpoint_results` GROUP BY `check_id`, substr(`at`, 1, 10);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -302,6 +302,13 @@
"when": 1792361094159,
"tag": "0042_silly_omega_red",
"breakpoints": true
},
{
"idx": 43,
"version": "6",
"when": 1792443011097,
"tag": "0043_puzzling_prism",
"breakpoints": true
}
]
}
+4
View File
@@ -50,6 +50,8 @@ import { alertTicketsRouter } from "./routes/alertTickets.js";
import { avatarsRouter } from "./routes/avatars.js";
import { heartbeatsRouter } from "./routes/heartbeats.js";
import { agendaRouter } from "./routes/agenda.js";
import { patchPlansRouter } from "./routes/patchPlans.js";
import { reportsRouter } from "./routes/reports.js";
import { publicCalendarRouter } from "./routes/publicCalendar.js";
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
@@ -113,6 +115,8 @@ export function createApp() {
app.use("/api/avatars", avatarsRouter);
app.use("/api/heartbeats", heartbeatsRouter);
app.use("/api/agenda", agendaRouter);
app.use("/api/patch-plans", patchPlansRouter);
app.use("/api/reports", reportsRouter);
app.use("/api/uptime", uptimeRouter);
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
+88 -1
View File
@@ -1,5 +1,5 @@
import { sql } from "drizzle-orm";
import { sqliteTable, text, integer, real, uniqueIndex, index } from "drizzle-orm/sqlite-core";
import { sqliteTable, text, integer, real, uniqueIndex, index, primaryKey } from "drizzle-orm/sqlite-core";
// ─── Users & roles ──────────────────────────────────────────────────────────
@@ -447,6 +447,10 @@ export const servers = sqliteTable("servers", {
// What the agent says it could not collect, as JSON: {items: [{check, status, reason?, detail?}], reportedAt} — see services/agentDiagnostics.ts. Null until an agent that can say so has.
agentDiagnostics: text("agent_diagnostics"),
// The ports this server was listening on when its agent first said so, as a JSON array like ["tcp:22","tcp:443"]. A port that is listening later and is
// not in this list, and not noted on the Ports page, is a "new port" (services/newPorts.ts). Null until the first report that carries ports.
portBaseline: text("port_baseline"),
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
hwManufacturer: text("hw_manufacturer"),
hwModel: text("hw_model"),
@@ -846,6 +850,89 @@ export const updateRuns = sqliteTable(
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
);
// ─── Patch plans: updates, on a schedule, one server at a time ─────────────
// "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure." A plan only says when, to whom and how; each server is updated by an
// ordinary update run (services/updateRuns.ts), and the plan waits for it to finish before the next — see services/patchPlans.ts.
export const patchPlans = sqliteTable("patch_plans", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
days: text("days").notNull(), // JSON: weekdays, 0 = Sunday … 6 = Saturday
startTime: text("start_time").notNull(), // "HH:MM" on the clock of `timezone`
timezone: text("timezone").notNull(),
tags: text("tags").notNull().default("[]"), // JSON string[]: servers that have any of these tags
serverIds: text("server_ids").notNull().default("[]"), // JSON number[]: and these servers by name, whatever their tags
integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK, like update runs
projectId: integer("project_id").notNull(),
templateId: integer("template_id").notNull(),
templateName: text("template_name").notNull(),
limitFrom: text("limit_from").notNull().default("name"), // what ansible's --limit is for a server: its name here, or its host name
stopOnFailure: integer("stop_on_failure", { mode: "boolean" }).notNull().default(true),
onlyWithUpdates: integer("only_with_updates", { mode: "boolean" }).notNull().default(true), // leave out a server that has nothing waiting
silenceDependents: integer("silence_dependents", { mode: "boolean" }).notNull().default(true), // what depends on a server is silenced while it is updated
windowMinutes: integer("window_minutes").notNull().default(90), // at most this long under maintenance per server
lastRunDate: text("last_run_date"), // the local day (YYYY-MM-DD) it last started on — so it starts once a day, even if the app is restarted
createdBy: text("created_by"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
});
// One time a plan ran, and how it went for each server: items is JSON [{serverId, name, status, updateRunId, message, windowIds}].
export const patchPlanRuns = sqliteTable(
"patch_plan_runs",
{
id: integer("id").primaryKey({ autoIncrement: true }),
planId: integer("plan_id")
.notNull()
.references(() => patchPlans.id, { onDelete: "cascade" }),
planName: text("plan_name").notNull(), // as it was, so a renamed plan's history still reads right
trigger: text("trigger").notNull(), // schedule | manual
startedBy: text("started_by"),
status: text("status").notNull(), // running | done | failed | cancelled
message: text("message"),
items: text("items").notNull().default("[]"),
startedAt: text("started_at").notNull(), // ISO
finishedAt: text("finished_at"), // ISO
},
(t) => [index("patch_plan_runs_plan_idx").on(t.planId, t.startedAt), index("patch_plan_runs_status_idx").on(t.status)],
);
// ─── The monthly report's long memory ───────────────────────────────────────
// One row per endpoint check per day: how many times it was asked, how many answered, and the answer times — kept for years, where the individual
// results are kept for two weeks. What the monthly report's availability is worked from (services/endpointChecks.ts).
export const endpointDaily = sqliteTable(
"endpoint_daily",
{
checkId: integer("check_id")
.notNull()
.references(() => endpointChecks.id, { onDelete: "cascade" }),
day: text("day").notNull(), // YYYY-MM-DD, UTC
total: integer("total").notNull().default(0),
ok: integer("ok").notNull().default(0),
msSum: integer("ms_sum").notNull().default(0), // of the answers that were good and timed
msCount: integer("ms_count").notNull().default(0),
},
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
);
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
// as far as the version that began watching (services/renewals.ts).
export const renewals = sqliteTable(
"renewals",
{
id: integer("id").primaryKey({ autoIncrement: true }),
kind: text("kind").notNull(), // certificate | domain | secret
subject: text("subject").notNull(), // its name, as it was
at: text("at").notNull(), // ISO — when it was seen
fromDate: text("from_date").notNull(), // YYYY-MM-DD — the expiry before
toDate: text("to_date").notNull(), // YYYY-MM-DD — and after
},
(t) => [index("renewals_at_idx").on(t.at)],
);
// ─── Inventory: the hardware behind it all ──────────────────────────────────
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;
+4
View File
@@ -13,6 +13,7 @@ import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js
import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js";
import { initMonthlyReportScheduler } from "./services/monthlyReportScheduler.js";
import { initMaintenanceEvents } from "./services/maintenanceEvents.js";
import { initHealthScheduler } from "./services/healthScheduler.js";
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
@@ -21,6 +22,7 @@ import { initMaintenanceScheduleScheduler } from "./services/maintenanceSchedule
import { initUpdateRuns } from "./services/updateRuns.js";
import { initEscalation } from "./services/escalation.js";
import { initEndpointMonitor } from "./services/endpointMonitor.js";
import { initPatchPlans } from "./services/patchPlans.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -51,6 +53,7 @@ await initProxmoxBackupScheduler();
await initPbsVerificationScheduler();
await initQuietHoursScheduler();
await initWeeklyDigestScheduler();
await initMonthlyReportScheduler();
initMaintenanceEvents();
await initHealthScheduler();
await initAlertTicketScheduler();
@@ -61,6 +64,7 @@ await initSelfBackupScheduler();
initIncidentRecorder(() => getAlerts(false));
initEscalation();
initEndpointMonitor();
initPatchPlans();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+196
View File
@@ -0,0 +1,196 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { patchPlans, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { isValidTimeZone } from "../services/maintenanceSchedules.js";
import { activeRunOf, cancelPlanRun, listPlanRuns, MAX_PLANS, parseItems, planDays, planOccurrences, planServerIds, planServers, planTags, PlanError, startPlanRun, type PlanRow, type PlanRunRow } from "../services/patchPlans.js";
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const patchPlansRouter = Router();
// Everyone signed in can see what is planned and how it went, like the Updates page. Making a plan is an administrator's (it will run on its own, at night);
// starting one now or stopping one that is running is an operator's, like an update run.
patchPlansRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const planSchema = z
.object({
name: z.string().trim().min(1).max(100),
enabled: z.boolean().default(true),
days: z.array(z.number().int().min(0).max(6)).min(1).max(7),
startTime: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 02:00"),
timezone: z.string().trim().min(1).max(60),
tags: z.array(z.string()).max(12).default([]),
serverIds: z.array(z.number().int().positive()).max(200).default([]),
integrationId: z.number().int().positive(),
projectId: z.number().int().positive(),
templateId: z.number().int().positive(),
templateName: z.string().trim().min(1).max(200),
limitFrom: z.enum(["name", "hostname"]).default("name"),
stopOnFailure: z.boolean().default(true),
onlyWithUpdates: z.boolean().default(true),
silenceDependents: z.boolean().default(true),
windowMinutes: z.number().int().min(15).max(480).default(90),
})
.refine((d) => isValidTimeZone(d.timezone), { message: "That isn't a time zone the system knows (use a name like Europe/Stockholm).", path: ["timezone"] })
.refine((d) => d.tags.length > 0 || d.serverIds.length > 0, { message: "Choose which servers: give a tag, or name at least one server.", path: ["tags"] });
function valuesOf(d: z.infer<typeof planSchema>) {
return {
name: d.name,
enabled: d.enabled,
days: JSON.stringify([...new Set(d.days)].sort()),
startTime: d.startTime,
timezone: d.timezone,
tags: JSON.stringify(normalizeTags(d.tags)),
serverIds: JSON.stringify([...new Set(d.serverIds)]),
integrationId: d.integrationId,
projectId: d.projectId,
templateId: d.templateId,
templateName: d.templateName,
limitFrom: d.limitFrom,
stopOnFailure: d.stopOnFailure,
onlyWithUpdates: d.onlyWithUpdates,
silenceDependents: d.silenceDependents,
windowMinutes: d.windowMinutes,
};
}
const shapeRun = (r: PlanRunRow) => ({
id: r.id,
planId: r.planId,
planName: r.planName,
trigger: r.trigger,
startedBy: r.startedBy,
status: r.status,
message: r.message,
startedAt: r.startedAt,
finishedAt: r.finishedAt,
items: parseItems(r.items).map(({ windowIds: _w, ...i }) => i),
});
async function shapePlan(p: PlanRow, all: { id: number; name: string; tags: string | null }[], now: Date) {
const covered = planServers(p, all);
const next = planOccurrences(p, now, new Date(now.getTime() + 15 * 86_400_000)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 3);
const active = await activeRunOf(p.id);
return {
id: p.id,
name: p.name,
enabled: p.enabled,
days: planDays(p),
startTime: p.startTime,
timezone: p.timezone,
tags: planTags(p),
serverIds: planServerIds(p),
integrationId: p.integrationId,
projectId: p.projectId,
templateId: p.templateId,
templateName: p.templateName,
limitFrom: p.limitFrom,
stopOnFailure: p.stopOnFailure,
onlyWithUpdates: p.onlyWithUpdates,
silenceDependents: p.silenceDependents,
windowMinutes: p.windowMinutes,
createdBy: p.createdBy,
servers: covered,
next: next.map((o) => o.start.toISOString()),
running: active !== null,
};
}
patchPlansRouter.get("/", asyncHandler(async (_req, res) => {
const now = new Date();
const rows = await db.select().from(patchPlans).orderBy(patchPlans.name);
const all = await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers);
const tags = [...new Set(all.flatMap((s) => parseTags(s.tags)))].sort((a, b) => a.localeCompare(b, "sv"));
res.json({
plans: await Promise.all(rows.map((p) => shapePlan(p, all, now))),
runs: (await listPlanRuns({ limit: 15 })).map(shapeRun),
servers: all.map((s) => ({ id: s.id, name: s.name, tags: parseTags(s.tags) })).sort((a, b) => a.name.localeCompare(b.name)),
tags,
});
}));
/** The message for a rejected body, as a sentence. */
const reason = (parsed: { error: z.ZodError }) => parsed.error.issues[0]?.message ?? "That isn't a plan.";
patchPlansRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = planSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: reason(parsed), details: parsed.error.flatten() });
if ((await db.select({ id: patchPlans.id }).from(patchPlans)).length >= MAX_PLANS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_PLANS} plans.` });
let values;
try {
values = valuesOf(parsed.data);
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
throw err;
}
const actor = req.currentUser!;
const [row] = await db.insert(patchPlans).values({ ...values, createdBy: actor.name ?? actor.email ?? actor.oidcSub }).returning();
await recordAudit({ actor, category: "patch_plan", action: "create", targetType: "patch_plan", targetId: row.id, detail: { name: row.name, days: planDays(row), time: row.startTime } });
res.status(201).json({ plan: await shapePlan(row, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers), new Date()) });
}));
patchPlansRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = planSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? "That isn't a plan." : reason(parsed) });
const [before] = await db.select().from(patchPlans).where(eq(patchPlans.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
let values;
try {
values = valuesOf(parsed.data);
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
throw err;
}
// A changed day or time is a new schedule: what it last started on no longer says whether today's has happened.
const rescheduled = values.days !== before.days || values.startTime !== before.startTime || values.timezone !== before.timezone;
const [row] = await db.update(patchPlans).set({ ...values, ...(rescheduled ? { lastRunDate: null } : {}) }).where(eq(patchPlans.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "patch_plan", action: "update", targetType: "patch_plan", targetId: id, detail: { name: row.name, days: planDays(row), time: row.startTime } });
res.json({ plan: await shapePlan(row, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers), new Date()) });
}));
patchPlansRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(patchPlans).where(eq(patchPlans.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (await activeRunOf(id)) return res.status(409).json({ error: "running", message: "It is running now. Stop the run first." });
await db.delete(patchPlans).where(eq(patchPlans.id, id));
await recordAudit({ actor: req.currentUser!, category: "patch_plan", action: "delete", targetType: "patch_plan", targetId: id, detail: { name: before.name } });
res.status(204).end();
}));
// Run it now, instead of at its time (it still starts again at its time).
patchPlansRouter.post("/:id/run", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
try {
const run = await startPlanRun(id, { trigger: "manual", actor: actor.name ?? actor.email ?? actor.oidcSub });
res.status(201).json({ run: shapeRun(run) });
} catch (err) {
if (err instanceof PlanError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
patchPlansRouter.post("/runs/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
try {
const run = await cancelPlanRun(id, actor.name ?? actor.email ?? actor.oidcSub);
if (!run) return res.status(404).json({ error: "not_found" });
res.json({ run: shapeRun(run) });
} catch (err) {
if (err instanceof PlanError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
+40
View File
@@ -0,0 +1,40 @@
import { Router } from "express";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { buildMonthlyReport, isMonthId, lastFullMonth, monthlyReportText, recentMonths } from "../services/monthlyReport.js";
import { recordAudit } from "../services/audit.js";
import { notifyNow } from "../services/notify.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const reportsRouter = Router();
reportsRouter.use(requireAuth);
/** The month asked for, or the last whole one; null when what was asked is not a month. */
async function monthFrom(query: unknown): Promise<{ id: string; months: { id: string; label: string }[] } | null> {
const tz = (await getSettings()).notifications.timezone || "UTC";
const months = recentMonths(new Date(), tz);
if (query === undefined || query === "") return { id: lastFullMonth(new Date(), tz), months };
if (!isMonthId(query) || !months.some((m) => m.id === query)) return null;
return { id: query, months };
}
// How a month went, for everyone signed in: it is only what the pages it comes from already show them. ?month=2026-09, the last whole month by default;
// ?format=text is the plain-text version a notification carries.
reportsRouter.get("/monthly", asyncHandler(async (req, res) => {
const picked = await monthFrom(req.query.month);
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
const report = await buildMonthlyReport(picked.id);
if (req.query.format === "text") return res.json(monthlyReportText(report));
res.json({ report, months: picked.months });
}));
// Send it now to the notification channels, as if it were the day. Administrators.
reportsRouter.post("/monthly/send", requireRole("admin"), asyncHandler(async (req, res) => {
const picked = await monthFrom((req.body as { month?: unknown } | undefined)?.month);
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
const { title, message } = monthlyReportText(await buildMonthlyReport(picked.id));
await notifyNow(title, message);
await recordAudit({ actor: req.currentUser!, category: "report", action: "send_monthly", targetType: "report", detail: { month: picked.id } });
res.json({ sent: true });
}));
+11
View File
@@ -7,6 +7,8 @@ import { requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { beginScan, endScan, MAX_SCAN_SPAN, resolveScanTarget, scanPorts, toRanges } from "../services/portScan.js";
import { type AgentPort, groupAgentPorts, parseJson } from "../services/agentPorts.js";
import { newListeningPorts, parseBaseline, portKey } from "../services/newPorts.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
// Mounted under /api/servers/:id/ports by the servers router, which has already required a signed-in user.
@@ -37,6 +39,8 @@ export interface PortEntry {
agent: { addresses: string[]; process: string | null; localOnly: boolean } | null;
/** "open" if anything is using it; "reserved" if it only has a note. */
state: "open" | "reserved";
/** Began listening after the first report and nobody has noted it: a "new port" alert is on until it is noted or stops listening. */
isNew: boolean;
}
async function buildPortList(serverId: number) {
@@ -46,6 +50,11 @@ async function buildPortList(serverId: number) {
const agentRaw = parseJson<AgentPort[] | null>(server.listeningPorts, null);
const agent = groupAgentPorts(agentRaw ?? []);
// What is new: listening, reachable from outside, not in the baseline and not noted — the same rule the alert uses.
const expected = new Set(rows.filter((r) => r.label || r.comment).map((r) => portKey(r.protocol, r.port)));
const alerting = (await getSettings()).healthChecks.newPortAlerts;
const fresh = new Set(alerting ? newListeningPorts(agentRaw, parseBaseline(server.portBaseline), expected).map((p) => portKey(p.protocol, p.port)) : []);
const entries = new Map<string, PortEntry>();
for (const row of rows) {
const key = `${row.protocol}:${row.port}`;
@@ -59,6 +68,7 @@ async function buildPortList(serverId: number) {
lastSeenOpenAt: row.lastSeenOpenAt,
agent: agent.get(key) ?? null,
state: "reserved",
isNew: fresh.has(key),
});
}
for (const [key, info] of agent) {
@@ -74,6 +84,7 @@ async function buildPortList(serverId: number) {
lastSeenOpenAt: null,
agent: info,
state: "reserved",
isNew: fresh.has(key),
});
}
for (const entry of entries.values()) {
+9 -1
View File
@@ -13,6 +13,7 @@ import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js"
import { purgeOldLogs } from "../services/logRetention.js";
import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
import { scheduleMonthlyReport } from "../services/monthlyReportScheduler.js";
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
import { enabledChannelCount } from "../services/notify.js";
import { CATEGORIES, CHANNEL_IDS } from "../services/notifyRouting.js";
@@ -143,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730) })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean() })
.partial()
.optional(),
quietHours: z
@@ -165,6 +166,10 @@ const updateSchema = z.object({
.object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
.partial()
.optional(),
monthlyReport: z
.object({ enabled: z.boolean(), day: z.number().int().min(1).max(28), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
.partial()
.optional(),
});
settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
@@ -200,6 +205,9 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
if (parsed.data.digest || parsed.data.notifications) {
await scheduleWeeklyDigest();
}
if (parsed.data.monthlyReport || parsed.data.notifications) {
await scheduleMonthlyReport();
}
await recordAudit({
actor: req.currentUser!,
+17 -2
View File
@@ -9,10 +9,11 @@
import { createHash } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, patchPlans, secrets, servers } from "../db/schema.js";
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
import { planDays, planOccurrences, planServers } from "./patchPlans.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { latestTests } from "./restoreTests.js";
import { getSettings } from "./settingsStore.js";
@@ -88,6 +89,8 @@ export interface AgendaData {
/** Windows that have been set up but not begun, one per target. */
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
/** Patch plans that are switched on, with how many servers each covers now. */
patchPlans: { id: number; name: string; days: string; startTime: string; timezone: string; servers: number }[];
/** The newest restore test of each subject. */
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
restoreLimitDays: number;
@@ -140,6 +143,10 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
}
const planRows = (await db.select().from(patchPlans)).filter((p) => p.enabled);
const allServers = planRows.length > 0 ? await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers) : [];
const plans: AgendaData["patchPlans"] = planRows.map((p) => ({ id: p.id, name: p.name, days: p.days, startTime: p.startTime, timezone: p.timezone, servers: planServers(p, allServers).length }));
const updates: AgendaData["updates"] = [];
for (const s of serverRows) {
const u = parsePendingUpdates(s.pendingUpdates);
@@ -163,6 +170,7 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
assets: assetRows,
windows,
schedules,
patchPlans: plans,
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
restoreLimitDays: settings.healthChecks.restoreTestDays,
updates,
@@ -223,7 +231,7 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
g.names.push(w.targetName);
groups.set(key, g);
}
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
const timed = (id: string, start: string, end: string | null, title: string, detail: string | null) =>
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
const names = (list: string[]) => {
const sorted = [...list].sort((a, b) => a.localeCompare(b));
@@ -236,6 +244,13 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
}
}
// A patch plan is maintenance too: when it will start and on how many servers.
for (const p of data.patchPlans) {
for (const occ of planOccurrences(p, now, new Date(lastMs)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 60)) {
timed(`plan:${p.id}:${occ.date}`, occ.start.toISOString(), null, `Patch plan: ${p.name}`, `updates ${plural(p.servers, "server")}, one at a time`);
}
}
// Updates that have been waiting: one line for all of them, from the longest wait.
if (data.updates.length > 0) {
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
+2 -2
View File
@@ -79,8 +79,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
/** Which kind of source, and which one, a health/automation condition key belongs to. */
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
+19
View File
@@ -54,6 +54,8 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
restore_test: "backups",
endpoint: "endpoints",
heartbeat: "endpoints",
patch_plan: "servers",
report: "other",
};
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
@@ -155,6 +157,22 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "report/send_monthly":
return { title: `Monthly report sent${str(detail.month) ? ` (${str(detail.month)})` : ""}`, tone: "neutral", detail: null };
case "patch_plan/create":
return { title: `Patch plan added${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "patch_plan/update":
return { title: `Patch plan changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "patch_plan/delete":
return { title: `Patch plan removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "patch_plan/run":
return { title: `Patch plan started${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.trigger) === "schedule" ? "by its schedule" : str(detail.by) ? `by ${str(detail.by)}` : null };
case "patch_plan/finished":
return { title: `Patch plan finished${name ? `: ${name}` : ""}`, tone: "good", detail: str(detail.message) };
case "patch_plan/failed":
return { title: `Patch plan didn't finish cleanly${name ? `: ${name}` : ""}`, tone: "bad", detail: str(detail.message) };
case "patch_plan/cancelled":
return { title: `Patch plan stopped${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.message) };
case "heartbeat/create":
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
case "heartbeat/update":
@@ -260,6 +278,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
targetName = endpointName.get(idNum) ?? targetName;
link = "/endpoints";
} else if (a.targetType === "heartbeat") link = "/heartbeats";
else if (a.targetType === "patch_plan") link = "/updates?view=plans";
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
else if (a.targetType === "restore_test") link = "/backups";
else if (a.targetType === "dependency") link = "/map?view=dependencies";
+3
View File
@@ -5,6 +5,7 @@ import { defaultDeps, lookupRegistration, normalizeDomain, resolveRegistration,
import { computeSecretStatus } from "./secretStatus.js";
import { notifyDomainExpiry } from "./notify.js";
import { getSettings } from "./settingsStore.js";
import { noteRenewal } from "./renewals.js";
export type DomainRow = typeof domains.$inferSelect;
@@ -48,6 +49,8 @@ export async function checkDomain(id: number, deps: LookupDeps = defaultDeps): P
if (!row) return null;
const result = await lookupRegistration(row.name, deps);
const [updated] = await db.update(domains).set(outcomeFields(result, row, new Date(deps.now()).toISOString())).where(eq(domains.id, id)).returning();
// A registration that now runs much longer than it did was renewed.
await noteRenewal("domain", row.name, row.expiresAt, updated.expiresAt, new Date(deps.now()));
return updated;
}
+28 -2
View File
@@ -12,9 +12,10 @@ import https from "node:https";
import net from "node:net";
import tls from "node:tls";
import type { TLSSocket } from "node:tls";
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointResults } from "../db/schema.js";
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
import { noteRenewal } from "./renewals.js";
import { resolveTarget } from "../utils/networkGuard.js";
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
@@ -244,9 +245,33 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
})
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
await addToDaily(c.id, iso, result);
// A certificate that is now good for much longer than it was has been renewed.
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
return result;
}
/** How long the daily figures are kept: years, where the individual results are kept two weeks. */
export const KEEP_DAILY_DAYS = 800;
/** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */
async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
const timed = result.ok && result.ms !== null;
const ok = result.ok ? 1 : 0;
const ms = timed ? (result.ms as number) : 0;
try {
await db
.insert(endpointDaily)
.values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 })
.onConflictDoUpdate({
target: [endpointDaily.checkId, endpointDaily.day],
set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` },
});
} catch (err) {
console.error("[endpoints] couldn't add to the daily figures:", err);
}
}
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
@@ -264,6 +289,7 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
export async function pruneResults(now: Date = new Date()): Promise<void> {
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
}
// ─── What the history says ─────────────────────────────────────────────────
+19 -2
View File
@@ -1,6 +1,6 @@
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, servers } from "../db/schema.js";
import { integrations, serverPorts, servers } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -11,6 +11,8 @@ import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js";
import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js";
import { parseJson, type AgentPort } from "./agentPorts.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
const STATE_FLAG = HEALTH_STATE_FLAG;
@@ -39,6 +41,8 @@ export interface ServerSnapshot {
health?: HostHealth | null;
/** This server's own thresholds, where it has any: they replace the global ones for it. */
thresholds?: ServerThresholds | null;
/** TCP ports that began listening after the baseline and that nobody has noted (services/newPorts.ts); empty when the check is off. */
newPorts?: NewPort[];
}
export interface ProxmoxSnapshot {
integrationId: number;
@@ -123,6 +127,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
if (s.lastSeenAt && !offline) {
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
for (const p of s.newPorts ?? []) out.push({ key: `new-port:server:${s.id}:${portKey(p.protocol, p.port)}`, source: "server", message: newPortMessage(s.name, p), severity: "warning" });
for (const d of s.disks) {
const p = usage(d.usedBytes, d.sizeBytes);
// A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one.
@@ -299,6 +304,17 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
const held = new Set<string>();
const snapshot: HealthSnapshot = { servers: [], proxmox: [], synology: [] };
const { healthChecks: checks } = await getSettings();
// The ports somebody has said are expected (a label or a comment on the Ports page); a row that only records an open scan says nothing.
const expectedByServer = new Map<number, Set<string>>();
if (checks.newPortAlerts) {
for (const r of await db.select().from(serverPorts)) {
if (!r.label && !r.comment) continue;
const set = expectedByServer.get(r.serverId) ?? new Set<string>();
set.add(portKey(r.protocol, r.port));
expectedByServer.set(r.serverId, set);
}
}
for (const s of await db.select().from(servers)) {
let disks: ServerSnapshot["disks"] = [];
try {
@@ -306,7 +322,8 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
} catch {
// a malformed report shouldn't stop every other server being checked
}
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds) });
const newPorts = checks.newPortAlerts ? newListeningPorts(parseJson<AgentPort[] | null>(s.listeningPorts, null), parseBaseline(s.portBaseline), expectedByServer.get(s.id) ?? new Set()) : [];
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts });
}
const rows = await db
+1 -1
View File
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
* so conditions persisted by an earlier version still map correctly.
*/
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
+303
View File
@@ -0,0 +1,303 @@
/**
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
* weekly digest.
*
* What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily
* figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the
* state now, not a history. A month with nothing to report for a part leaves that part out.
*/
import { and, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { listIncidents } from "./incidents.js";
import { zonedTime } from "./maintenanceSchedules.js";
import { renewalsBetween } from "./renewals.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { spanWords } from "./weeklyDigest.js";
const DAY_MS = 86_400_000;
const MAX_LISTED = 12;
export interface AvailabilityRow {
id: number;
name: string;
/** Share of the known time spent up, in percent; null when nothing is known for the month. */
percent: number | null;
downMinutes: number;
}
export interface EndpointRow {
id: number;
name: string;
percent: number | null;
avgMs: number | null;
checks: number;
}
export interface MonthlyReport {
/** "2026-09". */
month: string;
/** "September 2026". */
label: string;
/** ISO — the start of the month and its end (or now, for a month that is not over). */
from: string;
to: string;
/** The month is not over yet: everything is "so far". */
partial: boolean;
timezone: string;
servers: AvailabilityRow[];
integrations: AvailabilityRow[];
endpoints: EndpointRow[];
incidents: {
began: number;
resolved: number;
stillOpen: number;
/** Of those that began and ended in the month. */
totalMinutes: number;
averageMinutes: number | null;
longest: { id: number; message: string; minutes: number } | null;
byCategory: { category: string; count: number }[];
/** The longest, up to a dozen. */
list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[];
};
updates: {
runs: number;
done: number;
failed: number;
/** Updates the runs say they installed (before minus after, where both are known). */
installed: number;
byServer: { name: string; runs: number; failed: number; installed: number }[];
planRuns: { runs: number; done: number; failed: number };
};
restoreTests: { label: string; testedAt: string; result: string; note: string | null }[];
/** The backup state as of now; null when no coverage has been read yet. */
backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null;
renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[];
}
// ─── Months ────────────────────────────────────────────────────────────────
const pad = (n: number) => String(n).padStart(2, "0");
export function isMonthId(v: unknown): v is string {
return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v);
}
/** The year and month `now` is in, on the clock of `tz`. */
function monthOf(now: Date, tz: string): { year: number; month: number } {
const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now);
return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) };
}
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
const shift = (year: number, month: number, by: number) => {
const index = year * 12 + (month - 1) + by;
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
};
export function monthLabel(id: string): string {
const [y, m] = id.split("-").map(Number);
return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
}
/** The month before the one `now` is in: what a report sent at the start of a month is about. */
export function lastFullMonth(now: Date, tz: string): string {
const cur = monthOf(now, tz);
const prev = shift(cur.year, cur.month, -1);
return monthId(prev.year, prev.month);
}
/** This month and the eleven before it, newest first. */
export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] {
const cur = monthOf(now, tz);
return Array.from({ length: count }, (_, i) => {
const m = shift(cur.year, cur.month, -i);
const id = monthId(m.year, m.month);
return { id, label: monthLabel(id) };
});
}
/** The instants a month begins and ends at, on the clock of `tz`. */
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
const [y, m] = id.split("-").map(Number);
const next = shift(y, m, 1);
return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) };
}
// ─── Gathering ─────────────────────────────────────────────────────────────
const round1 = (n: number) => Math.round(n * 10) / 10;
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
const settings = await getSettings();
const tz = settings.notifications.timezone || "UTC";
const { start, end: fullEnd } = monthBounds(id, tz);
const partial = fullEnd.getTime() > now.getTime();
const end = partial ? now : fullEnd;
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
const startIso = start.toISOString();
const endIso = end.toISOString();
// Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does.
const uptime = await getUptime(end, days);
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
const rowsOf = (series: Record<number, { buckets: { k: number; d: number }[]; percent: number }>, names: Map<number, string>): AvailabilityRow[] =>
Object.entries(series)
.map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) }))
.filter((r) => names.has(r.id))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
const [yy, mm] = id.split("-").map(Number);
const following = shift(yy, mm, 1);
const firstDay = `${id}-01`;
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
for (const r of dailyRows) {
const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 };
t.total += r.total;
t.ok += r.ok;
t.msSum += r.msSum;
t.msCount += r.msCount;
perCheck.set(r.checkId, t);
}
const endpointRows: EndpointRow[] = [...perCheck]
.filter(([cid, t]) => checkNames.has(cid) && t.total > 0)
.map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total }))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Incidents.
const all = await listIncidents({ now: end, limit: 3000 });
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
const began = all.filter((i) => inMonth(i.openedAt));
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length;
const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000));
const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0);
const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs);
const categories = new Map<string, number>();
for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1);
// Updates, by hand and by plan.
const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso)));
const perServer = new Map<number, { runs: number; failed: number; installed: number }>();
let installed = 0;
for (const r of runRows) {
const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0;
installed += gained;
const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 };
t.runs++;
if (r.status === "failed") t.failed++;
t.installed += gained;
perServer.set(r.serverId, t);
}
const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso)));
// Restore tests, and the backups as they are now.
const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive)));
const coverage = await getCachedBackupCoverage(now);
const renewalRows = await renewalsBetween(start, end);
return {
month: id,
label: monthLabel(id),
from: startIso,
to: endIso,
partial,
timezone: tz,
servers: rowsOf(uptime.servers, serverNames),
integrations: rowsOf(uptime.integrations, integrationNames),
endpoints: endpointRows,
incidents: {
began: began.length,
resolved: resolved.length,
stillOpen,
totalMinutes,
averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null,
longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null,
byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)),
list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })),
},
updates: {
runs: runRows.length,
done: runRows.filter((r) => r.status === "done").length,
failed: runRows.filter((r) => r.status === "failed").length,
installed,
byServer: [...perServer]
.map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t }))
.sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)),
planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length },
},
restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)),
backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null,
renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })),
};
}
// ─── As a message ──────────────────────────────────────────────────────────
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
const RESULT_WORDS: Record<string, string> = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" };
function average(rows: AvailabilityRow[]): number | null {
const known = rows.filter((r) => r.percent !== null);
return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length);
}
/** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */
export function monthlyReportText(r: MonthlyReport): { title: string; message: string } {
const sections: string[] = [];
const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`);
const avail: string[] = [];
const serverAvg = average(r.servers);
if (serverAvg !== null) {
avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`);
for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`);
}
if (r.endpoints.length > 0) {
const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length);
avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`);
for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`);
}
const integrationAvg = average(r.integrations);
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
lines("Availability", avail);
const inc = r.incidents;
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`);
for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`);
lines("Incidents", body);
} else if (serverAvg !== null || r.endpoints.length > 0) {
lines("Incidents", ["None."]);
}
const u = r.updates;
const upd: string[] = [];
if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`);
if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`);
lines("Updates", upd);
const bk: string[] = [];
for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`);
if (r.restoreTests.length === 0) bk.push("No restore test was recorded.");
if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`);
lines("Backups", bk);
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
}
@@ -0,0 +1,43 @@
import schedule from "node-schedule";
import { getSettings } from "./settingsStore.js";
import { notify } from "./notify.js";
import { buildMonthlyReport, lastFullMonth, monthlyReportText } from "./monthlyReport.js";
let currentJob: schedule.Job | null = null;
/** The cron rule for "this day of the month at HH:MM" (1–28, so every month has it). */
export function cronForMonthlyReport(day: number, time: string): string {
const [h, m] = time.split(":").map(Number);
const d = Number.isInteger(day) && day >= 1 && day <= 28 ? day : 1;
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} ${d} * *`;
}
/** Builds last month's report and sends it through the channels (held with other notifications during quiet hours). */
export async function sendMonthlyReport(now: Date = new Date()): Promise<{ title: string; message: string }> {
const tz = (await getSettings()).notifications.timezone || "UTC";
const text = monthlyReportText(await buildMonthlyReport(lastFullMonth(now, tz), now));
await notify(text.title, text.message);
return text;
}
/** (Re)schedules the monthly report per the current settings. Call again after they change. */
export async function scheduleMonthlyReport(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { monthlyReport, notifications } = await getSettings();
if (!monthlyReport.enabled) {
console.log("[report] monthly report is off, nothing scheduled");
return;
}
const rule = cronForMonthlyReport(monthlyReport.day, monthlyReport.time);
currentJob = schedule.scheduleJob({ rule, tz: notifications.timezone }, () => {
sendMonthlyReport().catch((err) => console.error("[report] couldn't send the monthly report:", err));
});
console.log(`Monthly report scheduled: ${rule} (${notifications.timezone})`);
}
export async function initMonthlyReportScheduler(): Promise<void> {
await scheduleMonthlyReport();
}
+60
View File
@@ -0,0 +1,60 @@
/**
* New ports: a server that starts listening on something it did not before is worth a look — a service somebody started, or something that should not be
* there. The first report that carries ports is taken as the baseline: what the server was listening on, taken as normal. After that a TCP port that is
* listening on an address other than this machine's own, is not in the baseline, and has not been noted on the Ports page, is a warning (the Alerts page,
* and a notification) until the port is noted ("this is expected") or stops listening.
*
* Left out on purpose: UDP (clients and discovery open them all the time), ports in the dynamic range (49152 and up, where programs pick a free one), and
* anything bound only to the loopback address, which nothing else on the network can reach.
*/
import { isLoopback, type AgentPort } from "./agentPorts.js";
/** From here up, a port is usually one a program picked for itself, not a service. */
export const DYNAMIC_PORTS_FROM = 49152;
export interface NewPort {
protocol: "tcp";
port: number;
/** What is listening, as the agent says; null when it does not. */
process: string | null;
}
export const portKey = (protocol: string, port: number) => `${protocol}:${port}`;
/** What is stored, read back as a set of "tcp:443"; null when there is no baseline yet (or it can't be read). */
export function parseBaseline(stored: string | null): Set<string> | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? new Set(v.filter((k): k is string => typeof k === "string")) : null;
} catch {
return null;
}
}
/** The baseline a first report makes: everything it listens on, in any protocol, so nothing about it is ever "new". */
export function baselineFrom(listening: Pick<AgentPort, "protocol" | "port">[]): string {
return JSON.stringify([...new Set(listening.map((p) => portKey(p.protocol, p.port)))].sort());
}
/**
* Pure: the ports that are new — listening now, reachable from outside this machine, in neither the baseline nor the noted ports. `expected` is the set of
* "tcp:8080" somebody gave a label or a comment on the Ports page. With no baseline yet, nothing is new.
*/
export function newListeningPorts(listening: AgentPort[] | null | undefined, baseline: Set<string> | null, expected: Set<string>): NewPort[] {
if (!listening || baseline === null) return [];
const found = new Map<number, NewPort>();
for (const p of listening) {
if (p.protocol !== "tcp" || !Number.isInteger(p.port) || p.port <= 0 || p.port >= DYNAMIC_PORTS_FROM) continue;
if (isLoopback(p.address ?? "")) continue;
const key = portKey("tcp", p.port);
if (baseline.has(key) || expected.has(key)) continue;
const seen = found.get(p.port);
if (seen) seen.process = seen.process ?? (p.process || null);
else found.set(p.port, { protocol: "tcp", port: p.port, process: p.process || null });
}
return [...found.values()].sort((a, b) => a.port - b.port);
}
export const newPortMessage = (serverName: string, p: NewPort) =>
`${serverName}: a new port is listening — ${p.port}/tcp${p.process ? ` (${p.process})` : ""}. If it should be there, note it on the server's Ports card.`;
+10
View File
@@ -449,6 +449,16 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
);
}
/** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */
export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
await notify(
run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`,
`${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
}
export async function notifyPbsVerificationFailed(
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
): Promise<void> {
+2 -1
View File
@@ -107,11 +107,12 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
let category: AlertCategory | null = null;
if (key.startsWith("offline:")) category = "offline";
else if (key.startsWith("failed-units:")) category = "services";
else if (key.startsWith("new-port:")) category = "monitoring";
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
else if (key.startsWith("automation:")) category = "automation";
else if (key.startsWith("heartbeat")) category = "automation";
else if (key.startsWith("endpoint-cert:")) category = "expiry";
else if (key.startsWith("endpoint:")) category = "monitoring";
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
const server = /^(?:(?:offline|disk|failed-units|smart|zfs|new-port):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
}
+321
View File
@@ -0,0 +1,321 @@
/**
* Patch plans: "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure."
*
* A plan only says when, to whom and how. When it is due (or somebody presses Run now) it makes a *run*: a list of the servers it matches, and goes down
* the list one server at a time. Each server is updated by an ordinary update run (services/updateRuns.ts) — the server is put under maintenance, the
* Semaphore playbook is run limited to it, the app waits for it to report again — and the plan waits for that to finish before it starts the next. What
* depends on the server is silenced for the same time (it will look down while the server restarts). If a server's update fails and the plan says to stop,
* the servers after it are left alone and the run says why; otherwise the plan carries on. A message at the end says how each server went.
*
* A plan starts once a day at most: the day it last started on is kept, so a restart of this app at 02:10 neither misses it nor repeats it. If the app was
* down at the time, it still starts, up to half an hour late; later than that, that day is skipped.
*/
import { and, desc, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { maintenanceWindows, patchPlanRuns, patchPlans, servers, updateRuns } from "../db/schema.js";
import { invalidateAlertsCache } from "./alerts.js";
import { recordAudit } from "./audit.js";
import { impactOf } from "./dependencies.js";
import { startOrExtendWindow } from "./maintenance.js";
import { occurrencesBetween, type Occurrence } from "./maintenanceSchedules.js";
import { notifyPatchPlanFinished } from "./notify.js";
import { parseTags } from "./serverTags.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { ACTIVE_STATUSES, cancelUpdateRun, startUpdateRun, UpdateRunError } from "./updateRuns.js";
export type PlanRow = typeof patchPlans.$inferSelect;
export type PlanRunRow = typeof patchPlanRuns.$inferSelect;
export type ItemStatus = "pending" | "running" | "done" | "failed" | "skipped";
export type RunStatus = "running" | "done" | "failed" | "cancelled";
export interface PlanItem {
serverId: number;
name: string;
status: ItemStatus;
updateRunId: number | null;
message: string | null;
/** Windows made for what depends on this server, which end with it. */
windowIds: number[];
}
/** How late a plan may still start, if the app was not running at its time. */
export const START_GRACE_MS = 30 * 60_000;
export const POLL_INTERVAL_MS = 20_000;
export const MAX_PLANS = 30;
export class PlanError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
// ─── Reading what is stored ────────────────────────────────────────────────
function jsonArray<T>(stored: string | null | undefined): T[] {
if (!stored) return [];
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? (v as T[]) : [];
} catch {
return [];
}
}
export const parseItems = (stored: string | null | undefined): PlanItem[] => jsonArray<PlanItem>(stored).filter((i) => i && typeof i.serverId === "number");
export const planDays = (plan: Pick<PlanRow, "days">): number[] => jsonArray<number>(plan.days).filter((d) => Number.isInteger(d));
export const planTags = (plan: Pick<PlanRow, "tags">): string[] => parseTags(plan.tags);
export const planServerIds = (plan: Pick<PlanRow, "serverIds">): number[] => jsonArray<number>(plan.serverIds).filter((i) => Number.isInteger(i));
// ─── Who a plan is for, and when ───────────────────────────────────────────
/** Pure: the servers a plan covers — those with any of its tags, and those named — by name. */
export function planServers(plan: Pick<PlanRow, "tags" | "serverIds">, all: { id: number; name: string; tags: string | null }[]): { id: number; name: string }[] {
const tags = new Set(planTags(plan));
const ids = new Set(planServerIds(plan));
return all
.filter((s) => ids.has(s.id) || parseTags(s.tags).some((t) => tags.has(t)))
.map((s) => ({ id: s.id, name: s.name }))
.sort((a, b) => a.name.localeCompare(b.name));
}
/** The times a plan happens, from `from` to `to`. A plan has no length; the minute it starts is what matters. */
export function planOccurrences(plan: Pick<PlanRow, "days" | "startTime" | "timezone">, from: Date, to: Date): Occurrence[] {
return occurrencesBetween({ days: plan.days, startTime: plan.startTime, durationMinutes: 1, timezone: plan.timezone }, from, to);
}
/** Pure: the occurrence a plan should start for now — its time has come, it is no more than the grace late, and it has not started on that day. */
export function dueOccurrence(plan: Pick<PlanRow, "enabled" | "days" | "startTime" | "timezone" | "lastRunDate">, now: Date, graceMs: number = START_GRACE_MS): Occurrence | null {
if (!plan.enabled) return null;
const found = planOccurrences(plan, new Date(now.getTime() - graceMs - 60_000), new Date(now.getTime() + 1000)).filter((o) => o.start.getTime() <= now.getTime() && now.getTime() - o.start.getTime() <= graceMs && o.date !== plan.lastRunDate);
return found.length > 0 ? found[found.length - 1] : null;
}
// ─── Starting a run ────────────────────────────────────────────────────────
const ACTIVE_RUN: RunStatus[] = ["running"];
export async function activeRunOf(planId: number): Promise<PlanRunRow | null> {
const [run] = await db.select().from(patchPlanRuns).where(and(eq(patchPlanRuns.planId, planId), inArray(patchPlanRuns.status, ACTIVE_RUN))).limit(1);
return run ?? null;
}
export async function startPlanRun(planId: number, options: { trigger: "schedule" | "manual"; actor: string; now?: Date }): Promise<PlanRunRow> {
const now = options.now ?? new Date();
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId)).limit(1);
if (!plan) throw new PlanError(404, "not_found", "That plan doesn't exist.");
if (await activeRunOf(plan.id)) throw new PlanError(409, "already_running", `"${plan.name}" is already running.`);
const matched = planServers(plan, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers));
if (matched.length === 0) throw new PlanError(400, "no_servers", "No server has this plan's tags or is named in it, so there is nothing to update.");
const items: PlanItem[] = matched.map((s) => ({ serverId: s.id, name: s.name, status: "pending", updateRunId: null, message: null, windowIds: [] }));
const [run] = await db
.insert(patchPlanRuns)
.values({ planId: plan.id, planName: plan.name, trigger: options.trigger, startedBy: options.actor, status: "running", message: `Updating ${matched.length} server${matched.length === 1 ? "" : "s"}, one at a time.`, items: JSON.stringify(items), startedAt: now.toISOString() })
.returning();
await recordAudit({ category: "patch_plan", action: "run", targetType: "patch_plan", targetId: plan.id, detail: { name: plan.name, run: run.id, trigger: options.trigger, servers: matched.length, by: options.actor } });
return run;
}
// ─── Following a run ───────────────────────────────────────────────────────
async function endWindows(ids: number[], now: Date): Promise<void> {
if (ids.length === 0) return;
await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, ids));
invalidateAlertsCache();
}
/** Silences what depends on the server for as long as it is being updated; returns the windows it made (ones it only extended belong to someone else). */
async function silenceDependents(plan: PlanRow, serverId: number, serverName: string, actor: string, now: Date): Promise<number[]> {
if (!plan.silenceDependents) return [];
const made: number[] = [];
try {
const impact = await impactOf("server", serverId, now);
for (const d of impact?.dependents ?? []) {
const started = await startOrExtendWindow(d.type, d.id, plan.windowMinutes, `Depends on ${serverName} — patch plan "${plan.name}"`.slice(0, 200), actor, now);
if (started && !started.extended) made.push(started.window.id);
}
} catch (err) {
console.error("[patchPlans] couldn't silence what depends on a server:", err);
}
return made;
}
const words = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
/** What a finished run says, in a line. */
export function summariseItems(items: PlanItem[]): string {
const count = (s: ItemStatus) => items.filter((i) => i.status === s).length;
const parts = [`${count("done")} updated`];
if (count("failed") > 0) parts.push(`${count("failed")} failed`);
if (count("skipped") > 0) parts.push(`${count("skipped")} left alone`);
return `${parts.join(", ")} of ${words(items.length, "server")}.`;
}
async function saveRun(run: PlanRunRow, items: PlanItem[], extra: Partial<PlanRunRow> = {}): Promise<void> {
await db.update(patchPlanRuns).set({ items: JSON.stringify(items), ...extra }).where(eq(patchPlanRuns.id, run.id));
}
async function finishRun(run: PlanRunRow, plan: PlanRow | null, items: PlanItem[], status: RunStatus, now: Date, why?: string): Promise<void> {
const message = [summariseItems(items), why].filter(Boolean).join(" ");
await saveRun(run, items, { status, message, finishedAt: now.toISOString() });
await recordAudit({ category: "patch_plan", action: status === "done" ? "finished" : status, targetType: "patch_plan", targetId: run.planId, detail: { name: run.planName, run: run.id, message } });
if (status !== "cancelled") {
const lines = items.map((i) => `${i.status === "done" ? "✓" : i.status === "failed" ? "✕" : "–"} ${i.name}${i.message ? ` — ${i.message}` : ""}`);
await notifyPatchPlanFinished({ planName: run.planName, ok: status === "done", message, lines }).catch((err) => console.error("[patchPlans] couldn't send the result:", err));
}
void plan;
}
/** When a server's turn is over and it failed: stop (leaving the rest alone) or carry on. */
function afterFailure(plan: PlanRow, items: PlanItem[]): boolean {
if (!plan.stopOnFailure) return false;
for (const i of items) if (i.status === "pending") Object.assign(i, { status: "skipped", message: "Not started, because an earlier server failed." });
return true;
}
async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
const items = parseItems(run.items);
if (!plan) return; // deleted: the run goes with it
// The server whose turn it is: has its update run finished?
const current = items.find((i) => i.status === "running");
if (current) {
const [ur] = current.updateRunId !== null ? await db.select().from(updateRuns).where(eq(updateRuns.id, current.updateRunId)).limit(1) : [];
if (ur && ACTIVE_STATUSES.includes(ur.status as (typeof ACTIVE_STATUSES)[number])) return;
const ok = ur?.status === "done";
current.status = ok ? "done" : "failed";
current.message = ur?.message ?? "The update run is gone.";
await endWindows(current.windowIds, now);
if (!ok && afterFailure(plan, items)) {
await finishRun(run, plan, items, "failed", now, `Stopped after ${current.name}.`);
return;
}
await saveRun(run, items);
}
// The next one.
for (;;) {
const next = items.find((i) => i.status === "pending");
if (!next) {
await finishRun(run, plan, items, items.some((i) => i.status === "failed") ? "failed" : "done", now);
return;
}
const [server] = await db.select().from(servers).where(eq(servers.id, next.serverId)).limit(1);
if (!server) {
Object.assign(next, { status: "skipped", message: "The server was removed." });
continue;
}
if (plan.onlyWithUpdates) {
const updates = parsePendingUpdates(server.pendingUpdates);
if (!updates || updates.count === 0) {
Object.assign(next, { status: "skipped", message: "No updates were waiting." });
continue;
}
}
const actor = `Patch plan "${plan.name}"`;
try {
const ur = await startUpdateRun(
{
serverId: server.id,
integrationId: plan.integrationId,
projectId: plan.projectId,
templateId: plan.templateId,
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
windowMinutes: plan.windowMinutes,
actor,
},
now,
);
Object.assign(next, { status: "running", updateRunId: ur.id, message: "The update run has started.", windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
await saveRun(run, items);
return; // one at a time: this one is followed until it is over
} catch (err) {
const message = err instanceof UpdateRunError || err instanceof Error ? err.message : String(err);
Object.assign(next, { status: "failed", message });
if (afterFailure(plan, items)) {
await finishRun(run, plan, items, "failed", now, `Stopped at ${next.name}.`);
return;
}
}
}
}
/** One pass over every plan run that is under way. Returns how many it looked at (for the tests). */
export async function advancePlanRuns(now: Date = new Date()): Promise<number> {
const active = await db.select().from(patchPlanRuns).where(inArray(patchPlanRuns.status, ACTIVE_RUN));
for (const run of active) {
try {
await advanceRun(run, now);
} catch (err) {
console.error(`[patchPlans] run ${run.id}:`, err);
}
}
return active.length;
}
/** Starts the plans whose time has come. Returns how many it started. */
export async function startDuePlans(now: Date = new Date()): Promise<number> {
let started = 0;
for (const plan of await db.select().from(patchPlans)) {
const due = dueOccurrence(plan, now);
if (!due) continue;
if (await activeRunOf(plan.id)) continue; // still on the last one: today is skipped rather than overlapping
// Written down first, so that whatever happens next it is not tried again every twenty seconds for half an hour.
await db.update(patchPlans).set({ lastRunDate: due.date }).where(eq(patchPlans.id, plan.id));
try {
await startPlanRun(plan.id, { trigger: "schedule", actor: "the schedule", now });
started++;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
await db.insert(patchPlanRuns).values({ planId: plan.id, planName: plan.name, trigger: "schedule", startedBy: "the schedule", status: "failed", message: `Didn't start: ${message}`, items: "[]", startedAt: now.toISOString(), finishedAt: now.toISOString() });
}
}
return started;
}
export async function cancelPlanRun(id: number, actor: string, now: Date = new Date()): Promise<PlanRunRow | null> {
const [run] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
if (!run) return null;
if (run.status !== "running") throw new PlanError(409, "not_active", "That run is already over.");
const items = parseItems(run.items);
for (const i of items) {
if (i.status === "pending") Object.assign(i, { status: "skipped", message: `Not started: stopped by ${actor}.` });
else if (i.status === "running") {
if (i.updateRunId !== null) await cancelUpdateRun(i.updateRunId, actor, now).catch(() => undefined);
await endWindows(i.windowIds, now);
Object.assign(i, { status: "failed", message: `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.` });
}
}
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
await finishRun(run, plan ?? null, items, "cancelled", now, `Stopped by ${actor}.`);
const [after] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
return after;
}
/** Recent runs, those still going first. */
export async function listPlanRuns(options: { planId?: number; limit?: number } = {}): Promise<PlanRunRow[]> {
const rows = await db
.select()
.from(patchPlanRuns)
.where(options.planId !== undefined ? eq(patchPlanRuns.planId, options.planId) : undefined)
.orderBy(desc(patchPlanRuns.startedAt))
.limit(options.limit ?? 20);
return [...rows.filter((r) => r.status === "running"), ...rows.filter((r) => r.status !== "running")];
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Every twenty seconds: start what is due, follow what is going. A run that was under way when the app last stopped is picked up where it was. */
export function initPatchPlans(): void {
if (timer) return;
timer = setInterval(() => {
void startDuePlans()
.then(() => advancePlanRuns())
.catch((err) => console.error("[patchPlans]", err));
}, POLL_INTERVAL_MS);
timer.unref();
}
+44
View File
@@ -0,0 +1,44 @@
/**
* Renewals: a certificate, domain registration or secret whose expiry moved later was renewed. The app sees it happen when it re-reads the date — an endpoint
* check seeing a new certificate, the registry saying a domain now runs longer, a secret's live certificate being replaced — and writes it down, so the monthly
* report can say "certificates renewed: 3". It only goes back as far as the version that began watching; nothing is guessed about earlier.
*/
import { and, eq, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { renewals } from "../db/schema.js";
/** An expiry has to move at least this far for it to be a renewal (a registry rounding a date by a day is not one). */
export const MIN_RENEWAL_DAYS = 20;
const DAY_MS = 86_400_000;
export type RenewalKind = "certificate" | "domain" | "secret";
const dayOf = (value: string | null | undefined): string | null => (value && /^\d{4}-\d{2}-\d{2}/.test(value) ? value.slice(0, 10) : null);
/** Pure: whether the expiry went from one date to a clearly later one. Either may be a full ISO time; only the day counts. */
export function isRenewal(from: string | null | undefined, to: string | null | undefined): boolean {
const a = dayOf(from);
const b = dayOf(to);
if (!a || !b) return false;
return (Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / DAY_MS >= MIN_RENEWAL_DAYS;
}
/** Writes down a renewal if the dates say it was one (and the same one is not already written down). Never throws: it must not hold up the check that saw it. */
export async function noteRenewal(kind: RenewalKind, subject: string, from: string | null | undefined, to: string | null | undefined, now: Date = new Date()): Promise<boolean> {
try {
if (!isRenewal(from, to)) return false;
const fromDate = dayOf(from)!;
const toDate = dayOf(to)!;
const [same] = await db.select({ id: renewals.id }).from(renewals).where(and(eq(renewals.kind, kind), eq(renewals.subject, subject), eq(renewals.toDate, toDate))).limit(1);
if (same) return false;
await db.insert(renewals).values({ kind, subject, at: now.toISOString(), fromDate, toDate });
return true;
} catch (err) {
console.error("[renewals] couldn't record a renewal:", err);
return false;
}
}
export async function renewalsBetween(from: Date, to: Date) {
return db.select().from(renewals).where(and(gte(renewals.at, from.toISOString()), lt(renewals.at, to.toISOString()))).orderBy(renewals.at);
}
+14 -1
View File
@@ -113,6 +113,17 @@ export interface HealthCheckSettings {
backupStaleHours: number;
/** A restore test older than this many days (or none at all) is a problem for a guest that has backups; 0 = never (a failed test always is). */
restoreTestDays: number;
/** A server that starts listening on a TCP port it did not at first, and that is not noted on its Ports card, is a warning (services/newPorts.ts). */
newPortAlerts: boolean;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
export interface MonthlyReportSettings {
enabled: boolean;
/** Day of the month, 1–28 (every month has it), in notifications.timezone. */
day: number;
/** "HH:MM", in notifications.timezone. */
time: string;
}
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
@@ -225,6 +236,7 @@ export interface AppSettings {
quietHours: QuietHoursSettings;
healthChecks: HealthCheckSettings;
digest: DigestSettings;
monthlyReport: MonthlyReportSettings;
signIn: SignInSettings;
profilePictures: ProfilePictureSettings;
inventory: InventorySettings;
@@ -267,12 +279,13 @@ const DEFAULTS: AppSettings = {
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
quietHours: { enabled: false, start: "22:00", end: "07:00" },
digest: { enabled: false, day: 1, time: "08:00" },
monthlyReport: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0 },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },
+4 -1
View File
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
import { recordAudit } from "./audit.js";
import { baselineFrom } from "./newPorts.js";
import { fillInventoryFromServer } from "./inventoryAutofill.js";
import { nextDiagnostics, type AgentDiagnostic } from "./agentDiagnostics.js";
@@ -113,7 +114,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
const system = report.system;
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion, portBaseline: servers.portBaseline }).from(servers).where(eq(servers.id, serverId));
await db
.update(servers)
.set({
@@ -144,6 +145,8 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
: {}),
// Only stored when the agent reported it, so an agent that predates this doesn't wipe the field.
...(system?.listening_ports ? { listeningPorts: JSON.stringify(system.listening_ports) } : {}),
// The first report that carries ports says what is normal for this server; a port that appears later is "new" (services/newPorts.ts).
...(system?.listening_ports && before && before.portBaseline === null ? { portBaseline: baselineFrom(system.listening_ports) } : {}),
})
.where(eq(servers.id, serverId));
+3
View File
@@ -4,6 +4,7 @@ import { eq, isNotNull } from "drizzle-orm";
import { db } from "../db/client.js";
import { secrets } from "../db/schema.js";
import { resolveTarget } from "../utils/networkGuard.js";
import { noteRenewal } from "./renewals.js";
/**
* Opens a real TLS connection and returns the server certificate's expiry as
@@ -79,6 +80,8 @@ export async function refreshTlsSecret(id: number): Promise<TlsCheckResult | nul
...(expiryDate !== row.expiryDate ? { updatedAt: now } : {}),
})
.where(eq(secrets.id, id));
// A certificate that is now good for much longer than it was has been replaced.
await noteRenewal("certificate", row.name, row.expiryDate, expiryDate);
return { id, name: row.name, host: row.checkHost, port, ok: true, expiryDate };
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
+1 -1
View File
@@ -33,7 +33,7 @@ await runMigrations();
type Data = Parameters<typeof agendaFromData>[0];
const NOW = new Date("2026-10-19T12:00:00Z");
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], patchPlans: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
const titles = (data: Data, days?: number, tz = "UTC") => agendaFromData(data, NOW, tz, days).items.map((i) => [i.daysLeft, i.kind, i.title]);
describe("what is gathered, and how it is dated", () => {
+289
View File
@@ -0,0 +1,289 @@
/**
* The monthly report: which month is which, what renewals and daily figures are kept, what the report gathers for a month, how it reads as a message, and the
* routes and schedule around it.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-monthly-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, auditLog, alertIncidents, updateRuns, patchPlanRuns, patchPlans, restoreTests, renewals, endpointChecks, endpointDaily, endpointResults } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { isMonthId, lastFullMonth, recentMonths, monthBounds, monthLabel, buildMonthlyReport, monthlyReportText } = await import("../src/services/monthlyReport.js");
const { isRenewal, noteRenewal } = await import("../src/services/renewals.js");
const { runCheck, pruneResults } = await import("../src/services/endpointChecks.js");
const { cronForMonthlyReport } = await import("../src/services/monthlyReportScheduler.js");
const { reportsRouter } = await import("../src/routes/reports.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { sql, eq } = await import("drizzle-orm");
await runMigrations();
describe("which month is which", () => {
test("the last whole month, and the twelve to choose from, on the clock of the time zone", () => {
assert.equal(lastFullMonth(new Date("2026-10-19T12:00:00Z"), "UTC"), "2026-09");
assert.equal(lastFullMonth(new Date("2027-01-02T12:00:00Z"), "UTC"), "2026-12", "across the new year");
assert.equal(lastFullMonth(new Date("2026-10-31T23:30:00Z"), "Europe/Stockholm"), "2026-10", "it is already November in Stockholm");
const months = recentMonths(new Date("2026-10-19T12:00:00Z"), "UTC");
assert.equal(months.length, 12);
assert.deepEqual([months[0], months[1], months[11]], [{ id: "2026-10", label: "October 2026" }, { id: "2026-09", label: "September 2026" }, { id: "2025-11", label: "November 2025" }]);
assert.equal(monthLabel("2026-02"), "February 2026");
});
test("a month begins and ends where the clock of the time zone says, summer or winter", () => {
const sept = monthBounds("2026-09", "Europe/Stockholm");
assert.deepEqual([sept.start.toISOString(), sept.end.toISOString()], ["2026-08-31T22:00:00.000Z", "2026-09-30T22:00:00.000Z"]);
const oct = monthBounds("2026-10", "Europe/Stockholm");
assert.equal(oct.end.toISOString(), "2026-10-31T23:00:00.000Z", "the clocks went back on the last Sunday");
assert.equal(monthBounds("2026-12", "UTC").end.toISOString(), "2027-01-01T00:00:00.000Z");
});
test("only a real month is a month", () => {
for (const ok of ["2026-09", "1999-12"]) assert.equal(isMonthId(ok), true, ok);
for (const bad of ["2026-13", "2026-00", "2026-9", "26-09", "2026-09-01", "", undefined, 202609]) assert.equal(isMonthId(bad), false, String(bad));
});
test("the schedule is a day of the month, 1 to 28 so every month has it", () => {
assert.equal(cronForMonthlyReport(5, "07:30"), "30 7 5 * *");
assert.equal(cronForMonthlyReport(31, "07:30"), "30 7 1 * *");
assert.equal(cronForMonthlyReport(0, "bad"), "0 8 1 * *");
});
});
describe("renewals and the daily figures", () => {
test("an expiry that moves a clear step later is a renewal; a day's rounding, a move earlier and a gap in the data are not", () => {
assert.equal(isRenewal("2026-10-01", "2026-12-30"), true);
assert.equal(isRenewal("2026-10-01T08:00:00.000Z", "2026-11-20"), true, "a full time or a day, only the day counts");
assert.equal(isRenewal("2026-10-01", "2026-10-02"), false);
assert.equal(isRenewal("2026-10-01", "2026-10-20"), false, "nineteen days is not enough");
assert.equal(isRenewal("2026-10-21", "2026-10-01"), false);
assert.equal(isRenewal(null, "2026-12-01"), false, "there was nothing to compare with");
assert.equal(isRenewal("2026-10-01", "not a date"), false);
});
test("a renewal is written down once", async () => {
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2027-10-01"), true);
assert.equal(await noteRenewal("domain", "example.se", "2026-10-02", "2027-10-01"), false, "the same one seen again");
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2026-10-03"), false);
assert.equal((await db.select().from(renewals).where(eq(renewals.subject, "example.se"))).length, 1);
await db.delete(renewals);
});
test("each answer adds to its day; a certificate that is replaced is a renewal; old days go", async () => {
const [check] = await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning();
const answer = (ok: boolean, ms: number | null, cert?: string) => async () => ({ ok, ms, message: "m", certExpiresAt: cert });
const t = (h: number) => new Date(`2026-09-10T0${h}:00:00Z`);
let row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(1), answer(true, 100, "2026-10-05T00:00:00.000Z"));
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(2), answer(true, 300, "2026-10-05T00:00:00.000Z"));
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(3), answer(false, null, "2026-10-05T00:00:00.000Z"));
const [day] = await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id));
assert.deepEqual([day.day, day.total, day.ok, day.msSum, day.msCount], ["2026-09-10", 3, 2, 400, 2], "a failure counts as a check but adds no answer time");
assert.equal((await db.select().from(renewals)).length, 0, "the same certificate three times is not a renewal");
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
await runCheck(row, t(4), answer(true, 100, "2026-12-04T00:00:00.000Z"));
const [renewed] = await db.select().from(renewals);
assert.deepEqual([renewed.kind, renewed.subject, renewed.fromDate, renewed.toDate], ["certificate", "Gitea", "2026-10-05", "2026-12-04"]);
await db.insert(endpointDaily).values({ checkId: check.id, day: "2020-01-01", total: 1, ok: 1 });
await pruneResults(new Date("2026-10-19T00:00:00Z"));
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2020-01-01"))).length, 0, "kept for years, not for ever");
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2026-09-10"))).length, 1);
await db.delete(renewals);
});
test("the migration started the daily figures from the results that already existed", async () => {
const [check] = await db.insert(endpointChecks).values({ name: "Backfilled", kind: "tcp", target: "h", port: 22 }).returning();
await db.insert(endpointResults).values([
{ checkId: check.id, at: "2026-10-01T01:00:00.000Z", ok: true, ms: 10 },
{ checkId: check.id, at: "2026-10-01T02:00:00.000Z", ok: true, ms: 30 },
{ checkId: check.id, at: "2026-10-01T03:00:00.000Z", ok: false, ms: null },
{ checkId: check.id, at: "2026-10-02T01:00:00.000Z", ok: true, ms: 50 },
]);
const migration = readFileSync(new URL("../drizzle/0043_puzzling_prism.sql", import.meta.url), "utf8");
const statement = migration.split("--> statement-breakpoint").pop()!;
assert.match(statement, /INSERT INTO `endpoint_daily`/);
await db.run(sql.raw(statement));
const rows = (await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id))).sort((a, b) => a.day.localeCompare(b.day));
assert.deepEqual(rows.map((r) => [r.day, r.total, r.ok, r.msSum, r.msCount]), [["2026-10-01", 3, 2, 40, 2], ["2026-10-02", 1, 1, 50, 1]]);
});
});
describe("a month's report", () => {
let nasId = 0;
let checkId = 0;
const audit = (action: string, id: number, at: string) => ({ category: "server", action, targetType: "server", targetId: String(id), createdAt: at });
const incident = (extra: Partial<typeof alertIncidents.$inferInsert>) => ({ alertKey: "k", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "a disk", openedAt: "2026-09-02T00:00:00.000Z", lastSeenAt: "2026-09-02T00:00:00.000Z", ...extra });
before(async () => {
await updateSettings({ notifications: { timezone: "UTC" } });
await db.delete(endpointChecks); // the earlier tests left checks of their own, and their daily figures go with them
nasId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
const elsaId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "e", apiTokenPrefix: "e", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
void elsaId;
// nas was offline 13 hours in September, and a night in August that does not count
await db.insert(auditLog).values([audit("went_offline", nasId, "2026-09-10 08:00:00"), audit("back_online", nasId, "2026-09-10 21:00:00"), audit("went_offline", nasId, "2026-08-02 01:00:00"), audit("back_online", nasId, "2026-08-02 05:00:00")]);
await db.insert(alertIncidents).values([
incident({ message: "nas: / is 96% full", category: "disk", openedAt: "2026-09-03T10:00:00.000Z", resolvedAt: "2026-09-03T12:00:00.000Z", peakSeverity: "critical" }),
incident({ message: "elsa stopped reporting", category: "offline", openedAt: "2026-09-12T10:00:00.000Z", resolvedAt: "2026-09-12T10:30:00.000Z", targetType: "server", targetId: 2, targetName: "elsa" }),
incident({ message: "Backup of anna failed", category: "backup", openedAt: "2026-09-20T00:00:00.000Z" }),
incident({ message: "in August", openedAt: "2026-08-20T00:00:00.000Z", resolvedAt: "2026-08-20T01:00:00.000Z" }),
incident({ message: "began in August, ended in September", openedAt: "2026-08-31T20:00:00.000Z", resolvedAt: "2026-09-01T02:00:00.000Z" }),
]);
await db.insert(updateRuns).values([
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-09-05T02:00:00.000Z", updatesBefore: 12, updatesAfter: 2 },
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "failed", startedAt: "2026-09-12T02:00:00.000Z", updatesBefore: 3, updatesAfter: null },
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-10-05T02:00:00.000Z", updatesBefore: 9, updatesAfter: 0 },
]);
const [plan] = await db.insert(patchPlans).values({ name: "Lab", days: "[0]", startTime: "02:00", timezone: "UTC", integrationId: 1, projectId: 1, templateId: 1, templateName: "t" }).returning();
await db.insert(patchPlanRuns).values([
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "done", startedAt: "2026-09-06T02:00:00.000Z" },
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "failed", startedAt: "2026-09-13T02:00:00.000Z" },
]);
await db.insert(restoreTests).values([{ subjectKey: "custom:photos", label: "Photos share", testedAt: "2026-09-14", result: "ok" }, { subjectKey: "custom:old", label: "Old", testedAt: "2026-08-14", result: "ok" }]);
await db.insert(renewals).values([{ kind: "certificate", subject: "git.lab", at: "2026-09-12T03:00:00.000Z", fromDate: "2026-09-14", toDate: "2026-12-13" }, { kind: "domain", subject: "example.se", at: "2026-10-03T03:00:00.000Z", fromDate: "2026-10-04", toDate: "2027-10-04" }]);
checkId = (await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning({ id: endpointChecks.id }))[0].id;
await db.insert(endpointDaily).values([
{ checkId, day: "2026-09-01", total: 100, ok: 100, msSum: 10_000, msCount: 100 },
{ checkId, day: "2026-09-30", total: 100, ok: 90, msSum: 9_000, msCount: 90 },
{ checkId, day: "2026-08-31", total: 100, ok: 0 },
{ checkId, day: "2026-10-01", total: 100, ok: 0 },
]);
});
const NOW = new Date("2026-10-19T12:00:00Z");
test("availability: what was down and for how long, from what the audit log recorded; the neighbouring months are not in it", async () => {
const r = await buildMonthlyReport("2026-09", NOW);
assert.deepEqual([r.month, r.label, r.partial, r.timezone], ["2026-09", "September 2026", false, "UTC"]);
const nas = r.servers.find((s) => s.name === "nas")!;
assert.equal(nas.downMinutes, 13 * 60);
assert.equal(nas.percent, 98.2, "13 of 720 hours down");
assert.equal(r.servers[0].name, "nas", "the worst first");
assert.deepEqual(r.endpoints.map((e) => [e.name, e.percent, e.avgMs, e.checks]), [["Gitea", 95, 100, 200]], "the first and last day of the month, not the days either side");
});
test("incidents: how many began and ended, the time they took, the longest, by kind", async () => {
const i = (await buildMonthlyReport("2026-09", NOW)).incidents;
assert.deepEqual([i.began, i.resolved, i.stillOpen], [3, 3, 1], "the open one is still open; the August one that ended in September ended in it");
assert.equal(i.totalMinutes, 150, "2 h and 30 min, of those that began and ended in the month");
assert.equal(i.averageMinutes, 75);
assert.equal(i.longest!.message, "Backup of anna failed", "still open, so it has been going the longest");
assert.deepEqual(i.byCategory, [{ category: "backup", count: 1 }, { category: "disk", count: 1 }, { category: "offline", count: 1 }]);
assert.deepEqual(i.list.map((x) => x.message), ["Backup of anna failed", "nas: / is 96% full", "elsa stopped reporting"]);
assert.equal(i.list[2].where, "elsa");
});
test("updates, patch plans, restore tests and renewals for that month only", async () => {
const r = await buildMonthlyReport("2026-09", NOW);
assert.deepEqual([r.updates.runs, r.updates.done, r.updates.failed, r.updates.installed], [2, 1, 1, 10]);
assert.deepEqual(r.updates.byServer, [{ name: "nas", runs: 2, failed: 1, installed: 10 }]);
assert.deepEqual(r.updates.planRuns, { runs: 2, done: 1, failed: 1 });
assert.deepEqual(r.restoreTests.map((t) => t.label), ["Photos share"]);
assert.deepEqual(r.renewals.map((x) => [x.kind, x.subject]), [["certificate", "git.lab"]]);
assert.equal(r.backups, null, "no backup coverage has been read here");
const oct = await buildMonthlyReport("2026-10", NOW);
assert.equal(oct.partial, true, "October is not over");
assert.deepEqual(oct.renewals.map((x) => x.subject), ["example.se"]);
assert.equal(oct.updates.runs, 1);
});
test("as a message: a section only where there is something to say, with the numbers that matter", async () => {
const { title, message } = monthlyReportText(await buildMonthlyReport("2026-09", NOW));
assert.equal(title, "Homelab monthly report — September 2026");
assert.match(message, /^AVAILABILITY\nServers: 9\d\.\d% on average \(2 servers\)\.\n- nas: 9\d\.\d% \(13 h down\)/);
assert.match(message, /Endpoints: 95% on average \(1 check\)\.\n- Gitea: 95%, 100 ms on average/);
assert.match(message, /INCIDENTS\n3 incidents began, 3 ended, 1 still open\.\nA resolved incident took 1 h 15 min on average\./);
assert.match(message, /- Backup of anna failed — 11 d and still open/);
assert.match(message, /UPDATES\n2 update runs: 1 done, 1 failed — 10 updates installed\.\nPatch plans ran 2 times, 1 not cleanly\./);
assert.match(message, /BACKUPS\n- Restore test of Photos share \(2026-09-14\) passed/);
assert.match(message, /RENEWED\n- Certificate git\.lab: 2026-09-14 → 2026-12-13/);
const partial = monthlyReportText(await buildMonthlyReport("2026-10", NOW));
assert.equal(partial.title, "Homelab monthly report — October 2026 (so far)");
const empty = monthlyReportText(await buildMonthlyReport("2025-01", NOW));
assert.equal(empty.message.includes("Nothing was recorded") || !/INCIDENTS\n[1-9]/.test(empty.message), true);
});
});
describe("through the routes", () => {
let app: Server;
let hook: Server;
let base = "";
let who = "viewer";
const messages: string[] = [];
before(async () => {
await db.insert(users).values([
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
{ oidcSub: "admin", name: "Ada", role: "admin" },
]);
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/reports", reportsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/reports`;
});
after(() => {
app.close();
hook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("anyone signed in can read it: the last whole month by default, the twelve to choose from, or one of them; nothing else is a month", async () => {
const body = await (await fetch(`${base}/monthly`)).json();
assert.equal(body.months.length, 12);
assert.equal(body.report.month, body.months[1].id, "last month, not this one");
assert.equal((await (await fetch(`${base}/monthly?month=${body.months[0].id}`)).json()).report.partial, true);
for (const bad of ["2026-13", "2019-01", "abc", "2026-9"]) assert.equal((await fetch(`${base}/monthly?month=${bad}`)).status, 400, bad);
const text = await (await fetch(`${base}/monthly?format=text`)).json();
assert.match(text.title, /^Homelab monthly report — /);
assert.equal(typeof text.message, "string");
});
test("an administrator can send it now to the channels; nobody else can", async () => {
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 403);
who = "admin";
const res = await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
assert.equal(res.status, 200);
assert.equal(messages.length, 1);
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ month: "nope" }) })).status, 400);
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "send_monthly"))).length, 1);
});
});
+187
View File
@@ -0,0 +1,187 @@
/**
* New ports: what counts as one, that the first report is the baseline, and what the health check, the Alerts page, maintenance, routing and
* the server's Ports card do with them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-newports-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, serverPorts } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { newListeningPorts, baselineFrom, parseBaseline, newPortMessage } = await import("../src/services/newPorts.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { syncServerTasks } = await import("../src/services/taskSync.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { serverPortsRouter } = await import("../src/routes/serverPorts.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const sock = (port: number, extra: Partial<{ protocol: "tcp" | "udp"; address: string; process: string }> = {}) => ({ protocol: "tcp" as const, port, address: "0.0.0.0", ...extra });
const none = new Set<string>();
describe("what counts as a new port", () => {
const baseline = parseBaseline(baselineFrom([sock(22), sock(443), sock(53, { protocol: "udp" })]));
test("the baseline is everything the first report listened on, in any protocol", () => {
assert.deepEqual([...baseline!].sort(), ["tcp:22", "tcp:443", "udp:53"]);
assert.equal(parseBaseline(null), null);
assert.equal(parseBaseline("not json"), null);
assert.equal(parseBaseline(JSON.stringify({ not: "a list" })), null);
});
test("with no baseline yet, nothing is new", () => {
assert.deepEqual(newListeningPorts([sock(8080)], null, none), []);
assert.deepEqual(newListeningPorts(null, baseline, none), []);
});
test("a TCP port that is listening and was not in the baseline is new, with what is listening on it", () => {
const found = newListeningPorts([sock(22), sock(8080, { process: "node" }), sock(8080, { address: "::", process: "" }), sock(3000)], baseline, none);
assert.deepEqual(found, [{ protocol: "tcp", port: 3000, process: null }, { protocol: "tcp", port: 8080, process: "node" }]);
});
test("not UDP, not the dynamic range, not a port bound to this machine alone", () => {
const found = newListeningPorts([sock(5353, { protocol: "udp" }), sock(49152), sock(60000), sock(9000, { address: "127.0.0.1" }), sock(9001, { address: "::1" }), sock(9002, { address: "[::1]" }), sock(9003, { address: "127.0.0.1" }), sock(9003, { address: "10.0.0.5" })], baseline, none);
assert.deepEqual(found.map((p) => p.port), [9003], "one socket on a real address is enough");
assert.deepEqual(newListeningPorts([sock(49151)], baseline, none).map((p) => p.port), [49151], "the last port before the dynamic range is still a service");
});
test("a port somebody noted as expected is not new", () => {
assert.deepEqual(newListeningPorts([sock(8080), sock(9090)], baseline, new Set(["tcp:8080"])).map((p) => p.port), [9090]);
});
test("the message says what it is and what to do", () => {
assert.equal(newPortMessage("nas", { protocol: "tcp", port: 8080, process: "node" }), "nas: a new port is listening — 8080/tcp (node). If it should be there, note it on the server's Ports card.");
});
});
describe("the health check, the alert and the routing", () => {
const snap = (extra: object) => ({ servers: [{ id: 3, name: "nas", lastSeenAt: new Date().toISOString(), disks: [], ...extra }], proxmox: [], synology: [] });
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
test("each new port is a warning keyed by server and port; an offline server's are not judged", () => {
const list = evaluateHealth(snap({ newPorts: [{ protocol: "tcp", port: 8080, process: "node" }, { protocol: "tcp", port: 9090, process: null }] }), limits, Date.now());
assert.deepEqual(list.map((c) => [c.key, c.severity, c.source]), [["new-port:server:3:tcp:8080", "warning", "server"], ["new-port:server:3:tcp:9090", "warning", "server"]]);
assert.match(list[0].message, /nas: a new port is listening — 8080\/tcp \(node\)/);
const offline = evaluateHealth(snap({ lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString(), newPorts: [{ protocol: "tcp", port: 8080, process: null }] }), limits, Date.now());
assert.equal(offline.some((c) => c.key.startsWith("new-port")), false);
});
test("maintenance on the server holds it back, and routing treats it as monitoring with that server's tags", () => {
assert.equal(subjectOfConditionKey("new-port:server:3:tcp:8080"), "server:3");
assert.deepEqual(metaOfConditionKey("new-port:server:3:tcp:8080", "warning"), { category: "monitoring", severity: "warning", serverId: 3 });
});
});
describe("from a report to the Ports card", () => {
let app: Server;
let base = "";
let serverId = 0;
let who = "operator";
const report = (ports: ReturnType<typeof sock>[]) => ({ tasks: [], system: { listening_ports: ports } });
before(async () => {
await db.insert(users).values([
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown; currentUser: unknown }).session = { user: { sub: who } };
next();
});
// The real servers router puts a signed-in user on the request before this; here the role is read the same way the middleware does.
const { requireAuth } = await import("../src/auth/middleware.js");
a.use("/api/servers/:id/ports", requireAuth, serverPortsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers/${serverId}/ports`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const newKeys = async () => (await collectSnapshot()).snapshot.servers.find((s) => s.id === serverId)!.newPorts!.map((p) => p.port);
test("the first report that carries ports is the baseline: nothing in it is new, and it is never changed by a later one", async () => {
await syncServerTasks(serverId, report([sock(22), sock(443)]));
let [row] = await db.select().from(servers).where(eq(servers.id, serverId));
assert.deepEqual(JSON.parse(row.portBaseline!), ["tcp:22", "tcp:443"]);
assert.deepEqual(await newKeys(), []);
await syncServerTasks(serverId, report([sock(22), sock(443), sock(8080, { process: "node" })]));
[row] = await db.select().from(servers).where(eq(servers.id, serverId));
assert.deepEqual(JSON.parse(row.portBaseline!), ["tcp:22", "tcp:443"], "8080 is not folded into what is normal");
assert.deepEqual(await newKeys(), [8080]);
});
test("a report with no ports at all does not start a baseline", async () => {
const [other] = await db.insert(servers).values({ name: "bare", apiTokenHash: "b", apiTokenPrefix: "b" }).returning({ id: servers.id });
await syncServerTasks(other.id, { tasks: [], system: {} });
const [row] = await db.select().from(servers).where(eq(servers.id, other.id));
assert.equal(row.portBaseline, null);
});
test("it is an alert on the Alerts page, in monitoring, linked to the server — and silenced by maintenance on it", async () => {
invalidateAlertsCache();
const alert = (await getAlerts(true)).alerts.find((a) => a.message.includes("a new port is listening"))!;
assert.ok(alert, "the alert is there");
assert.deepEqual([alert.category, alert.severity, alert.silenced], ["monitoring", "warning", false]);
assert.deepEqual(alert.target && [alert.target.type, alert.target.id], ["server", serverId]);
await startOrExtendWindow("server", serverId, 30, "work", "Ada");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.find((a) => a.message.includes("a new port is listening"))!.silenced, true);
});
test("the Ports card marks it new, and noting it as expected ends the alert; a scan-only row does not", async () => {
who = "viewer";
let list = (await (await fetch(base)).json()).ports as { port: number; isNew: boolean }[];
assert.deepEqual(list.filter((p) => p.isNew).map((p) => p.port), [8080]);
await db.insert(serverPorts).values({ serverId, port: 8080, protocol: "tcp", open: true });
assert.deepEqual(await newKeys(), [8080], "a row that only records an open scan says nothing");
await db.delete(serverPorts).where(eq(serverPorts.port, 8080));
who = "operator";
const res = await fetch(base, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ port: 8080, protocol: "tcp", label: "Expected" }) });
assert.equal(res.status, 200);
list = (await res.json()).ports;
assert.equal(list.find((p) => p.port === 8080)!.isNew, false);
assert.deepEqual(await newKeys(), []);
});
test("it can be switched off, and then nothing is new", async () => {
await syncServerTasks(serverId, report([sock(22), sock(443), sock(8080), sock(9999)]));
assert.deepEqual(await newKeys(), [9999]);
await updateSettings({ healthChecks: { newPortAlerts: false } });
assert.deepEqual(await newKeys(), []);
who = "viewer";
assert.equal(((await (await fetch(base)).json()).ports as { isNew: boolean }[]).some((p) => p.isNew), false);
await updateSettings({ healthChecks: { newPortAlerts: true } });
});
});
+407
View File
@@ -0,0 +1,407 @@
/**
* Patch plans: who a plan covers, when it is due, that it goes one server at a time and waits for each, what it does when one fails, what it silences,
* the message at the end, and who may do what.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-patchplans-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, patchPlans, patchPlanRuns, updateRuns, maintenanceWindows } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { planServers, dueOccurrence, summariseItems, startPlanRun, advancePlanRuns, startDuePlans, cancelPlanRun, parseItems, PlanError } = await import("../src/services/patchPlans.js");
const { advanceUpdateRuns, setSemaphoreForTests } = await import("../src/services/updateRuns.js");
const { listActiveWindows } = await import("../src/services/maintenance.js");
const { agendaFromData } = await import("../src/services/agenda.js");
const { patchPlansRouter } = await import("../src/routes/patchPlans.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
// ── a Semaphore that answers as told ──
const fake = { runs: [] as { limit?: string }[], task: { id: 41, status: "waiting", created: null as string | null, start: null as string | null, end: null as string | null }, refuseFor: new Set<string>() };
setSemaphoreForTests(async (integrationId) =>
integrationId === 9
? {
ping: async () => ({ ok: true }),
listTemplatesWithStatus: async () => [],
checkTemplates: async () => ({ templates: [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "u.yml", lastTask: null }], failedProjectIds: [] }),
runTemplate: async (_p: number, _t: number, options?: { limit?: string }) => {
if (options?.limit && fake.refuseFor.has(options.limit)) throw new Error("Semaphore is having a moment");
fake.runs.push({ limit: options?.limit });
return { ...fake.task } as never;
},
getTask: async () => ({ ...fake.task }) as never,
}
: null,
);
const T0 = new Date("2026-10-18T02:05:00.000Z"); // a Sunday
const at = (m: number) => new Date(T0.getTime() + m * 60_000);
describe("who a plan covers, and when it is due", () => {
const all = [
{ id: 1, name: "web", tags: JSON.stringify(["lab"]) },
{ id: 2, name: "api", tags: JSON.stringify(["prod"]) },
{ id: 3, name: "ångström", tags: JSON.stringify(["lab", "prod"]) },
{ id: 4, name: "db", tags: null },
];
test("servers with any of the tags, and those named, by name; each only once", () => {
assert.deepEqual(planServers({ tags: JSON.stringify(["lab"]), serverIds: "[]" }, all).map((s) => s.name), ["ångström", "web"].sort((x, y) => x.localeCompare(y)));
assert.deepEqual(planServers({ tags: JSON.stringify(["lab"]), serverIds: JSON.stringify([4, 3]) }, all).map((s) => s.name), ["db", "ångström", "web"].sort((x, y) => x.localeCompare(y)), "in the order the app sorts names");
assert.deepEqual(planServers({ tags: "[]", serverIds: "[]" }, all), [], "a plan that names no one covers no one");
});
const plan = { enabled: true, days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", lastRunDate: null as string | null };
test("due from its time to half an hour after, once a day, only on its days", () => {
assert.equal(dueOccurrence(plan, new Date("2026-10-18T01:59:00Z")), null, "not yet");
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:00:00Z"))?.date, "2026-10-18");
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:29:00Z"))?.date, "2026-10-18", "the app may have been down for a while");
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:31:00Z")), null, "too late: that day is skipped");
assert.equal(dueOccurrence({ ...plan, lastRunDate: "2026-10-18" }, new Date("2026-10-18T02:10:00Z")), null, "already started today");
assert.equal(dueOccurrence(plan, new Date("2026-10-19T02:10:00Z")), null, "Monday is not a day it runs");
assert.equal(dueOccurrence({ ...plan, enabled: false }, new Date("2026-10-18T02:10:00Z")), null);
});
test("its time is on the clock of its time zone, winter and summer", () => {
const stockholm = { ...plan, timezone: "Europe/Stockholm" };
assert.equal(dueOccurrence(stockholm, new Date("2026-10-18T00:05:00Z"))?.date, "2026-10-18", "02:05 in Stockholm is 00:05 UTC while summer time lasts");
assert.equal(dueOccurrence(stockholm, new Date("2026-11-01T01:05:00Z"))?.date, "2026-11-01", "and 01:05 UTC after the clocks go back");
assert.equal(dueOccurrence(stockholm, new Date("2026-11-01T00:05:00Z")), null);
});
test("a finished run says how each went in one line", () => {
const item = (status: string) => ({ serverId: 1, name: "x", status, updateRunId: null, message: null, windowIds: [] }) as never;
assert.equal(summariseItems([item("done"), item("done"), item("failed"), item("skipped")]), "2 updated, 1 failed, 1 left alone of 4 servers.");
assert.equal(summariseItems([item("done")]), "1 updated of 1 server.");
});
test("a plan is in the agenda as maintenance, with how many servers it updates", () => {
const data = { secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], patchPlans: [{ id: 5, name: "Lab nights", days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", servers: 3 }] };
const items = agendaFromData(data, new Date("2026-10-19T12:00:00Z"), "UTC", 20).items.filter((i) => i.id.startsWith("plan:"));
assert.deepEqual(items.map((i) => [i.title, i.detail, i.at.slice(0, 16), i.kind]), [["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-10-25T02:00", "maintenance"], ["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-11-01T02:00", "maintenance"], ["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-11-08T02:00", "maintenance"]]);
});
});
describe("a run, from start to end", () => {
let hook: Server;
const messages: string[] = [];
const ids: Record<string, number> = {};
let planId = 0;
const plan = (extra: Partial<typeof patchPlans.$inferInsert> = {}) => ({
name: "Lab nights", days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", tags: JSON.stringify(["lab"]), serverIds: "[]", integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", limitFrom: "name", stopOnFailure: true, onlyWithUpdates: true, silenceDependents: true, windowMinutes: 90, ...extra,
});
const updates = (count: number, reportedAt = T0.toISOString()) => JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: false, reportedAt });
const runOf = async (id: number) => (await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)))[0];
const itemsOf = async (id: number) => parseItems((await runOf(id)).items);
/** The playbook for the server being updated finishes and the server reports again — as the update run follows. */
async function finishCurrent(server: string, minute: number) {
fake.task = { id: 41, status: "success", created: null, start: null, end: at(minute).toISOString() };
await advanceUpdateRuns(at(minute + 1));
await db.update(servers).set({ lastSeenAt: at(minute + 2).toISOString(), pendingUpdates: updates(0, at(minute + 2).toISOString()) }).where(eq(servers.id, ids[server]));
await advanceUpdateRuns(at(minute + 3));
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
}
async function reset(extra: Partial<typeof patchPlans.$inferInsert> = {}) {
await db.delete(patchPlans);
await db.delete(updateRuns);
await db.delete(maintenanceWindows); // the tests share one clock, so a window left by the last would still be open
fake.runs = [];
fake.refuseFor.clear();
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
for (const name of ["web", "api"]) await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: updates(4) }).where(eq(servers.id, ids[name]));
await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: updates(0) }).where(eq(servers.id, ids.idle));
messages.length = 0;
planId = (await db.insert(patchPlans).values(plan(extra)).returning({ id: patchPlans.id }))[0].id;
}
before(async () => {
for (const name of ["web", "api", "idle"]) {
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: JSON.stringify(["lab"]), lastSeenAt: T0.toISOString() }).returning({ id: servers.id }))[0].id;
}
ids.prod = (await db.insert(servers).values({ name: "prod", apiTokenHash: "p", apiTokenPrefix: "p", tags: JSON.stringify(["prod"]), lastSeenAt: T0.toISOString(), pendingUpdates: updates(9) }).returning({ id: servers.id }))[0].id;
// a guest that runs on web: what depends on web
ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: T0.toISOString(), parentServerId: ids.web }).returning({ id: servers.id }))[0].id;
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
});
after(() => {
hook.close();
setSemaphoreForTests(null);
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("one server at a time, each waited for, those with nothing waiting left alone, and a message at the end", async () => {
await reset();
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
assert.deepEqual((await itemsOf(run.id)).map((i) => [i.name, i.status]), [["api", "pending"], ["idle", "pending"], ["web", "pending"]], "by name");
await advancePlanRuns(T0);
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["running", "pending", "pending"]);
assert.deepEqual(fake.runs.map((r) => r.limit), ["api"], "only the first has been started");
assert.equal((await listActiveWindows(T0)).some((w) => w.targetType === "server" && w.targetId === ids.api), true, "and it is under maintenance");
await advancePlanRuns(at(1));
assert.equal(fake.runs.length, 1, "while it is still running, nothing else starts");
await finishCurrent("api", 5);
await advancePlanRuns(at(9));
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["done", "skipped", "running"], "idle has nothing waiting, so it is passed over, and web is next");
assert.deepEqual(fake.runs.map((r) => r.limit), ["api", "web"]);
assert.equal((await itemsOf(run.id))[1].message, "No updates were waiting.");
assert.equal((await listActiveWindows(at(9))).some((w) => w.targetId === ids.api), false, "api's maintenance ended with its update");
await finishCurrent("web", 15);
await advancePlanRuns(at(19));
const done = await runOf(run.id);
assert.equal(done.status, "done");
assert.equal(done.message, "2 updated, 1 left alone of 3 servers.");
assert.equal(messages.length >= 3, true, "each update run says how it went, and so does the plan");
const last = messages[messages.length - 1];
assert.match(last, /2 updated, 1 left alone of 3 servers\./);
assert.match(last, /✓ api/);
assert.match(last, /– idle — No updates were waiting\./);
assert.match(last, /✓ web/);
});
test("what depends on a server is silenced while it is updated, and let go when it is done", async () => {
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
const during = await listActiveWindows(T0);
assert.equal(during.some((w) => w.targetId === ids.guest), true, "the guest that runs on web");
assert.match(during.find((w) => w.targetId === ids.guest)!.reason ?? "", /Depends on web — patch plan "Lab nights"/);
assert.equal((await itemsOf(run.id))[0].windowIds.length, 1);
await finishCurrent("web", 5);
await advancePlanRuns(at(9));
assert.equal((await listActiveWindows(at(9))).some((w) => w.targetId === ids.guest), false);
assert.equal((await runOf(run.id)).status, "done");
});
test("it can be told not to silence what depends on a server", async () => {
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]), silenceDependents: false });
await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
assert.equal((await listActiveWindows(T0)).some((w) => w.targetId === ids.guest), false);
});
test("a failure stops the plan and leaves the rest alone, saying where it stopped", async () => {
await reset();
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
fake.task = { id: 41, status: "error", created: null, start: null, end: null };
await advanceUpdateRuns(at(2));
await advancePlanRuns(at(3));
const after = await runOf(run.id);
assert.equal(after.status, "failed");
assert.deepEqual((await itemsOf(run.id)).map((i) => [i.name, i.status]), [["api", "failed"], ["idle", "skipped"], ["web", "skipped"]]);
assert.equal((await itemsOf(run.id))[2].message, "Not started, because an earlier server failed.");
assert.match(after.message ?? "", /1 failed, 2 left alone of 3 servers\. Stopped after api\./);
assert.deepEqual(fake.runs.map((r) => r.limit), ["api"], "web was never touched");
assert.match(messages[messages.length - 1], /✕ api/);
assert.equal(messages.some((m) => m.includes("didn't finish cleanly")), true);
});
test("told to carry on, it does: the failed one is reported and the rest are updated", async () => {
await reset({ stopOnFailure: false });
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
fake.task = { id: 41, status: "error", created: null, start: null, end: null };
await advanceUpdateRuns(at(2));
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
await advancePlanRuns(at(3));
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["failed", "skipped", "running"], "web is next");
await finishCurrent("web", 10);
await advancePlanRuns(at(14));
const done = await runOf(run.id);
assert.equal(done.status, "failed", "something failed, so the run says so");
assert.equal(done.message, "1 updated, 1 failed, 1 left alone of 3 servers.");
});
test("a server that can't be started counts as a failure; a server with nothing to update can be updated all the same if asked", async () => {
await reset({ onlyWithUpdates: false, tags: "[]", serverIds: JSON.stringify([ids.idle]) });
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
assert.deepEqual(fake.runs.map((r) => r.limit), ["idle"], "it had nothing waiting but the plan was told not to mind");
await updateRunsCancelAll();
await cancelPlanRun(run.id, "Olle", at(1));
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
fake.refuseFor.add("web");
const refused = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
const items = await itemsOf(refused.id);
assert.equal(items[0].status, "failed");
assert.match(items[0].message ?? "", /Semaphore wouldn't start it: Semaphore is having a moment/);
assert.equal((await runOf(refused.id)).status, "failed");
});
async function updateRunsCancelAll() {
await db.update(updateRuns).set({ status: "cancelled" });
}
test("a plan does not overlap itself, and one with no servers says so rather than doing nothing silently", async () => {
await reset();
await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await assert.rejects(startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 }), (e: unknown) => e instanceof PlanError && e.code === "already_running");
await db.update(patchPlanRuns).set({ status: "cancelled" });
await db.update(patchPlans).set({ tags: JSON.stringify(["nothing-has-this"]) }).where(eq(patchPlans.id, planId));
await assert.rejects(startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 }), (e: unknown) => e instanceof PlanError && e.code === "no_servers");
});
test("stopping a run that is going leaves the rest alone, ends the maintenance it made and stops the update run being followed", async () => {
await reset();
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
await advancePlanRuns(T0);
const stopped = await cancelPlanRun(run.id, "Olga", at(2));
assert.equal(stopped!.status, "cancelled");
assert.deepEqual(parseItems(stopped!.items).map((i) => i.status), ["failed", "skipped", "skipped"]);
assert.equal((await listActiveWindows(at(2))).some((w) => w.targetId === ids.api), false);
assert.equal((await db.select().from(updateRuns))[0].status, "cancelled");
await assert.rejects(cancelPlanRun(run.id, "Olga", at(3)), (e: unknown) => e instanceof PlanError && e.code === "not_active");
assert.equal(await cancelPlanRun(99999, "Olga"), null);
});
test("the schedule starts a plan once on its day, and a restart of the app does not start it again", async () => {
await reset();
assert.equal(await startDuePlans(T0), 1);
const [row] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId));
assert.equal(row.lastRunDate, "2026-10-18");
assert.equal((await db.select().from(patchPlanRuns))[0].trigger, "schedule");
await db.update(patchPlanRuns).set({ status: "done" });
assert.equal(await startDuePlans(at(10)), 0, "not twice in a day");
assert.equal(await startDuePlans(new Date("2026-10-25T02:03:00Z")), 1, "but the next Sunday it does");
});
test("a plan that is due but has no servers leaves a failed run to say so", async () => {
await reset({ tags: JSON.stringify(["nothing-has-this"]) });
await db.delete(patchPlanRuns);
assert.equal(await startDuePlans(T0), 0);
const [run] = await db.select().from(patchPlanRuns);
assert.deepEqual([run.status, run.trigger], ["failed", "schedule"]);
assert.match(run.message ?? "", /^Didn't start: No server has this plan's tags/);
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "admin";
const body = (extra: object = {}) => ({ name: "Weekend patching", days: [6, 0], startTime: "03:30", timezone: "Europe/Stockholm", tags: ["Lab"], serverIds: [], integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", ...extra });
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/patch-plans", patchPlansRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/patch-plans`;
await db.delete(patchPlans);
await db.delete(patchPlanRuns);
});
after(() => app.close());
const send = (method: string, path: string, b?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: b === undefined ? undefined : JSON.stringify(b) });
let id = 0;
test("an administrator makes a plan, with its tags written the way tags are; a wrong one is refused with a reason; nobody else can", async () => {
const res = await send("POST", "/", body());
assert.equal(res.status, 201);
const plan = (await res.json()).plan;
id = plan.id;
assert.deepEqual([plan.tags, plan.days, plan.limitFrom, plan.stopOnFailure, plan.onlyWithUpdates, plan.windowMinutes], [["lab"], [0, 6], "name", true, true, 90]);
assert.deepEqual(plan.servers.map((s: { name: string }) => s.name), ["api", "idle", "web"], "the three tagged lab");
assert.equal(plan.next.length, 3, "its next times, so nobody has to work out when");
for (const bad of [{ days: [] }, { startTime: "3:30" }, { timezone: "Mars/Olympus" }, { tags: [], serverIds: [] }, { tags: ["bad tag!"] }, { windowMinutes: 5 }, { name: " " }, { integrationId: 0 }]) {
const r = await send("POST", "/", body(bad));
assert.equal(r.status, 400, JSON.stringify(bad));
assert.ok((await r.json()).message.length > 0);
}
who = "operator";
assert.equal((await send("POST", "/", body())).status, 403);
assert.equal((await send("PUT", `/${id}`, body())).status, 403);
assert.equal((await send("DELETE", `/${id}`)).status, 403);
who = "viewer";
const list = await (await send("GET", "/")).json();
assert.equal(list.plans.length, 1, "everyone can see what is planned");
assert.ok(list.tags.includes("lab") && list.servers.length >= 4);
});
test("changing when it happens forgets which day it last started on; changing anything else does not", async () => {
who = "admin";
await db.update(patchPlans).set({ lastRunDate: "2026-10-18" }).where(eq(patchPlans.id, id));
await send("PUT", `/${id}`, body({ name: "Renamed" }));
assert.equal((await db.select().from(patchPlans).where(eq(patchPlans.id, id)))[0].lastRunDate, "2026-10-18");
await send("PUT", `/${id}`, body({ name: "Renamed", startTime: "04:00" }));
assert.equal((await db.select().from(patchPlans).where(eq(patchPlans.id, id)))[0].lastRunDate, null);
assert.equal((await send("PUT", "/99999", body())).status, 404);
});
test("an operator can run it now and stop it; a viewer cannot; and it cannot be removed while it is running", async () => {
who = "viewer";
assert.equal((await send("POST", `/${id}/run`)).status, 403);
who = "operator";
const res = await send("POST", `/${id}/run`);
assert.equal(res.status, 201);
const run = (await res.json()).run;
assert.equal(run.status, "running");
assert.equal(run.trigger, "manual");
assert.ok(run.items.every((i: { windowIds?: unknown }) => i.windowIds === undefined), "the windows it made are its own business");
assert.equal((await send("POST", `/${id}/run`)).status, 409, "not twice at once");
who = "admin";
assert.equal((await send("DELETE", `/${id}`)).status, 409);
who = "operator";
const stopped = await send("POST", `/runs/${run.id}/cancel`);
assert.equal(stopped.status, 200);
assert.equal((await stopped.json()).run.status, "cancelled");
assert.equal((await send("POST", `/runs/${run.id}/cancel`)).status, 409);
assert.equal((await send("POST", "/runs/99999/cancel")).status, 404);
who = "admin";
assert.equal((await send("DELETE", `/${id}`)).status, 204);
assert.equal((await db.select().from(patchPlanRuns)).length, 0, "its history goes with it");
});
});