Patch plans, a monthly report and new-port alerts
Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an ordinary update run, stopping at the first failure by default, silencing what depends on a server while it is updated, with one message at the end. Monthly report (Operations -> Monthly report, and optionally a message on a day of the month): availability, incidents, updates, restore tests, backups and renewals for a calendar month, printable. Backed by daily endpoint figures and a record of renewals. New ports: the first report with ports is the baseline; a TCP port that begins listening later and is not noted on the server's Ports card is a warning, cleared by Mark as expected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ddd0aa622a
commit
be6c5552bd
46 files changed
+7836
-26
No files matched your search
@@ -0,0 +1,66 @@
|
||||
CREATE TABLE `endpoint_daily` (
|
||||
`check_id` integer NOT NULL,
|
||||
`day` text NOT NULL,
|
||||
`total` integer DEFAULT 0 NOT NULL,
|
||||
`ok` integer DEFAULT 0 NOT NULL,
|
||||
`ms_sum` integer DEFAULT 0 NOT NULL,
|
||||
`ms_count` integer DEFAULT 0 NOT NULL,
|
||||
PRIMARY KEY(`check_id`, `day`),
|
||||
FOREIGN KEY (`check_id`) REFERENCES `endpoint_checks`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `patch_plan_runs` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`plan_id` integer NOT NULL,
|
||||
`plan_name` text NOT NULL,
|
||||
`trigger` text NOT NULL,
|
||||
`started_by` text,
|
||||
`status` text NOT NULL,
|
||||
`message` text,
|
||||
`items` text DEFAULT '[]' NOT NULL,
|
||||
`started_at` text NOT NULL,
|
||||
`finished_at` text,
|
||||
FOREIGN KEY (`plan_id`) REFERENCES `patch_plans`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `patch_plan_runs_plan_idx` ON `patch_plan_runs` (`plan_id`,`started_at`);--> statement-breakpoint
|
||||
CREATE INDEX `patch_plan_runs_status_idx` ON `patch_plan_runs` (`status`);--> statement-breakpoint
|
||||
CREATE TABLE `patch_plans` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`enabled` integer DEFAULT true NOT NULL,
|
||||
`days` text NOT NULL,
|
||||
`start_time` text NOT NULL,
|
||||
`timezone` text NOT NULL,
|
||||
`tags` text DEFAULT '[]' NOT NULL,
|
||||
`server_ids` text DEFAULT '[]' NOT NULL,
|
||||
`integration_id` integer NOT NULL,
|
||||
`project_id` integer NOT NULL,
|
||||
`template_id` integer NOT NULL,
|
||||
`template_name` text NOT NULL,
|
||||
`limit_from` text DEFAULT 'name' NOT NULL,
|
||||
`stop_on_failure` integer DEFAULT true NOT NULL,
|
||||
`only_with_updates` integer DEFAULT true NOT NULL,
|
||||
`silence_dependents` integer DEFAULT true NOT NULL,
|
||||
`window_minutes` integer DEFAULT 90 NOT NULL,
|
||||
`last_run_date` text,
|
||||
`created_by` text,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `renewals` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`kind` text NOT NULL,
|
||||
`subject` text NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`from_date` text NOT NULL,
|
||||
`to_date` text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `renewals_at_idx` ON `renewals` (`at`);--> statement-breakpoint
|
||||
ALTER TABLE `servers` ADD `port_baseline` text;
|
||||
--> statement-breakpoint
|
||||
-- The daily rollup starts with what the endpoint results already hold (they are kept two weeks), so the first report is not blank.
|
||||
INSERT INTO `endpoint_daily` (`check_id`, `day`, `total`, `ok`, `ms_sum`, `ms_count`)
|
||||
SELECT `check_id`, substr(`at`, 1, 10), count(*), sum(`ok`), sum(CASE WHEN `ok` = 1 AND `ms` IS NOT NULL THEN `ms` ELSE 0 END), sum(CASE WHEN `ok` = 1 AND `ms` IS NOT NULL THEN 1 ELSE 0 END)
|
||||
FROM `endpoint_results` GROUP BY `check_id`, substr(`at`, 1, 10);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -302,6 +302,13 @@
|
||||
"when": 1792361094159,
|
||||
"tag": "0042_silly_omega_red",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 43,
|
||||
"version": "6",
|
||||
"when": 1792443011097,
|
||||
"tag": "0043_puzzling_prism",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -50,6 +50,8 @@ import { alertTicketsRouter } from "./routes/alertTickets.js";
|
||||
import { avatarsRouter } from "./routes/avatars.js";
|
||||
import { heartbeatsRouter } from "./routes/heartbeats.js";
|
||||
import { agendaRouter } from "./routes/agenda.js";
|
||||
import { patchPlansRouter } from "./routes/patchPlans.js";
|
||||
import { reportsRouter } from "./routes/reports.js";
|
||||
import { publicCalendarRouter } from "./routes/publicCalendar.js";
|
||||
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
|
||||
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
|
||||
@@ -113,6 +115,8 @@ export function createApp() {
|
||||
app.use("/api/avatars", avatarsRouter);
|
||||
app.use("/api/heartbeats", heartbeatsRouter);
|
||||
app.use("/api/agenda", agendaRouter);
|
||||
app.use("/api/patch-plans", patchPlansRouter);
|
||||
app.use("/api/reports", reportsRouter);
|
||||
app.use("/api/uptime", uptimeRouter);
|
||||
app.use("/api/share-links", shareLinksRouter);
|
||||
app.use("/api/runbooks", runbooksRouter);
|
||||
|
||||
+88
-1
@@ -1,5 +1,5 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { sqliteTable, text, integer, real, uniqueIndex, index } from "drizzle-orm/sqlite-core";
|
||||
import { sqliteTable, text, integer, real, uniqueIndex, index, primaryKey } from "drizzle-orm/sqlite-core";
|
||||
|
||||
// ─── Users & roles ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -447,6 +447,10 @@ export const servers = sqliteTable("servers", {
|
||||
// What the agent says it could not collect, as JSON: {items: [{check, status, reason?, detail?}], reportedAt} — see services/agentDiagnostics.ts. Null until an agent that can say so has.
|
||||
agentDiagnostics: text("agent_diagnostics"),
|
||||
|
||||
// The ports this server was listening on when its agent first said so, as a JSON array like ["tcp:22","tcp:443"]. A port that is listening later and is
|
||||
// not in this list, and not noted on the Ports page, is a "new port" (services/newPorts.ts). Null until the first report that carries ports.
|
||||
portBaseline: text("port_baseline"),
|
||||
|
||||
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
|
||||
hwManufacturer: text("hw_manufacturer"),
|
||||
hwModel: text("hw_model"),
|
||||
@@ -846,6 +850,89 @@ export const updateRuns = sqliteTable(
|
||||
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
|
||||
);
|
||||
|
||||
// ─── Patch plans: updates, on a schedule, one server at a time ─────────────
|
||||
|
||||
// "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure." A plan only says when, to whom and how; each server is updated by an
|
||||
// ordinary update run (services/updateRuns.ts), and the plan waits for it to finish before the next — see services/patchPlans.ts.
|
||||
export const patchPlans = sqliteTable("patch_plans", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
name: text("name").notNull(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
days: text("days").notNull(), // JSON: weekdays, 0 = Sunday … 6 = Saturday
|
||||
startTime: text("start_time").notNull(), // "HH:MM" on the clock of `timezone`
|
||||
timezone: text("timezone").notNull(),
|
||||
tags: text("tags").notNull().default("[]"), // JSON string[]: servers that have any of these tags
|
||||
serverIds: text("server_ids").notNull().default("[]"), // JSON number[]: and these servers by name, whatever their tags
|
||||
integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK, like update runs
|
||||
projectId: integer("project_id").notNull(),
|
||||
templateId: integer("template_id").notNull(),
|
||||
templateName: text("template_name").notNull(),
|
||||
limitFrom: text("limit_from").notNull().default("name"), // what ansible's --limit is for a server: its name here, or its host name
|
||||
stopOnFailure: integer("stop_on_failure", { mode: "boolean" }).notNull().default(true),
|
||||
onlyWithUpdates: integer("only_with_updates", { mode: "boolean" }).notNull().default(true), // leave out a server that has nothing waiting
|
||||
silenceDependents: integer("silence_dependents", { mode: "boolean" }).notNull().default(true), // what depends on a server is silenced while it is updated
|
||||
windowMinutes: integer("window_minutes").notNull().default(90), // at most this long under maintenance per server
|
||||
lastRunDate: text("last_run_date"), // the local day (YYYY-MM-DD) it last started on — so it starts once a day, even if the app is restarted
|
||||
createdBy: text("created_by"),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
});
|
||||
|
||||
// One time a plan ran, and how it went for each server: items is JSON [{serverId, name, status, updateRunId, message, windowIds}].
|
||||
export const patchPlanRuns = sqliteTable(
|
||||
"patch_plan_runs",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
planId: integer("plan_id")
|
||||
.notNull()
|
||||
.references(() => patchPlans.id, { onDelete: "cascade" }),
|
||||
planName: text("plan_name").notNull(), // as it was, so a renamed plan's history still reads right
|
||||
trigger: text("trigger").notNull(), // schedule | manual
|
||||
startedBy: text("started_by"),
|
||||
status: text("status").notNull(), // running | done | failed | cancelled
|
||||
message: text("message"),
|
||||
items: text("items").notNull().default("[]"),
|
||||
startedAt: text("started_at").notNull(), // ISO
|
||||
finishedAt: text("finished_at"), // ISO
|
||||
},
|
||||
(t) => [index("patch_plan_runs_plan_idx").on(t.planId, t.startedAt), index("patch_plan_runs_status_idx").on(t.status)],
|
||||
);
|
||||
|
||||
// ─── The monthly report's long memory ───────────────────────────────────────
|
||||
|
||||
// One row per endpoint check per day: how many times it was asked, how many answered, and the answer times — kept for years, where the individual
|
||||
// results are kept for two weeks. What the monthly report's availability is worked from (services/endpointChecks.ts).
|
||||
export const endpointDaily = sqliteTable(
|
||||
"endpoint_daily",
|
||||
{
|
||||
checkId: integer("check_id")
|
||||
.notNull()
|
||||
.references(() => endpointChecks.id, { onDelete: "cascade" }),
|
||||
day: text("day").notNull(), // YYYY-MM-DD, UTC
|
||||
total: integer("total").notNull().default(0),
|
||||
ok: integer("ok").notNull().default(0),
|
||||
msSum: integer("ms_sum").notNull().default(0), // of the answers that were good and timed
|
||||
msCount: integer("ms_count").notNull().default(0),
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
|
||||
);
|
||||
|
||||
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
|
||||
// as far as the version that began watching (services/renewals.ts).
|
||||
export const renewals = sqliteTable(
|
||||
"renewals",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
kind: text("kind").notNull(), // certificate | domain | secret
|
||||
subject: text("subject").notNull(), // its name, as it was
|
||||
at: text("at").notNull(), // ISO — when it was seen
|
||||
fromDate: text("from_date").notNull(), // YYYY-MM-DD — the expiry before
|
||||
toDate: text("to_date").notNull(), // YYYY-MM-DD — and after
|
||||
},
|
||||
(t) => [index("renewals_at_idx").on(t.at)],
|
||||
);
|
||||
|
||||
// ─── Inventory: the hardware behind it all ──────────────────────────────────
|
||||
|
||||
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;
|
||||
|
||||
@@ -13,6 +13,7 @@ import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js
|
||||
import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
|
||||
import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
|
||||
import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js";
|
||||
import { initMonthlyReportScheduler } from "./services/monthlyReportScheduler.js";
|
||||
import { initMaintenanceEvents } from "./services/maintenanceEvents.js";
|
||||
import { initHealthScheduler } from "./services/healthScheduler.js";
|
||||
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
|
||||
@@ -21,6 +22,7 @@ import { initMaintenanceScheduleScheduler } from "./services/maintenanceSchedule
|
||||
import { initUpdateRuns } from "./services/updateRuns.js";
|
||||
import { initEscalation } from "./services/escalation.js";
|
||||
import { initEndpointMonitor } from "./services/endpointMonitor.js";
|
||||
import { initPatchPlans } from "./services/patchPlans.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
import { initIncidentRecorder } from "./services/incidents.js";
|
||||
import { getAlerts } from "./services/alerts.js";
|
||||
@@ -51,6 +53,7 @@ await initProxmoxBackupScheduler();
|
||||
await initPbsVerificationScheduler();
|
||||
await initQuietHoursScheduler();
|
||||
await initWeeklyDigestScheduler();
|
||||
await initMonthlyReportScheduler();
|
||||
initMaintenanceEvents();
|
||||
await initHealthScheduler();
|
||||
await initAlertTicketScheduler();
|
||||
@@ -61,6 +64,7 @@ await initSelfBackupScheduler();
|
||||
initIncidentRecorder(() => getAlerts(false));
|
||||
initEscalation();
|
||||
initEndpointMonitor();
|
||||
initPatchPlans();
|
||||
|
||||
createApp().listen(env.port, () => {
|
||||
console.log(`homelab-manager listening on port ${env.port}`);
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { patchPlans, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { isValidTimeZone } from "../services/maintenanceSchedules.js";
|
||||
import { activeRunOf, cancelPlanRun, listPlanRuns, MAX_PLANS, parseItems, planDays, planOccurrences, planServerIds, planServers, planTags, PlanError, startPlanRun, type PlanRow, type PlanRunRow } from "../services/patchPlans.js";
|
||||
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const patchPlansRouter = Router();
|
||||
|
||||
// Everyone signed in can see what is planned and how it went, like the Updates page. Making a plan is an administrator's (it will run on its own, at night);
|
||||
// starting one now or stopping one that is running is an operator's, like an update run.
|
||||
patchPlansRouter.use(requireAuth);
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
const planSchema = z
|
||||
.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
enabled: z.boolean().default(true),
|
||||
days: z.array(z.number().int().min(0).max(6)).min(1).max(7),
|
||||
startTime: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 02:00"),
|
||||
timezone: z.string().trim().min(1).max(60),
|
||||
tags: z.array(z.string()).max(12).default([]),
|
||||
serverIds: z.array(z.number().int().positive()).max(200).default([]),
|
||||
integrationId: z.number().int().positive(),
|
||||
projectId: z.number().int().positive(),
|
||||
templateId: z.number().int().positive(),
|
||||
templateName: z.string().trim().min(1).max(200),
|
||||
limitFrom: z.enum(["name", "hostname"]).default("name"),
|
||||
stopOnFailure: z.boolean().default(true),
|
||||
onlyWithUpdates: z.boolean().default(true),
|
||||
silenceDependents: z.boolean().default(true),
|
||||
windowMinutes: z.number().int().min(15).max(480).default(90),
|
||||
})
|
||||
.refine((d) => isValidTimeZone(d.timezone), { message: "That isn't a time zone the system knows (use a name like Europe/Stockholm).", path: ["timezone"] })
|
||||
.refine((d) => d.tags.length > 0 || d.serverIds.length > 0, { message: "Choose which servers: give a tag, or name at least one server.", path: ["tags"] });
|
||||
|
||||
function valuesOf(d: z.infer<typeof planSchema>) {
|
||||
return {
|
||||
name: d.name,
|
||||
enabled: d.enabled,
|
||||
days: JSON.stringify([...new Set(d.days)].sort()),
|
||||
startTime: d.startTime,
|
||||
timezone: d.timezone,
|
||||
tags: JSON.stringify(normalizeTags(d.tags)),
|
||||
serverIds: JSON.stringify([...new Set(d.serverIds)]),
|
||||
integrationId: d.integrationId,
|
||||
projectId: d.projectId,
|
||||
templateId: d.templateId,
|
||||
templateName: d.templateName,
|
||||
limitFrom: d.limitFrom,
|
||||
stopOnFailure: d.stopOnFailure,
|
||||
onlyWithUpdates: d.onlyWithUpdates,
|
||||
silenceDependents: d.silenceDependents,
|
||||
windowMinutes: d.windowMinutes,
|
||||
};
|
||||
}
|
||||
|
||||
const shapeRun = (r: PlanRunRow) => ({
|
||||
id: r.id,
|
||||
planId: r.planId,
|
||||
planName: r.planName,
|
||||
trigger: r.trigger,
|
||||
startedBy: r.startedBy,
|
||||
status: r.status,
|
||||
message: r.message,
|
||||
startedAt: r.startedAt,
|
||||
finishedAt: r.finishedAt,
|
||||
items: parseItems(r.items).map(({ windowIds: _w, ...i }) => i),
|
||||
});
|
||||
|
||||
async function shapePlan(p: PlanRow, all: { id: number; name: string; tags: string | null }[], now: Date) {
|
||||
const covered = planServers(p, all);
|
||||
const next = planOccurrences(p, now, new Date(now.getTime() + 15 * 86_400_000)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 3);
|
||||
const active = await activeRunOf(p.id);
|
||||
return {
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
enabled: p.enabled,
|
||||
days: planDays(p),
|
||||
startTime: p.startTime,
|
||||
timezone: p.timezone,
|
||||
tags: planTags(p),
|
||||
serverIds: planServerIds(p),
|
||||
integrationId: p.integrationId,
|
||||
projectId: p.projectId,
|
||||
templateId: p.templateId,
|
||||
templateName: p.templateName,
|
||||
limitFrom: p.limitFrom,
|
||||
stopOnFailure: p.stopOnFailure,
|
||||
onlyWithUpdates: p.onlyWithUpdates,
|
||||
silenceDependents: p.silenceDependents,
|
||||
windowMinutes: p.windowMinutes,
|
||||
createdBy: p.createdBy,
|
||||
servers: covered,
|
||||
next: next.map((o) => o.start.toISOString()),
|
||||
running: active !== null,
|
||||
};
|
||||
}
|
||||
|
||||
patchPlansRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const now = new Date();
|
||||
const rows = await db.select().from(patchPlans).orderBy(patchPlans.name);
|
||||
const all = await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers);
|
||||
const tags = [...new Set(all.flatMap((s) => parseTags(s.tags)))].sort((a, b) => a.localeCompare(b, "sv"));
|
||||
res.json({
|
||||
plans: await Promise.all(rows.map((p) => shapePlan(p, all, now))),
|
||||
runs: (await listPlanRuns({ limit: 15 })).map(shapeRun),
|
||||
servers: all.map((s) => ({ id: s.id, name: s.name, tags: parseTags(s.tags) })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
tags,
|
||||
});
|
||||
}));
|
||||
|
||||
/** The message for a rejected body, as a sentence. */
|
||||
const reason = (parsed: { error: z.ZodError }) => parsed.error.issues[0]?.message ?? "That isn't a plan.";
|
||||
|
||||
patchPlansRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = planSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: reason(parsed), details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: patchPlans.id }).from(patchPlans)).length >= MAX_PLANS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_PLANS} plans.` });
|
||||
let values;
|
||||
try {
|
||||
values = valuesOf(parsed.data);
|
||||
} catch (err) {
|
||||
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
|
||||
throw err;
|
||||
}
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db.insert(patchPlans).values({ ...values, createdBy: actor.name ?? actor.email ?? actor.oidcSub }).returning();
|
||||
await recordAudit({ actor, category: "patch_plan", action: "create", targetType: "patch_plan", targetId: row.id, detail: { name: row.name, days: planDays(row), time: row.startTime } });
|
||||
res.status(201).json({ plan: await shapePlan(row, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers), new Date()) });
|
||||
}));
|
||||
|
||||
patchPlansRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = planSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? "That isn't a plan." : reason(parsed) });
|
||||
const [before] = await db.select().from(patchPlans).where(eq(patchPlans.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
let values;
|
||||
try {
|
||||
values = valuesOf(parsed.data);
|
||||
} catch (err) {
|
||||
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
|
||||
throw err;
|
||||
}
|
||||
// A changed day or time is a new schedule: what it last started on no longer says whether today's has happened.
|
||||
const rescheduled = values.days !== before.days || values.startTime !== before.startTime || values.timezone !== before.timezone;
|
||||
const [row] = await db.update(patchPlans).set({ ...values, ...(rescheduled ? { lastRunDate: null } : {}) }).where(eq(patchPlans.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "patch_plan", action: "update", targetType: "patch_plan", targetId: id, detail: { name: row.name, days: planDays(row), time: row.startTime } });
|
||||
res.json({ plan: await shapePlan(row, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers), new Date()) });
|
||||
}));
|
||||
|
||||
patchPlansRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(patchPlans).where(eq(patchPlans.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (await activeRunOf(id)) return res.status(409).json({ error: "running", message: "It is running now. Stop the run first." });
|
||||
await db.delete(patchPlans).where(eq(patchPlans.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "patch_plan", action: "delete", targetType: "patch_plan", targetId: id, detail: { name: before.name } });
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// Run it now, instead of at its time (it still starts again at its time).
|
||||
patchPlansRouter.post("/:id/run", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const run = await startPlanRun(id, { trigger: "manual", actor: actor.name ?? actor.email ?? actor.oidcSub });
|
||||
res.status(201).json({ run: shapeRun(run) });
|
||||
} catch (err) {
|
||||
if (err instanceof PlanError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
|
||||
patchPlansRouter.post("/runs/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const actor = req.currentUser!;
|
||||
try {
|
||||
const run = await cancelPlanRun(id, actor.name ?? actor.email ?? actor.oidcSub);
|
||||
if (!run) return res.status(404).json({ error: "not_found" });
|
||||
res.json({ run: shapeRun(run) });
|
||||
} catch (err) {
|
||||
if (err instanceof PlanError) return res.status(err.status).json({ error: err.code, message: err.message });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { buildMonthlyReport, isMonthId, lastFullMonth, monthlyReportText, recentMonths } from "../services/monthlyReport.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { notifyNow } from "../services/notify.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const reportsRouter = Router();
|
||||
|
||||
reportsRouter.use(requireAuth);
|
||||
|
||||
/** The month asked for, or the last whole one; null when what was asked is not a month. */
|
||||
async function monthFrom(query: unknown): Promise<{ id: string; months: { id: string; label: string }[] } | null> {
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const months = recentMonths(new Date(), tz);
|
||||
if (query === undefined || query === "") return { id: lastFullMonth(new Date(), tz), months };
|
||||
if (!isMonthId(query) || !months.some((m) => m.id === query)) return null;
|
||||
return { id: query, months };
|
||||
}
|
||||
|
||||
// How a month went, for everyone signed in: it is only what the pages it comes from already show them. ?month=2026-09, the last whole month by default;
|
||||
// ?format=text is the plain-text version a notification carries.
|
||||
reportsRouter.get("/monthly", asyncHandler(async (req, res) => {
|
||||
const picked = await monthFrom(req.query.month);
|
||||
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
|
||||
const report = await buildMonthlyReport(picked.id);
|
||||
if (req.query.format === "text") return res.json(monthlyReportText(report));
|
||||
res.json({ report, months: picked.months });
|
||||
}));
|
||||
|
||||
// Send it now to the notification channels, as if it were the day. Administrators.
|
||||
reportsRouter.post("/monthly/send", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const picked = await monthFrom((req.body as { month?: unknown } | undefined)?.month);
|
||||
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
|
||||
const { title, message } = monthlyReportText(await buildMonthlyReport(picked.id));
|
||||
await notifyNow(title, message);
|
||||
await recordAudit({ actor: req.currentUser!, category: "report", action: "send_monthly", targetType: "report", detail: { month: picked.id } });
|
||||
res.json({ sent: true });
|
||||
}));
|
||||
@@ -7,6 +7,8 @@ import { requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { beginScan, endScan, MAX_SCAN_SPAN, resolveScanTarget, scanPorts, toRanges } from "../services/portScan.js";
|
||||
import { type AgentPort, groupAgentPorts, parseJson } from "../services/agentPorts.js";
|
||||
import { newListeningPorts, parseBaseline, portKey } from "../services/newPorts.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
// Mounted under /api/servers/:id/ports by the servers router, which has already required a signed-in user.
|
||||
@@ -37,6 +39,8 @@ export interface PortEntry {
|
||||
agent: { addresses: string[]; process: string | null; localOnly: boolean } | null;
|
||||
/** "open" if anything is using it; "reserved" if it only has a note. */
|
||||
state: "open" | "reserved";
|
||||
/** Began listening after the first report and nobody has noted it: a "new port" alert is on until it is noted or stops listening. */
|
||||
isNew: boolean;
|
||||
}
|
||||
|
||||
async function buildPortList(serverId: number) {
|
||||
@@ -46,6 +50,11 @@ async function buildPortList(serverId: number) {
|
||||
const agentRaw = parseJson<AgentPort[] | null>(server.listeningPorts, null);
|
||||
const agent = groupAgentPorts(agentRaw ?? []);
|
||||
|
||||
// What is new: listening, reachable from outside, not in the baseline and not noted — the same rule the alert uses.
|
||||
const expected = new Set(rows.filter((r) => r.label || r.comment).map((r) => portKey(r.protocol, r.port)));
|
||||
const alerting = (await getSettings()).healthChecks.newPortAlerts;
|
||||
const fresh = new Set(alerting ? newListeningPorts(agentRaw, parseBaseline(server.portBaseline), expected).map((p) => portKey(p.protocol, p.port)) : []);
|
||||
|
||||
const entries = new Map<string, PortEntry>();
|
||||
for (const row of rows) {
|
||||
const key = `${row.protocol}:${row.port}`;
|
||||
@@ -59,6 +68,7 @@ async function buildPortList(serverId: number) {
|
||||
lastSeenOpenAt: row.lastSeenOpenAt,
|
||||
agent: agent.get(key) ?? null,
|
||||
state: "reserved",
|
||||
isNew: fresh.has(key),
|
||||
});
|
||||
}
|
||||
for (const [key, info] of agent) {
|
||||
@@ -74,6 +84,7 @@ async function buildPortList(serverId: number) {
|
||||
lastSeenOpenAt: null,
|
||||
agent: info,
|
||||
state: "reserved",
|
||||
isNew: fresh.has(key),
|
||||
});
|
||||
}
|
||||
for (const entry of entries.values()) {
|
||||
|
||||
@@ -13,6 +13,7 @@ import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js"
|
||||
import { purgeOldLogs } from "../services/logRetention.js";
|
||||
import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
|
||||
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
|
||||
import { scheduleMonthlyReport } from "../services/monthlyReportScheduler.js";
|
||||
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
|
||||
import { enabledChannelCount } from "../services/notify.js";
|
||||
import { CATEGORIES, CHANNEL_IDS } from "../services/notifyRouting.js";
|
||||
@@ -143,7 +144,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean() })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
@@ -165,6 +166,10 @@ const updateSchema = z.object({
|
||||
.object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
|
||||
.partial()
|
||||
.optional(),
|
||||
monthlyReport: z
|
||||
.object({ enabled: z.boolean(), day: z.number().int().min(1).max(28), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
|
||||
.partial()
|
||||
.optional(),
|
||||
});
|
||||
|
||||
settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
@@ -200,6 +205,9 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
if (parsed.data.digest || parsed.data.notifications) {
|
||||
await scheduleWeeklyDigest();
|
||||
}
|
||||
if (parsed.data.monthlyReport || parsed.data.notifications) {
|
||||
await scheduleMonthlyReport();
|
||||
}
|
||||
|
||||
await recordAudit({
|
||||
actor: req.currentUser!,
|
||||
|
||||
@@ -9,10 +9,11 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
|
||||
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, patchPlans, secrets, servers } from "../db/schema.js";
|
||||
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
|
||||
import { planDays, planOccurrences, planServers } from "./patchPlans.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
@@ -88,6 +89,8 @@ export interface AgendaData {
|
||||
/** Windows that have been set up but not begun, one per target. */
|
||||
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
|
||||
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
|
||||
/** Patch plans that are switched on, with how many servers each covers now. */
|
||||
patchPlans: { id: number; name: string; days: string; startTime: string; timezone: string; servers: number }[];
|
||||
/** The newest restore test of each subject. */
|
||||
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
|
||||
restoreLimitDays: number;
|
||||
@@ -140,6 +143,10 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
|
||||
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
|
||||
}
|
||||
|
||||
const planRows = (await db.select().from(patchPlans)).filter((p) => p.enabled);
|
||||
const allServers = planRows.length > 0 ? await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers) : [];
|
||||
const plans: AgendaData["patchPlans"] = planRows.map((p) => ({ id: p.id, name: p.name, days: p.days, startTime: p.startTime, timezone: p.timezone, servers: planServers(p, allServers).length }));
|
||||
|
||||
const updates: AgendaData["updates"] = [];
|
||||
for (const s of serverRows) {
|
||||
const u = parsePendingUpdates(s.pendingUpdates);
|
||||
@@ -163,6 +170,7 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
|
||||
assets: assetRows,
|
||||
windows,
|
||||
schedules,
|
||||
patchPlans: plans,
|
||||
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
|
||||
restoreLimitDays: settings.healthChecks.restoreTestDays,
|
||||
updates,
|
||||
@@ -223,7 +231,7 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
|
||||
g.names.push(w.targetName);
|
||||
groups.set(key, g);
|
||||
}
|
||||
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
|
||||
const timed = (id: string, start: string, end: string | null, title: string, detail: string | null) =>
|
||||
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
|
||||
const names = (list: string[]) => {
|
||||
const sorted = [...list].sort((a, b) => a.localeCompare(b));
|
||||
@@ -236,6 +244,13 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
|
||||
}
|
||||
}
|
||||
|
||||
// A patch plan is maintenance too: when it will start and on how many servers.
|
||||
for (const p of data.patchPlans) {
|
||||
for (const occ of planOccurrences(p, now, new Date(lastMs)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 60)) {
|
||||
timed(`plan:${p.id}:${occ.date}`, occ.start.toISOString(), null, `Patch plan: ${p.name}`, `updates ${plural(p.servers, "server")}, one at a time`);
|
||||
}
|
||||
}
|
||||
|
||||
// Updates that have been waiting: one line for all of them, from the longest wait.
|
||||
if (data.updates.length > 0) {
|
||||
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
|
||||
|
||||
@@ -79,8 +79,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
|
||||
|
||||
/** Which kind of source, and which one, a health/automation condition key belongs to. */
|
||||
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
|
||||
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
|
||||
|
||||
@@ -54,6 +54,8 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
restore_test: "backups",
|
||||
endpoint: "endpoints",
|
||||
heartbeat: "endpoints",
|
||||
patch_plan: "servers",
|
||||
report: "other",
|
||||
};
|
||||
|
||||
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
|
||||
@@ -155,6 +157,22 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
|
||||
case "endpoint/delete":
|
||||
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "report/send_monthly":
|
||||
return { title: `Monthly report sent${str(detail.month) ? ` (${str(detail.month)})` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/create":
|
||||
return { title: `Patch plan added${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/update":
|
||||
return { title: `Patch plan changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/delete":
|
||||
return { title: `Patch plan removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "patch_plan/run":
|
||||
return { title: `Patch plan started${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.trigger) === "schedule" ? "by its schedule" : str(detail.by) ? `by ${str(detail.by)}` : null };
|
||||
case "patch_plan/finished":
|
||||
return { title: `Patch plan finished${name ? `: ${name}` : ""}`, tone: "good", detail: str(detail.message) };
|
||||
case "patch_plan/failed":
|
||||
return { title: `Patch plan didn't finish cleanly${name ? `: ${name}` : ""}`, tone: "bad", detail: str(detail.message) };
|
||||
case "patch_plan/cancelled":
|
||||
return { title: `Patch plan stopped${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.message) };
|
||||
case "heartbeat/create":
|
||||
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
|
||||
case "heartbeat/update":
|
||||
@@ -260,6 +278,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
|
||||
targetName = endpointName.get(idNum) ?? targetName;
|
||||
link = "/endpoints";
|
||||
} else if (a.targetType === "heartbeat") link = "/heartbeats";
|
||||
else if (a.targetType === "patch_plan") link = "/updates?view=plans";
|
||||
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
else if (a.targetType === "restore_test") link = "/backups";
|
||||
else if (a.targetType === "dependency") link = "/map?view=dependencies";
|
||||
|
||||
@@ -5,6 +5,7 @@ import { defaultDeps, lookupRegistration, normalizeDomain, resolveRegistration,
|
||||
import { computeSecretStatus } from "./secretStatus.js";
|
||||
import { notifyDomainExpiry } from "./notify.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
|
||||
export type DomainRow = typeof domains.$inferSelect;
|
||||
|
||||
@@ -48,6 +49,8 @@ export async function checkDomain(id: number, deps: LookupDeps = defaultDeps): P
|
||||
if (!row) return null;
|
||||
const result = await lookupRegistration(row.name, deps);
|
||||
const [updated] = await db.update(domains).set(outcomeFields(result, row, new Date(deps.now()).toISOString())).where(eq(domains.id, id)).returning();
|
||||
// A registration that now runs much longer than it did was renewed.
|
||||
await noteRenewal("domain", row.name, row.expiresAt, updated.expiresAt, new Date(deps.now()));
|
||||
return updated;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,9 +12,10 @@ import https from "node:https";
|
||||
import net from "node:net";
|
||||
import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
|
||||
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointResults } from "../db/schema.js";
|
||||
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
|
||||
@@ -244,9 +245,33 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
|
||||
})
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
await addToDaily(c.id, iso, result);
|
||||
// A certificate that is now good for much longer than it was has been renewed.
|
||||
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** How long the daily figures are kept: years, where the individual results are kept two weeks. */
|
||||
export const KEEP_DAILY_DAYS = 800;
|
||||
|
||||
/** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */
|
||||
async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
|
||||
const timed = result.ok && result.ms !== null;
|
||||
const ok = result.ok ? 1 : 0;
|
||||
const ms = timed ? (result.ms as number) : 0;
|
||||
try {
|
||||
await db
|
||||
.insert(endpointDaily)
|
||||
.values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 })
|
||||
.onConflictDoUpdate({
|
||||
target: [endpointDaily.checkId, endpointDaily.day],
|
||||
set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[endpoints] couldn't add to the daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
@@ -264,6 +289,7 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
|
||||
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, servers } from "../db/schema.js";
|
||||
import { integrations, serverPorts, servers } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -11,6 +11,8 @@ import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js";
|
||||
import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js";
|
||||
import { parseJson, type AgentPort } from "./agentPorts.js";
|
||||
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
|
||||
|
||||
const STATE_FLAG = HEALTH_STATE_FLAG;
|
||||
@@ -39,6 +41,8 @@ export interface ServerSnapshot {
|
||||
health?: HostHealth | null;
|
||||
/** This server's own thresholds, where it has any: they replace the global ones for it. */
|
||||
thresholds?: ServerThresholds | null;
|
||||
/** TCP ports that began listening after the baseline and that nobody has noted (services/newPorts.ts); empty when the check is off. */
|
||||
newPorts?: NewPort[];
|
||||
}
|
||||
export interface ProxmoxSnapshot {
|
||||
integrationId: number;
|
||||
@@ -123,6 +127,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
|
||||
if (s.lastSeenAt && !offline) {
|
||||
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
|
||||
for (const p of s.newPorts ?? []) out.push({ key: `new-port:server:${s.id}:${portKey(p.protocol, p.port)}`, source: "server", message: newPortMessage(s.name, p), severity: "warning" });
|
||||
for (const d of s.disks) {
|
||||
const p = usage(d.usedBytes, d.sizeBytes);
|
||||
// A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one.
|
||||
@@ -299,6 +304,17 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
const held = new Set<string>();
|
||||
const snapshot: HealthSnapshot = { servers: [], proxmox: [], synology: [] };
|
||||
|
||||
const { healthChecks: checks } = await getSettings();
|
||||
// The ports somebody has said are expected (a label or a comment on the Ports page); a row that only records an open scan says nothing.
|
||||
const expectedByServer = new Map<number, Set<string>>();
|
||||
if (checks.newPortAlerts) {
|
||||
for (const r of await db.select().from(serverPorts)) {
|
||||
if (!r.label && !r.comment) continue;
|
||||
const set = expectedByServer.get(r.serverId) ?? new Set<string>();
|
||||
set.add(portKey(r.protocol, r.port));
|
||||
expectedByServer.set(r.serverId, set);
|
||||
}
|
||||
}
|
||||
for (const s of await db.select().from(servers)) {
|
||||
let disks: ServerSnapshot["disks"] = [];
|
||||
try {
|
||||
@@ -306,7 +322,8 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
} catch {
|
||||
// a malformed report shouldn't stop every other server being checked
|
||||
}
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds) });
|
||||
const newPorts = checks.newPortAlerts ? newListeningPorts(parseJson<AgentPort[] | null>(s.listeningPorts, null), parseBaseline(s.portBaseline), expectedByServer.get(s.id) ?? new Set()) : [];
|
||||
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts });
|
||||
}
|
||||
|
||||
const rows = await db
|
||||
|
||||
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
* so conditions persisted by an earlier version still map correctly.
|
||||
*/
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
/**
|
||||
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
|
||||
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
|
||||
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
|
||||
* weekly digest.
|
||||
*
|
||||
* What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily
|
||||
* figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the
|
||||
* state now, not a history. A month with nothing to report for a part leaves that part out.
|
||||
*/
|
||||
import { and, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js";
|
||||
import { getCachedBackupCoverage } from "./backupCoverage.js";
|
||||
import { listIncidents } from "./incidents.js";
|
||||
import { zonedTime } from "./maintenanceSchedules.js";
|
||||
import { renewalsBetween } from "./renewals.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
import { spanWords } from "./weeklyDigest.js";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const MAX_LISTED = 12;
|
||||
|
||||
export interface AvailabilityRow {
|
||||
id: number;
|
||||
name: string;
|
||||
/** Share of the known time spent up, in percent; null when nothing is known for the month. */
|
||||
percent: number | null;
|
||||
downMinutes: number;
|
||||
}
|
||||
|
||||
export interface EndpointRow {
|
||||
id: number;
|
||||
name: string;
|
||||
percent: number | null;
|
||||
avgMs: number | null;
|
||||
checks: number;
|
||||
}
|
||||
|
||||
export interface MonthlyReport {
|
||||
/** "2026-09". */
|
||||
month: string;
|
||||
/** "September 2026". */
|
||||
label: string;
|
||||
/** ISO — the start of the month and its end (or now, for a month that is not over). */
|
||||
from: string;
|
||||
to: string;
|
||||
/** The month is not over yet: everything is "so far". */
|
||||
partial: boolean;
|
||||
timezone: string;
|
||||
servers: AvailabilityRow[];
|
||||
integrations: AvailabilityRow[];
|
||||
endpoints: EndpointRow[];
|
||||
incidents: {
|
||||
began: number;
|
||||
resolved: number;
|
||||
stillOpen: number;
|
||||
/** Of those that began and ended in the month. */
|
||||
totalMinutes: number;
|
||||
averageMinutes: number | null;
|
||||
longest: { id: number; message: string; minutes: number } | null;
|
||||
byCategory: { category: string; count: number }[];
|
||||
/** The longest, up to a dozen. */
|
||||
list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[];
|
||||
};
|
||||
updates: {
|
||||
runs: number;
|
||||
done: number;
|
||||
failed: number;
|
||||
/** Updates the runs say they installed (before minus after, where both are known). */
|
||||
installed: number;
|
||||
byServer: { name: string; runs: number; failed: number; installed: number }[];
|
||||
planRuns: { runs: number; done: number; failed: number };
|
||||
};
|
||||
restoreTests: { label: string; testedAt: string; result: string; note: string | null }[];
|
||||
/** The backup state as of now; null when no coverage has been read yet. */
|
||||
backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null;
|
||||
renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[];
|
||||
}
|
||||
|
||||
// ─── Months ────────────────────────────────────────────────────────────────
|
||||
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
|
||||
export function isMonthId(v: unknown): v is string {
|
||||
return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v);
|
||||
}
|
||||
|
||||
/** The year and month `now` is in, on the clock of `tz`. */
|
||||
function monthOf(now: Date, tz: string): { year: number; month: number } {
|
||||
const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now);
|
||||
return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) };
|
||||
}
|
||||
|
||||
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
|
||||
const shift = (year: number, month: number, by: number) => {
|
||||
const index = year * 12 + (month - 1) + by;
|
||||
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
|
||||
};
|
||||
|
||||
export function monthLabel(id: string): string {
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
|
||||
}
|
||||
|
||||
/** The month before the one `now` is in: what a report sent at the start of a month is about. */
|
||||
export function lastFullMonth(now: Date, tz: string): string {
|
||||
const cur = monthOf(now, tz);
|
||||
const prev = shift(cur.year, cur.month, -1);
|
||||
return monthId(prev.year, prev.month);
|
||||
}
|
||||
|
||||
/** This month and the eleven before it, newest first. */
|
||||
export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] {
|
||||
const cur = monthOf(now, tz);
|
||||
return Array.from({ length: count }, (_, i) => {
|
||||
const m = shift(cur.year, cur.month, -i);
|
||||
const id = monthId(m.year, m.month);
|
||||
return { id, label: monthLabel(id) };
|
||||
});
|
||||
}
|
||||
|
||||
/** The instants a month begins and ends at, on the clock of `tz`. */
|
||||
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
|
||||
const [y, m] = id.split("-").map(Number);
|
||||
const next = shift(y, m, 1);
|
||||
return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) };
|
||||
}
|
||||
|
||||
// ─── Gathering ─────────────────────────────────────────────────────────────
|
||||
|
||||
const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
|
||||
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
|
||||
const settings = await getSettings();
|
||||
const tz = settings.notifications.timezone || "UTC";
|
||||
const { start, end: fullEnd } = monthBounds(id, tz);
|
||||
const partial = fullEnd.getTime() > now.getTime();
|
||||
const end = partial ? now : fullEnd;
|
||||
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
|
||||
const startIso = start.toISOString();
|
||||
const endIso = end.toISOString();
|
||||
|
||||
// Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does.
|
||||
const uptime = await getUptime(end, days);
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
|
||||
const rowsOf = (series: Record<number, { buckets: { k: number; d: number }[]; percent: number }>, names: Map<number, string>): AvailabilityRow[] =>
|
||||
Object.entries(series)
|
||||
.map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) }))
|
||||
.filter((r) => names.has(r.id))
|
||||
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
|
||||
|
||||
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
|
||||
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
|
||||
const [yy, mm] = id.split("-").map(Number);
|
||||
const following = shift(yy, mm, 1);
|
||||
const firstDay = `${id}-01`;
|
||||
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
|
||||
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
|
||||
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
|
||||
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
|
||||
for (const r of dailyRows) {
|
||||
const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 };
|
||||
t.total += r.total;
|
||||
t.ok += r.ok;
|
||||
t.msSum += r.msSum;
|
||||
t.msCount += r.msCount;
|
||||
perCheck.set(r.checkId, t);
|
||||
}
|
||||
const endpointRows: EndpointRow[] = [...perCheck]
|
||||
.filter(([cid, t]) => checkNames.has(cid) && t.total > 0)
|
||||
.map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total }))
|
||||
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
|
||||
|
||||
// Incidents.
|
||||
const all = await listIncidents({ now: end, limit: 3000 });
|
||||
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
|
||||
const began = all.filter((i) => inMonth(i.openedAt));
|
||||
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
|
||||
const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length;
|
||||
const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
|
||||
const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000));
|
||||
const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0);
|
||||
const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs);
|
||||
const categories = new Map<string, number>();
|
||||
for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1);
|
||||
|
||||
// Updates, by hand and by plan.
|
||||
const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso)));
|
||||
const perServer = new Map<number, { runs: number; failed: number; installed: number }>();
|
||||
let installed = 0;
|
||||
for (const r of runRows) {
|
||||
const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0;
|
||||
installed += gained;
|
||||
const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 };
|
||||
t.runs++;
|
||||
if (r.status === "failed") t.failed++;
|
||||
t.installed += gained;
|
||||
perServer.set(r.serverId, t);
|
||||
}
|
||||
const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso)));
|
||||
|
||||
// Restore tests, and the backups as they are now.
|
||||
const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive)));
|
||||
const coverage = await getCachedBackupCoverage(now);
|
||||
|
||||
const renewalRows = await renewalsBetween(start, end);
|
||||
|
||||
return {
|
||||
month: id,
|
||||
label: monthLabel(id),
|
||||
from: startIso,
|
||||
to: endIso,
|
||||
partial,
|
||||
timezone: tz,
|
||||
servers: rowsOf(uptime.servers, serverNames),
|
||||
integrations: rowsOf(uptime.integrations, integrationNames),
|
||||
endpoints: endpointRows,
|
||||
incidents: {
|
||||
began: began.length,
|
||||
resolved: resolved.length,
|
||||
stillOpen,
|
||||
totalMinutes,
|
||||
averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null,
|
||||
longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null,
|
||||
byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)),
|
||||
list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })),
|
||||
},
|
||||
updates: {
|
||||
runs: runRows.length,
|
||||
done: runRows.filter((r) => r.status === "done").length,
|
||||
failed: runRows.filter((r) => r.status === "failed").length,
|
||||
installed,
|
||||
byServer: [...perServer]
|
||||
.map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t }))
|
||||
.sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)),
|
||||
planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length },
|
||||
},
|
||||
restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)),
|
||||
backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null,
|
||||
renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── As a message ──────────────────────────────────────────────────────────
|
||||
|
||||
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
const RESULT_WORDS: Record<string, string> = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" };
|
||||
|
||||
function average(rows: AvailabilityRow[]): number | null {
|
||||
const known = rows.filter((r) => r.percent !== null);
|
||||
return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length);
|
||||
}
|
||||
|
||||
/** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */
|
||||
export function monthlyReportText(r: MonthlyReport): { title: string; message: string } {
|
||||
const sections: string[] = [];
|
||||
const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`);
|
||||
|
||||
const avail: string[] = [];
|
||||
const serverAvg = average(r.servers);
|
||||
if (serverAvg !== null) {
|
||||
avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`);
|
||||
for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`);
|
||||
}
|
||||
if (r.endpoints.length > 0) {
|
||||
const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length);
|
||||
avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`);
|
||||
for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`);
|
||||
}
|
||||
const integrationAvg = average(r.integrations);
|
||||
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
|
||||
lines("Availability", avail);
|
||||
|
||||
const inc = r.incidents;
|
||||
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
|
||||
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
|
||||
if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`);
|
||||
for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`);
|
||||
lines("Incidents", body);
|
||||
} else if (serverAvg !== null || r.endpoints.length > 0) {
|
||||
lines("Incidents", ["None."]);
|
||||
}
|
||||
|
||||
const u = r.updates;
|
||||
const upd: string[] = [];
|
||||
if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`);
|
||||
if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`);
|
||||
lines("Updates", upd);
|
||||
|
||||
const bk: string[] = [];
|
||||
for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`);
|
||||
if (r.restoreTests.length === 0) bk.push("No restore test was recorded.");
|
||||
if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`);
|
||||
lines("Backups", bk);
|
||||
|
||||
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
|
||||
|
||||
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
|
||||
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import schedule from "node-schedule";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { notify } from "./notify.js";
|
||||
import { buildMonthlyReport, lastFullMonth, monthlyReportText } from "./monthlyReport.js";
|
||||
|
||||
let currentJob: schedule.Job | null = null;
|
||||
|
||||
/** The cron rule for "this day of the month at HH:MM" (1–28, so every month has it). */
|
||||
export function cronForMonthlyReport(day: number, time: string): string {
|
||||
const [h, m] = time.split(":").map(Number);
|
||||
const d = Number.isInteger(day) && day >= 1 && day <= 28 ? day : 1;
|
||||
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} ${d} * *`;
|
||||
}
|
||||
|
||||
/** Builds last month's report and sends it through the channels (held with other notifications during quiet hours). */
|
||||
export async function sendMonthlyReport(now: Date = new Date()): Promise<{ title: string; message: string }> {
|
||||
const tz = (await getSettings()).notifications.timezone || "UTC";
|
||||
const text = monthlyReportText(await buildMonthlyReport(lastFullMonth(now, tz), now));
|
||||
await notify(text.title, text.message);
|
||||
return text;
|
||||
}
|
||||
|
||||
/** (Re)schedules the monthly report per the current settings. Call again after they change. */
|
||||
export async function scheduleMonthlyReport(): Promise<void> {
|
||||
if (currentJob) {
|
||||
currentJob.cancel();
|
||||
currentJob = null;
|
||||
}
|
||||
const { monthlyReport, notifications } = await getSettings();
|
||||
if (!monthlyReport.enabled) {
|
||||
console.log("[report] monthly report is off, nothing scheduled");
|
||||
return;
|
||||
}
|
||||
const rule = cronForMonthlyReport(monthlyReport.day, monthlyReport.time);
|
||||
currentJob = schedule.scheduleJob({ rule, tz: notifications.timezone }, () => {
|
||||
sendMonthlyReport().catch((err) => console.error("[report] couldn't send the monthly report:", err));
|
||||
});
|
||||
console.log(`Monthly report scheduled: ${rule} (${notifications.timezone})`);
|
||||
}
|
||||
|
||||
export async function initMonthlyReportScheduler(): Promise<void> {
|
||||
await scheduleMonthlyReport();
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* New ports: a server that starts listening on something it did not before is worth a look — a service somebody started, or something that should not be
|
||||
* there. The first report that carries ports is taken as the baseline: what the server was listening on, taken as normal. After that a TCP port that is
|
||||
* listening on an address other than this machine's own, is not in the baseline, and has not been noted on the Ports page, is a warning (the Alerts page,
|
||||
* and a notification) until the port is noted ("this is expected") or stops listening.
|
||||
*
|
||||
* Left out on purpose: UDP (clients and discovery open them all the time), ports in the dynamic range (49152 and up, where programs pick a free one), and
|
||||
* anything bound only to the loopback address, which nothing else on the network can reach.
|
||||
*/
|
||||
import { isLoopback, type AgentPort } from "./agentPorts.js";
|
||||
|
||||
/** From here up, a port is usually one a program picked for itself, not a service. */
|
||||
export const DYNAMIC_PORTS_FROM = 49152;
|
||||
|
||||
export interface NewPort {
|
||||
protocol: "tcp";
|
||||
port: number;
|
||||
/** What is listening, as the agent says; null when it does not. */
|
||||
process: string | null;
|
||||
}
|
||||
|
||||
export const portKey = (protocol: string, port: number) => `${protocol}:${port}`;
|
||||
|
||||
/** What is stored, read back as a set of "tcp:443"; null when there is no baseline yet (or it can't be read). */
|
||||
export function parseBaseline(stored: string | null): Set<string> | null {
|
||||
if (!stored) return null;
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? new Set(v.filter((k): k is string => typeof k === "string")) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The baseline a first report makes: everything it listens on, in any protocol, so nothing about it is ever "new". */
|
||||
export function baselineFrom(listening: Pick<AgentPort, "protocol" | "port">[]): string {
|
||||
return JSON.stringify([...new Set(listening.map((p) => portKey(p.protocol, p.port)))].sort());
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the ports that are new — listening now, reachable from outside this machine, in neither the baseline nor the noted ports. `expected` is the set of
|
||||
* "tcp:8080" somebody gave a label or a comment on the Ports page. With no baseline yet, nothing is new.
|
||||
*/
|
||||
export function newListeningPorts(listening: AgentPort[] | null | undefined, baseline: Set<string> | null, expected: Set<string>): NewPort[] {
|
||||
if (!listening || baseline === null) return [];
|
||||
const found = new Map<number, NewPort>();
|
||||
for (const p of listening) {
|
||||
if (p.protocol !== "tcp" || !Number.isInteger(p.port) || p.port <= 0 || p.port >= DYNAMIC_PORTS_FROM) continue;
|
||||
if (isLoopback(p.address ?? "")) continue;
|
||||
const key = portKey("tcp", p.port);
|
||||
if (baseline.has(key) || expected.has(key)) continue;
|
||||
const seen = found.get(p.port);
|
||||
if (seen) seen.process = seen.process ?? (p.process || null);
|
||||
else found.set(p.port, { protocol: "tcp", port: p.port, process: p.process || null });
|
||||
}
|
||||
return [...found.values()].sort((a, b) => a.port - b.port);
|
||||
}
|
||||
|
||||
export const newPortMessage = (serverName: string, p: NewPort) =>
|
||||
`${serverName}: a new port is listening — ${p.port}/tcp${p.process ? ` (${p.process})` : ""}. If it should be there, note it on the server's Ports card.`;
|
||||
@@ -449,6 +449,16 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
|
||||
);
|
||||
}
|
||||
|
||||
/** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */
|
||||
export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise<void> {
|
||||
if (!(await eventEnabled("updateRunResults"))) return;
|
||||
await notify(
|
||||
run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`,
|
||||
`${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`,
|
||||
{ category: "updates", severity: run.ok ? "warning" : "critical" },
|
||||
);
|
||||
}
|
||||
|
||||
export async function notifyPbsVerificationFailed(
|
||||
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
|
||||
): Promise<void> {
|
||||
|
||||
@@ -107,11 +107,12 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
let category: AlertCategory | null = null;
|
||||
if (key.startsWith("offline:")) category = "offline";
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (key.startsWith("new-port:")) category = "monitoring";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
else if (key.startsWith("heartbeat")) category = "automation";
|
||||
else if (key.startsWith("endpoint-cert:")) category = "expiry";
|
||||
else if (key.startsWith("endpoint:")) category = "monitoring";
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs|new-port):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
/**
|
||||
* Patch plans: "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure."
|
||||
*
|
||||
* A plan only says when, to whom and how. When it is due (or somebody presses Run now) it makes a *run*: a list of the servers it matches, and goes down
|
||||
* the list one server at a time. Each server is updated by an ordinary update run (services/updateRuns.ts) — the server is put under maintenance, the
|
||||
* Semaphore playbook is run limited to it, the app waits for it to report again — and the plan waits for that to finish before it starts the next. What
|
||||
* depends on the server is silenced for the same time (it will look down while the server restarts). If a server's update fails and the plan says to stop,
|
||||
* the servers after it are left alone and the run says why; otherwise the plan carries on. A message at the end says how each server went.
|
||||
*
|
||||
* A plan starts once a day at most: the day it last started on is kept, so a restart of this app at 02:10 neither misses it nor repeats it. If the app was
|
||||
* down at the time, it still starts, up to half an hour late; later than that, that day is skipped.
|
||||
*/
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { maintenanceWindows, patchPlanRuns, patchPlans, servers, updateRuns } from "../db/schema.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { impactOf } from "./dependencies.js";
|
||||
import { startOrExtendWindow } from "./maintenance.js";
|
||||
import { occurrencesBetween, type Occurrence } from "./maintenanceSchedules.js";
|
||||
import { notifyPatchPlanFinished } from "./notify.js";
|
||||
import { parseTags } from "./serverTags.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { ACTIVE_STATUSES, cancelUpdateRun, startUpdateRun, UpdateRunError } from "./updateRuns.js";
|
||||
|
||||
export type PlanRow = typeof patchPlans.$inferSelect;
|
||||
export type PlanRunRow = typeof patchPlanRuns.$inferSelect;
|
||||
export type ItemStatus = "pending" | "running" | "done" | "failed" | "skipped";
|
||||
export type RunStatus = "running" | "done" | "failed" | "cancelled";
|
||||
|
||||
export interface PlanItem {
|
||||
serverId: number;
|
||||
name: string;
|
||||
status: ItemStatus;
|
||||
updateRunId: number | null;
|
||||
message: string | null;
|
||||
/** Windows made for what depends on this server, which end with it. */
|
||||
windowIds: number[];
|
||||
}
|
||||
|
||||
/** How late a plan may still start, if the app was not running at its time. */
|
||||
export const START_GRACE_MS = 30 * 60_000;
|
||||
export const POLL_INTERVAL_MS = 20_000;
|
||||
export const MAX_PLANS = 30;
|
||||
|
||||
export class PlanError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Reading what is stored ────────────────────────────────────────────────
|
||||
|
||||
function jsonArray<T>(stored: string | null | undefined): T[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export const parseItems = (stored: string | null | undefined): PlanItem[] => jsonArray<PlanItem>(stored).filter((i) => i && typeof i.serverId === "number");
|
||||
export const planDays = (plan: Pick<PlanRow, "days">): number[] => jsonArray<number>(plan.days).filter((d) => Number.isInteger(d));
|
||||
export const planTags = (plan: Pick<PlanRow, "tags">): string[] => parseTags(plan.tags);
|
||||
export const planServerIds = (plan: Pick<PlanRow, "serverIds">): number[] => jsonArray<number>(plan.serverIds).filter((i) => Number.isInteger(i));
|
||||
|
||||
// ─── Who a plan is for, and when ───────────────────────────────────────────
|
||||
|
||||
/** Pure: the servers a plan covers — those with any of its tags, and those named — by name. */
|
||||
export function planServers(plan: Pick<PlanRow, "tags" | "serverIds">, all: { id: number; name: string; tags: string | null }[]): { id: number; name: string }[] {
|
||||
const tags = new Set(planTags(plan));
|
||||
const ids = new Set(planServerIds(plan));
|
||||
return all
|
||||
.filter((s) => ids.has(s.id) || parseTags(s.tags).some((t) => tags.has(t)))
|
||||
.map((s) => ({ id: s.id, name: s.name }))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/** The times a plan happens, from `from` to `to`. A plan has no length; the minute it starts is what matters. */
|
||||
export function planOccurrences(plan: Pick<PlanRow, "days" | "startTime" | "timezone">, from: Date, to: Date): Occurrence[] {
|
||||
return occurrencesBetween({ days: plan.days, startTime: plan.startTime, durationMinutes: 1, timezone: plan.timezone }, from, to);
|
||||
}
|
||||
|
||||
/** Pure: the occurrence a plan should start for now — its time has come, it is no more than the grace late, and it has not started on that day. */
|
||||
export function dueOccurrence(plan: Pick<PlanRow, "enabled" | "days" | "startTime" | "timezone" | "lastRunDate">, now: Date, graceMs: number = START_GRACE_MS): Occurrence | null {
|
||||
if (!plan.enabled) return null;
|
||||
const found = planOccurrences(plan, new Date(now.getTime() - graceMs - 60_000), new Date(now.getTime() + 1000)).filter((o) => o.start.getTime() <= now.getTime() && now.getTime() - o.start.getTime() <= graceMs && o.date !== plan.lastRunDate);
|
||||
return found.length > 0 ? found[found.length - 1] : null;
|
||||
}
|
||||
|
||||
// ─── Starting a run ────────────────────────────────────────────────────────
|
||||
|
||||
const ACTIVE_RUN: RunStatus[] = ["running"];
|
||||
|
||||
export async function activeRunOf(planId: number): Promise<PlanRunRow | null> {
|
||||
const [run] = await db.select().from(patchPlanRuns).where(and(eq(patchPlanRuns.planId, planId), inArray(patchPlanRuns.status, ACTIVE_RUN))).limit(1);
|
||||
return run ?? null;
|
||||
}
|
||||
|
||||
export async function startPlanRun(planId: number, options: { trigger: "schedule" | "manual"; actor: string; now?: Date }): Promise<PlanRunRow> {
|
||||
const now = options.now ?? new Date();
|
||||
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId)).limit(1);
|
||||
if (!plan) throw new PlanError(404, "not_found", "That plan doesn't exist.");
|
||||
if (await activeRunOf(plan.id)) throw new PlanError(409, "already_running", `"${plan.name}" is already running.`);
|
||||
const matched = planServers(plan, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers));
|
||||
if (matched.length === 0) throw new PlanError(400, "no_servers", "No server has this plan's tags or is named in it, so there is nothing to update.");
|
||||
const items: PlanItem[] = matched.map((s) => ({ serverId: s.id, name: s.name, status: "pending", updateRunId: null, message: null, windowIds: [] }));
|
||||
const [run] = await db
|
||||
.insert(patchPlanRuns)
|
||||
.values({ planId: plan.id, planName: plan.name, trigger: options.trigger, startedBy: options.actor, status: "running", message: `Updating ${matched.length} server${matched.length === 1 ? "" : "s"}, one at a time.`, items: JSON.stringify(items), startedAt: now.toISOString() })
|
||||
.returning();
|
||||
await recordAudit({ category: "patch_plan", action: "run", targetType: "patch_plan", targetId: plan.id, detail: { name: plan.name, run: run.id, trigger: options.trigger, servers: matched.length, by: options.actor } });
|
||||
return run;
|
||||
}
|
||||
|
||||
// ─── Following a run ───────────────────────────────────────────────────────
|
||||
|
||||
async function endWindows(ids: number[], now: Date): Promise<void> {
|
||||
if (ids.length === 0) return;
|
||||
await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, ids));
|
||||
invalidateAlertsCache();
|
||||
}
|
||||
|
||||
/** Silences what depends on the server for as long as it is being updated; returns the windows it made (ones it only extended belong to someone else). */
|
||||
async function silenceDependents(plan: PlanRow, serverId: number, serverName: string, actor: string, now: Date): Promise<number[]> {
|
||||
if (!plan.silenceDependents) return [];
|
||||
const made: number[] = [];
|
||||
try {
|
||||
const impact = await impactOf("server", serverId, now);
|
||||
for (const d of impact?.dependents ?? []) {
|
||||
const started = await startOrExtendWindow(d.type, d.id, plan.windowMinutes, `Depends on ${serverName} — patch plan "${plan.name}"`.slice(0, 200), actor, now);
|
||||
if (started && !started.extended) made.push(started.window.id);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[patchPlans] couldn't silence what depends on a server:", err);
|
||||
}
|
||||
return made;
|
||||
}
|
||||
|
||||
const words = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
|
||||
/** What a finished run says, in a line. */
|
||||
export function summariseItems(items: PlanItem[]): string {
|
||||
const count = (s: ItemStatus) => items.filter((i) => i.status === s).length;
|
||||
const parts = [`${count("done")} updated`];
|
||||
if (count("failed") > 0) parts.push(`${count("failed")} failed`);
|
||||
if (count("skipped") > 0) parts.push(`${count("skipped")} left alone`);
|
||||
return `${parts.join(", ")} of ${words(items.length, "server")}.`;
|
||||
}
|
||||
|
||||
async function saveRun(run: PlanRunRow, items: PlanItem[], extra: Partial<PlanRunRow> = {}): Promise<void> {
|
||||
await db.update(patchPlanRuns).set({ items: JSON.stringify(items), ...extra }).where(eq(patchPlanRuns.id, run.id));
|
||||
}
|
||||
|
||||
async function finishRun(run: PlanRunRow, plan: PlanRow | null, items: PlanItem[], status: RunStatus, now: Date, why?: string): Promise<void> {
|
||||
const message = [summariseItems(items), why].filter(Boolean).join(" ");
|
||||
await saveRun(run, items, { status, message, finishedAt: now.toISOString() });
|
||||
await recordAudit({ category: "patch_plan", action: status === "done" ? "finished" : status, targetType: "patch_plan", targetId: run.planId, detail: { name: run.planName, run: run.id, message } });
|
||||
if (status !== "cancelled") {
|
||||
const lines = items.map((i) => `${i.status === "done" ? "✓" : i.status === "failed" ? "✕" : "–"} ${i.name}${i.message ? ` — ${i.message}` : ""}`);
|
||||
await notifyPatchPlanFinished({ planName: run.planName, ok: status === "done", message, lines }).catch((err) => console.error("[patchPlans] couldn't send the result:", err));
|
||||
}
|
||||
void plan;
|
||||
}
|
||||
|
||||
/** When a server's turn is over and it failed: stop (leaving the rest alone) or carry on. */
|
||||
function afterFailure(plan: PlanRow, items: PlanItem[]): boolean {
|
||||
if (!plan.stopOnFailure) return false;
|
||||
for (const i of items) if (i.status === "pending") Object.assign(i, { status: "skipped", message: "Not started, because an earlier server failed." });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
|
||||
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
|
||||
const items = parseItems(run.items);
|
||||
if (!plan) return; // deleted: the run goes with it
|
||||
|
||||
// The server whose turn it is: has its update run finished?
|
||||
const current = items.find((i) => i.status === "running");
|
||||
if (current) {
|
||||
const [ur] = current.updateRunId !== null ? await db.select().from(updateRuns).where(eq(updateRuns.id, current.updateRunId)).limit(1) : [];
|
||||
if (ur && ACTIVE_STATUSES.includes(ur.status as (typeof ACTIVE_STATUSES)[number])) return;
|
||||
const ok = ur?.status === "done";
|
||||
current.status = ok ? "done" : "failed";
|
||||
current.message = ur?.message ?? "The update run is gone.";
|
||||
await endWindows(current.windowIds, now);
|
||||
if (!ok && afterFailure(plan, items)) {
|
||||
await finishRun(run, plan, items, "failed", now, `Stopped after ${current.name}.`);
|
||||
return;
|
||||
}
|
||||
await saveRun(run, items);
|
||||
}
|
||||
|
||||
// The next one.
|
||||
for (;;) {
|
||||
const next = items.find((i) => i.status === "pending");
|
||||
if (!next) {
|
||||
await finishRun(run, plan, items, items.some((i) => i.status === "failed") ? "failed" : "done", now);
|
||||
return;
|
||||
}
|
||||
const [server] = await db.select().from(servers).where(eq(servers.id, next.serverId)).limit(1);
|
||||
if (!server) {
|
||||
Object.assign(next, { status: "skipped", message: "The server was removed." });
|
||||
continue;
|
||||
}
|
||||
if (plan.onlyWithUpdates) {
|
||||
const updates = parsePendingUpdates(server.pendingUpdates);
|
||||
if (!updates || updates.count === 0) {
|
||||
Object.assign(next, { status: "skipped", message: "No updates were waiting." });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const actor = `Patch plan "${plan.name}"`;
|
||||
try {
|
||||
const ur = await startUpdateRun(
|
||||
{
|
||||
serverId: server.id,
|
||||
integrationId: plan.integrationId,
|
||||
projectId: plan.projectId,
|
||||
templateId: plan.templateId,
|
||||
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
|
||||
windowMinutes: plan.windowMinutes,
|
||||
actor,
|
||||
},
|
||||
now,
|
||||
);
|
||||
Object.assign(next, { status: "running", updateRunId: ur.id, message: "The update run has started.", windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
|
||||
await saveRun(run, items);
|
||||
return; // one at a time: this one is followed until it is over
|
||||
} catch (err) {
|
||||
const message = err instanceof UpdateRunError || err instanceof Error ? err.message : String(err);
|
||||
Object.assign(next, { status: "failed", message });
|
||||
if (afterFailure(plan, items)) {
|
||||
await finishRun(run, plan, items, "failed", now, `Stopped at ${next.name}.`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** One pass over every plan run that is under way. Returns how many it looked at (for the tests). */
|
||||
export async function advancePlanRuns(now: Date = new Date()): Promise<number> {
|
||||
const active = await db.select().from(patchPlanRuns).where(inArray(patchPlanRuns.status, ACTIVE_RUN));
|
||||
for (const run of active) {
|
||||
try {
|
||||
await advanceRun(run, now);
|
||||
} catch (err) {
|
||||
console.error(`[patchPlans] run ${run.id}:`, err);
|
||||
}
|
||||
}
|
||||
return active.length;
|
||||
}
|
||||
|
||||
/** Starts the plans whose time has come. Returns how many it started. */
|
||||
export async function startDuePlans(now: Date = new Date()): Promise<number> {
|
||||
let started = 0;
|
||||
for (const plan of await db.select().from(patchPlans)) {
|
||||
const due = dueOccurrence(plan, now);
|
||||
if (!due) continue;
|
||||
if (await activeRunOf(plan.id)) continue; // still on the last one: today is skipped rather than overlapping
|
||||
// Written down first, so that whatever happens next it is not tried again every twenty seconds for half an hour.
|
||||
await db.update(patchPlans).set({ lastRunDate: due.date }).where(eq(patchPlans.id, plan.id));
|
||||
try {
|
||||
await startPlanRun(plan.id, { trigger: "schedule", actor: "the schedule", now });
|
||||
started++;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await db.insert(patchPlanRuns).values({ planId: plan.id, planName: plan.name, trigger: "schedule", startedBy: "the schedule", status: "failed", message: `Didn't start: ${message}`, items: "[]", startedAt: now.toISOString(), finishedAt: now.toISOString() });
|
||||
}
|
||||
}
|
||||
return started;
|
||||
}
|
||||
|
||||
export async function cancelPlanRun(id: number, actor: string, now: Date = new Date()): Promise<PlanRunRow | null> {
|
||||
const [run] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
|
||||
if (!run) return null;
|
||||
if (run.status !== "running") throw new PlanError(409, "not_active", "That run is already over.");
|
||||
const items = parseItems(run.items);
|
||||
for (const i of items) {
|
||||
if (i.status === "pending") Object.assign(i, { status: "skipped", message: `Not started: stopped by ${actor}.` });
|
||||
else if (i.status === "running") {
|
||||
if (i.updateRunId !== null) await cancelUpdateRun(i.updateRunId, actor, now).catch(() => undefined);
|
||||
await endWindows(i.windowIds, now);
|
||||
Object.assign(i, { status: "failed", message: `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.` });
|
||||
}
|
||||
}
|
||||
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
|
||||
await finishRun(run, plan ?? null, items, "cancelled", now, `Stopped by ${actor}.`);
|
||||
const [after] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
|
||||
return after;
|
||||
}
|
||||
|
||||
/** Recent runs, those still going first. */
|
||||
export async function listPlanRuns(options: { planId?: number; limit?: number } = {}): Promise<PlanRunRow[]> {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(patchPlanRuns)
|
||||
.where(options.planId !== undefined ? eq(patchPlanRuns.planId, options.planId) : undefined)
|
||||
.orderBy(desc(patchPlanRuns.startedAt))
|
||||
.limit(options.limit ?? 20);
|
||||
return [...rows.filter((r) => r.status === "running"), ...rows.filter((r) => r.status !== "running")];
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
/** Every twenty seconds: start what is due, follow what is going. A run that was under way when the app last stopped is picked up where it was. */
|
||||
export function initPatchPlans(): void {
|
||||
if (timer) return;
|
||||
timer = setInterval(() => {
|
||||
void startDuePlans()
|
||||
.then(() => advancePlanRuns())
|
||||
.catch((err) => console.error("[patchPlans]", err));
|
||||
}, POLL_INTERVAL_MS);
|
||||
timer.unref();
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Renewals: a certificate, domain registration or secret whose expiry moved later was renewed. The app sees it happen when it re-reads the date — an endpoint
|
||||
* check seeing a new certificate, the registry saying a domain now runs longer, a secret's live certificate being replaced — and writes it down, so the monthly
|
||||
* report can say "certificates renewed: 3". It only goes back as far as the version that began watching; nothing is guessed about earlier.
|
||||
*/
|
||||
import { and, eq, gte, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { renewals } from "../db/schema.js";
|
||||
|
||||
/** An expiry has to move at least this far for it to be a renewal (a registry rounding a date by a day is not one). */
|
||||
export const MIN_RENEWAL_DAYS = 20;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
export type RenewalKind = "certificate" | "domain" | "secret";
|
||||
|
||||
const dayOf = (value: string | null | undefined): string | null => (value && /^\d{4}-\d{2}-\d{2}/.test(value) ? value.slice(0, 10) : null);
|
||||
|
||||
/** Pure: whether the expiry went from one date to a clearly later one. Either may be a full ISO time; only the day counts. */
|
||||
export function isRenewal(from: string | null | undefined, to: string | null | undefined): boolean {
|
||||
const a = dayOf(from);
|
||||
const b = dayOf(to);
|
||||
if (!a || !b) return false;
|
||||
return (Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / DAY_MS >= MIN_RENEWAL_DAYS;
|
||||
}
|
||||
|
||||
/** Writes down a renewal if the dates say it was one (and the same one is not already written down). Never throws: it must not hold up the check that saw it. */
|
||||
export async function noteRenewal(kind: RenewalKind, subject: string, from: string | null | undefined, to: string | null | undefined, now: Date = new Date()): Promise<boolean> {
|
||||
try {
|
||||
if (!isRenewal(from, to)) return false;
|
||||
const fromDate = dayOf(from)!;
|
||||
const toDate = dayOf(to)!;
|
||||
const [same] = await db.select({ id: renewals.id }).from(renewals).where(and(eq(renewals.kind, kind), eq(renewals.subject, subject), eq(renewals.toDate, toDate))).limit(1);
|
||||
if (same) return false;
|
||||
await db.insert(renewals).values({ kind, subject, at: now.toISOString(), fromDate, toDate });
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error("[renewals] couldn't record a renewal:", err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function renewalsBetween(from: Date, to: Date) {
|
||||
return db.select().from(renewals).where(and(gte(renewals.at, from.toISOString()), lt(renewals.at, to.toISOString()))).orderBy(renewals.at);
|
||||
}
|
||||
@@ -113,6 +113,17 @@ export interface HealthCheckSettings {
|
||||
backupStaleHours: number;
|
||||
/** A restore test older than this many days (or none at all) is a problem for a guest that has backups; 0 = never (a failed test always is). */
|
||||
restoreTestDays: number;
|
||||
/** A server that starts listening on a TCP port it did not at first, and that is not noted on its Ports card, is a warning (services/newPorts.ts). */
|
||||
newPortAlerts: boolean;
|
||||
}
|
||||
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
export interface MonthlyReportSettings {
|
||||
enabled: boolean;
|
||||
/** Day of the month, 1–28 (every month has it), in notifications.timezone. */
|
||||
day: number;
|
||||
/** "HH:MM", in notifications.timezone. */
|
||||
time: string;
|
||||
}
|
||||
|
||||
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
|
||||
@@ -225,6 +236,7 @@ export interface AppSettings {
|
||||
quietHours: QuietHoursSettings;
|
||||
healthChecks: HealthCheckSettings;
|
||||
digest: DigestSettings;
|
||||
monthlyReport: MonthlyReportSettings;
|
||||
signIn: SignInSettings;
|
||||
profilePictures: ProfilePictureSettings;
|
||||
inventory: InventorySettings;
|
||||
@@ -267,12 +279,13 @@ const DEFAULTS: AppSettings = {
|
||||
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
|
||||
quietHours: { enabled: false, start: "22:00", end: "07:00" },
|
||||
digest: { enabled: false, day: 1, time: "08:00" },
|
||||
monthlyReport: { enabled: false, day: 1, time: "08:00" },
|
||||
inventory: { currency: "", pricePerKwh: null },
|
||||
agents: { autoUpdate: false },
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
|
||||
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
|
||||
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { baselineFrom } from "./newPorts.js";
|
||||
import { fillInventoryFromServer } from "./inventoryAutofill.js";
|
||||
import { nextDiagnostics, type AgentDiagnostic } from "./agentDiagnostics.js";
|
||||
|
||||
@@ -113,7 +114,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
}
|
||||
|
||||
const system = report.system;
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
|
||||
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion, portBaseline: servers.portBaseline }).from(servers).where(eq(servers.id, serverId));
|
||||
await db
|
||||
.update(servers)
|
||||
.set({
|
||||
@@ -144,6 +145,8 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
|
||||
: {}),
|
||||
// Only stored when the agent reported it, so an agent that predates this doesn't wipe the field.
|
||||
...(system?.listening_ports ? { listeningPorts: JSON.stringify(system.listening_ports) } : {}),
|
||||
// The first report that carries ports says what is normal for this server; a port that appears later is "new" (services/newPorts.ts).
|
||||
...(system?.listening_ports && before && before.portBaseline === null ? { portBaseline: baselineFrom(system.listening_ports) } : {}),
|
||||
})
|
||||
.where(eq(servers.id, serverId));
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import { eq, isNotNull } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { secrets } from "../db/schema.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
|
||||
/**
|
||||
* Opens a real TLS connection and returns the server certificate's expiry as
|
||||
@@ -79,6 +80,8 @@ export async function refreshTlsSecret(id: number): Promise<TlsCheckResult | nul
|
||||
...(expiryDate !== row.expiryDate ? { updatedAt: now } : {}),
|
||||
})
|
||||
.where(eq(secrets.id, id));
|
||||
// A certificate that is now good for much longer than it was has been replaced.
|
||||
await noteRenewal("certificate", row.name, row.expiryDate, expiryDate);
|
||||
return { id, name: row.name, host: row.checkHost, port, ok: true, expiryDate };
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
|
||||
@@ -33,7 +33,7 @@ await runMigrations();
|
||||
|
||||
type Data = Parameters<typeof agendaFromData>[0];
|
||||
const NOW = new Date("2026-10-19T12:00:00Z");
|
||||
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
|
||||
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], patchPlans: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
|
||||
const titles = (data: Data, days?: number, tz = "UTC") => agendaFromData(data, NOW, tz, days).items.map((i) => [i.daysLeft, i.kind, i.title]);
|
||||
|
||||
describe("what is gathered, and how it is dated", () => {
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
/**
|
||||
* The monthly report: which month is which, what renewals and daily figures are kept, what the report gathers for a month, how it reads as a message, and the
|
||||
* routes and schedule around it.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-monthly-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, auditLog, alertIncidents, updateRuns, patchPlanRuns, patchPlans, restoreTests, renewals, endpointChecks, endpointDaily, endpointResults } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { isMonthId, lastFullMonth, recentMonths, monthBounds, monthLabel, buildMonthlyReport, monthlyReportText } = await import("../src/services/monthlyReport.js");
|
||||
const { isRenewal, noteRenewal } = await import("../src/services/renewals.js");
|
||||
const { runCheck, pruneResults } = await import("../src/services/endpointChecks.js");
|
||||
const { cronForMonthlyReport } = await import("../src/services/monthlyReportScheduler.js");
|
||||
const { reportsRouter } = await import("../src/routes/reports.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { sql, eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("which month is which", () => {
|
||||
test("the last whole month, and the twelve to choose from, on the clock of the time zone", () => {
|
||||
assert.equal(lastFullMonth(new Date("2026-10-19T12:00:00Z"), "UTC"), "2026-09");
|
||||
assert.equal(lastFullMonth(new Date("2027-01-02T12:00:00Z"), "UTC"), "2026-12", "across the new year");
|
||||
assert.equal(lastFullMonth(new Date("2026-10-31T23:30:00Z"), "Europe/Stockholm"), "2026-10", "it is already November in Stockholm");
|
||||
const months = recentMonths(new Date("2026-10-19T12:00:00Z"), "UTC");
|
||||
assert.equal(months.length, 12);
|
||||
assert.deepEqual([months[0], months[1], months[11]], [{ id: "2026-10", label: "October 2026" }, { id: "2026-09", label: "September 2026" }, { id: "2025-11", label: "November 2025" }]);
|
||||
assert.equal(monthLabel("2026-02"), "February 2026");
|
||||
});
|
||||
|
||||
test("a month begins and ends where the clock of the time zone says, summer or winter", () => {
|
||||
const sept = monthBounds("2026-09", "Europe/Stockholm");
|
||||
assert.deepEqual([sept.start.toISOString(), sept.end.toISOString()], ["2026-08-31T22:00:00.000Z", "2026-09-30T22:00:00.000Z"]);
|
||||
const oct = monthBounds("2026-10", "Europe/Stockholm");
|
||||
assert.equal(oct.end.toISOString(), "2026-10-31T23:00:00.000Z", "the clocks went back on the last Sunday");
|
||||
assert.equal(monthBounds("2026-12", "UTC").end.toISOString(), "2027-01-01T00:00:00.000Z");
|
||||
});
|
||||
|
||||
test("only a real month is a month", () => {
|
||||
for (const ok of ["2026-09", "1999-12"]) assert.equal(isMonthId(ok), true, ok);
|
||||
for (const bad of ["2026-13", "2026-00", "2026-9", "26-09", "2026-09-01", "", undefined, 202609]) assert.equal(isMonthId(bad), false, String(bad));
|
||||
});
|
||||
|
||||
test("the schedule is a day of the month, 1 to 28 so every month has it", () => {
|
||||
assert.equal(cronForMonthlyReport(5, "07:30"), "30 7 5 * *");
|
||||
assert.equal(cronForMonthlyReport(31, "07:30"), "30 7 1 * *");
|
||||
assert.equal(cronForMonthlyReport(0, "bad"), "0 8 1 * *");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renewals and the daily figures", () => {
|
||||
test("an expiry that moves a clear step later is a renewal; a day's rounding, a move earlier and a gap in the data are not", () => {
|
||||
assert.equal(isRenewal("2026-10-01", "2026-12-30"), true);
|
||||
assert.equal(isRenewal("2026-10-01T08:00:00.000Z", "2026-11-20"), true, "a full time or a day, only the day counts");
|
||||
assert.equal(isRenewal("2026-10-01", "2026-10-02"), false);
|
||||
assert.equal(isRenewal("2026-10-01", "2026-10-20"), false, "nineteen days is not enough");
|
||||
assert.equal(isRenewal("2026-10-21", "2026-10-01"), false);
|
||||
assert.equal(isRenewal(null, "2026-12-01"), false, "there was nothing to compare with");
|
||||
assert.equal(isRenewal("2026-10-01", "not a date"), false);
|
||||
});
|
||||
|
||||
test("a renewal is written down once", async () => {
|
||||
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2027-10-01"), true);
|
||||
assert.equal(await noteRenewal("domain", "example.se", "2026-10-02", "2027-10-01"), false, "the same one seen again");
|
||||
assert.equal(await noteRenewal("domain", "example.se", "2026-10-01", "2026-10-03"), false);
|
||||
assert.equal((await db.select().from(renewals).where(eq(renewals.subject, "example.se"))).length, 1);
|
||||
await db.delete(renewals);
|
||||
});
|
||||
|
||||
test("each answer adds to its day; a certificate that is replaced is a renewal; old days go", async () => {
|
||||
const [check] = await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning();
|
||||
const answer = (ok: boolean, ms: number | null, cert?: string) => async () => ({ ok, ms, message: "m", certExpiresAt: cert });
|
||||
const t = (h: number) => new Date(`2026-09-10T0${h}:00:00Z`);
|
||||
let row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(1), answer(true, 100, "2026-10-05T00:00:00.000Z"));
|
||||
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(2), answer(true, 300, "2026-10-05T00:00:00.000Z"));
|
||||
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(3), answer(false, null, "2026-10-05T00:00:00.000Z"));
|
||||
const [day] = await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id));
|
||||
assert.deepEqual([day.day, day.total, day.ok, day.msSum, day.msCount], ["2026-09-10", 3, 2, 400, 2], "a failure counts as a check but adds no answer time");
|
||||
assert.equal((await db.select().from(renewals)).length, 0, "the same certificate three times is not a renewal");
|
||||
|
||||
row = (await db.select().from(endpointChecks).where(eq(endpointChecks.id, check.id)))[0];
|
||||
await runCheck(row, t(4), answer(true, 100, "2026-12-04T00:00:00.000Z"));
|
||||
const [renewed] = await db.select().from(renewals);
|
||||
assert.deepEqual([renewed.kind, renewed.subject, renewed.fromDate, renewed.toDate], ["certificate", "Gitea", "2026-10-05", "2026-12-04"]);
|
||||
|
||||
await db.insert(endpointDaily).values({ checkId: check.id, day: "2020-01-01", total: 1, ok: 1 });
|
||||
await pruneResults(new Date("2026-10-19T00:00:00Z"));
|
||||
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2020-01-01"))).length, 0, "kept for years, not for ever");
|
||||
assert.equal((await db.select().from(endpointDaily).where(eq(endpointDaily.day, "2026-09-10"))).length, 1);
|
||||
await db.delete(renewals);
|
||||
});
|
||||
|
||||
test("the migration started the daily figures from the results that already existed", async () => {
|
||||
const [check] = await db.insert(endpointChecks).values({ name: "Backfilled", kind: "tcp", target: "h", port: 22 }).returning();
|
||||
await db.insert(endpointResults).values([
|
||||
{ checkId: check.id, at: "2026-10-01T01:00:00.000Z", ok: true, ms: 10 },
|
||||
{ checkId: check.id, at: "2026-10-01T02:00:00.000Z", ok: true, ms: 30 },
|
||||
{ checkId: check.id, at: "2026-10-01T03:00:00.000Z", ok: false, ms: null },
|
||||
{ checkId: check.id, at: "2026-10-02T01:00:00.000Z", ok: true, ms: 50 },
|
||||
]);
|
||||
const migration = readFileSync(new URL("../drizzle/0043_puzzling_prism.sql", import.meta.url), "utf8");
|
||||
const statement = migration.split("--> statement-breakpoint").pop()!;
|
||||
assert.match(statement, /INSERT INTO `endpoint_daily`/);
|
||||
await db.run(sql.raw(statement));
|
||||
const rows = (await db.select().from(endpointDaily).where(eq(endpointDaily.checkId, check.id))).sort((a, b) => a.day.localeCompare(b.day));
|
||||
assert.deepEqual(rows.map((r) => [r.day, r.total, r.ok, r.msSum, r.msCount]), [["2026-10-01", 3, 2, 40, 2], ["2026-10-02", 1, 1, 50, 1]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a month's report", () => {
|
||||
let nasId = 0;
|
||||
let checkId = 0;
|
||||
const audit = (action: string, id: number, at: string) => ({ category: "server", action, targetType: "server", targetId: String(id), createdAt: at });
|
||||
const incident = (extra: Partial<typeof alertIncidents.$inferInsert>) => ({ alertKey: "k", severity: "warning", peakSeverity: "warning", category: "disk", source: "Server", message: "a disk", openedAt: "2026-09-02T00:00:00.000Z", lastSeenAt: "2026-09-02T00:00:00.000Z", ...extra });
|
||||
|
||||
before(async () => {
|
||||
await updateSettings({ notifications: { timezone: "UTC" } });
|
||||
await db.delete(endpointChecks); // the earlier tests left checks of their own, and their daily figures go with them
|
||||
nasId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
|
||||
const elsaId = (await db.insert(servers).values({ name: "elsa", apiTokenHash: "e", apiTokenPrefix: "e", lastSeenAt: "2026-10-01T00:00:00.000Z", createdAt: "2026-01-01 00:00:00" }).returning({ id: servers.id }))[0].id;
|
||||
void elsaId;
|
||||
// nas was offline 13 hours in September, and a night in August that does not count
|
||||
await db.insert(auditLog).values([audit("went_offline", nasId, "2026-09-10 08:00:00"), audit("back_online", nasId, "2026-09-10 21:00:00"), audit("went_offline", nasId, "2026-08-02 01:00:00"), audit("back_online", nasId, "2026-08-02 05:00:00")]);
|
||||
await db.insert(alertIncidents).values([
|
||||
incident({ message: "nas: / is 96% full", category: "disk", openedAt: "2026-09-03T10:00:00.000Z", resolvedAt: "2026-09-03T12:00:00.000Z", peakSeverity: "critical" }),
|
||||
incident({ message: "elsa stopped reporting", category: "offline", openedAt: "2026-09-12T10:00:00.000Z", resolvedAt: "2026-09-12T10:30:00.000Z", targetType: "server", targetId: 2, targetName: "elsa" }),
|
||||
incident({ message: "Backup of anna failed", category: "backup", openedAt: "2026-09-20T00:00:00.000Z" }),
|
||||
incident({ message: "in August", openedAt: "2026-08-20T00:00:00.000Z", resolvedAt: "2026-08-20T01:00:00.000Z" }),
|
||||
incident({ message: "began in August, ended in September", openedAt: "2026-08-31T20:00:00.000Z", resolvedAt: "2026-09-01T02:00:00.000Z" }),
|
||||
]);
|
||||
await db.insert(updateRuns).values([
|
||||
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-09-05T02:00:00.000Z", updatesBefore: 12, updatesAfter: 2 },
|
||||
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "failed", startedAt: "2026-09-12T02:00:00.000Z", updatesBefore: 3, updatesAfter: null },
|
||||
{ serverId: nasId, integrationId: 1, projectId: 1, templateId: 1, templateName: "t", hostLimit: "nas", status: "done", startedAt: "2026-10-05T02:00:00.000Z", updatesBefore: 9, updatesAfter: 0 },
|
||||
]);
|
||||
const [plan] = await db.insert(patchPlans).values({ name: "Lab", days: "[0]", startTime: "02:00", timezone: "UTC", integrationId: 1, projectId: 1, templateId: 1, templateName: "t" }).returning();
|
||||
await db.insert(patchPlanRuns).values([
|
||||
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "done", startedAt: "2026-09-06T02:00:00.000Z" },
|
||||
{ planId: plan.id, planName: "Lab", trigger: "schedule", status: "failed", startedAt: "2026-09-13T02:00:00.000Z" },
|
||||
]);
|
||||
await db.insert(restoreTests).values([{ subjectKey: "custom:photos", label: "Photos share", testedAt: "2026-09-14", result: "ok" }, { subjectKey: "custom:old", label: "Old", testedAt: "2026-08-14", result: "ok" }]);
|
||||
await db.insert(renewals).values([{ kind: "certificate", subject: "git.lab", at: "2026-09-12T03:00:00.000Z", fromDate: "2026-09-14", toDate: "2026-12-13" }, { kind: "domain", subject: "example.se", at: "2026-10-03T03:00:00.000Z", fromDate: "2026-10-04", toDate: "2027-10-04" }]);
|
||||
checkId = (await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/" }).returning({ id: endpointChecks.id }))[0].id;
|
||||
await db.insert(endpointDaily).values([
|
||||
{ checkId, day: "2026-09-01", total: 100, ok: 100, msSum: 10_000, msCount: 100 },
|
||||
{ checkId, day: "2026-09-30", total: 100, ok: 90, msSum: 9_000, msCount: 90 },
|
||||
{ checkId, day: "2026-08-31", total: 100, ok: 0 },
|
||||
{ checkId, day: "2026-10-01", total: 100, ok: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
const NOW = new Date("2026-10-19T12:00:00Z");
|
||||
|
||||
test("availability: what was down and for how long, from what the audit log recorded; the neighbouring months are not in it", async () => {
|
||||
const r = await buildMonthlyReport("2026-09", NOW);
|
||||
assert.deepEqual([r.month, r.label, r.partial, r.timezone], ["2026-09", "September 2026", false, "UTC"]);
|
||||
const nas = r.servers.find((s) => s.name === "nas")!;
|
||||
assert.equal(nas.downMinutes, 13 * 60);
|
||||
assert.equal(nas.percent, 98.2, "13 of 720 hours down");
|
||||
assert.equal(r.servers[0].name, "nas", "the worst first");
|
||||
assert.deepEqual(r.endpoints.map((e) => [e.name, e.percent, e.avgMs, e.checks]), [["Gitea", 95, 100, 200]], "the first and last day of the month, not the days either side");
|
||||
});
|
||||
|
||||
test("incidents: how many began and ended, the time they took, the longest, by kind", async () => {
|
||||
const i = (await buildMonthlyReport("2026-09", NOW)).incidents;
|
||||
assert.deepEqual([i.began, i.resolved, i.stillOpen], [3, 3, 1], "the open one is still open; the August one that ended in September ended in it");
|
||||
assert.equal(i.totalMinutes, 150, "2 h and 30 min, of those that began and ended in the month");
|
||||
assert.equal(i.averageMinutes, 75);
|
||||
assert.equal(i.longest!.message, "Backup of anna failed", "still open, so it has been going the longest");
|
||||
assert.deepEqual(i.byCategory, [{ category: "backup", count: 1 }, { category: "disk", count: 1 }, { category: "offline", count: 1 }]);
|
||||
assert.deepEqual(i.list.map((x) => x.message), ["Backup of anna failed", "nas: / is 96% full", "elsa stopped reporting"]);
|
||||
assert.equal(i.list[2].where, "elsa");
|
||||
});
|
||||
|
||||
test("updates, patch plans, restore tests and renewals for that month only", async () => {
|
||||
const r = await buildMonthlyReport("2026-09", NOW);
|
||||
assert.deepEqual([r.updates.runs, r.updates.done, r.updates.failed, r.updates.installed], [2, 1, 1, 10]);
|
||||
assert.deepEqual(r.updates.byServer, [{ name: "nas", runs: 2, failed: 1, installed: 10 }]);
|
||||
assert.deepEqual(r.updates.planRuns, { runs: 2, done: 1, failed: 1 });
|
||||
assert.deepEqual(r.restoreTests.map((t) => t.label), ["Photos share"]);
|
||||
assert.deepEqual(r.renewals.map((x) => [x.kind, x.subject]), [["certificate", "git.lab"]]);
|
||||
assert.equal(r.backups, null, "no backup coverage has been read here");
|
||||
const oct = await buildMonthlyReport("2026-10", NOW);
|
||||
assert.equal(oct.partial, true, "October is not over");
|
||||
assert.deepEqual(oct.renewals.map((x) => x.subject), ["example.se"]);
|
||||
assert.equal(oct.updates.runs, 1);
|
||||
});
|
||||
|
||||
test("as a message: a section only where there is something to say, with the numbers that matter", async () => {
|
||||
const { title, message } = monthlyReportText(await buildMonthlyReport("2026-09", NOW));
|
||||
assert.equal(title, "Homelab monthly report — September 2026");
|
||||
assert.match(message, /^AVAILABILITY\nServers: 9\d\.\d% on average \(2 servers\)\.\n- nas: 9\d\.\d% \(13 h down\)/);
|
||||
assert.match(message, /Endpoints: 95% on average \(1 check\)\.\n- Gitea: 95%, 100 ms on average/);
|
||||
assert.match(message, /INCIDENTS\n3 incidents began, 3 ended, 1 still open\.\nA resolved incident took 1 h 15 min on average\./);
|
||||
assert.match(message, /- Backup of anna failed — 11 d and still open/);
|
||||
assert.match(message, /UPDATES\n2 update runs: 1 done, 1 failed — 10 updates installed\.\nPatch plans ran 2 times, 1 not cleanly\./);
|
||||
assert.match(message, /BACKUPS\n- Restore test of Photos share \(2026-09-14\) passed/);
|
||||
assert.match(message, /RENEWED\n- Certificate git\.lab: 2026-09-14 → 2026-12-13/);
|
||||
const partial = monthlyReportText(await buildMonthlyReport("2026-10", NOW));
|
||||
assert.equal(partial.title, "Homelab monthly report — October 2026 (so far)");
|
||||
const empty = monthlyReportText(await buildMonthlyReport("2025-01", NOW));
|
||||
assert.equal(empty.message.includes("Nothing was recorded") || !/INCIDENTS\n[1-9]/.test(empty.message), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let hook: Server;
|
||||
let base = "";
|
||||
let who = "viewer";
|
||||
const messages: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
]);
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/reports", reportsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/reports`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("anyone signed in can read it: the last whole month by default, the twelve to choose from, or one of them; nothing else is a month", async () => {
|
||||
const body = await (await fetch(`${base}/monthly`)).json();
|
||||
assert.equal(body.months.length, 12);
|
||||
assert.equal(body.report.month, body.months[1].id, "last month, not this one");
|
||||
assert.equal((await (await fetch(`${base}/monthly?month=${body.months[0].id}`)).json()).report.partial, true);
|
||||
for (const bad of ["2026-13", "2019-01", "abc", "2026-9"]) assert.equal((await fetch(`${base}/monthly?month=${bad}`)).status, 400, bad);
|
||||
const text = await (await fetch(`${base}/monthly?format=text`)).json();
|
||||
assert.match(text.title, /^Homelab monthly report — /);
|
||||
assert.equal(typeof text.message, "string");
|
||||
});
|
||||
|
||||
test("an administrator can send it now to the channels; nobody else can", async () => {
|
||||
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 403);
|
||||
who = "admin";
|
||||
const res = await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal((await fetch(`${base}/monthly/send`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ month: "nope" }) })).status, 400);
|
||||
assert.equal((await db.select().from(auditLog).where(eq(auditLog.action, "send_monthly"))).length, 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* New ports: what counts as one, that the first report is the baseline, and what the health check, the Alerts page, maintenance, routing and
|
||||
* the server's Ports card do with them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-newports-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, serverPorts } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { newListeningPorts, baselineFrom, parseBaseline, newPortMessage } = await import("../src/services/newPorts.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { syncServerTasks } = await import("../src/services/taskSync.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
|
||||
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { serverPortsRouter } = await import("../src/routes/serverPorts.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const sock = (port: number, extra: Partial<{ protocol: "tcp" | "udp"; address: string; process: string }> = {}) => ({ protocol: "tcp" as const, port, address: "0.0.0.0", ...extra });
|
||||
const none = new Set<string>();
|
||||
|
||||
describe("what counts as a new port", () => {
|
||||
const baseline = parseBaseline(baselineFrom([sock(22), sock(443), sock(53, { protocol: "udp" })]));
|
||||
|
||||
test("the baseline is everything the first report listened on, in any protocol", () => {
|
||||
assert.deepEqual([...baseline!].sort(), ["tcp:22", "tcp:443", "udp:53"]);
|
||||
assert.equal(parseBaseline(null), null);
|
||||
assert.equal(parseBaseline("not json"), null);
|
||||
assert.equal(parseBaseline(JSON.stringify({ not: "a list" })), null);
|
||||
});
|
||||
|
||||
test("with no baseline yet, nothing is new", () => {
|
||||
assert.deepEqual(newListeningPorts([sock(8080)], null, none), []);
|
||||
assert.deepEqual(newListeningPorts(null, baseline, none), []);
|
||||
});
|
||||
|
||||
test("a TCP port that is listening and was not in the baseline is new, with what is listening on it", () => {
|
||||
const found = newListeningPorts([sock(22), sock(8080, { process: "node" }), sock(8080, { address: "::", process: "" }), sock(3000)], baseline, none);
|
||||
assert.deepEqual(found, [{ protocol: "tcp", port: 3000, process: null }, { protocol: "tcp", port: 8080, process: "node" }]);
|
||||
});
|
||||
|
||||
test("not UDP, not the dynamic range, not a port bound to this machine alone", () => {
|
||||
const found = newListeningPorts([sock(5353, { protocol: "udp" }), sock(49152), sock(60000), sock(9000, { address: "127.0.0.1" }), sock(9001, { address: "::1" }), sock(9002, { address: "[::1]" }), sock(9003, { address: "127.0.0.1" }), sock(9003, { address: "10.0.0.5" })], baseline, none);
|
||||
assert.deepEqual(found.map((p) => p.port), [9003], "one socket on a real address is enough");
|
||||
assert.deepEqual(newListeningPorts([sock(49151)], baseline, none).map((p) => p.port), [49151], "the last port before the dynamic range is still a service");
|
||||
});
|
||||
|
||||
test("a port somebody noted as expected is not new", () => {
|
||||
assert.deepEqual(newListeningPorts([sock(8080), sock(9090)], baseline, new Set(["tcp:8080"])).map((p) => p.port), [9090]);
|
||||
});
|
||||
|
||||
test("the message says what it is and what to do", () => {
|
||||
assert.equal(newPortMessage("nas", { protocol: "tcp", port: 8080, process: "node" }), "nas: a new port is listening — 8080/tcp (node). If it should be there, note it on the server's Ports card.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health check, the alert and the routing", () => {
|
||||
const snap = (extra: object) => ({ servers: [{ id: 3, name: "nas", lastSeenAt: new Date().toISOString(), disks: [], ...extra }], proxmox: [], synology: [] });
|
||||
const limits = { serverOfflineMinutes: 60, diskUsagePercent: 90 };
|
||||
|
||||
test("each new port is a warning keyed by server and port; an offline server's are not judged", () => {
|
||||
const list = evaluateHealth(snap({ newPorts: [{ protocol: "tcp", port: 8080, process: "node" }, { protocol: "tcp", port: 9090, process: null }] }), limits, Date.now());
|
||||
assert.deepEqual(list.map((c) => [c.key, c.severity, c.source]), [["new-port:server:3:tcp:8080", "warning", "server"], ["new-port:server:3:tcp:9090", "warning", "server"]]);
|
||||
assert.match(list[0].message, /nas: a new port is listening — 8080\/tcp \(node\)/);
|
||||
const offline = evaluateHealth(snap({ lastSeenAt: new Date(Date.now() - 5 * 3600_000).toISOString(), newPorts: [{ protocol: "tcp", port: 8080, process: null }] }), limits, Date.now());
|
||||
assert.equal(offline.some((c) => c.key.startsWith("new-port")), false);
|
||||
});
|
||||
|
||||
test("maintenance on the server holds it back, and routing treats it as monitoring with that server's tags", () => {
|
||||
assert.equal(subjectOfConditionKey("new-port:server:3:tcp:8080"), "server:3");
|
||||
assert.deepEqual(metaOfConditionKey("new-port:server:3:tcp:8080", "warning"), { category: "monitoring", severity: "warning", serverId: 3 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("from a report to the Ports card", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let serverId = 0;
|
||||
let who = "operator";
|
||||
const report = (ports: ReturnType<typeof sock>[]) => ({ tasks: [], system: { listening_ports: ports } });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown; currentUser: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
// The real servers router puts a signed-in user on the request before this; here the role is read the same way the middleware does.
|
||||
const { requireAuth } = await import("../src/auth/middleware.js");
|
||||
a.use("/api/servers/:id/ports", requireAuth, serverPortsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/servers/${serverId}/ports`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const newKeys = async () => (await collectSnapshot()).snapshot.servers.find((s) => s.id === serverId)!.newPorts!.map((p) => p.port);
|
||||
|
||||
test("the first report that carries ports is the baseline: nothing in it is new, and it is never changed by a later one", async () => {
|
||||
await syncServerTasks(serverId, report([sock(22), sock(443)]));
|
||||
let [row] = await db.select().from(servers).where(eq(servers.id, serverId));
|
||||
assert.deepEqual(JSON.parse(row.portBaseline!), ["tcp:22", "tcp:443"]);
|
||||
assert.deepEqual(await newKeys(), []);
|
||||
await syncServerTasks(serverId, report([sock(22), sock(443), sock(8080, { process: "node" })]));
|
||||
[row] = await db.select().from(servers).where(eq(servers.id, serverId));
|
||||
assert.deepEqual(JSON.parse(row.portBaseline!), ["tcp:22", "tcp:443"], "8080 is not folded into what is normal");
|
||||
assert.deepEqual(await newKeys(), [8080]);
|
||||
});
|
||||
|
||||
test("a report with no ports at all does not start a baseline", async () => {
|
||||
const [other] = await db.insert(servers).values({ name: "bare", apiTokenHash: "b", apiTokenPrefix: "b" }).returning({ id: servers.id });
|
||||
await syncServerTasks(other.id, { tasks: [], system: {} });
|
||||
const [row] = await db.select().from(servers).where(eq(servers.id, other.id));
|
||||
assert.equal(row.portBaseline, null);
|
||||
});
|
||||
|
||||
test("it is an alert on the Alerts page, in monitoring, linked to the server — and silenced by maintenance on it", async () => {
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((a) => a.message.includes("a new port is listening"))!;
|
||||
assert.ok(alert, "the alert is there");
|
||||
assert.deepEqual([alert.category, alert.severity, alert.silenced], ["monitoring", "warning", false]);
|
||||
assert.deepEqual(alert.target && [alert.target.type, alert.target.id], ["server", serverId]);
|
||||
await startOrExtendWindow("server", serverId, 30, "work", "Ada");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((a) => a.message.includes("a new port is listening"))!.silenced, true);
|
||||
});
|
||||
|
||||
test("the Ports card marks it new, and noting it as expected ends the alert; a scan-only row does not", async () => {
|
||||
who = "viewer";
|
||||
let list = (await (await fetch(base)).json()).ports as { port: number; isNew: boolean }[];
|
||||
assert.deepEqual(list.filter((p) => p.isNew).map((p) => p.port), [8080]);
|
||||
await db.insert(serverPorts).values({ serverId, port: 8080, protocol: "tcp", open: true });
|
||||
assert.deepEqual(await newKeys(), [8080], "a row that only records an open scan says nothing");
|
||||
await db.delete(serverPorts).where(eq(serverPorts.port, 8080));
|
||||
|
||||
who = "operator";
|
||||
const res = await fetch(base, { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ port: 8080, protocol: "tcp", label: "Expected" }) });
|
||||
assert.equal(res.status, 200);
|
||||
list = (await res.json()).ports;
|
||||
assert.equal(list.find((p) => p.port === 8080)!.isNew, false);
|
||||
assert.deepEqual(await newKeys(), []);
|
||||
});
|
||||
|
||||
test("it can be switched off, and then nothing is new", async () => {
|
||||
await syncServerTasks(serverId, report([sock(22), sock(443), sock(8080), sock(9999)]));
|
||||
assert.deepEqual(await newKeys(), [9999]);
|
||||
await updateSettings({ healthChecks: { newPortAlerts: false } });
|
||||
assert.deepEqual(await newKeys(), []);
|
||||
who = "viewer";
|
||||
assert.equal(((await (await fetch(base)).json()).ports as { isNew: boolean }[]).some((p) => p.isNew), false);
|
||||
await updateSettings({ healthChecks: { newPortAlerts: true } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,407 @@
|
||||
/**
|
||||
* Patch plans: who a plan covers, when it is due, that it goes one server at a time and waits for each, what it does when one fails, what it silences,
|
||||
* the message at the end, and who may do what.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-patchplans-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, patchPlans, patchPlanRuns, updateRuns, maintenanceWindows } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { planServers, dueOccurrence, summariseItems, startPlanRun, advancePlanRuns, startDuePlans, cancelPlanRun, parseItems, PlanError } = await import("../src/services/patchPlans.js");
|
||||
const { advanceUpdateRuns, setSemaphoreForTests } = await import("../src/services/updateRuns.js");
|
||||
const { listActiveWindows } = await import("../src/services/maintenance.js");
|
||||
const { agendaFromData } = await import("../src/services/agenda.js");
|
||||
const { patchPlansRouter } = await import("../src/routes/patchPlans.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
// ── a Semaphore that answers as told ──
|
||||
const fake = { runs: [] as { limit?: string }[], task: { id: 41, status: "waiting", created: null as string | null, start: null as string | null, end: null as string | null }, refuseFor: new Set<string>() };
|
||||
setSemaphoreForTests(async (integrationId) =>
|
||||
integrationId === 9
|
||||
? {
|
||||
ping: async () => ({ ok: true }),
|
||||
listTemplatesWithStatus: async () => [],
|
||||
checkTemplates: async () => ({ templates: [{ id: 7, projectId: 2, projectName: "Homelab", name: "Apply updates", playbook: "u.yml", lastTask: null }], failedProjectIds: [] }),
|
||||
runTemplate: async (_p: number, _t: number, options?: { limit?: string }) => {
|
||||
if (options?.limit && fake.refuseFor.has(options.limit)) throw new Error("Semaphore is having a moment");
|
||||
fake.runs.push({ limit: options?.limit });
|
||||
return { ...fake.task } as never;
|
||||
},
|
||||
getTask: async () => ({ ...fake.task }) as never,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const T0 = new Date("2026-10-18T02:05:00.000Z"); // a Sunday
|
||||
const at = (m: number) => new Date(T0.getTime() + m * 60_000);
|
||||
|
||||
describe("who a plan covers, and when it is due", () => {
|
||||
const all = [
|
||||
{ id: 1, name: "web", tags: JSON.stringify(["lab"]) },
|
||||
{ id: 2, name: "api", tags: JSON.stringify(["prod"]) },
|
||||
{ id: 3, name: "ångström", tags: JSON.stringify(["lab", "prod"]) },
|
||||
{ id: 4, name: "db", tags: null },
|
||||
];
|
||||
|
||||
test("servers with any of the tags, and those named, by name; each only once", () => {
|
||||
assert.deepEqual(planServers({ tags: JSON.stringify(["lab"]), serverIds: "[]" }, all).map((s) => s.name), ["ångström", "web"].sort((x, y) => x.localeCompare(y)));
|
||||
assert.deepEqual(planServers({ tags: JSON.stringify(["lab"]), serverIds: JSON.stringify([4, 3]) }, all).map((s) => s.name), ["db", "ångström", "web"].sort((x, y) => x.localeCompare(y)), "in the order the app sorts names");
|
||||
assert.deepEqual(planServers({ tags: "[]", serverIds: "[]" }, all), [], "a plan that names no one covers no one");
|
||||
});
|
||||
|
||||
const plan = { enabled: true, days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", lastRunDate: null as string | null };
|
||||
|
||||
test("due from its time to half an hour after, once a day, only on its days", () => {
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T01:59:00Z")), null, "not yet");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:00:00Z"))?.date, "2026-10-18");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:29:00Z"))?.date, "2026-10-18", "the app may have been down for a while");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-18T02:31:00Z")), null, "too late: that day is skipped");
|
||||
assert.equal(dueOccurrence({ ...plan, lastRunDate: "2026-10-18" }, new Date("2026-10-18T02:10:00Z")), null, "already started today");
|
||||
assert.equal(dueOccurrence(plan, new Date("2026-10-19T02:10:00Z")), null, "Monday is not a day it runs");
|
||||
assert.equal(dueOccurrence({ ...plan, enabled: false }, new Date("2026-10-18T02:10:00Z")), null);
|
||||
});
|
||||
|
||||
test("its time is on the clock of its time zone, winter and summer", () => {
|
||||
const stockholm = { ...plan, timezone: "Europe/Stockholm" };
|
||||
assert.equal(dueOccurrence(stockholm, new Date("2026-10-18T00:05:00Z"))?.date, "2026-10-18", "02:05 in Stockholm is 00:05 UTC while summer time lasts");
|
||||
assert.equal(dueOccurrence(stockholm, new Date("2026-11-01T01:05:00Z"))?.date, "2026-11-01", "and 01:05 UTC after the clocks go back");
|
||||
assert.equal(dueOccurrence(stockholm, new Date("2026-11-01T00:05:00Z")), null);
|
||||
});
|
||||
|
||||
test("a finished run says how each went in one line", () => {
|
||||
const item = (status: string) => ({ serverId: 1, name: "x", status, updateRunId: null, message: null, windowIds: [] }) as never;
|
||||
assert.equal(summariseItems([item("done"), item("done"), item("failed"), item("skipped")]), "2 updated, 1 failed, 1 left alone of 4 servers.");
|
||||
assert.equal(summariseItems([item("done")]), "1 updated of 1 server.");
|
||||
});
|
||||
|
||||
test("a plan is in the agenda as maintenance, with how many servers it updates", () => {
|
||||
const data = { secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], patchPlans: [{ id: 5, name: "Lab nights", days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", servers: 3 }] };
|
||||
const items = agendaFromData(data, new Date("2026-10-19T12:00:00Z"), "UTC", 20).items.filter((i) => i.id.startsWith("plan:"));
|
||||
assert.deepEqual(items.map((i) => [i.title, i.detail, i.at.slice(0, 16), i.kind]), [["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-10-25T02:00", "maintenance"], ["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-11-01T02:00", "maintenance"], ["Patch plan: Lab nights", "updates 3 servers, one at a time", "2026-11-08T02:00", "maintenance"]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a run, from start to end", () => {
|
||||
let hook: Server;
|
||||
const messages: string[] = [];
|
||||
const ids: Record<string, number> = {};
|
||||
let planId = 0;
|
||||
|
||||
const plan = (extra: Partial<typeof patchPlans.$inferInsert> = {}) => ({
|
||||
name: "Lab nights", days: JSON.stringify([0]), startTime: "02:00", timezone: "UTC", tags: JSON.stringify(["lab"]), serverIds: "[]", integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", limitFrom: "name", stopOnFailure: true, onlyWithUpdates: true, silenceDependents: true, windowMinutes: 90, ...extra,
|
||||
});
|
||||
const updates = (count: number, reportedAt = T0.toISOString()) => JSON.stringify({ manager: "apt", count, security: 0, rebootRequired: false, reportedAt });
|
||||
const runOf = async (id: number) => (await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)))[0];
|
||||
const itemsOf = async (id: number) => parseItems((await runOf(id)).items);
|
||||
|
||||
/** The playbook for the server being updated finishes and the server reports again — as the update run follows. */
|
||||
async function finishCurrent(server: string, minute: number) {
|
||||
fake.task = { id: 41, status: "success", created: null, start: null, end: at(minute).toISOString() };
|
||||
await advanceUpdateRuns(at(minute + 1));
|
||||
await db.update(servers).set({ lastSeenAt: at(minute + 2).toISOString(), pendingUpdates: updates(0, at(minute + 2).toISOString()) }).where(eq(servers.id, ids[server]));
|
||||
await advanceUpdateRuns(at(minute + 3));
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
}
|
||||
|
||||
async function reset(extra: Partial<typeof patchPlans.$inferInsert> = {}) {
|
||||
await db.delete(patchPlans);
|
||||
await db.delete(updateRuns);
|
||||
await db.delete(maintenanceWindows); // the tests share one clock, so a window left by the last would still be open
|
||||
fake.runs = [];
|
||||
fake.refuseFor.clear();
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
for (const name of ["web", "api"]) await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: updates(4) }).where(eq(servers.id, ids[name]));
|
||||
await db.update(servers).set({ lastSeenAt: T0.toISOString(), pendingUpdates: updates(0) }).where(eq(servers.id, ids.idle));
|
||||
messages.length = 0;
|
||||
planId = (await db.insert(patchPlans).values(plan(extra)).returning({ id: patchPlans.id }))[0].id;
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
for (const name of ["web", "api", "idle"]) {
|
||||
ids[name] = (await db.insert(servers).values({ name, apiTokenHash: name, apiTokenPrefix: name, tags: JSON.stringify(["lab"]), lastSeenAt: T0.toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
}
|
||||
ids.prod = (await db.insert(servers).values({ name: "prod", apiTokenHash: "p", apiTokenPrefix: "p", tags: JSON.stringify(["prod"]), lastSeenAt: T0.toISOString(), pendingUpdates: updates(9) }).returning({ id: servers.id }))[0].id;
|
||||
// a guest that runs on web: what depends on web
|
||||
ids.guest = (await db.insert(servers).values({ name: "guest", apiTokenHash: "g", apiTokenPrefix: "g", lastSeenAt: T0.toISOString(), parentServerId: ids.web }).returning({ id: servers.id }))[0].id;
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
});
|
||||
|
||||
after(() => {
|
||||
hook.close();
|
||||
setSemaphoreForTests(null);
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("one server at a time, each waited for, those with nothing waiting left alone, and a message at the end", async () => {
|
||||
await reset();
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => [i.name, i.status]), [["api", "pending"], ["idle", "pending"], ["web", "pending"]], "by name");
|
||||
|
||||
await advancePlanRuns(T0);
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["running", "pending", "pending"]);
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["api"], "only the first has been started");
|
||||
assert.equal((await listActiveWindows(T0)).some((w) => w.targetType === "server" && w.targetId === ids.api), true, "and it is under maintenance");
|
||||
|
||||
await advancePlanRuns(at(1));
|
||||
assert.equal(fake.runs.length, 1, "while it is still running, nothing else starts");
|
||||
|
||||
await finishCurrent("api", 5);
|
||||
await advancePlanRuns(at(9));
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["done", "skipped", "running"], "idle has nothing waiting, so it is passed over, and web is next");
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["api", "web"]);
|
||||
assert.equal((await itemsOf(run.id))[1].message, "No updates were waiting.");
|
||||
assert.equal((await listActiveWindows(at(9))).some((w) => w.targetId === ids.api), false, "api's maintenance ended with its update");
|
||||
|
||||
await finishCurrent("web", 15);
|
||||
await advancePlanRuns(at(19));
|
||||
const done = await runOf(run.id);
|
||||
assert.equal(done.status, "done");
|
||||
assert.equal(done.message, "2 updated, 1 left alone of 3 servers.");
|
||||
assert.equal(messages.length >= 3, true, "each update run says how it went, and so does the plan");
|
||||
const last = messages[messages.length - 1];
|
||||
assert.match(last, /2 updated, 1 left alone of 3 servers\./);
|
||||
assert.match(last, /✓ api/);
|
||||
assert.match(last, /– idle — No updates were waiting\./);
|
||||
assert.match(last, /✓ web/);
|
||||
});
|
||||
|
||||
test("what depends on a server is silenced while it is updated, and let go when it is done", async () => {
|
||||
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
const during = await listActiveWindows(T0);
|
||||
assert.equal(during.some((w) => w.targetId === ids.guest), true, "the guest that runs on web");
|
||||
assert.match(during.find((w) => w.targetId === ids.guest)!.reason ?? "", /Depends on web — patch plan "Lab nights"/);
|
||||
assert.equal((await itemsOf(run.id))[0].windowIds.length, 1);
|
||||
await finishCurrent("web", 5);
|
||||
await advancePlanRuns(at(9));
|
||||
assert.equal((await listActiveWindows(at(9))).some((w) => w.targetId === ids.guest), false);
|
||||
assert.equal((await runOf(run.id)).status, "done");
|
||||
});
|
||||
|
||||
test("it can be told not to silence what depends on a server", async () => {
|
||||
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]), silenceDependents: false });
|
||||
await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
assert.equal((await listActiveWindows(T0)).some((w) => w.targetId === ids.guest), false);
|
||||
});
|
||||
|
||||
test("a failure stops the plan and leaves the rest alone, saying where it stopped", async () => {
|
||||
await reset();
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
fake.task = { id: 41, status: "error", created: null, start: null, end: null };
|
||||
await advanceUpdateRuns(at(2));
|
||||
await advancePlanRuns(at(3));
|
||||
const after = await runOf(run.id);
|
||||
assert.equal(after.status, "failed");
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => [i.name, i.status]), [["api", "failed"], ["idle", "skipped"], ["web", "skipped"]]);
|
||||
assert.equal((await itemsOf(run.id))[2].message, "Not started, because an earlier server failed.");
|
||||
assert.match(after.message ?? "", /1 failed, 2 left alone of 3 servers\. Stopped after api\./);
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["api"], "web was never touched");
|
||||
assert.match(messages[messages.length - 1], /✕ api/);
|
||||
assert.equal(messages.some((m) => m.includes("didn't finish cleanly")), true);
|
||||
});
|
||||
|
||||
test("told to carry on, it does: the failed one is reported and the rest are updated", async () => {
|
||||
await reset({ stopOnFailure: false });
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
fake.task = { id: 41, status: "error", created: null, start: null, end: null };
|
||||
await advanceUpdateRuns(at(2));
|
||||
fake.task = { id: 41, status: "waiting", created: null, start: null, end: null };
|
||||
await advancePlanRuns(at(3));
|
||||
assert.deepEqual((await itemsOf(run.id)).map((i) => i.status), ["failed", "skipped", "running"], "web is next");
|
||||
await finishCurrent("web", 10);
|
||||
await advancePlanRuns(at(14));
|
||||
const done = await runOf(run.id);
|
||||
assert.equal(done.status, "failed", "something failed, so the run says so");
|
||||
assert.equal(done.message, "1 updated, 1 failed, 1 left alone of 3 servers.");
|
||||
});
|
||||
|
||||
test("a server that can't be started counts as a failure; a server with nothing to update can be updated all the same if asked", async () => {
|
||||
await reset({ onlyWithUpdates: false, tags: "[]", serverIds: JSON.stringify([ids.idle]) });
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
assert.deepEqual(fake.runs.map((r) => r.limit), ["idle"], "it had nothing waiting but the plan was told not to mind");
|
||||
await updateRunsCancelAll();
|
||||
await cancelPlanRun(run.id, "Olle", at(1));
|
||||
|
||||
await reset({ tags: "[]", serverIds: JSON.stringify([ids.web]) });
|
||||
fake.refuseFor.add("web");
|
||||
const refused = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
const items = await itemsOf(refused.id);
|
||||
assert.equal(items[0].status, "failed");
|
||||
assert.match(items[0].message ?? "", /Semaphore wouldn't start it: Semaphore is having a moment/);
|
||||
assert.equal((await runOf(refused.id)).status, "failed");
|
||||
});
|
||||
|
||||
async function updateRunsCancelAll() {
|
||||
await db.update(updateRuns).set({ status: "cancelled" });
|
||||
}
|
||||
|
||||
test("a plan does not overlap itself, and one with no servers says so rather than doing nothing silently", async () => {
|
||||
await reset();
|
||||
await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await assert.rejects(startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 }), (e: unknown) => e instanceof PlanError && e.code === "already_running");
|
||||
await db.update(patchPlanRuns).set({ status: "cancelled" });
|
||||
|
||||
await db.update(patchPlans).set({ tags: JSON.stringify(["nothing-has-this"]) }).where(eq(patchPlans.id, planId));
|
||||
await assert.rejects(startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 }), (e: unknown) => e instanceof PlanError && e.code === "no_servers");
|
||||
});
|
||||
|
||||
test("stopping a run that is going leaves the rest alone, ends the maintenance it made and stops the update run being followed", async () => {
|
||||
await reset();
|
||||
const run = await startPlanRun(planId, { trigger: "manual", actor: "Olle", now: T0 });
|
||||
await advancePlanRuns(T0);
|
||||
const stopped = await cancelPlanRun(run.id, "Olga", at(2));
|
||||
assert.equal(stopped!.status, "cancelled");
|
||||
assert.deepEqual(parseItems(stopped!.items).map((i) => i.status), ["failed", "skipped", "skipped"]);
|
||||
assert.equal((await listActiveWindows(at(2))).some((w) => w.targetId === ids.api), false);
|
||||
assert.equal((await db.select().from(updateRuns))[0].status, "cancelled");
|
||||
await assert.rejects(cancelPlanRun(run.id, "Olga", at(3)), (e: unknown) => e instanceof PlanError && e.code === "not_active");
|
||||
assert.equal(await cancelPlanRun(99999, "Olga"), null);
|
||||
});
|
||||
|
||||
test("the schedule starts a plan once on its day, and a restart of the app does not start it again", async () => {
|
||||
await reset();
|
||||
assert.equal(await startDuePlans(T0), 1);
|
||||
const [row] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId));
|
||||
assert.equal(row.lastRunDate, "2026-10-18");
|
||||
assert.equal((await db.select().from(patchPlanRuns))[0].trigger, "schedule");
|
||||
await db.update(patchPlanRuns).set({ status: "done" });
|
||||
assert.equal(await startDuePlans(at(10)), 0, "not twice in a day");
|
||||
assert.equal(await startDuePlans(new Date("2026-10-25T02:03:00Z")), 1, "but the next Sunday it does");
|
||||
});
|
||||
|
||||
test("a plan that is due but has no servers leaves a failed run to say so", async () => {
|
||||
await reset({ tags: JSON.stringify(["nothing-has-this"]) });
|
||||
await db.delete(patchPlanRuns);
|
||||
assert.equal(await startDuePlans(T0), 0);
|
||||
const [run] = await db.select().from(patchPlanRuns);
|
||||
assert.deepEqual([run.status, run.trigger], ["failed", "schedule"]);
|
||||
assert.match(run.message ?? "", /^Didn't start: No server has this plan's tags/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
const body = (extra: object = {}) => ({ name: "Weekend patching", days: [6, 0], startTime: "03:30", timezone: "Europe/Stockholm", tags: ["Lab"], serverIds: [], integrationId: 9, projectId: 2, templateId: 7, templateName: "Apply updates", ...extra });
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/patch-plans", patchPlansRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/patch-plans`;
|
||||
await db.delete(patchPlans);
|
||||
await db.delete(patchPlanRuns);
|
||||
});
|
||||
|
||||
after(() => app.close());
|
||||
|
||||
const send = (method: string, path: string, b?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: b === undefined ? undefined : JSON.stringify(b) });
|
||||
let id = 0;
|
||||
|
||||
test("an administrator makes a plan, with its tags written the way tags are; a wrong one is refused with a reason; nobody else can", async () => {
|
||||
const res = await send("POST", "/", body());
|
||||
assert.equal(res.status, 201);
|
||||
const plan = (await res.json()).plan;
|
||||
id = plan.id;
|
||||
assert.deepEqual([plan.tags, plan.days, plan.limitFrom, plan.stopOnFailure, plan.onlyWithUpdates, plan.windowMinutes], [["lab"], [0, 6], "name", true, true, 90]);
|
||||
assert.deepEqual(plan.servers.map((s: { name: string }) => s.name), ["api", "idle", "web"], "the three tagged lab");
|
||||
assert.equal(plan.next.length, 3, "its next times, so nobody has to work out when");
|
||||
for (const bad of [{ days: [] }, { startTime: "3:30" }, { timezone: "Mars/Olympus" }, { tags: [], serverIds: [] }, { tags: ["bad tag!"] }, { windowMinutes: 5 }, { name: " " }, { integrationId: 0 }]) {
|
||||
const r = await send("POST", "/", body(bad));
|
||||
assert.equal(r.status, 400, JSON.stringify(bad));
|
||||
assert.ok((await r.json()).message.length > 0);
|
||||
}
|
||||
who = "operator";
|
||||
assert.equal((await send("POST", "/", body())).status, 403);
|
||||
assert.equal((await send("PUT", `/${id}`, body())).status, 403);
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 403);
|
||||
who = "viewer";
|
||||
const list = await (await send("GET", "/")).json();
|
||||
assert.equal(list.plans.length, 1, "everyone can see what is planned");
|
||||
assert.ok(list.tags.includes("lab") && list.servers.length >= 4);
|
||||
});
|
||||
|
||||
test("changing when it happens forgets which day it last started on; changing anything else does not", async () => {
|
||||
who = "admin";
|
||||
await db.update(patchPlans).set({ lastRunDate: "2026-10-18" }).where(eq(patchPlans.id, id));
|
||||
await send("PUT", `/${id}`, body({ name: "Renamed" }));
|
||||
assert.equal((await db.select().from(patchPlans).where(eq(patchPlans.id, id)))[0].lastRunDate, "2026-10-18");
|
||||
await send("PUT", `/${id}`, body({ name: "Renamed", startTime: "04:00" }));
|
||||
assert.equal((await db.select().from(patchPlans).where(eq(patchPlans.id, id)))[0].lastRunDate, null);
|
||||
assert.equal((await send("PUT", "/99999", body())).status, 404);
|
||||
});
|
||||
|
||||
test("an operator can run it now and stop it; a viewer cannot; and it cannot be removed while it is running", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", `/${id}/run`)).status, 403);
|
||||
who = "operator";
|
||||
const res = await send("POST", `/${id}/run`);
|
||||
assert.equal(res.status, 201);
|
||||
const run = (await res.json()).run;
|
||||
assert.equal(run.status, "running");
|
||||
assert.equal(run.trigger, "manual");
|
||||
assert.ok(run.items.every((i: { windowIds?: unknown }) => i.windowIds === undefined), "the windows it made are its own business");
|
||||
assert.equal((await send("POST", `/${id}/run`)).status, 409, "not twice at once");
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 409);
|
||||
who = "operator";
|
||||
const stopped = await send("POST", `/runs/${run.id}/cancel`);
|
||||
assert.equal(stopped.status, 200);
|
||||
assert.equal((await stopped.json()).run.status, "cancelled");
|
||||
assert.equal((await send("POST", `/runs/${run.id}/cancel`)).status, 409);
|
||||
assert.equal((await send("POST", "/runs/99999/cancel")).status, 404);
|
||||
who = "admin";
|
||||
assert.equal((await send("DELETE", `/${id}`)).status, 204);
|
||||
assert.equal((await db.select().from(patchPlanRuns)).length, 0, "its history goes with it");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user