Patch plans, a monthly report and new-port alerts

Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an
ordinary update run, stopping at the first failure by default, silencing what depends on a
server while it is updated, with one message at the end.

Monthly report (Operations -> Monthly report, and optionally a message on a day of the month):
availability, incidents, updates, restore tests, backups and renewals for a calendar month,
printable. Backed by daily endpoint figures and a record of renewals.

New ports: the first report with ports is the baseline; a TCP port that begins listening later and
is not noted on the server's Ports card is a warning, cleared by Mark as expected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 23:16:14 +02:00
1 parent ddd0aa622a
commit be6c5552bd
46 files changed
+7836 -26

No files matched your search

+4
View File
@@ -50,6 +50,8 @@ import { alertTicketsRouter } from "./routes/alertTickets.js";
import { avatarsRouter } from "./routes/avatars.js";
import { heartbeatsRouter } from "./routes/heartbeats.js";
import { agendaRouter } from "./routes/agenda.js";
import { patchPlansRouter } from "./routes/patchPlans.js";
import { reportsRouter } from "./routes/reports.js";
import { publicCalendarRouter } from "./routes/publicCalendar.js";
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
@@ -113,6 +115,8 @@ export function createApp() {
app.use("/api/avatars", avatarsRouter);
app.use("/api/heartbeats", heartbeatsRouter);
app.use("/api/agenda", agendaRouter);
app.use("/api/patch-plans", patchPlansRouter);
app.use("/api/reports", reportsRouter);
app.use("/api/uptime", uptimeRouter);
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
+88 -1
View File
@@ -1,5 +1,5 @@
import { sql } from "drizzle-orm";
import { sqliteTable, text, integer, real, uniqueIndex, index } from "drizzle-orm/sqlite-core";
import { sqliteTable, text, integer, real, uniqueIndex, index, primaryKey } from "drizzle-orm/sqlite-core";
// ─── Users & roles ──────────────────────────────────────────────────────────
@@ -447,6 +447,10 @@ export const servers = sqliteTable("servers", {
// What the agent says it could not collect, as JSON: {items: [{check, status, reason?, detail?}], reportedAt} — see services/agentDiagnostics.ts. Null until an agent that can say so has.
agentDiagnostics: text("agent_diagnostics"),
// The ports this server was listening on when its agent first said so, as a JSON array like ["tcp:22","tcp:443"]. A port that is listening later and is
// not in this list, and not noted on the Ports page, is a "new port" (services/newPorts.ts). Null until the first report that carries ports.
portBaseline: text("port_baseline"),
// What the machine is, as its firmware says (agents report it): maker, model and serial number. Used to fill in the inventory.
hwManufacturer: text("hw_manufacturer"),
hwModel: text("hw_model"),
@@ -846,6 +850,89 @@ export const updateRuns = sqliteTable(
(t) => [index("update_runs_server_idx").on(t.serverId, t.startedAt), index("update_runs_status_idx").on(t.status)],
);
// ─── Patch plans: updates, on a schedule, one server at a time ─────────────
// "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure." A plan only says when, to whom and how; each server is updated by an
// ordinary update run (services/updateRuns.ts), and the plan waits for it to finish before the next — see services/patchPlans.ts.
export const patchPlans = sqliteTable("patch_plans", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
days: text("days").notNull(), // JSON: weekdays, 0 = Sunday … 6 = Saturday
startTime: text("start_time").notNull(), // "HH:MM" on the clock of `timezone`
timezone: text("timezone").notNull(),
tags: text("tags").notNull().default("[]"), // JSON string[]: servers that have any of these tags
serverIds: text("server_ids").notNull().default("[]"), // JSON number[]: and these servers by name, whatever their tags
integrationId: integer("integration_id").notNull(), // the Semaphore integration; no FK, like update runs
projectId: integer("project_id").notNull(),
templateId: integer("template_id").notNull(),
templateName: text("template_name").notNull(),
limitFrom: text("limit_from").notNull().default("name"), // what ansible's --limit is for a server: its name here, or its host name
stopOnFailure: integer("stop_on_failure", { mode: "boolean" }).notNull().default(true),
onlyWithUpdates: integer("only_with_updates", { mode: "boolean" }).notNull().default(true), // leave out a server that has nothing waiting
silenceDependents: integer("silence_dependents", { mode: "boolean" }).notNull().default(true), // what depends on a server is silenced while it is updated
windowMinutes: integer("window_minutes").notNull().default(90), // at most this long under maintenance per server
lastRunDate: text("last_run_date"), // the local day (YYYY-MM-DD) it last started on — so it starts once a day, even if the app is restarted
createdBy: text("created_by"),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
});
// One time a plan ran, and how it went for each server: items is JSON [{serverId, name, status, updateRunId, message, windowIds}].
export const patchPlanRuns = sqliteTable(
"patch_plan_runs",
{
id: integer("id").primaryKey({ autoIncrement: true }),
planId: integer("plan_id")
.notNull()
.references(() => patchPlans.id, { onDelete: "cascade" }),
planName: text("plan_name").notNull(), // as it was, so a renamed plan's history still reads right
trigger: text("trigger").notNull(), // schedule | manual
startedBy: text("started_by"),
status: text("status").notNull(), // running | done | failed | cancelled
message: text("message"),
items: text("items").notNull().default("[]"),
startedAt: text("started_at").notNull(), // ISO
finishedAt: text("finished_at"), // ISO
},
(t) => [index("patch_plan_runs_plan_idx").on(t.planId, t.startedAt), index("patch_plan_runs_status_idx").on(t.status)],
);
// ─── The monthly report's long memory ───────────────────────────────────────
// One row per endpoint check per day: how many times it was asked, how many answered, and the answer times — kept for years, where the individual
// results are kept for two weeks. What the monthly report's availability is worked from (services/endpointChecks.ts).
export const endpointDaily = sqliteTable(
"endpoint_daily",
{
checkId: integer("check_id")
.notNull()
.references(() => endpointChecks.id, { onDelete: "cascade" }),
day: text("day").notNull(), // YYYY-MM-DD, UTC
total: integer("total").notNull().default(0),
ok: integer("ok").notNull().default(0),
msSum: integer("ms_sum").notNull().default(0), // of the answers that were good and timed
msCount: integer("ms_count").notNull().default(0),
},
(t) => [primaryKey({ columns: [t.checkId, t.day] })],
);
// A certificate, domain registration or secret whose expiry moved later: it was renewed. Written when the app sees it happen, so it only goes back
// as far as the version that began watching (services/renewals.ts).
export const renewals = sqliteTable(
"renewals",
{
id: integer("id").primaryKey({ autoIncrement: true }),
kind: text("kind").notNull(), // certificate | domain | secret
subject: text("subject").notNull(), // its name, as it was
at: text("at").notNull(), // ISO — when it was seen
fromDate: text("from_date").notNull(), // YYYY-MM-DD — the expiry before
toDate: text("to_date").notNull(), // YYYY-MM-DD — and after
},
(t) => [index("renewals_at_idx").on(t.at)],
);
// ─── Inventory: the hardware behind it all ──────────────────────────────────
export const assetKinds = ["server", "storage", "network", "power", "client", "other"] as const;
+4
View File
@@ -13,6 +13,7 @@ import { initProxmoxBackupScheduler } from "./services/proxmoxBackupScheduler.js
import { initPbsVerificationScheduler } from "./services/pbsVerificationScheduler.js";
import { initQuietHoursScheduler } from "./services/quietHoursScheduler.js";
import { initWeeklyDigestScheduler } from "./services/weeklyDigestScheduler.js";
import { initMonthlyReportScheduler } from "./services/monthlyReportScheduler.js";
import { initMaintenanceEvents } from "./services/maintenanceEvents.js";
import { initHealthScheduler } from "./services/healthScheduler.js";
import { initAlertTicketScheduler } from "./services/alertTicketScheduler.js";
@@ -21,6 +22,7 @@ import { initMaintenanceScheduleScheduler } from "./services/maintenanceSchedule
import { initUpdateRuns } from "./services/updateRuns.js";
import { initEscalation } from "./services/escalation.js";
import { initEndpointMonitor } from "./services/endpointMonitor.js";
import { initPatchPlans } from "./services/patchPlans.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
import { initIncidentRecorder } from "./services/incidents.js";
import { getAlerts } from "./services/alerts.js";
@@ -51,6 +53,7 @@ await initProxmoxBackupScheduler();
await initPbsVerificationScheduler();
await initQuietHoursScheduler();
await initWeeklyDigestScheduler();
await initMonthlyReportScheduler();
initMaintenanceEvents();
await initHealthScheduler();
await initAlertTicketScheduler();
@@ -61,6 +64,7 @@ await initSelfBackupScheduler();
initIncidentRecorder(() => getAlerts(false));
initEscalation();
initEndpointMonitor();
initPatchPlans();
createApp().listen(env.port, () => {
console.log(`homelab-manager listening on port ${env.port}`);
+196
View File
@@ -0,0 +1,196 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { patchPlans, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { isValidTimeZone } from "../services/maintenanceSchedules.js";
import { activeRunOf, cancelPlanRun, listPlanRuns, MAX_PLANS, parseItems, planDays, planOccurrences, planServerIds, planServers, planTags, PlanError, startPlanRun, type PlanRow, type PlanRunRow } from "../services/patchPlans.js";
import { InvalidTagError, normalizeTags, parseTags } from "../services/serverTags.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const patchPlansRouter = Router();
// Everyone signed in can see what is planned and how it went, like the Updates page. Making a plan is an administrator's (it will run on its own, at night);
// starting one now or stopping one that is running is an operator's, like an update run.
patchPlansRouter.use(requireAuth);
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const planSchema = z
.object({
name: z.string().trim().min(1).max(100),
enabled: z.boolean().default(true),
days: z.array(z.number().int().min(0).max(6)).min(1).max(7),
startTime: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 02:00"),
timezone: z.string().trim().min(1).max(60),
tags: z.array(z.string()).max(12).default([]),
serverIds: z.array(z.number().int().positive()).max(200).default([]),
integrationId: z.number().int().positive(),
projectId: z.number().int().positive(),
templateId: z.number().int().positive(),
templateName: z.string().trim().min(1).max(200),
limitFrom: z.enum(["name", "hostname"]).default("name"),
stopOnFailure: z.boolean().default(true),
onlyWithUpdates: z.boolean().default(true),
silenceDependents: z.boolean().default(true),
windowMinutes: z.number().int().min(15).max(480).default(90),
})
.refine((d) => isValidTimeZone(d.timezone), { message: "That isn't a time zone the system knows (use a name like Europe/Stockholm).", path: ["timezone"] })
.refine((d) => d.tags.length > 0 || d.serverIds.length > 0, { message: "Choose which servers: give a tag, or name at least one server.", path: ["tags"] });
function valuesOf(d: z.infer<typeof planSchema>) {
return {
name: d.name,
enabled: d.enabled,
days: JSON.stringify([...new Set(d.days)].sort()),
startTime: d.startTime,
timezone: d.timezone,
tags: JSON.stringify(normalizeTags(d.tags)),
serverIds: JSON.stringify([...new Set(d.serverIds)]),
integrationId: d.integrationId,
projectId: d.projectId,
templateId: d.templateId,
templateName: d.templateName,
limitFrom: d.limitFrom,
stopOnFailure: d.stopOnFailure,
onlyWithUpdates: d.onlyWithUpdates,
silenceDependents: d.silenceDependents,
windowMinutes: d.windowMinutes,
};
}
const shapeRun = (r: PlanRunRow) => ({
id: r.id,
planId: r.planId,
planName: r.planName,
trigger: r.trigger,
startedBy: r.startedBy,
status: r.status,
message: r.message,
startedAt: r.startedAt,
finishedAt: r.finishedAt,
items: parseItems(r.items).map(({ windowIds: _w, ...i }) => i),
});
async function shapePlan(p: PlanRow, all: { id: number; name: string; tags: string | null }[], now: Date) {
const covered = planServers(p, all);
const next = planOccurrences(p, now, new Date(now.getTime() + 15 * 86_400_000)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 3);
const active = await activeRunOf(p.id);
return {
id: p.id,
name: p.name,
enabled: p.enabled,
days: planDays(p),
startTime: p.startTime,
timezone: p.timezone,
tags: planTags(p),
serverIds: planServerIds(p),
integrationId: p.integrationId,
projectId: p.projectId,
templateId: p.templateId,
templateName: p.templateName,
limitFrom: p.limitFrom,
stopOnFailure: p.stopOnFailure,
onlyWithUpdates: p.onlyWithUpdates,
silenceDependents: p.silenceDependents,
windowMinutes: p.windowMinutes,
createdBy: p.createdBy,
servers: covered,
next: next.map((o) => o.start.toISOString()),
running: active !== null,
};
}
patchPlansRouter.get("/", asyncHandler(async (_req, res) => {
const now = new Date();
const rows = await db.select().from(patchPlans).orderBy(patchPlans.name);
const all = await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers);
const tags = [...new Set(all.flatMap((s) => parseTags(s.tags)))].sort((a, b) => a.localeCompare(b, "sv"));
res.json({
plans: await Promise.all(rows.map((p) => shapePlan(p, all, now))),
runs: (await listPlanRuns({ limit: 15 })).map(shapeRun),
servers: all.map((s) => ({ id: s.id, name: s.name, tags: parseTags(s.tags) })).sort((a, b) => a.name.localeCompare(b.name)),
tags,
});
}));
/** The message for a rejected body, as a sentence. */
const reason = (parsed: { error: z.ZodError }) => parsed.error.issues[0]?.message ?? "That isn't a plan.";
patchPlansRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = planSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: reason(parsed), details: parsed.error.flatten() });
if ((await db.select({ id: patchPlans.id }).from(patchPlans)).length >= MAX_PLANS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_PLANS} plans.` });
let values;
try {
values = valuesOf(parsed.data);
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
throw err;
}
const actor = req.currentUser!;
const [row] = await db.insert(patchPlans).values({ ...values, createdBy: actor.name ?? actor.email ?? actor.oidcSub }).returning();
await recordAudit({ actor, category: "patch_plan", action: "create", targetType: "patch_plan", targetId: row.id, detail: { name: row.name, days: planDays(row), time: row.startTime } });
res.status(201).json({ plan: await shapePlan(row, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers), new Date()) });
}));
patchPlansRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = planSchema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? "That isn't a plan." : reason(parsed) });
const [before] = await db.select().from(patchPlans).where(eq(patchPlans.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
let values;
try {
values = valuesOf(parsed.data);
} catch (err) {
if (err instanceof InvalidTagError) return res.status(400).json({ error: "invalid_tag", message: err.message });
throw err;
}
// A changed day or time is a new schedule: what it last started on no longer says whether today's has happened.
const rescheduled = values.days !== before.days || values.startTime !== before.startTime || values.timezone !== before.timezone;
const [row] = await db.update(patchPlans).set({ ...values, ...(rescheduled ? { lastRunDate: null } : {}) }).where(eq(patchPlans.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "patch_plan", action: "update", targetType: "patch_plan", targetId: id, detail: { name: row.name, days: planDays(row), time: row.startTime } });
res.json({ plan: await shapePlan(row, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers), new Date()) });
}));
patchPlansRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(patchPlans).where(eq(patchPlans.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (await activeRunOf(id)) return res.status(409).json({ error: "running", message: "It is running now. Stop the run first." });
await db.delete(patchPlans).where(eq(patchPlans.id, id));
await recordAudit({ actor: req.currentUser!, category: "patch_plan", action: "delete", targetType: "patch_plan", targetId: id, detail: { name: before.name } });
res.status(204).end();
}));
// Run it now, instead of at its time (it still starts again at its time).
patchPlansRouter.post("/:id/run", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
try {
const run = await startPlanRun(id, { trigger: "manual", actor: actor.name ?? actor.email ?? actor.oidcSub });
res.status(201).json({ run: shapeRun(run) });
} catch (err) {
if (err instanceof PlanError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
patchPlansRouter.post("/runs/:id/cancel", requireRole("operator"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const actor = req.currentUser!;
try {
const run = await cancelPlanRun(id, actor.name ?? actor.email ?? actor.oidcSub);
if (!run) return res.status(404).json({ error: "not_found" });
res.json({ run: shapeRun(run) });
} catch (err) {
if (err instanceof PlanError) return res.status(err.status).json({ error: err.code, message: err.message });
throw err;
}
}));
+40
View File
@@ -0,0 +1,40 @@
import { Router } from "express";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { buildMonthlyReport, isMonthId, lastFullMonth, monthlyReportText, recentMonths } from "../services/monthlyReport.js";
import { recordAudit } from "../services/audit.js";
import { notifyNow } from "../services/notify.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const reportsRouter = Router();
reportsRouter.use(requireAuth);
/** The month asked for, or the last whole one; null when what was asked is not a month. */
async function monthFrom(query: unknown): Promise<{ id: string; months: { id: string; label: string }[] } | null> {
const tz = (await getSettings()).notifications.timezone || "UTC";
const months = recentMonths(new Date(), tz);
if (query === undefined || query === "") return { id: lastFullMonth(new Date(), tz), months };
if (!isMonthId(query) || !months.some((m) => m.id === query)) return null;
return { id: query, months };
}
// How a month went, for everyone signed in: it is only what the pages it comes from already show them. ?month=2026-09, the last whole month by default;
// ?format=text is the plain-text version a notification carries.
reportsRouter.get("/monthly", asyncHandler(async (req, res) => {
const picked = await monthFrom(req.query.month);
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
const report = await buildMonthlyReport(picked.id);
if (req.query.format === "text") return res.json(monthlyReportText(report));
res.json({ report, months: picked.months });
}));
// Send it now to the notification channels, as if it were the day. Administrators.
reportsRouter.post("/monthly/send", requireRole("admin"), asyncHandler(async (req, res) => {
const picked = await monthFrom((req.body as { month?: unknown } | undefined)?.month);
if (!picked) return res.status(400).json({ error: "invalid_month", message: "Ask for one of the last twelve months, like 2026-09." });
const { title, message } = monthlyReportText(await buildMonthlyReport(picked.id));
await notifyNow(title, message);
await recordAudit({ actor: req.currentUser!, category: "report", action: "send_monthly", targetType: "report", detail: { month: picked.id } });
res.json({ sent: true });
}));
+11
View File
@@ -7,6 +7,8 @@ import { requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { beginScan, endScan, MAX_SCAN_SPAN, resolveScanTarget, scanPorts, toRanges } from "../services/portScan.js";
import { type AgentPort, groupAgentPorts, parseJson } from "../services/agentPorts.js";
import { newListeningPorts, parseBaseline, portKey } from "../services/newPorts.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
// Mounted under /api/servers/:id/ports by the servers router, which has already required a signed-in user.
@@ -37,6 +39,8 @@ export interface PortEntry {
agent: { addresses: string[]; process: string | null; localOnly: boolean } | null;
/** "open" if anything is using it; "reserved" if it only has a note. */
state: "open" | "reserved";
/** Began listening after the first report and nobody has noted it: a "new port" alert is on until it is noted or stops listening. */
isNew: boolean;
}
async function buildPortList(serverId: number) {
@@ -46,6 +50,11 @@ async function buildPortList(serverId: number) {
const agentRaw = parseJson<AgentPort[] | null>(server.listeningPorts, null);
const agent = groupAgentPorts(agentRaw ?? []);
// What is new: listening, reachable from outside, not in the baseline and not noted — the same rule the alert uses.
const expected = new Set(rows.filter((r) => r.label || r.comment).map((r) => portKey(r.protocol, r.port)));
const alerting = (await getSettings()).healthChecks.newPortAlerts;
const fresh = new Set(alerting ? newListeningPorts(agentRaw, parseBaseline(server.portBaseline), expected).map((p) => portKey(p.protocol, p.port)) : []);
const entries = new Map<string, PortEntry>();
for (const row of rows) {
const key = `${row.protocol}:${row.port}`;
@@ -59,6 +68,7 @@ async function buildPortList(serverId: number) {
lastSeenOpenAt: row.lastSeenOpenAt,
agent: agent.get(key) ?? null,
state: "reserved",
isNew: fresh.has(key),
});
}
for (const [key, info] of agent) {
@@ -74,6 +84,7 @@ async function buildPortList(serverId: number) {
lastSeenOpenAt: null,
agent: info,
state: "reserved",
isNew: fresh.has(key),
});
}
for (const entry of entries.values()) {
+9 -1
View File
@@ -13,6 +13,7 @@ import { scheduleLogRetentionPurge } from "../services/logRetentionScheduler.js"
import { purgeOldLogs } from "../services/logRetention.js";
import { scheduleQuietHoursFlush } from "../services/quietHoursScheduler.js";
import { scheduleWeeklyDigest } from "../services/weeklyDigestScheduler.js";
import { scheduleMonthlyReport } from "../services/monthlyReportScheduler.js";
import { buildDigest, sendDigest } from "../services/weeklyDigest.js";
import { enabledChannelCount } from "../services/notify.js";
import { CATEGORIES, CHANNEL_IDS } from "../services/notifyRouting.js";
@@ -143,7 +144,7 @@ const updateSchema = z.object({
.partial()
.optional(),
healthChecks: z
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730) })
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730), newPortAlerts: z.boolean() })
.partial()
.optional(),
quietHours: z
@@ -165,6 +166,10 @@ const updateSchema = z.object({
.object({ enabled: z.boolean(), day: z.number().int().min(0).max(6), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
.partial()
.optional(),
monthlyReport: z
.object({ enabled: z.boolean(), day: z.number().int().min(1).max(28), time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/) })
.partial()
.optional(),
});
settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
@@ -200,6 +205,9 @@ settingsRouter.put("/", requireRole("admin"), asyncHandler(async (req, res) => {
if (parsed.data.digest || parsed.data.notifications) {
await scheduleWeeklyDigest();
}
if (parsed.data.monthlyReport || parsed.data.notifications) {
await scheduleMonthlyReport();
}
await recordAudit({
actor: req.currentUser!,
+17 -2
View File
@@ -9,10 +9,11 @@
import { createHash } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, patchPlans, secrets, servers } from "../db/schema.js";
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
import { planDays, planOccurrences, planServers } from "./patchPlans.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { latestTests } from "./restoreTests.js";
import { getSettings } from "./settingsStore.js";
@@ -88,6 +89,8 @@ export interface AgendaData {
/** Windows that have been set up but not begun, one per target. */
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
/** Patch plans that are switched on, with how many servers each covers now. */
patchPlans: { id: number; name: string; days: string; startTime: string; timezone: string; servers: number }[];
/** The newest restore test of each subject. */
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
restoreLimitDays: number;
@@ -140,6 +143,10 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
}
const planRows = (await db.select().from(patchPlans)).filter((p) => p.enabled);
const allServers = planRows.length > 0 ? await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers) : [];
const plans: AgendaData["patchPlans"] = planRows.map((p) => ({ id: p.id, name: p.name, days: p.days, startTime: p.startTime, timezone: p.timezone, servers: planServers(p, allServers).length }));
const updates: AgendaData["updates"] = [];
for (const s of serverRows) {
const u = parsePendingUpdates(s.pendingUpdates);
@@ -163,6 +170,7 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
assets: assetRows,
windows,
schedules,
patchPlans: plans,
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
restoreLimitDays: settings.healthChecks.restoreTestDays,
updates,
@@ -223,7 +231,7 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
g.names.push(w.targetName);
groups.set(key, g);
}
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
const timed = (id: string, start: string, end: string | null, title: string, detail: string | null) =>
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
const names = (list: string[]) => {
const sorted = [...list].sort((a, b) => a.localeCompare(b));
@@ -236,6 +244,13 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
}
}
// A patch plan is maintenance too: when it will start and on how many servers.
for (const p of data.patchPlans) {
for (const occ of planOccurrences(p, now, new Date(lastMs)).filter((o) => o.start.getTime() > now.getTime()).slice(0, 60)) {
timed(`plan:${p.id}:${occ.date}`, occ.start.toISOString(), null, `Patch plan: ${p.name}`, `updates ${plural(p.servers, "server")}, one at a time`);
}
}
// Updates that have been waiting: one line for all of them, from the longest wait.
if (data.updates.length > 0) {
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
+2 -2
View File
@@ -79,8 +79,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
/** Which kind of source, and which one, a health/automation condition key belongs to. */
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : "disk" };
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return { type: "proxmox", id: Number(m[1]), category: "disk" };
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
+19
View File
@@ -54,6 +54,8 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
restore_test: "backups",
endpoint: "endpoints",
heartbeat: "endpoints",
patch_plan: "servers",
report: "other",
};
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
@@ -155,6 +157,22 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "report/send_monthly":
return { title: `Monthly report sent${str(detail.month) ? ` (${str(detail.month)})` : ""}`, tone: "neutral", detail: null };
case "patch_plan/create":
return { title: `Patch plan added${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "patch_plan/update":
return { title: `Patch plan changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "patch_plan/delete":
return { title: `Patch plan removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "patch_plan/run":
return { title: `Patch plan started${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.trigger) === "schedule" ? "by its schedule" : str(detail.by) ? `by ${str(detail.by)}` : null };
case "patch_plan/finished":
return { title: `Patch plan finished${name ? `: ${name}` : ""}`, tone: "good", detail: str(detail.message) };
case "patch_plan/failed":
return { title: `Patch plan didn't finish cleanly${name ? `: ${name}` : ""}`, tone: "bad", detail: str(detail.message) };
case "patch_plan/cancelled":
return { title: `Patch plan stopped${name ? `: ${name}` : ""}`, tone: "neutral", detail: str(detail.message) };
case "heartbeat/create":
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
case "heartbeat/update":
@@ -260,6 +278,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
targetName = endpointName.get(idNum) ?? targetName;
link = "/endpoints";
} else if (a.targetType === "heartbeat") link = "/heartbeats";
else if (a.targetType === "patch_plan") link = "/updates?view=plans";
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
else if (a.targetType === "restore_test") link = "/backups";
else if (a.targetType === "dependency") link = "/map?view=dependencies";
+3
View File
@@ -5,6 +5,7 @@ import { defaultDeps, lookupRegistration, normalizeDomain, resolveRegistration,
import { computeSecretStatus } from "./secretStatus.js";
import { notifyDomainExpiry } from "./notify.js";
import { getSettings } from "./settingsStore.js";
import { noteRenewal } from "./renewals.js";
export type DomainRow = typeof domains.$inferSelect;
@@ -48,6 +49,8 @@ export async function checkDomain(id: number, deps: LookupDeps = defaultDeps): P
if (!row) return null;
const result = await lookupRegistration(row.name, deps);
const [updated] = await db.update(domains).set(outcomeFields(result, row, new Date(deps.now()).toISOString())).where(eq(domains.id, id)).returning();
// A registration that now runs much longer than it did was renewed.
await noteRenewal("domain", row.name, row.expiresAt, updated.expiresAt, new Date(deps.now()));
return updated;
}
+28 -2
View File
@@ -12,9 +12,10 @@ import https from "node:https";
import net from "node:net";
import tls from "node:tls";
import type { TLSSocket } from "node:tls";
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointResults } from "../db/schema.js";
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
import { noteRenewal } from "./renewals.js";
import { resolveTarget } from "../utils/networkGuard.js";
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
@@ -244,9 +245,33 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
})
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
await addToDaily(c.id, iso, result);
// A certificate that is now good for much longer than it was has been renewed.
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
return result;
}
/** How long the daily figures are kept: years, where the individual results are kept two weeks. */
export const KEEP_DAILY_DAYS = 800;
/** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */
async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
const timed = result.ok && result.ms !== null;
const ok = result.ok ? 1 : 0;
const ms = timed ? (result.ms as number) : 0;
try {
await db
.insert(endpointDaily)
.values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 })
.onConflictDoUpdate({
target: [endpointDaily.checkId, endpointDaily.day],
set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` },
});
} catch (err) {
console.error("[endpoints] couldn't add to the daily figures:", err);
}
}
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
@@ -264,6 +289,7 @@ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRo
export async function pruneResults(now: Date = new Date()): Promise<void> {
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
}
// ─── What the history says ─────────────────────────────────────────────────
+19 -2
View File
@@ -1,6 +1,6 @@
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, servers } from "../db/schema.js";
import { integrations, serverPorts, servers } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -11,6 +11,8 @@ import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
import { getFailingSources } from "./integrationHealthMonitor.js";
import { hostHealthConditions, parseHostHealth, withoutIgnoredServices, type HostHealth } from "./hostHealth.js";
import { newListeningPorts, newPortMessage, parseBaseline, portKey, type NewPort } from "./newPorts.js";
import { parseJson, type AgentPort } from "./agentPorts.js";
import { diskLimit, parseThresholds, type ServerThresholds } from "./serverThresholds.js";
const STATE_FLAG = HEALTH_STATE_FLAG;
@@ -39,6 +41,8 @@ export interface ServerSnapshot {
health?: HostHealth | null;
/** This server's own thresholds, where it has any: they replace the global ones for it. */
thresholds?: ServerThresholds | null;
/** TCP ports that began listening after the baseline and that nobody has noted (services/newPorts.ts); empty when the check is off. */
newPorts?: NewPort[];
}
export interface ProxmoxSnapshot {
integrationId: number;
@@ -123,6 +127,7 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// A server that never reported has no agent to go quiet; and an offline one's disk figures are stale, so neither is judged on disks.
if (s.lastSeenAt && !offline) {
for (const c of hostHealthConditions(s.id, s.name, s.health ?? null)) out.push({ key: c.key, source: "server", message: c.message, severity: c.severity });
for (const p of s.newPorts ?? []) out.push({ key: `new-port:server:${s.id}:${portKey(p.protocol, p.port)}`, source: "server", message: newPortMessage(s.name, p), severity: "warning" });
for (const d of s.disks) {
const p = usage(d.usedBytes, d.sizeBytes);
// A disk can have its own limit, or be left alone altogether; otherwise the server's, then the global one.
@@ -299,6 +304,17 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
const held = new Set<string>();
const snapshot: HealthSnapshot = { servers: [], proxmox: [], synology: [] };
const { healthChecks: checks } = await getSettings();
// The ports somebody has said are expected (a label or a comment on the Ports page); a row that only records an open scan says nothing.
const expectedByServer = new Map<number, Set<string>>();
if (checks.newPortAlerts) {
for (const r of await db.select().from(serverPorts)) {
if (!r.label && !r.comment) continue;
const set = expectedByServer.get(r.serverId) ?? new Set<string>();
set.add(portKey(r.protocol, r.port));
expectedByServer.set(r.serverId, set);
}
}
for (const s of await db.select().from(servers)) {
let disks: ServerSnapshot["disks"] = [];
try {
@@ -306,7 +322,8 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
} catch {
// a malformed report shouldn't stop every other server being checked
}
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds) });
const newPorts = checks.newPortAlerts ? newListeningPorts(parseJson<AgentPort[] | null>(s.listeningPorts, null), parseBaseline(s.portBaseline), expectedByServer.get(s.id) ?? new Set()) : [];
snapshot.servers.push({ id: s.id, name: s.name, lastSeenAt: s.lastSeenAt, disks, health: withoutIgnoredServices(parseHostHealth(s.hostHealth), parseThresholds(s.thresholds)?.ignoreServices), thresholds: parseThresholds(s.thresholds), newPorts });
}
const rows = await db
+1 -1
View File
@@ -23,7 +23,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
* so conditions persisted by an earlier version still map correctly.
*/
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
+303
View File
@@ -0,0 +1,303 @@
/**
* The monthly report: how a month went — availability of servers, integrations and endpoints, incidents and how long they took, updates applied (by hand and by
* patch plans), restore tests and the state of the backups, and what was renewed. Everything in it the app already knows; this gathers it for one calendar month
* (on the clock of the notification time zone) into one structure that the page shows, prints and exports, and into a plain-text version that can be sent like the
* weekly digest.
*
* What it can't say, it says: servers' and integrations' availability only goes back as far as the audit log records it; endpoint availability from the daily
* figures (kept since they were added, and started from the last two weeks of results); renewals only from when the app began noticing them; the backups are the
* state now, not a history. A month with nothing to report for a part leaves that part out.
*/
import { and, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, integrations, patchPlanRuns, restoreTests, servers, updateRuns } from "../db/schema.js";
import { getCachedBackupCoverage } from "./backupCoverage.js";
import { listIncidents } from "./incidents.js";
import { zonedTime } from "./maintenanceSchedules.js";
import { renewalsBetween } from "./renewals.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
import { spanWords } from "./weeklyDigest.js";
const DAY_MS = 86_400_000;
const MAX_LISTED = 12;
export interface AvailabilityRow {
id: number;
name: string;
/** Share of the known time spent up, in percent; null when nothing is known for the month. */
percent: number | null;
downMinutes: number;
}
export interface EndpointRow {
id: number;
name: string;
percent: number | null;
avgMs: number | null;
checks: number;
}
export interface MonthlyReport {
/** "2026-09". */
month: string;
/** "September 2026". */
label: string;
/** ISO — the start of the month and its end (or now, for a month that is not over). */
from: string;
to: string;
/** The month is not over yet: everything is "so far". */
partial: boolean;
timezone: string;
servers: AvailabilityRow[];
integrations: AvailabilityRow[];
endpoints: EndpointRow[];
incidents: {
began: number;
resolved: number;
stillOpen: number;
/** Of those that began and ended in the month. */
totalMinutes: number;
averageMinutes: number | null;
longest: { id: number; message: string; minutes: number } | null;
byCategory: { category: string; count: number }[];
/** The longest, up to a dozen. */
list: { id: number; message: string; where: string; severity: string; openedAt: string; resolvedAt: string | null; minutes: number }[];
};
updates: {
runs: number;
done: number;
failed: number;
/** Updates the runs say they installed (before minus after, where both are known). */
installed: number;
byServer: { name: string; runs: number; failed: number; installed: number }[];
planRuns: { runs: number; done: number; failed: number };
};
restoreTests: { label: string; testedAt: string; result: string; note: string | null }[];
/** The backup state as of now; null when no coverage has been read yet. */
backups: { ok: number; stale: number; failed: number; none: number; unknown: number; total: number; asOf: string } | null;
renewals: { kind: string; subject: string; at: string; fromDate: string; toDate: string }[];
}
// ─── Months ────────────────────────────────────────────────────────────────
const pad = (n: number) => String(n).padStart(2, "0");
export function isMonthId(v: unknown): v is string {
return typeof v === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(v);
}
/** The year and month `now` is in, on the clock of `tz`. */
function monthOf(now: Date, tz: string): { year: number; month: number } {
const parts = new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit" }).formatToParts(now);
return { year: Number(parts.find((p) => p.type === "year")?.value), month: Number(parts.find((p) => p.type === "month")?.value) };
}
export const monthId = (year: number, month: number) => `${year}-${pad(month)}`;
const shift = (year: number, month: number, by: number) => {
const index = year * 12 + (month - 1) + by;
return { year: Math.floor(index / 12), month: (index % 12) + 1 };
};
export function monthLabel(id: string): string {
const [y, m] = id.split("-").map(Number);
return new Date(Date.UTC(y, m - 1, 15)).toLocaleDateString("en-GB", { month: "long", year: "numeric", timeZone: "UTC" });
}
/** The month before the one `now` is in: what a report sent at the start of a month is about. */
export function lastFullMonth(now: Date, tz: string): string {
const cur = monthOf(now, tz);
const prev = shift(cur.year, cur.month, -1);
return monthId(prev.year, prev.month);
}
/** This month and the eleven before it, newest first. */
export function recentMonths(now: Date, tz: string, count = 12): { id: string; label: string }[] {
const cur = monthOf(now, tz);
return Array.from({ length: count }, (_, i) => {
const m = shift(cur.year, cur.month, -i);
const id = monthId(m.year, m.month);
return { id, label: monthLabel(id) };
});
}
/** The instants a month begins and ends at, on the clock of `tz`. */
export function monthBounds(id: string, tz: string): { start: Date; end: Date } {
const [y, m] = id.split("-").map(Number);
const next = shift(y, m, 1);
return { start: zonedTime(y, m, 1, 0, 0, tz), end: zonedTime(next.year, next.month, 1, 0, 0, tz) };
}
// ─── Gathering ─────────────────────────────────────────────────────────────
const round1 = (n: number) => Math.round(n * 10) / 10;
export async function buildMonthlyReport(id: string, now: Date = new Date()): Promise<MonthlyReport> {
const settings = await getSettings();
const tz = settings.notifications.timezone || "UTC";
const { start, end: fullEnd } = monthBounds(id, tz);
const partial = fullEnd.getTime() > now.getTime();
const end = partial ? now : fullEnd;
const days = Math.max(1, Math.ceil((end.getTime() - start.getTime()) / DAY_MS));
const startIso = start.toISOString();
const endIso = end.toISOString();
// Availability. Servers and integrations from the audit log's record of going away and coming back; laid over the month's days ending where the month does.
const uptime = await getUptime(end, days);
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
const integrationNames = new Map((await db.select({ id: integrations.id, name: integrations.name }).from(integrations)).map((i) => [i.id, i.name]));
const rowsOf = (series: Record<number, { buckets: { k: number; d: number }[]; percent: number }>, names: Map<number, string>): AvailabilityRow[] =>
Object.entries(series)
.map(([key, s]) => ({ id: Number(key), name: names.get(Number(key)) ?? `#${key}`, percent: s.percent, downMinutes: s.buckets.reduce((n, b) => n + b.d, 0) }))
.filter((r) => names.has(r.id))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Endpoints: from the daily figures, for the days the month covers. The figures are per UTC day, so a day at either end can belong to a neighbouring month
// by a few hours (more than the time zone is ahead of UTC); a month that is not over takes in today.
const [yy, mm] = id.split("-").map(Number);
const following = shift(yy, mm, 1);
const firstDay = `${id}-01`;
const lastDayExclusive = partial ? new Date(now.getTime() + DAY_MS).toISOString().slice(0, 10) : `${monthId(following.year, following.month)}-01`;
const dailyRows = await db.select().from(endpointDaily).where(and(gte(endpointDaily.day, firstDay), lt(endpointDaily.day, lastDayExclusive)));
const checkNames = new Map((await db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks)).map((c) => [c.id, c.name]));
const perCheck = new Map<number, { total: number; ok: number; msSum: number; msCount: number }>();
for (const r of dailyRows) {
const t = perCheck.get(r.checkId) ?? { total: 0, ok: 0, msSum: 0, msCount: 0 };
t.total += r.total;
t.ok += r.ok;
t.msSum += r.msSum;
t.msCount += r.msCount;
perCheck.set(r.checkId, t);
}
const endpointRows: EndpointRow[] = [...perCheck]
.filter(([cid, t]) => checkNames.has(cid) && t.total > 0)
.map(([cid, t]) => ({ id: cid, name: checkNames.get(cid)!, percent: round1((t.ok / t.total) * 100), avgMs: t.msCount > 0 ? Math.round(t.msSum / t.msCount) : null, checks: t.total }))
.sort((a, b) => (a.percent ?? 101) - (b.percent ?? 101) || a.name.localeCompare(b.name));
// Incidents.
const all = await listIncidents({ now: end, limit: 3000 });
const inMonth = (iso: string) => iso >= startIso && iso < endIso;
const began = all.filter((i) => inMonth(i.openedAt));
const resolved = all.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const stillOpen = all.filter((i) => i.openedAt < endIso && (i.resolvedAt === null || i.resolvedAt >= endIso)).length;
const closedHere = began.filter((i) => i.resolvedAt !== null && inMonth(i.resolvedAt));
const minutesOf = (ms: number) => Math.max(1, Math.round(ms / 60_000));
const totalMinutes = closedHere.reduce((n, i) => n + minutesOf(i.durationMs), 0);
const byDuration = [...began].sort((a, b) => b.durationMs - a.durationMs);
const categories = new Map<string, number>();
for (const i of began) categories.set(i.category, (categories.get(i.category) ?? 0) + 1);
// Updates, by hand and by plan.
const runRows = await db.select().from(updateRuns).where(and(gte(updateRuns.startedAt, startIso), lt(updateRuns.startedAt, endIso)));
const perServer = new Map<number, { runs: number; failed: number; installed: number }>();
let installed = 0;
for (const r of runRows) {
const gained = r.status === "done" && r.updatesBefore !== null && r.updatesAfter !== null ? Math.max(0, r.updatesBefore - r.updatesAfter) : 0;
installed += gained;
const t = perServer.get(r.serverId) ?? { runs: 0, failed: 0, installed: 0 };
t.runs++;
if (r.status === "failed") t.failed++;
t.installed += gained;
perServer.set(r.serverId, t);
}
const planRows = await db.select().from(patchPlanRuns).where(and(gte(patchPlanRuns.startedAt, startIso), lt(patchPlanRuns.startedAt, endIso)));
// Restore tests, and the backups as they are now.
const tests = await db.select().from(restoreTests).where(and(gte(restoreTests.testedAt, firstDay), lt(restoreTests.testedAt, lastDayExclusive)));
const coverage = await getCachedBackupCoverage(now);
const renewalRows = await renewalsBetween(start, end);
return {
month: id,
label: monthLabel(id),
from: startIso,
to: endIso,
partial,
timezone: tz,
servers: rowsOf(uptime.servers, serverNames),
integrations: rowsOf(uptime.integrations, integrationNames),
endpoints: endpointRows,
incidents: {
began: began.length,
resolved: resolved.length,
stillOpen,
totalMinutes,
averageMinutes: closedHere.length > 0 ? Math.round(totalMinutes / closedHere.length) : null,
longest: byDuration[0] ? { id: byDuration[0].id, message: byDuration[0].message, minutes: minutesOf(byDuration[0].durationMs) } : null,
byCategory: [...categories].map(([category, count]) => ({ category, count })).sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)),
list: byDuration.slice(0, MAX_LISTED).map((i) => ({ id: i.id, message: i.message, where: i.target?.name ?? i.source, severity: i.peakSeverity, openedAt: i.openedAt, resolvedAt: i.resolvedAt, minutes: minutesOf(i.durationMs) })),
},
updates: {
runs: runRows.length,
done: runRows.filter((r) => r.status === "done").length,
failed: runRows.filter((r) => r.status === "failed").length,
installed,
byServer: [...perServer]
.map(([sid, t]) => ({ name: serverNames.get(sid) ?? `#${sid}`, ...t }))
.sort((a, b) => b.runs - a.runs || a.name.localeCompare(b.name)),
planRuns: { runs: planRows.length, done: planRows.filter((r) => r.status === "done").length, failed: planRows.filter((r) => r.status === "failed").length },
},
restoreTests: tests.map((t) => ({ label: t.label, testedAt: t.testedAt, result: t.result, note: t.note })).sort((a, b) => a.testedAt.localeCompare(b.testedAt)),
backups: coverage ? { ok: coverage.counts.ok, stale: coverage.counts.stale, failed: coverage.counts.failed, none: coverage.counts.none, unknown: coverage.counts.unknown, total: coverage.counts.total, asOf: coverage.generatedAt } : null,
renewals: renewalRows.map((r) => ({ kind: r.kind, subject: r.subject, at: r.at, fromDate: r.fromDate, toDate: r.toDate })),
};
}
// ─── As a message ──────────────────────────────────────────────────────────
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
const RESULT_WORDS: Record<string, string> = { ok: "passed", partial: "partly worked", failed: "failed", not_needed: "marked as not needed" };
function average(rows: AvailabilityRow[]): number | null {
const known = rows.filter((r) => r.percent !== null);
return known.length === 0 ? null : round1(known.reduce((n, r) => n + (r.percent as number), 0) / known.length);
}
/** The report as plain text, for a notification or an email: short, a topic only where there is something to say. */
export function monthlyReportText(r: MonthlyReport): { title: string; message: string } {
const sections: string[] = [];
const lines = (heading: string, body: string[]) => body.length > 0 && sections.push(`${heading.toUpperCase()}\n${body.join("\n")}`);
const avail: string[] = [];
const serverAvg = average(r.servers);
if (serverAvg !== null) {
avail.push(`Servers: ${serverAvg}% on average (${plural(r.servers.length, "server")}).`);
for (const s of r.servers.filter((x) => x.downMinutes > 0).slice(0, 6)) avail.push(`- ${s.name}: ${s.percent}% (${spanWords(s.downMinutes)} down)`);
}
if (r.endpoints.length > 0) {
const avg = round1(r.endpoints.reduce((n, e) => n + (e.percent ?? 0), 0) / r.endpoints.length);
avail.push(`Endpoints: ${avg}% on average (${plural(r.endpoints.length, "check")}).`);
for (const e of r.endpoints.filter((x) => (x.percent ?? 100) < 100).slice(0, 6)) avail.push(`- ${e.name}: ${e.percent}%${e.avgMs !== null ? `, ${e.avgMs} ms on average` : ""}`);
}
const integrationAvg = average(r.integrations);
if (integrationAvg !== null) avail.push(`Integrations: ${integrationAvg}% on average.`);
lines("Availability", avail);
const inc = r.incidents;
if (inc.began > 0 || inc.resolved > 0 || inc.stillOpen > 0) {
const body = [`${plural(inc.began, "incident")} began, ${inc.resolved} ended, ${inc.stillOpen} still open.`];
if (inc.averageMinutes !== null) body.push(`A resolved incident took ${spanWords(inc.averageMinutes)} on average.`);
for (const i of inc.list.slice(0, 3)) body.push(`- ${i.message} — ${spanWords(i.minutes)}${i.resolvedAt ? "" : " and still open"}`);
lines("Incidents", body);
} else if (serverAvg !== null || r.endpoints.length > 0) {
lines("Incidents", ["None."]);
}
const u = r.updates;
const upd: string[] = [];
if (u.runs > 0) upd.push(`${plural(u.runs, "update run")}: ${u.done} done${u.failed > 0 ? `, ${u.failed} failed` : ""}${u.installed > 0 ? ` — ${plural(u.installed, "update")} installed` : ""}.`);
if (u.planRuns.runs > 0) upd.push(`Patch plans ran ${plural(u.planRuns.runs, "time")}${u.planRuns.failed > 0 ? `, ${u.planRuns.failed} not cleanly` : ", all cleanly"}.`);
lines("Updates", upd);
const bk: string[] = [];
for (const t of r.restoreTests) bk.push(`- Restore test of ${t.label} (${t.testedAt}) ${RESULT_WORDS[t.result] ?? t.result}`);
if (r.restoreTests.length === 0) bk.push("No restore test was recorded.");
if (r.backups) bk.push(`Backups now: ${r.backups.ok} of ${plural(r.backups.total, "guest")} recent${r.backups.failed + r.backups.stale + r.backups.none > 0 ? `; ${r.backups.failed} failed, ${r.backups.stale} stale, ${r.backups.none} without a backup` : ""}.`);
lines("Backups", bk);
lines("Renewed", r.renewals.slice(0, 10).map((x) => `- ${x.kind === "domain" ? "Domain" : "Certificate"} ${x.subject}: ${x.fromDate} → ${x.toDate}`));
const title = `Homelab monthly report — ${r.label}${r.partial ? " (so far)" : ""}`;
return { title, message: sections.length > 0 ? sections.join("\n\n") : "Nothing was recorded for this month." };
}
@@ -0,0 +1,43 @@
import schedule from "node-schedule";
import { getSettings } from "./settingsStore.js";
import { notify } from "./notify.js";
import { buildMonthlyReport, lastFullMonth, monthlyReportText } from "./monthlyReport.js";
let currentJob: schedule.Job | null = null;
/** The cron rule for "this day of the month at HH:MM" (1–28, so every month has it). */
export function cronForMonthlyReport(day: number, time: string): string {
const [h, m] = time.split(":").map(Number);
const d = Number.isInteger(day) && day >= 1 && day <= 28 ? day : 1;
return `${Number.isFinite(m) ? m : 0} ${Number.isFinite(h) ? h : 8} ${d} * *`;
}
/** Builds last month's report and sends it through the channels (held with other notifications during quiet hours). */
export async function sendMonthlyReport(now: Date = new Date()): Promise<{ title: string; message: string }> {
const tz = (await getSettings()).notifications.timezone || "UTC";
const text = monthlyReportText(await buildMonthlyReport(lastFullMonth(now, tz), now));
await notify(text.title, text.message);
return text;
}
/** (Re)schedules the monthly report per the current settings. Call again after they change. */
export async function scheduleMonthlyReport(): Promise<void> {
if (currentJob) {
currentJob.cancel();
currentJob = null;
}
const { monthlyReport, notifications } = await getSettings();
if (!monthlyReport.enabled) {
console.log("[report] monthly report is off, nothing scheduled");
return;
}
const rule = cronForMonthlyReport(monthlyReport.day, monthlyReport.time);
currentJob = schedule.scheduleJob({ rule, tz: notifications.timezone }, () => {
sendMonthlyReport().catch((err) => console.error("[report] couldn't send the monthly report:", err));
});
console.log(`Monthly report scheduled: ${rule} (${notifications.timezone})`);
}
export async function initMonthlyReportScheduler(): Promise<void> {
await scheduleMonthlyReport();
}
+60
View File
@@ -0,0 +1,60 @@
/**
* New ports: a server that starts listening on something it did not before is worth a look — a service somebody started, or something that should not be
* there. The first report that carries ports is taken as the baseline: what the server was listening on, taken as normal. After that a TCP port that is
* listening on an address other than this machine's own, is not in the baseline, and has not been noted on the Ports page, is a warning (the Alerts page,
* and a notification) until the port is noted ("this is expected") or stops listening.
*
* Left out on purpose: UDP (clients and discovery open them all the time), ports in the dynamic range (49152 and up, where programs pick a free one), and
* anything bound only to the loopback address, which nothing else on the network can reach.
*/
import { isLoopback, type AgentPort } from "./agentPorts.js";
/** From here up, a port is usually one a program picked for itself, not a service. */
export const DYNAMIC_PORTS_FROM = 49152;
export interface NewPort {
protocol: "tcp";
port: number;
/** What is listening, as the agent says; null when it does not. */
process: string | null;
}
export const portKey = (protocol: string, port: number) => `${protocol}:${port}`;
/** What is stored, read back as a set of "tcp:443"; null when there is no baseline yet (or it can't be read). */
export function parseBaseline(stored: string | null): Set<string> | null {
if (!stored) return null;
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? new Set(v.filter((k): k is string => typeof k === "string")) : null;
} catch {
return null;
}
}
/** The baseline a first report makes: everything it listens on, in any protocol, so nothing about it is ever "new". */
export function baselineFrom(listening: Pick<AgentPort, "protocol" | "port">[]): string {
return JSON.stringify([...new Set(listening.map((p) => portKey(p.protocol, p.port)))].sort());
}
/**
* Pure: the ports that are new — listening now, reachable from outside this machine, in neither the baseline nor the noted ports. `expected` is the set of
* "tcp:8080" somebody gave a label or a comment on the Ports page. With no baseline yet, nothing is new.
*/
export function newListeningPorts(listening: AgentPort[] | null | undefined, baseline: Set<string> | null, expected: Set<string>): NewPort[] {
if (!listening || baseline === null) return [];
const found = new Map<number, NewPort>();
for (const p of listening) {
if (p.protocol !== "tcp" || !Number.isInteger(p.port) || p.port <= 0 || p.port >= DYNAMIC_PORTS_FROM) continue;
if (isLoopback(p.address ?? "")) continue;
const key = portKey("tcp", p.port);
if (baseline.has(key) || expected.has(key)) continue;
const seen = found.get(p.port);
if (seen) seen.process = seen.process ?? (p.process || null);
else found.set(p.port, { protocol: "tcp", port: p.port, process: p.process || null });
}
return [...found.values()].sort((a, b) => a.port - b.port);
}
export const newPortMessage = (serverName: string, p: NewPort) =>
`${serverName}: a new port is listening — ${p.port}/tcp${p.process ? ` (${p.process})` : ""}. If it should be there, note it on the server's Ports card.`;
+10
View File
@@ -449,6 +449,16 @@ export async function notifyUpdateRunFinished(run: { serverName: string; templat
);
}
/** How a patch plan's run ended: one message, with a line for each server. Held back or sent with the other update-run results. */
export async function notifyPatchPlanFinished(run: { planName: string; ok: boolean; message: string; lines: string[] }): Promise<void> {
if (!(await eventEnabled("updateRunResults"))) return;
await notify(
run.ok ? `Homelab Manager — Patch plan "${run.planName}" finished` : `Homelab Manager — Patch plan "${run.planName}" didn't finish cleanly`,
`${run.message}${run.lines.length > 0 ? `\n\n${run.lines.join("\n")}` : ""}`,
{ category: "updates", severity: run.ok ? "warning" : "critical" },
);
}
export async function notifyPbsVerificationFailed(
failures: { integrationName: string; datastore: string; failedCount: number; error: string | null }[],
): Promise<void> {
+2 -1
View File
@@ -107,11 +107,12 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
let category: AlertCategory | null = null;
if (key.startsWith("offline:")) category = "offline";
else if (key.startsWith("failed-units:")) category = "services";
else if (key.startsWith("new-port:")) category = "monitoring";
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
else if (key.startsWith("automation:")) category = "automation";
else if (key.startsWith("heartbeat")) category = "automation";
else if (key.startsWith("endpoint-cert:")) category = "expiry";
else if (key.startsWith("endpoint:")) category = "monitoring";
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
const server = /^(?:(?:offline|disk|failed-units|smart|zfs|new-port):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
}
+321
View File
@@ -0,0 +1,321 @@
/**
* Patch plans: "Sundays at 02:00, the servers tagged lab, one at a time, stop at the first failure."
*
* A plan only says when, to whom and how. When it is due (or somebody presses Run now) it makes a *run*: a list of the servers it matches, and goes down
* the list one server at a time. Each server is updated by an ordinary update run (services/updateRuns.ts) — the server is put under maintenance, the
* Semaphore playbook is run limited to it, the app waits for it to report again — and the plan waits for that to finish before it starts the next. What
* depends on the server is silenced for the same time (it will look down while the server restarts). If a server's update fails and the plan says to stop,
* the servers after it are left alone and the run says why; otherwise the plan carries on. A message at the end says how each server went.
*
* A plan starts once a day at most: the day it last started on is kept, so a restart of this app at 02:10 neither misses it nor repeats it. If the app was
* down at the time, it still starts, up to half an hour late; later than that, that day is skipped.
*/
import { and, desc, eq, inArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { maintenanceWindows, patchPlanRuns, patchPlans, servers, updateRuns } from "../db/schema.js";
import { invalidateAlertsCache } from "./alerts.js";
import { recordAudit } from "./audit.js";
import { impactOf } from "./dependencies.js";
import { startOrExtendWindow } from "./maintenance.js";
import { occurrencesBetween, type Occurrence } from "./maintenanceSchedules.js";
import { notifyPatchPlanFinished } from "./notify.js";
import { parseTags } from "./serverTags.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { ACTIVE_STATUSES, cancelUpdateRun, startUpdateRun, UpdateRunError } from "./updateRuns.js";
export type PlanRow = typeof patchPlans.$inferSelect;
export type PlanRunRow = typeof patchPlanRuns.$inferSelect;
export type ItemStatus = "pending" | "running" | "done" | "failed" | "skipped";
export type RunStatus = "running" | "done" | "failed" | "cancelled";
export interface PlanItem {
serverId: number;
name: string;
status: ItemStatus;
updateRunId: number | null;
message: string | null;
/** Windows made for what depends on this server, which end with it. */
windowIds: number[];
}
/** How late a plan may still start, if the app was not running at its time. */
export const START_GRACE_MS = 30 * 60_000;
export const POLL_INTERVAL_MS = 20_000;
export const MAX_PLANS = 30;
export class PlanError extends Error {
constructor(
readonly status: number,
readonly code: string,
message: string,
) {
super(message);
}
}
// ─── Reading what is stored ────────────────────────────────────────────────
function jsonArray<T>(stored: string | null | undefined): T[] {
if (!stored) return [];
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? (v as T[]) : [];
} catch {
return [];
}
}
export const parseItems = (stored: string | null | undefined): PlanItem[] => jsonArray<PlanItem>(stored).filter((i) => i && typeof i.serverId === "number");
export const planDays = (plan: Pick<PlanRow, "days">): number[] => jsonArray<number>(plan.days).filter((d) => Number.isInteger(d));
export const planTags = (plan: Pick<PlanRow, "tags">): string[] => parseTags(plan.tags);
export const planServerIds = (plan: Pick<PlanRow, "serverIds">): number[] => jsonArray<number>(plan.serverIds).filter((i) => Number.isInteger(i));
// ─── Who a plan is for, and when ───────────────────────────────────────────
/** Pure: the servers a plan covers — those with any of its tags, and those named — by name. */
export function planServers(plan: Pick<PlanRow, "tags" | "serverIds">, all: { id: number; name: string; tags: string | null }[]): { id: number; name: string }[] {
const tags = new Set(planTags(plan));
const ids = new Set(planServerIds(plan));
return all
.filter((s) => ids.has(s.id) || parseTags(s.tags).some((t) => tags.has(t)))
.map((s) => ({ id: s.id, name: s.name }))
.sort((a, b) => a.name.localeCompare(b.name));
}
/** The times a plan happens, from `from` to `to`. A plan has no length; the minute it starts is what matters. */
export function planOccurrences(plan: Pick<PlanRow, "days" | "startTime" | "timezone">, from: Date, to: Date): Occurrence[] {
return occurrencesBetween({ days: plan.days, startTime: plan.startTime, durationMinutes: 1, timezone: plan.timezone }, from, to);
}
/** Pure: the occurrence a plan should start for now — its time has come, it is no more than the grace late, and it has not started on that day. */
export function dueOccurrence(plan: Pick<PlanRow, "enabled" | "days" | "startTime" | "timezone" | "lastRunDate">, now: Date, graceMs: number = START_GRACE_MS): Occurrence | null {
if (!plan.enabled) return null;
const found = planOccurrences(plan, new Date(now.getTime() - graceMs - 60_000), new Date(now.getTime() + 1000)).filter((o) => o.start.getTime() <= now.getTime() && now.getTime() - o.start.getTime() <= graceMs && o.date !== plan.lastRunDate);
return found.length > 0 ? found[found.length - 1] : null;
}
// ─── Starting a run ────────────────────────────────────────────────────────
const ACTIVE_RUN: RunStatus[] = ["running"];
export async function activeRunOf(planId: number): Promise<PlanRunRow | null> {
const [run] = await db.select().from(patchPlanRuns).where(and(eq(patchPlanRuns.planId, planId), inArray(patchPlanRuns.status, ACTIVE_RUN))).limit(1);
return run ?? null;
}
export async function startPlanRun(planId: number, options: { trigger: "schedule" | "manual"; actor: string; now?: Date }): Promise<PlanRunRow> {
const now = options.now ?? new Date();
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, planId)).limit(1);
if (!plan) throw new PlanError(404, "not_found", "That plan doesn't exist.");
if (await activeRunOf(plan.id)) throw new PlanError(409, "already_running", `"${plan.name}" is already running.`);
const matched = planServers(plan, await db.select({ id: servers.id, name: servers.name, tags: servers.tags }).from(servers));
if (matched.length === 0) throw new PlanError(400, "no_servers", "No server has this plan's tags or is named in it, so there is nothing to update.");
const items: PlanItem[] = matched.map((s) => ({ serverId: s.id, name: s.name, status: "pending", updateRunId: null, message: null, windowIds: [] }));
const [run] = await db
.insert(patchPlanRuns)
.values({ planId: plan.id, planName: plan.name, trigger: options.trigger, startedBy: options.actor, status: "running", message: `Updating ${matched.length} server${matched.length === 1 ? "" : "s"}, one at a time.`, items: JSON.stringify(items), startedAt: now.toISOString() })
.returning();
await recordAudit({ category: "patch_plan", action: "run", targetType: "patch_plan", targetId: plan.id, detail: { name: plan.name, run: run.id, trigger: options.trigger, servers: matched.length, by: options.actor } });
return run;
}
// ─── Following a run ───────────────────────────────────────────────────────
async function endWindows(ids: number[], now: Date): Promise<void> {
if (ids.length === 0) return;
await db.update(maintenanceWindows).set({ endsAt: now.toISOString() }).where(inArray(maintenanceWindows.id, ids));
invalidateAlertsCache();
}
/** Silences what depends on the server for as long as it is being updated; returns the windows it made (ones it only extended belong to someone else). */
async function silenceDependents(plan: PlanRow, serverId: number, serverName: string, actor: string, now: Date): Promise<number[]> {
if (!plan.silenceDependents) return [];
const made: number[] = [];
try {
const impact = await impactOf("server", serverId, now);
for (const d of impact?.dependents ?? []) {
const started = await startOrExtendWindow(d.type, d.id, plan.windowMinutes, `Depends on ${serverName} — patch plan "${plan.name}"`.slice(0, 200), actor, now);
if (started && !started.extended) made.push(started.window.id);
}
} catch (err) {
console.error("[patchPlans] couldn't silence what depends on a server:", err);
}
return made;
}
const words = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
/** What a finished run says, in a line. */
export function summariseItems(items: PlanItem[]): string {
const count = (s: ItemStatus) => items.filter((i) => i.status === s).length;
const parts = [`${count("done")} updated`];
if (count("failed") > 0) parts.push(`${count("failed")} failed`);
if (count("skipped") > 0) parts.push(`${count("skipped")} left alone`);
return `${parts.join(", ")} of ${words(items.length, "server")}.`;
}
async function saveRun(run: PlanRunRow, items: PlanItem[], extra: Partial<PlanRunRow> = {}): Promise<void> {
await db.update(patchPlanRuns).set({ items: JSON.stringify(items), ...extra }).where(eq(patchPlanRuns.id, run.id));
}
async function finishRun(run: PlanRunRow, plan: PlanRow | null, items: PlanItem[], status: RunStatus, now: Date, why?: string): Promise<void> {
const message = [summariseItems(items), why].filter(Boolean).join(" ");
await saveRun(run, items, { status, message, finishedAt: now.toISOString() });
await recordAudit({ category: "patch_plan", action: status === "done" ? "finished" : status, targetType: "patch_plan", targetId: run.planId, detail: { name: run.planName, run: run.id, message } });
if (status !== "cancelled") {
const lines = items.map((i) => `${i.status === "done" ? "✓" : i.status === "failed" ? "✕" : "–"} ${i.name}${i.message ? ` — ${i.message}` : ""}`);
await notifyPatchPlanFinished({ planName: run.planName, ok: status === "done", message, lines }).catch((err) => console.error("[patchPlans] couldn't send the result:", err));
}
void plan;
}
/** When a server's turn is over and it failed: stop (leaving the rest alone) or carry on. */
function afterFailure(plan: PlanRow, items: PlanItem[]): boolean {
if (!plan.stopOnFailure) return false;
for (const i of items) if (i.status === "pending") Object.assign(i, { status: "skipped", message: "Not started, because an earlier server failed." });
return true;
}
async function advanceRun(run: PlanRunRow, now: Date): Promise<void> {
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
const items = parseItems(run.items);
if (!plan) return; // deleted: the run goes with it
// The server whose turn it is: has its update run finished?
const current = items.find((i) => i.status === "running");
if (current) {
const [ur] = current.updateRunId !== null ? await db.select().from(updateRuns).where(eq(updateRuns.id, current.updateRunId)).limit(1) : [];
if (ur && ACTIVE_STATUSES.includes(ur.status as (typeof ACTIVE_STATUSES)[number])) return;
const ok = ur?.status === "done";
current.status = ok ? "done" : "failed";
current.message = ur?.message ?? "The update run is gone.";
await endWindows(current.windowIds, now);
if (!ok && afterFailure(plan, items)) {
await finishRun(run, plan, items, "failed", now, `Stopped after ${current.name}.`);
return;
}
await saveRun(run, items);
}
// The next one.
for (;;) {
const next = items.find((i) => i.status === "pending");
if (!next) {
await finishRun(run, plan, items, items.some((i) => i.status === "failed") ? "failed" : "done", now);
return;
}
const [server] = await db.select().from(servers).where(eq(servers.id, next.serverId)).limit(1);
if (!server) {
Object.assign(next, { status: "skipped", message: "The server was removed." });
continue;
}
if (plan.onlyWithUpdates) {
const updates = parsePendingUpdates(server.pendingUpdates);
if (!updates || updates.count === 0) {
Object.assign(next, { status: "skipped", message: "No updates were waiting." });
continue;
}
}
const actor = `Patch plan "${plan.name}"`;
try {
const ur = await startUpdateRun(
{
serverId: server.id,
integrationId: plan.integrationId,
projectId: plan.projectId,
templateId: plan.templateId,
limit: plan.limitFrom === "hostname" && server.hostname ? server.hostname : server.name,
windowMinutes: plan.windowMinutes,
actor,
},
now,
);
Object.assign(next, { status: "running", updateRunId: ur.id, message: "The update run has started.", windowIds: await silenceDependents(plan, server.id, server.name, actor, now) });
await saveRun(run, items);
return; // one at a time: this one is followed until it is over
} catch (err) {
const message = err instanceof UpdateRunError || err instanceof Error ? err.message : String(err);
Object.assign(next, { status: "failed", message });
if (afterFailure(plan, items)) {
await finishRun(run, plan, items, "failed", now, `Stopped at ${next.name}.`);
return;
}
}
}
}
/** One pass over every plan run that is under way. Returns how many it looked at (for the tests). */
export async function advancePlanRuns(now: Date = new Date()): Promise<number> {
const active = await db.select().from(patchPlanRuns).where(inArray(patchPlanRuns.status, ACTIVE_RUN));
for (const run of active) {
try {
await advanceRun(run, now);
} catch (err) {
console.error(`[patchPlans] run ${run.id}:`, err);
}
}
return active.length;
}
/** Starts the plans whose time has come. Returns how many it started. */
export async function startDuePlans(now: Date = new Date()): Promise<number> {
let started = 0;
for (const plan of await db.select().from(patchPlans)) {
const due = dueOccurrence(plan, now);
if (!due) continue;
if (await activeRunOf(plan.id)) continue; // still on the last one: today is skipped rather than overlapping
// Written down first, so that whatever happens next it is not tried again every twenty seconds for half an hour.
await db.update(patchPlans).set({ lastRunDate: due.date }).where(eq(patchPlans.id, plan.id));
try {
await startPlanRun(plan.id, { trigger: "schedule", actor: "the schedule", now });
started++;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
await db.insert(patchPlanRuns).values({ planId: plan.id, planName: plan.name, trigger: "schedule", startedBy: "the schedule", status: "failed", message: `Didn't start: ${message}`, items: "[]", startedAt: now.toISOString(), finishedAt: now.toISOString() });
}
}
return started;
}
export async function cancelPlanRun(id: number, actor: string, now: Date = new Date()): Promise<PlanRunRow | null> {
const [run] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
if (!run) return null;
if (run.status !== "running") throw new PlanError(409, "not_active", "That run is already over.");
const items = parseItems(run.items);
for (const i of items) {
if (i.status === "pending") Object.assign(i, { status: "skipped", message: `Not started: stopped by ${actor}.` });
else if (i.status === "running") {
if (i.updateRunId !== null) await cancelUpdateRun(i.updateRunId, actor, now).catch(() => undefined);
await endWindows(i.windowIds, now);
Object.assign(i, { status: "failed", message: `Stopped being followed by ${actor}. A playbook already started keeps running in Semaphore.` });
}
}
const [plan] = await db.select().from(patchPlans).where(eq(patchPlans.id, run.planId)).limit(1);
await finishRun(run, plan ?? null, items, "cancelled", now, `Stopped by ${actor}.`);
const [after] = await db.select().from(patchPlanRuns).where(eq(patchPlanRuns.id, id)).limit(1);
return after;
}
/** Recent runs, those still going first. */
export async function listPlanRuns(options: { planId?: number; limit?: number } = {}): Promise<PlanRunRow[]> {
const rows = await db
.select()
.from(patchPlanRuns)
.where(options.planId !== undefined ? eq(patchPlanRuns.planId, options.planId) : undefined)
.orderBy(desc(patchPlanRuns.startedAt))
.limit(options.limit ?? 20);
return [...rows.filter((r) => r.status === "running"), ...rows.filter((r) => r.status !== "running")];
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Every twenty seconds: start what is due, follow what is going. A run that was under way when the app last stopped is picked up where it was. */
export function initPatchPlans(): void {
if (timer) return;
timer = setInterval(() => {
void startDuePlans()
.then(() => advancePlanRuns())
.catch((err) => console.error("[patchPlans]", err));
}, POLL_INTERVAL_MS);
timer.unref();
}
+44
View File
@@ -0,0 +1,44 @@
/**
* Renewals: a certificate, domain registration or secret whose expiry moved later was renewed. The app sees it happen when it re-reads the date — an endpoint
* check seeing a new certificate, the registry saying a domain now runs longer, a secret's live certificate being replaced — and writes it down, so the monthly
* report can say "certificates renewed: 3". It only goes back as far as the version that began watching; nothing is guessed about earlier.
*/
import { and, eq, gte, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { renewals } from "../db/schema.js";
/** An expiry has to move at least this far for it to be a renewal (a registry rounding a date by a day is not one). */
export const MIN_RENEWAL_DAYS = 20;
const DAY_MS = 86_400_000;
export type RenewalKind = "certificate" | "domain" | "secret";
const dayOf = (value: string | null | undefined): string | null => (value && /^\d{4}-\d{2}-\d{2}/.test(value) ? value.slice(0, 10) : null);
/** Pure: whether the expiry went from one date to a clearly later one. Either may be a full ISO time; only the day counts. */
export function isRenewal(from: string | null | undefined, to: string | null | undefined): boolean {
const a = dayOf(from);
const b = dayOf(to);
if (!a || !b) return false;
return (Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / DAY_MS >= MIN_RENEWAL_DAYS;
}
/** Writes down a renewal if the dates say it was one (and the same one is not already written down). Never throws: it must not hold up the check that saw it. */
export async function noteRenewal(kind: RenewalKind, subject: string, from: string | null | undefined, to: string | null | undefined, now: Date = new Date()): Promise<boolean> {
try {
if (!isRenewal(from, to)) return false;
const fromDate = dayOf(from)!;
const toDate = dayOf(to)!;
const [same] = await db.select({ id: renewals.id }).from(renewals).where(and(eq(renewals.kind, kind), eq(renewals.subject, subject), eq(renewals.toDate, toDate))).limit(1);
if (same) return false;
await db.insert(renewals).values({ kind, subject, at: now.toISOString(), fromDate, toDate });
return true;
} catch (err) {
console.error("[renewals] couldn't record a renewal:", err);
return false;
}
}
export async function renewalsBetween(from: Date, to: Date) {
return db.select().from(renewals).where(and(gte(renewals.at, from.toISOString()), lt(renewals.at, to.toISOString()))).orderBy(renewals.at);
}
+14 -1
View File
@@ -113,6 +113,17 @@ export interface HealthCheckSettings {
backupStaleHours: number;
/** A restore test older than this many days (or none at all) is a problem for a guest that has backups; 0 = never (a failed test always is). */
restoreTestDays: number;
/** A server that starts listening on a TCP port it did not at first, and that is not noted on its Ports card, is a warning (services/newPorts.ts). */
newPortAlerts: boolean;
}
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
export interface MonthlyReportSettings {
enabled: boolean;
/** Day of the month, 1–28 (every month has it), in notifications.timezone. */
day: number;
/** "HH:MM", in notifications.timezone. */
time: string;
}
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
@@ -225,6 +236,7 @@ export interface AppSettings {
quietHours: QuietHoursSettings;
healthChecks: HealthCheckSettings;
digest: DigestSettings;
monthlyReport: MonthlyReportSettings;
signIn: SignInSettings;
profilePictures: ProfilePictureSettings;
inventory: InventorySettings;
@@ -267,12 +279,13 @@ const DEFAULTS: AppSettings = {
logRetention: { enabled: false, retentionDays: 90, intervalHours: 24 },
quietHours: { enabled: false, start: "22:00", end: "07:00" },
digest: { enabled: false, day: 1, time: "08:00" },
monthlyReport: { enabled: false, day: 1, time: "08:00" },
inventory: { currency: "", pricePerKwh: null },
agents: { autoUpdate: false },
selfBackup: { enabled: true, time: "03:30", keep: 14 },
profilePictures: { gravatar: false, identityProvider: false },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0 },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },
+4 -1
View File
@@ -5,6 +5,7 @@ import { recordServerMetrics } from "./serverMetrics.js";
import { nextUpdateState, type IncomingUpdates } from "./pendingUpdates.js";
import { nextHostHealth, type IncomingHostHealth } from "./hostHealth.js";
import { recordAudit } from "./audit.js";
import { baselineFrom } from "./newPorts.js";
import { fillInventoryFromServer } from "./inventoryAutofill.js";
import { nextDiagnostics, type AgentDiagnostic } from "./agentDiagnostics.js";
@@ -113,7 +114,7 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
}
const system = report.system;
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion }).from(servers).where(eq(servers.id, serverId));
const [before] = await db.select({ pendingUpdates: servers.pendingUpdates, updatesPendingSince: servers.updatesPendingSince, hostHealth: servers.hostHealth, agentVersion: servers.agentVersion, portBaseline: servers.portBaseline }).from(servers).where(eq(servers.id, serverId));
await db
.update(servers)
.set({
@@ -144,6 +145,8 @@ export async function syncServerTasks(serverId: number, report: AgentReport) {
: {}),
// Only stored when the agent reported it, so an agent that predates this doesn't wipe the field.
...(system?.listening_ports ? { listeningPorts: JSON.stringify(system.listening_ports) } : {}),
// The first report that carries ports says what is normal for this server; a port that appears later is "new" (services/newPorts.ts).
...(system?.listening_ports && before && before.portBaseline === null ? { portBaseline: baselineFrom(system.listening_ports) } : {}),
})
.where(eq(servers.id, serverId));
+3
View File
@@ -4,6 +4,7 @@ import { eq, isNotNull } from "drizzle-orm";
import { db } from "../db/client.js";
import { secrets } from "../db/schema.js";
import { resolveTarget } from "../utils/networkGuard.js";
import { noteRenewal } from "./renewals.js";
/**
* Opens a real TLS connection and returns the server certificate's expiry as
@@ -79,6 +80,8 @@ export async function refreshTlsSecret(id: number): Promise<TlsCheckResult | nul
...(expiryDate !== row.expiryDate ? { updatedAt: now } : {}),
})
.where(eq(secrets.id, id));
// A certificate that is now good for much longer than it was has been replaced.
await noteRenewal("certificate", row.name, row.expiryDate, expiryDate);
return { id, name: row.name, host: row.checkHost, port, ok: true, expiryDate };
} catch (err) {
const error = err instanceof Error ? err.message : String(err);