From bff0c2169e6c19166e6ae0e49897b9c5cc6b6b30 Mon Sep 17 00:00:00 2001 From: Bobban Rydh Date: Fri, 16 Oct 2026 20:58:01 +0200 Subject: [PATCH] Start maintenance on everything, all servers or all integrations at once The Start maintenance list gets a "Many at once" group: Everything, All servers, All integrations, All DNS providers (enabled ones only). Each target still gets its own window, existing ones are extended, and the action is one audit entry. "Everything" asks for confirmation first, and "End all" on the Active list ends every running window together. The top banner and the wall display summarise long lists instead of printing every name. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 6 +- server/src/routes/maintenance.ts | 62 +++++++++++- server/test/maintenanceBulk.test.ts | 147 ++++++++++++++++++++++++++++ web/src/api/client.ts | 6 ++ web/src/layout/AppShell.tsx | 7 +- web/src/pages/Maintenance.tsx | 60 ++++++++++++ web/src/pages/Wall.tsx | 3 +- 7 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 server/test/maintenanceBulk.test.ts diff --git a/README.md b/README.md index 5e52370..5d7311b 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,11 @@ this server's IP address, allowed to create tickets), entered on the osTicket in provider while you work on it (server offline / disk, storage and Synology health, Proxmox backup alerts, Proxmox Backup Server verification alerts, and "integration down" for that service type). -Every window has a fixed end (5 minutes to 7 days) and expires on its own, and +For a power cut or a rack reboot, the "Many at once" choices at the top of the +list start a window on every server, every (enabled) integration, every +(enabled) DNS provider, or all of them in one go — asking first before +"Everything" — and "End all" on the Active list closes them together. Each is +still its own window, so any can be ended alone. Every window has a fixed end (5 minutes to 7 days) and expires on its own, and a problem that began during a window and is still present when it ends alerts then — a forgotten window can't hide an outage. A banner shows what's currently silenced to every signed-in user. If you also run Uptime Kuma, the Maintenance diff --git a/server/src/routes/maintenance.ts b/server/src/routes/maintenance.ts index 50fe91d..4d8f5f1 100644 --- a/server/src/routes/maintenance.ts +++ b/server/src/routes/maintenance.ts @@ -1,5 +1,5 @@ import { Router } from "express"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { z } from "zod"; import { db } from "../db/client.js"; import { dnsProviders, integrations, maintenanceAliases, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js"; @@ -208,6 +208,66 @@ maintenanceRouter.delete("/aliases/:id", requireRole("operator"), asyncHandler(a res.status(204).end(); })); +const bulkSchema = z.object({ + scope: z.enum(["everything", "servers", "integrations", "dns_providers"]), + minutes: z.number().int().min(5).max(7 * 24 * 60), + reason: z.string().max(200).optional(), +}); + +/** + * Starts a window on every server, every integration, every DNS provider, or all of them, in one go — for a power cut, a network move, + * a rack reboot. It is the same window as one started by hand for each target (so each can still be ended on its own); only the + * asking is done once. Disabled integrations and providers are left out: they raise no alerts to silence. + */ +maintenanceRouter.post("/bulk", requireRole("operator"), asyncHandler(async (req, res) => { + const parsed = bulkSchema.safeParse(req.body); + if (!parsed.success) { + return res.status(400).json({ error: "invalid_body", message: "Choose what to put in maintenance and for how long.", details: parsed.error.flatten() }); + } + const { scope, minutes, reason } = parsed.data; + + const targets: { type: "server" | "integration" | "dns_provider"; id: number }[] = []; + if (scope === "everything" || scope === "servers") { + for (const s of await db.select({ id: servers.id }).from(servers)) targets.push({ type: "server", id: s.id }); + } + if (scope === "everything" || scope === "integrations") { + for (const i of await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.enabled, true))) targets.push({ type: "integration", id: i.id }); + } + if (scope === "everything" || scope === "dns_providers") { + for (const p of await db.select({ id: dnsProviders.id }).from(dnsProviders).where(eq(dnsProviders.enabled, true))) targets.push({ type: "dns_provider", id: p.id }); + } + if (targets.length === 0) { + return res.status(400).json({ error: "nothing_to_silence", message: "There is nothing of that kind to put in maintenance yet." }); + } + + const actor = req.currentUser!; + const createdBy = actor.name ?? actor.email ?? actor.oidcSub; + const why = reason ?? "Everything in maintenance"; + let started = 0; + let extended = 0; + for (const t of targets) { + const result = await startOrExtendWindow(t.type, t.id, minutes, why, createdBy); + if (!result) continue; // removed while this ran + if (result.extended) extended++; + else started++; + } + + await recordAudit({ actor, category: "maintenance", action: "start_bulk", detail: { scope, minutes, reason: why, started, extended } }); + invalidateAlertsCache(); + res.status(201).json({ started, extended, total: started + extended }); +})); + +/** Ends every window that is running now — the way out after starting many. Scheduled ones that haven't begun are left as they are. */ +maintenanceRouter.post("/end-all", requireRole("operator"), asyncHandler(async (req, res) => { + const active = await listActiveWindows(); + if (active.length > 0) { + await db.update(maintenanceWindows).set({ endsAt: new Date().toISOString() }).where(inArray(maintenanceWindows.id, active.map((w) => w.id))); + } + await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "end_all", detail: { ended: active.length } }); + invalidateAlertsCache(); + res.json({ ended: active.length }); +})); + const syncUptimeKumaSchema = z.object({ integrationId: z.number().int().positive() }); /** Reads Uptime Kuma's maintenance schedule now and brings the mirrored windows in line with it (the same pass that runs every 15 minutes when switched on). */ diff --git a/server/test/maintenanceBulk.test.ts b/server/test/maintenanceBulk.test.ts new file mode 100644 index 0000000..1870761 --- /dev/null +++ b/server/test/maintenanceBulk.test.ts @@ -0,0 +1,147 @@ +/** + * Putting everything in maintenance at once, and ending it all again. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-bulk-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, integrations, dnsProviders, maintenanceWindows, auditLog } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { maintenanceRouter } = await import("../src/routes/maintenance.js"); +const { listActiveWindows } = await import("../src/services/maintenance.js"); + +await runMigrations(); + +describe("bulk maintenance", () => { + let app: Server; + let base = ""; + let who = "operator"; + + before(async () => { + await db.insert(users).values([ + { oidcSub: "operator", name: "Olga", role: "operator" }, + { oidcSub: "viewer", name: "Vic", role: "viewer" }, + ]); + await db.insert(servers).values([ + { name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }, + { name: "anna", apiTokenHash: "b", apiTokenPrefix: "b" }, + { name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c" }, + ]); + await db.insert(integrations).values([ + { type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}" }, + { type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" }, + { type: "synology", name: "Old NAS", baseUrl: "https://nas.lab", config: "{}", enabled: false }, + ]); + await db.insert(dnsProviders).values([ + { providerType: "technitium", name: "Ginny", config: "{}" }, + { providerType: "pihole", name: "Off pihole", config: "{}", enabled: false }, + ]); + + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: who } }; + next(); + }); + a.use("/api/maintenance", maintenanceRouter); + app = await new Promise((resolve) => { + const s = a.listen(0, "127.0.0.1", () => resolve(s)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/maintenance`; + }); + + after(() => { + app.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + const post = (path: string, body?: unknown) => + fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); + const byType = async () => { + const counts: Record = {}; + for (const w of await listActiveWindows()) counts[w.targetType] = (counts[w.targetType] ?? 0) + 1; + return counts; + }; + + test("all servers starts one window per server and nothing else", async () => { + const res = await post("/bulk", { scope: "servers", minutes: 60 }); + assert.equal(res.status, 201); + assert.deepEqual(await res.json(), { started: 3, extended: 0, total: 3 }); + assert.deepEqual(await byType(), { server: 3 }); + }); + + test("everything adds the enabled integrations and DNS providers, and extends the servers already in maintenance", async () => { + const res = await post("/bulk", { scope: "everything", minutes: 120, reason: "Power work" }); + assert.deepEqual(await res.json(), { started: 3, extended: 3, total: 6 }, "2 integrations + 1 provider are new; the 3 servers are extended, not doubled"); + assert.deepEqual(await byType(), { server: 3, integration: 2, dns_provider: 1 }); + const windows = await listActiveWindows(); + assert.ok(windows.every((w) => w.reason === "Power work" || w.reason === "Everything in maintenance")); + assert.ok(windows.every((w) => w.createdBy === "Olga")); + }); + + test("disabled integrations and providers are left out", async () => { + const ids = new Set((await listActiveWindows()).filter((w) => w.targetType === "integration").map((w) => w.targetId)); + assert.equal(ids.size, 2); + }); + + test("it is one entry in the audit log, saying what it covered", async () => { + const entries = (await db.select().from(auditLog)).filter((e) => e.action === "start_bulk"); + assert.equal(entries.length, 2); + assert.match(entries[1].detail ?? "", /"scope":"everything"/); + }); + + test("a viewer can't, and a bad request is refused", async () => { + who = "viewer"; + assert.equal((await post("/bulk", { scope: "servers", minutes: 60 })).status, 403); + assert.equal((await post("/end-all")).status, 403); + who = "operator"; + assert.equal((await post("/bulk", { scope: "the moon", minutes: 60 })).status, 400); + assert.equal((await post("/bulk", { scope: "servers", minutes: 1 })).status, 400); + assert.equal((await post("/bulk", { scope: "servers", minutes: 60 * 24 * 30 })).status, 400); + }); + + test("end-all ends what is running, and leaves a scheduled window alone", async () => { + const [anyServer] = await db.select().from(servers); + await db.insert(maintenanceWindows).values({ + targetType: "server", + targetId: anyServer.id, + reason: "tonight", + startedAt: new Date(Date.now() + 5 * 3600_000).toISOString(), + endsAt: new Date(Date.now() + 6 * 3600_000).toISOString(), + }); + const res = await post("/end-all"); + assert.deepEqual(await res.json(), { ended: 6 }); + assert.equal((await listActiveWindows()).length, 0); + const upcoming = (await db.select().from(maintenanceWindows)).filter((w) => w.reason === "tonight"); + assert.equal(upcoming.length, 1); + assert.ok(upcoming[0].endsAt > upcoming[0].startedAt, "the scheduled one is untouched"); + }); + + test("end-all with nothing running is fine", async () => { + assert.deepEqual(await (await post("/end-all")).json(), { ended: 0 }); + }); + + test("a group with nothing in it says so instead of doing nothing silently", async () => { + await db.delete(dnsProviders); + const res = await post("/bulk", { scope: "dns_providers", minutes: 60 }); + assert.equal(res.status, 400); + assert.equal((await res.json()).error, "nothing_to_silence"); + }); +}); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index c7d785b..23ac73c 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -34,6 +34,9 @@ export interface SessionSummary { export type MaintenanceTargetType = "server" | "integration" | "dns_provider"; +/** A whole group at once: every server, every enabled integration, every enabled DNS provider, or all of those. */ +export type MaintenanceBulkScope = "everything" | "servers" | "integrations" | "dns_providers"; + export interface MaintenanceWindow { id: number; targetType: MaintenanceTargetType; @@ -1127,6 +1130,9 @@ export const api = { start: (data: { targetType: MaintenanceTargetType; targetId: number; minutes: number; reason?: string }) => request<{ window: MaintenanceWindow }>("/api/maintenance", { method: "POST", body: JSON.stringify(data) }), end: (id: number) => request(`/api/maintenance/${id}`, { method: "DELETE" }), + startBulk: (data: { scope: MaintenanceBulkScope; minutes: number; reason?: string }) => + request<{ started: number; extended: number; total: number }>("/api/maintenance/bulk", { method: "POST", body: JSON.stringify(data) }), + endAll: () => request<{ ended: number }>("/api/maintenance/end-all", { method: "POST" }), syncUptimeKuma: (data: { integrationId: number }) => request("/api/maintenance/sync/uptimekuma", { method: "POST", body: JSON.stringify(data) }), importUptimeKuma: (data: { integrationId: number; minutes: number }) => diff --git a/web/src/layout/AppShell.tsx b/web/src/layout/AppShell.tsx index 244db85..4c83bc1 100644 --- a/web/src/layout/AppShell.tsx +++ b/web/src/layout/AppShell.tsx @@ -344,7 +344,12 @@ export default function AppShell({ user, children }: { user: CurrentUser; childr
Maintenance — alerts silenced for{" "} - {maintenance.map((w) => `${w.targetName} (${formatRemaining(w.endsAt)} left)`).join(", ")}.{" "} + {maintenance.length <= 3 + ? maintenance.map((w) => `${w.targetName} (${formatRemaining(w.endsAt)} left)`).join(", ") + : `${maintenance.slice(0, 3).map((w) => w.targetName).join(", ")} and ${maintenance.length - 3} more (the last ends in ${formatRemaining( + maintenance.reduce((latest, w) => (w.endsAt > latest ? w.endsAt : latest), maintenance[0].endsAt), + )})`} + .{" "} Manage
diff --git a/web/src/pages/Maintenance.tsx b/web/src/pages/Maintenance.tsx index 92fbd0e..0185d3b 100644 --- a/web/src/pages/Maintenance.tsx +++ b/web/src/pages/Maintenance.tsx @@ -4,6 +4,7 @@ import { type CurrentUser, type IntegrationSummary, type MaintenanceAlias, + type MaintenanceBulkScope, type MaintenanceTargets, type MaintenanceTargetType, type MaintenanceWindow, @@ -15,6 +16,7 @@ import { formatRemaining } from "../utils/duration"; import { readableError } from "../utils/errors"; import { ErrorNotice } from "../components/Notice"; import { toast } from "../utils/toast"; +import { confirmDialog } from "../utils/dialogs"; const DURATIONS: { minutes: number; label: string }[] = [ { minutes: 30, label: "30 minutes" }, @@ -89,8 +91,51 @@ export default function Maintenance({ user }: { user: CurrentUser }) { return () => clearInterval(tick); }, []); + async function startBulk(scope: MaintenanceBulkScope) { + const words = DURATIONS.find((d) => d.minutes === minutes)?.label ?? `${minutes} minutes`; + // Everything is the one choice that can hide a real outage, so it asks first. + if ( + scope === "everything" && + !(await confirmDialog({ + title: "Silence everything?", + message: `This silences every alert about every server, integration and DNS provider for ${words}. Anything still wrong when it ends alerts then.`, + confirmLabel: "Silence everything", + danger: true, + })) + ) + return; + setError(null); + setBusy(true); + try { + const res = await api.maintenance.startBulk({ scope, minutes, reason: reason.trim() || undefined }); + setTarget(""); + setReason(""); + toast.success(`Maintenance started on ${res.total} item${res.total === 1 ? "" : "s"} for ${words}. End them all from the Active list.`); + load(); + announceChange(); + } catch (err) { + setError(readableError(err)); + } finally { + setBusy(false); + } + } + + async function endAll() { + if (!(await confirmDialog({ title: "End all maintenance", message: `End all ${windows?.length ?? 0} windows now? Alerts about them become active again.`, confirmLabel: "End all", danger: true }))) return; + setError(null); + try { + const res = await api.maintenance.endAll(); + toast.success(`Ended ${res.ended} maintenance window${res.ended === 1 ? "" : "s"}. Alerts are active again.`); + load(); + announceChange(); + } catch (err) { + setError(readableError(err)); + } + } + async function start(e: React.FormEvent) { e.preventDefault(); + if (target.startsWith("all:")) return startBulk(target.slice(4) as MaintenanceBulkScope); const [targetType, id] = target.split(":"); if (!targetType || !id) return; setError(null); @@ -211,6 +256,14 @@ export default function Maintenance({ user }: { user: CurrentUser }) {