diff --git a/README.md b/README.md index 5e52370..5d7311b 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,11 @@ this server's IP address, allowed to create tickets), entered on the osTicket in provider while you work on it (server offline / disk, storage and Synology health, Proxmox backup alerts, Proxmox Backup Server verification alerts, and "integration down" for that service type). -Every window has a fixed end (5 minutes to 7 days) and expires on its own, and +For a power cut or a rack reboot, the "Many at once" choices at the top of the +list start a window on every server, every (enabled) integration, every +(enabled) DNS provider, or all of them in one go — asking first before +"Everything" — and "End all" on the Active list closes them together. Each is +still its own window, so any can be ended alone. Every window has a fixed end (5 minutes to 7 days) and expires on its own, and a problem that began during a window and is still present when it ends alerts then — a forgotten window can't hide an outage. A banner shows what's currently silenced to every signed-in user. If you also run Uptime Kuma, the Maintenance diff --git a/server/src/routes/maintenance.ts b/server/src/routes/maintenance.ts index 50fe91d..4d8f5f1 100644 --- a/server/src/routes/maintenance.ts +++ b/server/src/routes/maintenance.ts @@ -1,5 +1,5 @@ import { Router } from "express"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { z } from "zod"; import { db } from "../db/client.js"; import { dnsProviders, integrations, maintenanceAliases, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js"; @@ -208,6 +208,66 @@ maintenanceRouter.delete("/aliases/:id", requireRole("operator"), asyncHandler(a res.status(204).end(); })); +const bulkSchema = z.object({ + scope: z.enum(["everything", "servers", "integrations", "dns_providers"]), + minutes: z.number().int().min(5).max(7 * 24 * 60), + reason: z.string().max(200).optional(), +}); + +/** + * Starts a window on every server, every integration, every DNS provider, or all of them, in one go — for a power cut, a network move, + * a rack reboot. It is the same window as one started by hand for each target (so each can still be ended on its own); only the + * asking is done once. Disabled integrations and providers are left out: they raise no alerts to silence. + */ +maintenanceRouter.post("/bulk", requireRole("operator"), asyncHandler(async (req, res) => { + const parsed = bulkSchema.safeParse(req.body); + if (!parsed.success) { + return res.status(400).json({ error: "invalid_body", message: "Choose what to put in maintenance and for how long.", details: parsed.error.flatten() }); + } + const { scope, minutes, reason } = parsed.data; + + const targets: { type: "server" | "integration" | "dns_provider"; id: number }[] = []; + if (scope === "everything" || scope === "servers") { + for (const s of await db.select({ id: servers.id }).from(servers)) targets.push({ type: "server", id: s.id }); + } + if (scope === "everything" || scope === "integrations") { + for (const i of await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.enabled, true))) targets.push({ type: "integration", id: i.id }); + } + if (scope === "everything" || scope === "dns_providers") { + for (const p of await db.select({ id: dnsProviders.id }).from(dnsProviders).where(eq(dnsProviders.enabled, true))) targets.push({ type: "dns_provider", id: p.id }); + } + if (targets.length === 0) { + return res.status(400).json({ error: "nothing_to_silence", message: "There is nothing of that kind to put in maintenance yet." }); + } + + const actor = req.currentUser!; + const createdBy = actor.name ?? actor.email ?? actor.oidcSub; + const why = reason ?? "Everything in maintenance"; + let started = 0; + let extended = 0; + for (const t of targets) { + const result = await startOrExtendWindow(t.type, t.id, minutes, why, createdBy); + if (!result) continue; // removed while this ran + if (result.extended) extended++; + else started++; + } + + await recordAudit({ actor, category: "maintenance", action: "start_bulk", detail: { scope, minutes, reason: why, started, extended } }); + invalidateAlertsCache(); + res.status(201).json({ started, extended, total: started + extended }); +})); + +/** Ends every window that is running now — the way out after starting many. Scheduled ones that haven't begun are left as they are. */ +maintenanceRouter.post("/end-all", requireRole("operator"), asyncHandler(async (req, res) => { + const active = await listActiveWindows(); + if (active.length > 0) { + await db.update(maintenanceWindows).set({ endsAt: new Date().toISOString() }).where(inArray(maintenanceWindows.id, active.map((w) => w.id))); + } + await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "end_all", detail: { ended: active.length } }); + invalidateAlertsCache(); + res.json({ ended: active.length }); +})); + const syncUptimeKumaSchema = z.object({ integrationId: z.number().int().positive() }); /** Reads Uptime Kuma's maintenance schedule now and brings the mirrored windows in line with it (the same pass that runs every 15 minutes when switched on). */ diff --git a/server/test/maintenanceBulk.test.ts b/server/test/maintenanceBulk.test.ts new file mode 100644 index 0000000..1870761 --- /dev/null +++ b/server/test/maintenanceBulk.test.ts @@ -0,0 +1,147 @@ +/** + * Putting everything in maintenance at once, and ending it all again. + * + * Run with: npm test --workspace server + */ +import { test, describe, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import type { Server } from "node:http"; + +const dir = mkdtempSync(join(tmpdir(), "hm-bulk-")); +process.env.DATABASE_PATH = join(dir, "test.sqlite"); +process.env.SESSION_DIR = join(dir, "sessions"); +process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); + +const express = (await import("express")).default; +const { db } = await import("../src/db/client.js"); +const { users, servers, integrations, dnsProviders, maintenanceWindows, auditLog } = await import("../src/db/schema.js"); +const { runMigrations } = await import("../src/db/migrate.js"); +const { maintenanceRouter } = await import("../src/routes/maintenance.js"); +const { listActiveWindows } = await import("../src/services/maintenance.js"); + +await runMigrations(); + +describe("bulk maintenance", () => { + let app: Server; + let base = ""; + let who = "operator"; + + before(async () => { + await db.insert(users).values([ + { oidcSub: "operator", name: "Olga", role: "operator" }, + { oidcSub: "viewer", name: "Vic", role: "viewer" }, + ]); + await db.insert(servers).values([ + { name: "elsa", apiTokenHash: "a", apiTokenPrefix: "a" }, + { name: "anna", apiTokenHash: "b", apiTokenPrefix: "b" }, + { name: "zoe", apiTokenHash: "c", apiTokenPrefix: "c" }, + ]); + await db.insert(integrations).values([ + { type: "proxmox", name: "Felicia", baseUrl: "https://pve.lab", config: "{}" }, + { type: "gitea", name: "Gitea", baseUrl: "https://git.lab", config: "{}" }, + { type: "synology", name: "Old NAS", baseUrl: "https://nas.lab", config: "{}", enabled: false }, + ]); + await db.insert(dnsProviders).values([ + { providerType: "technitium", name: "Ginny", config: "{}" }, + { providerType: "pihole", name: "Off pihole", config: "{}", enabled: false }, + ]); + + const a = express(); + a.use(express.json()); + a.use((req, _res, next) => { + (req as unknown as { session: unknown }).session = { user: { sub: who } }; + next(); + }); + a.use("/api/maintenance", maintenanceRouter); + app = await new Promise((resolve) => { + const s = a.listen(0, "127.0.0.1", () => resolve(s)); + }); + base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/maintenance`; + }); + + after(() => { + app.close(); + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + // Windows keeps the database file locked until the process exits. + } + }); + + const post = (path: string, body?: unknown) => + fetch(`${base}${path}`, { method: "POST", headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); + const byType = async () => { + const counts: Record = {}; + for (const w of await listActiveWindows()) counts[w.targetType] = (counts[w.targetType] ?? 0) + 1; + return counts; + }; + + test("all servers starts one window per server and nothing else", async () => { + const res = await post("/bulk", { scope: "servers", minutes: 60 }); + assert.equal(res.status, 201); + assert.deepEqual(await res.json(), { started: 3, extended: 0, total: 3 }); + assert.deepEqual(await byType(), { server: 3 }); + }); + + test("everything adds the enabled integrations and DNS providers, and extends the servers already in maintenance", async () => { + const res = await post("/bulk", { scope: "everything", minutes: 120, reason: "Power work" }); + assert.deepEqual(await res.json(), { started: 3, extended: 3, total: 6 }, "2 integrations + 1 provider are new; the 3 servers are extended, not doubled"); + assert.deepEqual(await byType(), { server: 3, integration: 2, dns_provider: 1 }); + const windows = await listActiveWindows(); + assert.ok(windows.every((w) => w.reason === "Power work" || w.reason === "Everything in maintenance")); + assert.ok(windows.every((w) => w.createdBy === "Olga")); + }); + + test("disabled integrations and providers are left out", async () => { + const ids = new Set((await listActiveWindows()).filter((w) => w.targetType === "integration").map((w) => w.targetId)); + assert.equal(ids.size, 2); + }); + + test("it is one entry in the audit log, saying what it covered", async () => { + const entries = (await db.select().from(auditLog)).filter((e) => e.action === "start_bulk"); + assert.equal(entries.length, 2); + assert.match(entries[1].detail ?? "", /"scope":"everything"/); + }); + + test("a viewer can't, and a bad request is refused", async () => { + who = "viewer"; + assert.equal((await post("/bulk", { scope: "servers", minutes: 60 })).status, 403); + assert.equal((await post("/end-all")).status, 403); + who = "operator"; + assert.equal((await post("/bulk", { scope: "the moon", minutes: 60 })).status, 400); + assert.equal((await post("/bulk", { scope: "servers", minutes: 1 })).status, 400); + assert.equal((await post("/bulk", { scope: "servers", minutes: 60 * 24 * 30 })).status, 400); + }); + + test("end-all ends what is running, and leaves a scheduled window alone", async () => { + const [anyServer] = await db.select().from(servers); + await db.insert(maintenanceWindows).values({ + targetType: "server", + targetId: anyServer.id, + reason: "tonight", + startedAt: new Date(Date.now() + 5 * 3600_000).toISOString(), + endsAt: new Date(Date.now() + 6 * 3600_000).toISOString(), + }); + const res = await post("/end-all"); + assert.deepEqual(await res.json(), { ended: 6 }); + assert.equal((await listActiveWindows()).length, 0); + const upcoming = (await db.select().from(maintenanceWindows)).filter((w) => w.reason === "tonight"); + assert.equal(upcoming.length, 1); + assert.ok(upcoming[0].endsAt > upcoming[0].startedAt, "the scheduled one is untouched"); + }); + + test("end-all with nothing running is fine", async () => { + assert.deepEqual(await (await post("/end-all")).json(), { ended: 0 }); + }); + + test("a group with nothing in it says so instead of doing nothing silently", async () => { + await db.delete(dnsProviders); + const res = await post("/bulk", { scope: "dns_providers", minutes: 60 }); + assert.equal(res.status, 400); + assert.equal((await res.json()).error, "nothing_to_silence"); + }); +}); diff --git a/web/src/api/client.ts b/web/src/api/client.ts index c7d785b..23ac73c 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -34,6 +34,9 @@ export interface SessionSummary { export type MaintenanceTargetType = "server" | "integration" | "dns_provider"; +/** A whole group at once: every server, every enabled integration, every enabled DNS provider, or all of those. */ +export type MaintenanceBulkScope = "everything" | "servers" | "integrations" | "dns_providers"; + export interface MaintenanceWindow { id: number; targetType: MaintenanceTargetType; @@ -1127,6 +1130,9 @@ export const api = { start: (data: { targetType: MaintenanceTargetType; targetId: number; minutes: number; reason?: string }) => request<{ window: MaintenanceWindow }>("/api/maintenance", { method: "POST", body: JSON.stringify(data) }), end: (id: number) => request(`/api/maintenance/${id}`, { method: "DELETE" }), + startBulk: (data: { scope: MaintenanceBulkScope; minutes: number; reason?: string }) => + request<{ started: number; extended: number; total: number }>("/api/maintenance/bulk", { method: "POST", body: JSON.stringify(data) }), + endAll: () => request<{ ended: number }>("/api/maintenance/end-all", { method: "POST" }), syncUptimeKuma: (data: { integrationId: number }) => request("/api/maintenance/sync/uptimekuma", { method: "POST", body: JSON.stringify(data) }), importUptimeKuma: (data: { integrationId: number; minutes: number }) => diff --git a/web/src/layout/AppShell.tsx b/web/src/layout/AppShell.tsx index 244db85..4c83bc1 100644 --- a/web/src/layout/AppShell.tsx +++ b/web/src/layout/AppShell.tsx @@ -344,7 +344,12 @@ export default function AppShell({ user, children }: { user: CurrentUser; childr
Maintenance — alerts silenced for{" "} - {maintenance.map((w) => `${w.targetName} (${formatRemaining(w.endsAt)} left)`).join(", ")}.{" "} + {maintenance.length <= 3 + ? maintenance.map((w) => `${w.targetName} (${formatRemaining(w.endsAt)} left)`).join(", ") + : `${maintenance.slice(0, 3).map((w) => w.targetName).join(", ")} and ${maintenance.length - 3} more (the last ends in ${formatRemaining( + maintenance.reduce((latest, w) => (w.endsAt > latest ? w.endsAt : latest), maintenance[0].endsAt), + )})`} + .{" "} Manage
diff --git a/web/src/pages/Maintenance.tsx b/web/src/pages/Maintenance.tsx index 92fbd0e..0185d3b 100644 --- a/web/src/pages/Maintenance.tsx +++ b/web/src/pages/Maintenance.tsx @@ -4,6 +4,7 @@ import { type CurrentUser, type IntegrationSummary, type MaintenanceAlias, + type MaintenanceBulkScope, type MaintenanceTargets, type MaintenanceTargetType, type MaintenanceWindow, @@ -15,6 +16,7 @@ import { formatRemaining } from "../utils/duration"; import { readableError } from "../utils/errors"; import { ErrorNotice } from "../components/Notice"; import { toast } from "../utils/toast"; +import { confirmDialog } from "../utils/dialogs"; const DURATIONS: { minutes: number; label: string }[] = [ { minutes: 30, label: "30 minutes" }, @@ -89,8 +91,51 @@ export default function Maintenance({ user }: { user: CurrentUser }) { return () => clearInterval(tick); }, []); + async function startBulk(scope: MaintenanceBulkScope) { + const words = DURATIONS.find((d) => d.minutes === minutes)?.label ?? `${minutes} minutes`; + // Everything is the one choice that can hide a real outage, so it asks first. + if ( + scope === "everything" && + !(await confirmDialog({ + title: "Silence everything?", + message: `This silences every alert about every server, integration and DNS provider for ${words}. Anything still wrong when it ends alerts then.`, + confirmLabel: "Silence everything", + danger: true, + })) + ) + return; + setError(null); + setBusy(true); + try { + const res = await api.maintenance.startBulk({ scope, minutes, reason: reason.trim() || undefined }); + setTarget(""); + setReason(""); + toast.success(`Maintenance started on ${res.total} item${res.total === 1 ? "" : "s"} for ${words}. End them all from the Active list.`); + load(); + announceChange(); + } catch (err) { + setError(readableError(err)); + } finally { + setBusy(false); + } + } + + async function endAll() { + if (!(await confirmDialog({ title: "End all maintenance", message: `End all ${windows?.length ?? 0} windows now? Alerts about them become active again.`, confirmLabel: "End all", danger: true }))) return; + setError(null); + try { + const res = await api.maintenance.endAll(); + toast.success(`Ended ${res.ended} maintenance window${res.ended === 1 ? "" : "s"}. Alerts are active again.`); + load(); + announceChange(); + } catch (err) { + setError(readableError(err)); + } + } + async function start(e: React.FormEvent) { e.preventDefault(); + if (target.startsWith("all:")) return startBulk(target.slice(4) as MaintenanceBulkScope); const [targetType, id] = target.split(":"); if (!targetType || !id) return; setError(null); @@ -211,6 +256,14 @@ export default function Maintenance({ user }: { user: CurrentUser }) {