import { Router } from "express"; import { and, eq, inArray } from "drizzle-orm"; import { z } from "zod"; import { db } from "../db/client.js"; import { dnsProviders, integrations, maintenanceAliases, maintenanceSchedules, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js"; import { deleteSchedule, isValidTimeZone, materializeSchedules, occurrencesBetween, removeFutureWindows, targetsOf } from "../services/maintenanceSchedules.js"; import { hostOfAddress } from "../services/kumaTargets.js"; import { invalidateAlertsCache } from "../services/alerts.js"; import { requireAuth, requireRole } from "../auth/middleware.js"; import { recordAudit } from "../services/audit.js"; import { describeTarget, listActiveWindows, listUpcomingWindows, startOrExtendWindow } from "../services/maintenance.js"; import { impactOf } from "../services/dependencies.js"; import { KumaSyncError, syncKumaMaintenance } from "../services/kumaMaintenanceSync.js"; import { loadIntegrationConfig } from "../integrations/loadIntegration.js"; import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js"; import { attachMatchedServers, toMatchableServers } from "../services/uptimeKumaMatch.js"; import { asyncHandler } from "../utils/asyncHandler.js"; export const maintenanceRouter = Router(); // Anyone signed in can see what's currently silenced (so nobody is surprised by missing alerts); // starting and ending a window is an operator action, like the other things that change how the homelab behaves. maintenanceRouter.use(requireAuth); maintenanceRouter.get("/", asyncHandler(async (_req, res) => { const describe = async (rows: Awaited>) => { const out = []; for (const w of rows) { const target = await describeTarget(w.targetType, w.targetId); if (!target) continue; // target was deleted — nothing left to silence out.push({ ...w, targetName: target.name, targetKind: target.kind }); } return out; }; const windows = await describe(await listActiveWindows()); windows.sort((a, b) => a.endsAt.localeCompare(b.endsAt)); const upcoming = await describe(await listUpcomingWindows()); upcoming.sort((a, b) => a.startedAt.localeCompare(b.startedAt)); const [serverRows, integrationRows, providerRows] = await Promise.all([ db.select({ id: servers.id, name: servers.name }).from(servers).orderBy(servers.name), db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).orderBy(integrations.name), db.select({ id: dnsProviders.id, name: dnsProviders.name, providerType: dnsProviders.providerType }).from(dnsProviders).orderBy(dnsProviders.name), ]); const aliases = []; for (const a of await db.select().from(maintenanceAliases).orderBy(maintenanceAliases.address)) { const target = await describeTarget(a.targetType, a.targetId); if (!target) continue; // target was deleted — the alias has nothing left to point at aliases.push({ id: a.id, address: a.address, targetType: a.targetType, targetId: a.targetId, targetName: target.name, targetKind: target.kind }); } res.json({ windows, upcoming, aliases, targets: { servers: serverRows, integrations: integrationRows, dnsProviders: providerRows } }); })); const startSchema = z.object({ targetType: z.enum(maintenanceTargetTypes), targetId: z.number().int().positive(), // Required and capped: an open-ended window is the way this feature could quietly hide a real outage. minutes: z.number().int().min(5).max(7 * 24 * 60), reason: z.string().max(200).optional(), // Also put what depends on the target under maintenance, for the same time: it will look down while the target is. withDependents: z.boolean().optional(), }); maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => { const parsed = startSchema.safeParse(req.body); if (!parsed.success) { return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() }); } const { targetType, targetId, minutes, reason, withDependents } = parsed.data; const actor = req.currentUser!; const createdBy = actor.name ?? actor.email ?? actor.oidcSub; // Starting maintenance on something already in maintenance restarts its clock rather than stacking windows. const result = await startOrExtendWindow(targetType, targetId, minutes, reason ?? null, createdBy); if (!result) { return res.status(404).json({ error: "not_found", message: "That target doesn't exist." }); } // What depends on it goes down with it, so it is silenced for the same time (and says why). const alsoSilenced: { type: "server" | "integration"; id: number; name: string }[] = []; if (withDependents && (targetType === "server" || targetType === "integration")) { const impact = await impactOf(targetType, targetId); for (const d of impact?.dependents ?? []) { const also = await startOrExtendWindow(d.type, d.id, minutes, `Depends on ${result.target.name}${reason ? ` — ${reason}` : ""}`.slice(0, 200), createdBy); if (also) { alsoSilenced.push({ type: d.type, id: d.id, name: also.target.name }); await recordAudit({ actor, category: "maintenance", action: also.extended ? "extend" : "start", targetType: d.type, targetId: d.id, detail: { name: also.target.name, minutes, reason: `depends on ${result.target.name}`, dependsOn: result.target.name } }); } } } await recordAudit({ actor, category: "maintenance", action: result.extended ? "extend" : "start", targetType, targetId, detail: { name: result.target.name, minutes, reason: reason ?? null }, }); invalidateAlertsCache(); res.status(201).json({ window: { ...result.window, targetName: result.target.name, targetKind: result.target.kind }, alsoSilenced }); })); const importUptimeKumaSchema = z.object({ integrationId: z.number().int().positive(), // Same bounds as a manual window: required and capped, so an import can't quietly create an open-ended one. minutes: z.number().int().min(5).max(7 * 24 * 60), }); /** * Uptime Kuma has no scheduled-maintenance API to read (see the integration's own notes) — only each monitor's * *current* status, which is "maintenance" for exactly as long as a window is active there. So rather than * mirroring Kuma's schedule, this reads what's in maintenance right now and starts (or extends) a matching * window here for that long, on whichever server the monitor's target matches. Re-running it while Kuma is * still in maintenance just extends the same window rather than stacking a new one. */ maintenanceRouter.post("/import/uptimekuma", requireRole("operator"), asyncHandler(async (req, res) => { const parsed = importUptimeKumaSchema.safeParse(req.body); if (!parsed.success) { return res.status(400).json({ error: "invalid_body", message: "Choose an integration and a duration.", details: parsed.error.flatten() }); } const { integrationId, minutes } = parsed.data; const loaded = await loadIntegrationConfig(integrationId); if (!loaded) return res.status(404).json({ error: "not_found", message: "That Uptime Kuma integration no longer exists." }); if (loaded.integration.type !== "uptimekuma") return res.status(400).json({ error: "wrong_type", message: "That integration isn't an Uptime Kuma one." }); if (!loaded.integration.enabled) return res.status(400).json({ error: "integration_disabled", message: "That integration is disabled. Enable it under Integrations first." }); let monitors; try { monitors = await createUptimeKumaAdapter(loaded.config as any).listMonitors(); } catch (err) { const message = err instanceof Error ? err.message : String(err); return res.status(502).json({ error: "uptimekuma_unreachable", message: `Couldn't read Uptime Kuma: ${message}` }); } const serverRows = await db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ips: servers.ipAddresses }).from(servers); const inMaintenance = attachMatchedServers(monitors, toMatchableServers(serverRows)).filter((m) => m.status === "maintenance"); const actor = req.currentUser!; const createdBy = actor.name ?? actor.email ?? actor.oidcSub; const imported: { serverId: number; serverName: string; monitorName: string; extended: boolean }[] = []; const unmatched: string[] = []; // Two monitors on the same server would otherwise start, then immediately re-extend, the same window — // harmless, but the response would misleadingly list the server twice. const seenServers = new Set(); for (const m of inMaintenance) { if (!m.matchedServer) { unmatched.push(m.name); continue; } if (seenServers.has(m.matchedServer.id)) continue; seenServers.add(m.matchedServer.id); const reason = `Imported from Uptime Kuma (${loaded.integration.name}): ${m.name}`; const result = await startOrExtendWindow("server", m.matchedServer.id, minutes, reason, createdBy); if (!result) continue; // the server was removed between the query above and now await recordAudit({ actor, category: "maintenance", action: result.extended ? "extend" : "start", targetType: "server", targetId: m.matchedServer.id, detail: { name: result.target.name, minutes, reason }, }); imported.push({ serverId: m.matchedServer.id, serverName: result.target.name, monitorName: m.name, extended: result.extended }); } invalidateAlertsCache(); res.json({ imported, unmatched, monitorsInMaintenance: inMaintenance.length, monitorsChecked: monitors.length }); })); const aliasSchema = z.object({ address: z.string().min(1).max(253), targetType: z.enum(maintenanceTargetTypes), targetId: z.number().int().positive(), }); maintenanceRouter.post("/aliases", requireRole("operator"), asyncHandler(async (req, res) => { const parsed = aliasSchema.safeParse(req.body); if (!parsed.success) { return res.status(400).json({ error: "invalid_body", message: "Give an address and say what it belongs to.", details: parsed.error.flatten() }); } const host = hostOfAddress(parsed.data.address); if (!host) { return res.status(400).json({ error: "invalid_address", message: "That isn't an address — use a host name or IP, like 100.64.0.7 or pve.example.lan." }); } const target = await describeTarget(parsed.data.targetType, parsed.data.targetId); if (!target) return res.status(404).json({ error: "not_found", message: "That target doesn't exist." }); const [existing] = await db .select() .from(maintenanceAliases) .where(and(eq(maintenanceAliases.address, host), eq(maintenanceAliases.targetType, parsed.data.targetType), eq(maintenanceAliases.targetId, parsed.data.targetId))) .limit(1); if (existing) return res.status(409).json({ error: "exists", message: `${host} is already an alias of ${target.name}.` }); const [created] = await db.insert(maintenanceAliases).values({ address: host, targetType: parsed.data.targetType, targetId: parsed.data.targetId }).returning(); await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "alias_add", targetType: parsed.data.targetType, targetId: parsed.data.targetId, detail: { name: target.name, address: host }, }); res.status(201).json({ alias: { ...created, targetName: target.name, targetKind: target.kind } }); })); maintenanceRouter.delete("/aliases/:id", requireRole("operator"), asyncHandler(async (req, res) => { const id = Number(req.params.id); const [existing] = await db.select().from(maintenanceAliases).where(eq(maintenanceAliases.id, id)).limit(1); if (!existing) return res.status(404).json({ error: "not_found" }); await db.delete(maintenanceAliases).where(eq(maintenanceAliases.id, id)); await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "alias_remove", targetType: existing.targetType, targetId: existing.targetId, detail: { address: existing.address }, }); res.status(204).end(); })); const bulkSchema = z.object({ scope: z.enum(["everything", "servers", "integrations", "dns_providers"]), minutes: z.number().int().min(5).max(7 * 24 * 60), reason: z.string().max(200).optional(), }); /** * Starts a window on every server, every integration, every DNS provider, or all of them, in one go — for a power cut, a network move, * a rack reboot. It is the same window as one started by hand for each target (so each can still be ended on its own); only the * asking is done once. Disabled integrations and providers are left out: they raise no alerts to silence. */ maintenanceRouter.post("/bulk", requireRole("operator"), asyncHandler(async (req, res) => { const parsed = bulkSchema.safeParse(req.body); if (!parsed.success) { return res.status(400).json({ error: "invalid_body", message: "Choose what to put in maintenance and for how long.", details: parsed.error.flatten() }); } const { scope, minutes, reason } = parsed.data; const targets: { type: "server" | "integration" | "dns_provider"; id: number }[] = []; if (scope === "everything" || scope === "servers") { for (const s of await db.select({ id: servers.id }).from(servers)) targets.push({ type: "server", id: s.id }); } if (scope === "everything" || scope === "integrations") { for (const i of await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.enabled, true))) targets.push({ type: "integration", id: i.id }); } if (scope === "everything" || scope === "dns_providers") { for (const p of await db.select({ id: dnsProviders.id }).from(dnsProviders).where(eq(dnsProviders.enabled, true))) targets.push({ type: "dns_provider", id: p.id }); } if (targets.length === 0) { return res.status(400).json({ error: "nothing_to_silence", message: "There is nothing of that kind to put in maintenance yet." }); } const actor = req.currentUser!; const createdBy = actor.name ?? actor.email ?? actor.oidcSub; const why = reason ?? "Everything in maintenance"; let started = 0; let extended = 0; for (const t of targets) { const result = await startOrExtendWindow(t.type, t.id, minutes, why, createdBy); if (!result) continue; // removed while this ran if (result.extended) extended++; else started++; } await recordAudit({ actor, category: "maintenance", action: "start_bulk", detail: { scope, minutes, reason: why, started, extended } }); invalidateAlertsCache(); res.status(201).json({ started, extended, total: started + extended }); })); /** Ends every window that is running now — the way out after starting many. Scheduled ones that haven't begun are left as they are. */ maintenanceRouter.post("/end-all", requireRole("operator"), asyncHandler(async (req, res) => { const active = await listActiveWindows(); if (active.length > 0) { await db.update(maintenanceWindows).set({ endsAt: new Date().toISOString() }).where(inArray(maintenanceWindows.id, active.map((w) => w.id))); } await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "end_all", detail: { ended: active.length } }); invalidateAlertsCache(); res.json({ ended: active.length }); })); // ─── Recurring schedules ──────────────────────────────────────────────────── const scheduleSchema = z.object({ name: z.string().trim().min(1).max(80), reason: z.string().trim().max(200).optional(), targets: z.array(z.object({ type: z.enum(maintenanceTargetTypes), id: z.number().int() })).min(1).max(100), days: z.array(z.number().int().min(0).max(6)).min(1).max(7), startTime: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 02:00"), durationMinutes: z.number().int().min(15).max(3 * 24 * 60), timezone: z.string().min(1).max(64), }); maintenanceRouter.get("/schedules", asyncHandler(async (_req, res) => { const rows = await db.select().from(maintenanceSchedules).orderBy(maintenanceSchedules.name); const now = new Date(); const out = []; for (const s of rows) { const targets = []; for (const t of targetsOf(s)) { const d = await describeTarget(t.type, t.id); if (d) targets.push({ type: t.type, id: t.id, name: d.name, kind: d.kind }); } const next = s.enabled ? occurrencesBetween(s, now, new Date(now.getTime() + 15 * 24 * 3600_000))[0] : undefined; out.push({ id: s.id, name: s.name, reason: s.reason, days: JSON.parse(s.days) as number[], startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, enabled: s.enabled, createdBy: s.createdBy, targets, nextStart: next ? next.start.toISOString() : null, }); } res.json({ schedules: out }); })); type ScheduleBody = z.infer; /** Checks a schedule's fields the same way for making one and for changing one: the shape, the time zone, and that what it covers exists (once each). */ async function checkScheduleBody(body: unknown): Promise<{ data: ScheduleBody; targets: ScheduleBody["targets"] } | { status: number; json: object }> { const parsed = scheduleSchema.safeParse(body); if (!parsed.success) { return { status: 400, json: { error: "invalid_body", message: "Name it, choose what it covers, the days, a start time and how long.", details: parsed.error.flatten() } }; } const data = parsed.data; if (!isValidTimeZone(data.timezone)) return { status: 400, json: { error: "invalid_timezone", message: "That isn't a time zone this server knows." } }; const seen = new Set(); const targets: ScheduleBody["targets"] = []; for (const t of data.targets) { const key = `${t.type}:${t.id}`; if (seen.has(key)) continue; seen.add(key); if (!(await describeTarget(t.type, t.id))) return { status: 400, json: { error: "unknown_target", message: "One of the things chosen doesn't exist any more." } }; targets.push(t); } return { data, targets }; } maintenanceRouter.post("/schedules", requireRole("operator"), asyncHandler(async (req, res) => { const checked = await checkScheduleBody(req.body); if ("status" in checked) return res.status(checked.status).json(checked.json); const { data, targets } = checked; const actor = req.currentUser!; const [created] = await db .insert(maintenanceSchedules) .values({ name: data.name, reason: data.reason || null, targets: JSON.stringify(targets), days: JSON.stringify([...new Set(data.days)].sort()), startTime: data.startTime, durationMinutes: data.durationMinutes, timezone: data.timezone, createdBy: actor.name ?? actor.email ?? actor.oidcSub, }) .returning(); const made = await materializeSchedules(new Date(), undefined, created.id); await recordAudit({ actor, category: "maintenance", action: "create_schedule", targetType: "maintenance_schedule", targetId: created.id, detail: { name: created.name, targets: targets.length, days: data.days, startTime: data.startTime, durationMinutes: data.durationMinutes } }); res.status(201).json({ id: created.id, windowsMade: made }); })); /** * Changes a schedule's name, days, time, length or what it covers. Windows it made that haven't begun are taken back and made again from the * new settings; one already under way keeps the times it started with, and ends on its own. */ maintenanceRouter.put("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => { const id = Number(req.params.id); if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" }); const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1); if (!row) return res.status(404).json({ error: "not_found" }); const checked = await checkScheduleBody(req.body); if ("status" in checked) return res.status(checked.status).json(checked.json); const { data, targets } = checked; await db .update(maintenanceSchedules) .set({ name: data.name, reason: data.reason || null, targets: JSON.stringify(targets), days: JSON.stringify([...new Set(data.days)].sort()), startTime: data.startTime, durationMinutes: data.durationMinutes, timezone: data.timezone, }) .where(eq(maintenanceSchedules.id, id)); await removeFutureWindows(id); const made = row.enabled ? await materializeSchedules(new Date(), undefined, id) : 0; await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "update_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: data.name, targets: targets.length, days: data.days, startTime: data.startTime, durationMinutes: data.durationMinutes } }); res.json({ windowsMade: made }); })); maintenanceRouter.patch("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => { const id = Number(req.params.id); const parsed = z.object({ enabled: z.boolean() }).safeParse(req.body); if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" }); const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1); if (!row) return res.status(404).json({ error: "not_found" }); await db.update(maintenanceSchedules).set({ enabled: parsed.data.enabled }).where(eq(maintenanceSchedules.id, id)); if (parsed.data.enabled) await materializeSchedules(new Date(), undefined, id); else await removeFutureWindows(id); await recordAudit({ actor: req.currentUser!, category: "maintenance", action: parsed.data.enabled ? "enable_schedule" : "disable_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: row.name } }); res.json({ enabled: parsed.data.enabled }); })); maintenanceRouter.delete("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => { const id = Number(req.params.id); if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" }); const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1); if (!row) return res.status(404).json({ error: "not_found" }); await deleteSchedule(id); await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "delete_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: row.name } }); res.status(204).end(); })); const syncUptimeKumaSchema = z.object({ integrationId: z.number().int().positive() }); /** Reads Uptime Kuma's maintenance schedule now and brings the mirrored windows in line with it (the same pass that runs every 15 minutes when switched on). */ maintenanceRouter.post("/sync/uptimekuma", requireRole("operator"), asyncHandler(async (req, res) => { const parsed = syncUptimeKumaSchema.safeParse(req.body); if (!parsed.success) { return res.status(400).json({ error: "invalid_body", message: "Choose an Uptime Kuma integration.", details: parsed.error.flatten() }); } try { const result = await syncKumaMaintenance(parsed.data.integrationId); await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "sync", targetType: "integration", targetId: parsed.data.integrationId, detail: { started: result.started, updated: result.updated, removed: result.removed }, }); invalidateAlertsCache(); res.json(result); } catch (err) { if (err instanceof KumaSyncError) { const status = err.code === "not_found" ? 404 : err.code === "unreachable" ? 502 : 400; return res.status(status).json({ error: err.code, message: err.message }); } throw err; } })); maintenanceRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => { const id = Number(req.params.id); const [existing] = await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, id)).limit(1); if (!existing) { return res.status(404).json({ error: "not_found" }); } // Ending moves the end to now, so it stops applying immediately; the row is pruned later like any expired one. // A window that hasn't begun is cancelled instead: its end goes back to its start, so it never applies. const now = new Date().toISOString(); await db.update(maintenanceWindows).set({ endsAt: existing.startedAt > now ? existing.startedAt : now }).where(eq(maintenanceWindows.id, id)); const target = await describeTarget(existing.targetType, existing.targetId); await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "end", targetType: existing.targetType, targetId: existing.targetId, detail: { name: target?.name ?? null }, }); invalidateAlertsCache(); res.status(204).end(); }));