Files
Homelab-manager/server/src/routes/maintenance.ts
T
bobbanandClaude Sonnet 5.5 9caa414829 Dependency map, impact preview in maintenance, and a calmer Alerts page
- Map: a Dependencies view showing what would go down with a picked box and what it relies on
- Maintenance: lists what depends on a target and can silence those too (same time, audited); the recurring form and update drawer show the same note
- Alerts: a headline on where things stand, held-back alerts nested under their cause, severity edge on rows; caused-by now carries the explaining alert's id

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-18 19:17:09 +02:00

478 lines
24 KiB
TypeScript

import { Router } from "express";
import { and, eq, inArray } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { dnsProviders, integrations, maintenanceAliases, maintenanceSchedules, maintenanceTargetTypes, maintenanceWindows, servers } from "../db/schema.js";
import { deleteSchedule, isValidTimeZone, materializeSchedules, occurrencesBetween, removeFutureWindows, targetsOf } from "../services/maintenanceSchedules.js";
import { hostOfAddress } from "../services/kumaTargets.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { describeTarget, listActiveWindows, listUpcomingWindows, startOrExtendWindow } from "../services/maintenance.js";
import { impactOf } from "../services/dependencies.js";
import { KumaSyncError, syncKumaMaintenance } from "../services/kumaMaintenanceSync.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createUptimeKumaAdapter } from "../integrations/uptimekuma/adapter.js";
import { attachMatchedServers, toMatchableServers } from "../services/uptimeKumaMatch.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const maintenanceRouter = Router();
// Anyone signed in can see what's currently silenced (so nobody is surprised by missing alerts);
// starting and ending a window is an operator action, like the other things that change how the homelab behaves.
maintenanceRouter.use(requireAuth);
maintenanceRouter.get("/", asyncHandler(async (_req, res) => {
const describe = async (rows: Awaited<ReturnType<typeof listActiveWindows>>) => {
const out = [];
for (const w of rows) {
const target = await describeTarget(w.targetType, w.targetId);
if (!target) continue; // target was deleted — nothing left to silence
out.push({ ...w, targetName: target.name, targetKind: target.kind });
}
return out;
};
const windows = await describe(await listActiveWindows());
windows.sort((a, b) => a.endsAt.localeCompare(b.endsAt));
const upcoming = await describe(await listUpcomingWindows());
upcoming.sort((a, b) => a.startedAt.localeCompare(b.startedAt));
const [serverRows, integrationRows, providerRows] = await Promise.all([
db.select({ id: servers.id, name: servers.name }).from(servers).orderBy(servers.name),
db.select({ id: integrations.id, name: integrations.name, type: integrations.type }).from(integrations).orderBy(integrations.name),
db.select({ id: dnsProviders.id, name: dnsProviders.name, providerType: dnsProviders.providerType }).from(dnsProviders).orderBy(dnsProviders.name),
]);
const aliases = [];
for (const a of await db.select().from(maintenanceAliases).orderBy(maintenanceAliases.address)) {
const target = await describeTarget(a.targetType, a.targetId);
if (!target) continue; // target was deleted — the alias has nothing left to point at
aliases.push({ id: a.id, address: a.address, targetType: a.targetType, targetId: a.targetId, targetName: target.name, targetKind: target.kind });
}
res.json({ windows, upcoming, aliases, targets: { servers: serverRows, integrations: integrationRows, dnsProviders: providerRows } });
}));
const startSchema = z.object({
targetType: z.enum(maintenanceTargetTypes),
targetId: z.number().int().positive(),
// Required and capped: an open-ended window is the way this feature could quietly hide a real outage.
minutes: z.number().int().min(5).max(7 * 24 * 60),
reason: z.string().max(200).optional(),
// Also put what depends on the target under maintenance, for the same time: it will look down while the target is.
withDependents: z.boolean().optional(),
});
maintenanceRouter.post("/", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = startSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", details: parsed.error.flatten() });
}
const { targetType, targetId, minutes, reason, withDependents } = parsed.data;
const actor = req.currentUser!;
const createdBy = actor.name ?? actor.email ?? actor.oidcSub;
// Starting maintenance on something already in maintenance restarts its clock rather than stacking windows.
const result = await startOrExtendWindow(targetType, targetId, minutes, reason ?? null, createdBy);
if (!result) {
return res.status(404).json({ error: "not_found", message: "That target doesn't exist." });
}
// What depends on it goes down with it, so it is silenced for the same time (and says why).
const alsoSilenced: { type: "server" | "integration"; id: number; name: string }[] = [];
if (withDependents && (targetType === "server" || targetType === "integration")) {
const impact = await impactOf(targetType, targetId);
for (const d of impact?.dependents ?? []) {
const also = await startOrExtendWindow(d.type, d.id, minutes, `Depends on ${result.target.name}${reason ? ` — ${reason}` : ""}`.slice(0, 200), createdBy);
if (also) {
alsoSilenced.push({ type: d.type, id: d.id, name: also.target.name });
await recordAudit({ actor, category: "maintenance", action: also.extended ? "extend" : "start", targetType: d.type, targetId: d.id, detail: { name: also.target.name, minutes, reason: `depends on ${result.target.name}`, dependsOn: result.target.name } });
}
}
}
await recordAudit({
actor,
category: "maintenance",
action: result.extended ? "extend" : "start",
targetType,
targetId,
detail: { name: result.target.name, minutes, reason: reason ?? null },
});
invalidateAlertsCache();
res.status(201).json({ window: { ...result.window, targetName: result.target.name, targetKind: result.target.kind }, alsoSilenced });
}));
const importUptimeKumaSchema = z.object({
integrationId: z.number().int().positive(),
// Same bounds as a manual window: required and capped, so an import can't quietly create an open-ended one.
minutes: z.number().int().min(5).max(7 * 24 * 60),
});
/**
* Uptime Kuma has no scheduled-maintenance API to read (see the integration's own notes) — only each monitor's
* *current* status, which is "maintenance" for exactly as long as a window is active there. So rather than
* mirroring Kuma's schedule, this reads what's in maintenance right now and starts (or extends) a matching
* window here for that long, on whichever server the monitor's target matches. Re-running it while Kuma is
* still in maintenance just extends the same window rather than stacking a new one.
*/
maintenanceRouter.post("/import/uptimekuma", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = importUptimeKumaSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: "Choose an integration and a duration.", details: parsed.error.flatten() });
}
const { integrationId, minutes } = parsed.data;
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded) return res.status(404).json({ error: "not_found", message: "That Uptime Kuma integration no longer exists." });
if (loaded.integration.type !== "uptimekuma") return res.status(400).json({ error: "wrong_type", message: "That integration isn't an Uptime Kuma one." });
if (!loaded.integration.enabled) return res.status(400).json({ error: "integration_disabled", message: "That integration is disabled. Enable it under Integrations first." });
let monitors;
try {
monitors = await createUptimeKumaAdapter(loaded.config as any).listMonitors();
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
return res.status(502).json({ error: "uptimekuma_unreachable", message: `Couldn't read Uptime Kuma: ${message}` });
}
const serverRows = await db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ips: servers.ipAddresses }).from(servers);
const inMaintenance = attachMatchedServers(monitors, toMatchableServers(serverRows)).filter((m) => m.status === "maintenance");
const actor = req.currentUser!;
const createdBy = actor.name ?? actor.email ?? actor.oidcSub;
const imported: { serverId: number; serverName: string; monitorName: string; extended: boolean }[] = [];
const unmatched: string[] = [];
// Two monitors on the same server would otherwise start, then immediately re-extend, the same window —
// harmless, but the response would misleadingly list the server twice.
const seenServers = new Set<number>();
for (const m of inMaintenance) {
if (!m.matchedServer) {
unmatched.push(m.name);
continue;
}
if (seenServers.has(m.matchedServer.id)) continue;
seenServers.add(m.matchedServer.id);
const reason = `Imported from Uptime Kuma (${loaded.integration.name}): ${m.name}`;
const result = await startOrExtendWindow("server", m.matchedServer.id, minutes, reason, createdBy);
if (!result) continue; // the server was removed between the query above and now
await recordAudit({
actor,
category: "maintenance",
action: result.extended ? "extend" : "start",
targetType: "server",
targetId: m.matchedServer.id,
detail: { name: result.target.name, minutes, reason },
});
imported.push({ serverId: m.matchedServer.id, serverName: result.target.name, monitorName: m.name, extended: result.extended });
}
invalidateAlertsCache();
res.json({ imported, unmatched, monitorsInMaintenance: inMaintenance.length, monitorsChecked: monitors.length });
}));
const aliasSchema = z.object({
address: z.string().min(1).max(253),
targetType: z.enum(maintenanceTargetTypes),
targetId: z.number().int().positive(),
});
maintenanceRouter.post("/aliases", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = aliasSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: "Give an address and say what it belongs to.", details: parsed.error.flatten() });
}
const host = hostOfAddress(parsed.data.address);
if (!host) {
return res.status(400).json({ error: "invalid_address", message: "That isn't an address — use a host name or IP, like 100.64.0.7 or pve.example.lan." });
}
const target = await describeTarget(parsed.data.targetType, parsed.data.targetId);
if (!target) return res.status(404).json({ error: "not_found", message: "That target doesn't exist." });
const [existing] = await db
.select()
.from(maintenanceAliases)
.where(and(eq(maintenanceAliases.address, host), eq(maintenanceAliases.targetType, parsed.data.targetType), eq(maintenanceAliases.targetId, parsed.data.targetId)))
.limit(1);
if (existing) return res.status(409).json({ error: "exists", message: `${host} is already an alias of ${target.name}.` });
const [created] = await db.insert(maintenanceAliases).values({ address: host, targetType: parsed.data.targetType, targetId: parsed.data.targetId }).returning();
await recordAudit({
actor: req.currentUser!,
category: "maintenance",
action: "alias_add",
targetType: parsed.data.targetType,
targetId: parsed.data.targetId,
detail: { name: target.name, address: host },
});
res.status(201).json({ alias: { ...created, targetName: target.name, targetKind: target.kind } });
}));
maintenanceRouter.delete("/aliases/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
const [existing] = await db.select().from(maintenanceAliases).where(eq(maintenanceAliases.id, id)).limit(1);
if (!existing) return res.status(404).json({ error: "not_found" });
await db.delete(maintenanceAliases).where(eq(maintenanceAliases.id, id));
await recordAudit({
actor: req.currentUser!,
category: "maintenance",
action: "alias_remove",
targetType: existing.targetType,
targetId: existing.targetId,
detail: { address: existing.address },
});
res.status(204).end();
}));
const bulkSchema = z.object({
scope: z.enum(["everything", "servers", "integrations", "dns_providers"]),
minutes: z.number().int().min(5).max(7 * 24 * 60),
reason: z.string().max(200).optional(),
});
/**
* Starts a window on every server, every integration, every DNS provider, or all of them, in one go — for a power cut, a network move,
* a rack reboot. It is the same window as one started by hand for each target (so each can still be ended on its own); only the
* asking is done once. Disabled integrations and providers are left out: they raise no alerts to silence.
*/
maintenanceRouter.post("/bulk", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = bulkSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: "Choose what to put in maintenance and for how long.", details: parsed.error.flatten() });
}
const { scope, minutes, reason } = parsed.data;
const targets: { type: "server" | "integration" | "dns_provider"; id: number }[] = [];
if (scope === "everything" || scope === "servers") {
for (const s of await db.select({ id: servers.id }).from(servers)) targets.push({ type: "server", id: s.id });
}
if (scope === "everything" || scope === "integrations") {
for (const i of await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.enabled, true))) targets.push({ type: "integration", id: i.id });
}
if (scope === "everything" || scope === "dns_providers") {
for (const p of await db.select({ id: dnsProviders.id }).from(dnsProviders).where(eq(dnsProviders.enabled, true))) targets.push({ type: "dns_provider", id: p.id });
}
if (targets.length === 0) {
return res.status(400).json({ error: "nothing_to_silence", message: "There is nothing of that kind to put in maintenance yet." });
}
const actor = req.currentUser!;
const createdBy = actor.name ?? actor.email ?? actor.oidcSub;
const why = reason ?? "Everything in maintenance";
let started = 0;
let extended = 0;
for (const t of targets) {
const result = await startOrExtendWindow(t.type, t.id, minutes, why, createdBy);
if (!result) continue; // removed while this ran
if (result.extended) extended++;
else started++;
}
await recordAudit({ actor, category: "maintenance", action: "start_bulk", detail: { scope, minutes, reason: why, started, extended } });
invalidateAlertsCache();
res.status(201).json({ started, extended, total: started + extended });
}));
/** Ends every window that is running now — the way out after starting many. Scheduled ones that haven't begun are left as they are. */
maintenanceRouter.post("/end-all", requireRole("operator"), asyncHandler(async (req, res) => {
const active = await listActiveWindows();
if (active.length > 0) {
await db.update(maintenanceWindows).set({ endsAt: new Date().toISOString() }).where(inArray(maintenanceWindows.id, active.map((w) => w.id)));
}
await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "end_all", detail: { ended: active.length } });
invalidateAlertsCache();
res.json({ ended: active.length });
}));
// ─── Recurring schedules ────────────────────────────────────────────────────
const scheduleSchema = z.object({
name: z.string().trim().min(1).max(80),
reason: z.string().trim().max(200).optional(),
targets: z.array(z.object({ type: z.enum(maintenanceTargetTypes), id: z.number().int() })).min(1).max(100),
days: z.array(z.number().int().min(0).max(6)).min(1).max(7),
startTime: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/, "Use a time like 02:00"),
durationMinutes: z.number().int().min(15).max(3 * 24 * 60),
timezone: z.string().min(1).max(64),
});
maintenanceRouter.get("/schedules", asyncHandler(async (_req, res) => {
const rows = await db.select().from(maintenanceSchedules).orderBy(maintenanceSchedules.name);
const now = new Date();
const out = [];
for (const s of rows) {
const targets = [];
for (const t of targetsOf(s)) {
const d = await describeTarget(t.type, t.id);
if (d) targets.push({ type: t.type, id: t.id, name: d.name, kind: d.kind });
}
const next = s.enabled ? occurrencesBetween(s, now, new Date(now.getTime() + 15 * 24 * 3600_000))[0] : undefined;
out.push({
id: s.id,
name: s.name,
reason: s.reason,
days: JSON.parse(s.days) as number[],
startTime: s.startTime,
durationMinutes: s.durationMinutes,
timezone: s.timezone,
enabled: s.enabled,
createdBy: s.createdBy,
targets,
nextStart: next ? next.start.toISOString() : null,
});
}
res.json({ schedules: out });
}));
type ScheduleBody = z.infer<typeof scheduleSchema>;
/** Checks a schedule's fields the same way for making one and for changing one: the shape, the time zone, and that what it covers exists (once each). */
async function checkScheduleBody(body: unknown): Promise<{ data: ScheduleBody; targets: ScheduleBody["targets"] } | { status: number; json: object }> {
const parsed = scheduleSchema.safeParse(body);
if (!parsed.success) {
return { status: 400, json: { error: "invalid_body", message: "Name it, choose what it covers, the days, a start time and how long.", details: parsed.error.flatten() } };
}
const data = parsed.data;
if (!isValidTimeZone(data.timezone)) return { status: 400, json: { error: "invalid_timezone", message: "That isn't a time zone this server knows." } };
const seen = new Set<string>();
const targets: ScheduleBody["targets"] = [];
for (const t of data.targets) {
const key = `${t.type}:${t.id}`;
if (seen.has(key)) continue;
seen.add(key);
if (!(await describeTarget(t.type, t.id))) return { status: 400, json: { error: "unknown_target", message: "One of the things chosen doesn't exist any more." } };
targets.push(t);
}
return { data, targets };
}
maintenanceRouter.post("/schedules", requireRole("operator"), asyncHandler(async (req, res) => {
const checked = await checkScheduleBody(req.body);
if ("status" in checked) return res.status(checked.status).json(checked.json);
const { data, targets } = checked;
const actor = req.currentUser!;
const [created] = await db
.insert(maintenanceSchedules)
.values({
name: data.name,
reason: data.reason || null,
targets: JSON.stringify(targets),
days: JSON.stringify([...new Set(data.days)].sort()),
startTime: data.startTime,
durationMinutes: data.durationMinutes,
timezone: data.timezone,
createdBy: actor.name ?? actor.email ?? actor.oidcSub,
})
.returning();
const made = await materializeSchedules(new Date(), undefined, created.id);
await recordAudit({ actor, category: "maintenance", action: "create_schedule", targetType: "maintenance_schedule", targetId: created.id, detail: { name: created.name, targets: targets.length, days: data.days, startTime: data.startTime, durationMinutes: data.durationMinutes } });
res.status(201).json({ id: created.id, windowsMade: made });
}));
/**
* Changes a schedule's name, days, time, length or what it covers. Windows it made that haven't begun are taken back and made again from the
* new settings; one already under way keeps the times it started with, and ends on its own.
*/
maintenanceRouter.put("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
const checked = await checkScheduleBody(req.body);
if ("status" in checked) return res.status(checked.status).json(checked.json);
const { data, targets } = checked;
await db
.update(maintenanceSchedules)
.set({
name: data.name,
reason: data.reason || null,
targets: JSON.stringify(targets),
days: JSON.stringify([...new Set(data.days)].sort()),
startTime: data.startTime,
durationMinutes: data.durationMinutes,
timezone: data.timezone,
})
.where(eq(maintenanceSchedules.id, id));
await removeFutureWindows(id);
const made = row.enabled ? await materializeSchedules(new Date(), undefined, id) : 0;
await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "update_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: data.name, targets: targets.length, days: data.days, startTime: data.startTime, durationMinutes: data.durationMinutes } });
res.json({ windowsMade: made });
}));
maintenanceRouter.patch("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
const parsed = z.object({ enabled: z.boolean() }).safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body" });
const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
await db.update(maintenanceSchedules).set({ enabled: parsed.data.enabled }).where(eq(maintenanceSchedules.id, id));
if (parsed.data.enabled) await materializeSchedules(new Date(), undefined, id);
else await removeFutureWindows(id);
await recordAudit({ actor: req.currentUser!, category: "maintenance", action: parsed.data.enabled ? "enable_schedule" : "disable_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: row.name } });
res.json({ enabled: parsed.data.enabled });
}));
maintenanceRouter.delete("/schedules/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select().from(maintenanceSchedules).where(eq(maintenanceSchedules.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
await deleteSchedule(id);
await recordAudit({ actor: req.currentUser!, category: "maintenance", action: "delete_schedule", targetType: "maintenance_schedule", targetId: id, detail: { name: row.name } });
res.status(204).end();
}));
const syncUptimeKumaSchema = z.object({ integrationId: z.number().int().positive() });
/** Reads Uptime Kuma's maintenance schedule now and brings the mirrored windows in line with it (the same pass that runs every 15 minutes when switched on). */
maintenanceRouter.post("/sync/uptimekuma", requireRole("operator"), asyncHandler(async (req, res) => {
const parsed = syncUptimeKumaSchema.safeParse(req.body);
if (!parsed.success) {
return res.status(400).json({ error: "invalid_body", message: "Choose an Uptime Kuma integration.", details: parsed.error.flatten() });
}
try {
const result = await syncKumaMaintenance(parsed.data.integrationId);
await recordAudit({
actor: req.currentUser!,
category: "maintenance",
action: "sync",
targetType: "integration",
targetId: parsed.data.integrationId,
detail: { started: result.started, updated: result.updated, removed: result.removed },
});
invalidateAlertsCache();
res.json(result);
} catch (err) {
if (err instanceof KumaSyncError) {
const status = err.code === "not_found" ? 404 : err.code === "unreachable" ? 502 : 400;
return res.status(status).json({ error: err.code, message: err.message });
}
throw err;
}
}));
maintenanceRouter.delete("/:id", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
const [existing] = await db.select().from(maintenanceWindows).where(eq(maintenanceWindows.id, id)).limit(1);
if (!existing) {
return res.status(404).json({ error: "not_found" });
}
// Ending moves the end to now, so it stops applying immediately; the row is pruned later like any expired one.
// A window that hasn't begun is cancelled instead: its end goes back to its start, so it never applies.
const now = new Date().toISOString();
await db.update(maintenanceWindows).set({ endsAt: existing.startedAt > now ? existing.startedAt : now }).where(eq(maintenanceWindows.id, id));
const target = await describeTarget(existing.targetType, existing.targetId);
await recordAudit({
actor: req.currentUser!,
category: "maintenance",
action: "end",
targetType: existing.targetType,
targetId: existing.targetId,
detail: { name: target?.name ?? null },
});
invalidateAlertsCache();
res.status(204).end();
}));