Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -0,0 +1,111 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { heartbeats, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { dueBy, heartbeatState, listHeartbeats, loadRecent, MAX_GRACE_MINUTES, MAX_PERIOD_MINUTES, newToken } from "../services/heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const heartbeatsRouter = Router();
|
||||
|
||||
heartbeatsRouter.use(requireAuth);
|
||||
|
||||
const MAX_HEARTBEATS = 100;
|
||||
|
||||
const schema = z.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
periodMinutes: z.number().int().min(1).max(MAX_PERIOD_MINUTES),
|
||||
graceMinutes: z.number().int().min(0).max(MAX_GRACE_MINUTES).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
notes: z.string().max(500).nullable().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
const pathOf = (token: string) => `/api/heartbeat/${token}`;
|
||||
|
||||
async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Everyone signed in can see how the jobs are doing, like the Alerts page. The address with its token is shown only to administrators: it
|
||||
// is what lets anyone say "I'm alive" for the job.
|
||||
heartbeatsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const now = new Date();
|
||||
const isAdmin = req.currentUser!.role === "admin";
|
||||
const rows = await listHeartbeats();
|
||||
const recent = await loadRecent(rows.map((r) => r.id));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
baseUrl: isAdmin ? env.appBaseUrl.replace(/\/+$/, "") : null,
|
||||
heartbeats: rows.map(({ token, ...r }) => ({
|
||||
...r,
|
||||
path: isAdmin ? pathOf(token) : null,
|
||||
state: heartbeatState(r, now),
|
||||
dueBy: dueBy(r).toISOString(),
|
||||
server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null,
|
||||
recent: recent.get(r.id) ?? [],
|
||||
})),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
heartbeatsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = schema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: heartbeats.id }).from(heartbeats)).length >= MAX_HEARTBEATS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_HEARTBEATS} heartbeats.` });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const d = parsed.data;
|
||||
const [row] = await db
|
||||
.insert(heartbeats)
|
||||
.values({ name: d.name, token: newToken(), periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? 10, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? true })
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "create", targetType: "heartbeat", targetId: row.id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
|
||||
const { token, ...rest } = row;
|
||||
res.status(201).json({ heartbeat: { ...rest, path: pathOf(token) } });
|
||||
}));
|
||||
|
||||
heartbeatsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = schema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const d = parsed.data;
|
||||
const [row] = await db
|
||||
.update(heartbeats)
|
||||
.set({ name: d.name, periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? before.graceMinutes, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? before.enabled })
|
||||
.where(eq(heartbeats.id, id))
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "update", targetType: "heartbeat", targetId: id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
|
||||
invalidateAlertsCache();
|
||||
const { token, ...rest } = row;
|
||||
res.json({ heartbeat: { ...rest, path: pathOf(token) } });
|
||||
}));
|
||||
|
||||
// A new address, when the old one has got out. The old one stops working at once.
|
||||
heartbeatsRouter.post("/:id/new-address", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.update(heartbeats).set({ token: newToken() }).where(eq(heartbeats.id, id)).returning();
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "new_address", targetType: "heartbeat", targetId: id, detail: { name: row.name } });
|
||||
res.json({ path: pathOf(row.token) });
|
||||
}));
|
||||
|
||||
heartbeatsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(heartbeats).where(eq(heartbeats.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "delete", targetType: "heartbeat", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
Reference in new issue
Block a user