Heartbeats, Coming up and suggested checks; Operations menu tidied

Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 21:42:34 +02:00
1 parent c05c847d17
commit 4e0c6927b2
44 files changed
+7092 -75

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { buildAgenda, DEFAULT_DAYS, MAX_DAYS } from "../services/agenda.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const agendaRouter = Router();
agendaRouter.use(requireAuth);
// Everything with a date ahead, for everyone signed in: it is only what the pages it comes from already show them. `days` is how far ahead.
agendaRouter.get("/", asyncHandler(async (req, res) => {
const asked = Number(req.query.days);
const days = Number.isInteger(asked) && asked >= 1 && asked <= MAX_DAYS ? asked : DEFAULT_DAYS;
res.json(await buildAgenda(new Date(), days));
}));
+48 -1
View File
@@ -7,6 +7,7 @@ import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const endpointsRouter = Router();
@@ -84,8 +85,54 @@ async function serverExists(id: number | null | undefined) {
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
}
// Checks worth setting up, from what the app already knows: a web port a server is listening on, a secret that reads a certificate. Reads what is
// stored, asks nobody. Administrators only, like the checks themselves.
endpointsRouter.get("/suggestions", requireRole("admin"), asyncHandler(async (_req, res) => {
res.json(await loadSuggestions());
}));
const keysSchema = z.object({ keys: z.array(z.string().min(1).max(600)).min(1).max(MAX_CHECKS) });
// Makes the checks for the suggestions named. What each one is comes from the server's own list, not from the request, so a request can only
// ever add what was offered.
endpointsRouter.post("/suggestions/add", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = keysSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to add." });
const offered = new Map((await loadSuggestions()).suggestions.map((s) => [s.key, s]));
let have = (await db.select({ id: endpointChecks.id }).from(endpointChecks)).length;
const added: { id: number; name: string }[] = [];
for (const key of new Set(parsed.data.keys)) {
const s = offered.get(key);
if (!s) continue;
if (have >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.`, added });
const [row] = await db
.insert(endpointChecks)
.values(valuesOf({ name: s.name, kind: s.kind, target: s.target, port: s.port, serverId: s.serverId }))
.returning();
have++;
added.push({ id: row.id, name: row.name });
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target, suggested: true } });
}
res.status(201).json({ added });
}));
// "Not for me": what is dismissed stops being offered, and can be brought back.
endpointsRouter.post("/suggestions/dismiss", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = keysSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to dismiss." });
const keys = await loadDismissed();
for (const k of parsed.data.keys) keys.add(k);
await saveDismissed(keys);
res.json(await loadSuggestions());
}));
endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(async (_req, res) => {
await saveDismissed(new Set());
res.json(await loadSuggestions());
}));
// Asking things on the network is an administrator's to set up.
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = checkSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
+32
View File
@@ -0,0 +1,32 @@
import express, { Router } from "express";
import { cleanMessage, recordPing, type PingKind } from "../services/heartbeats.js";
import { rateLimit } from "../utils/rateLimit.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/**
* The address a job calls: /api/heartbeat/<token>, /api/heartbeat/<token>/start, /api/heartbeat/<token>/fail. No session and no sign-in — the
* token is the whole of the access, and all it allows is to say "I'm alive". Mounted ahead of the browser-only middleware (a script isn't a
* browser and has no cookie), so it answers GET, HEAD, POST and PUT alike: whatever `curl` or `wget` happens to send.
*
* It answers a token that doesn't exist, and a path that isn't one of the three, in the same two words, and is limited per address.
*/
export const heartbeatPingRouter = Router();
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
// Whatever a script sends as the body ("3.2 GB copied") is a note; it is cut down to a line.
const body = express.text({ type: () => true, limit: "100kb" });
const KINDS: Record<string, PingKind> = { "": "ok", ok: "ok", start: "start", fail: "fail" };
heartbeatPingRouter.all("/:token/:kind?", limiter, body, asyncHandler(async (req, res) => {
res.setHeader("Cache-Control", "no-store");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
if (!["GET", "HEAD", "POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
const kind = KINDS[(req.params.kind ?? "").toLowerCase()];
if (!kind) return res.status(404).type("text/plain").send("Not found");
const fromQuery = typeof req.query.msg === "string" ? req.query.msg : null;
const message = cleanMessage(fromQuery ?? (typeof req.body === "string" ? req.body : null));
const heartbeat = await recordPing(req.params.token, kind, message);
if (!heartbeat) return res.status(404).type("text/plain").send("Not found");
res.type("text/plain").send("OK");
}));
+111
View File
@@ -0,0 +1,111 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { heartbeats, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { dueBy, heartbeatState, listHeartbeats, loadRecent, MAX_GRACE_MINUTES, MAX_PERIOD_MINUTES, newToken } from "../services/heartbeats.js";
import { env } from "../env.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const heartbeatsRouter = Router();
heartbeatsRouter.use(requireAuth);
const MAX_HEARTBEATS = 100;
const schema = z.object({
name: z.string().trim().min(1).max(100),
periodMinutes: z.number().int().min(1).max(MAX_PERIOD_MINUTES),
graceMinutes: z.number().int().min(0).max(MAX_GRACE_MINUTES).optional(),
serverId: z.number().int().positive().nullable().optional(),
notes: z.string().max(500).nullable().optional(),
enabled: z.boolean().optional(),
});
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const pathOf = (token: string) => `/api/heartbeat/${token}`;
async function serverExists(id: number | null | undefined) {
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
}
// Everyone signed in can see how the jobs are doing, like the Alerts page. The address with its token is shown only to administrators: it
// is what lets anyone say "I'm alive" for the job.
heartbeatsRouter.get("/", asyncHandler(async (req, res) => {
const now = new Date();
const isAdmin = req.currentUser!.role === "admin";
const rows = await listHeartbeats();
const recent = await loadRecent(rows.map((r) => r.id));
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
res.json({
baseUrl: isAdmin ? env.appBaseUrl.replace(/\/+$/, "") : null,
heartbeats: rows.map(({ token, ...r }) => ({
...r,
path: isAdmin ? pathOf(token) : null,
state: heartbeatState(r, now),
dueBy: dueBy(r).toISOString(),
server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null,
recent: recent.get(r.id) ?? [],
})),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
});
}));
heartbeatsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = schema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.error.flatten() });
if ((await db.select({ id: heartbeats.id }).from(heartbeats)).length >= MAX_HEARTBEATS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_HEARTBEATS} heartbeats.` });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const d = parsed.data;
const [row] = await db
.insert(heartbeats)
.values({ name: d.name, token: newToken(), periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? 10, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? true })
.returning();
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "create", targetType: "heartbeat", targetId: row.id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
const { token, ...rest } = row;
res.status(201).json({ heartbeat: { ...rest, path: pathOf(token) } });
}));
heartbeatsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = schema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.success ? undefined : parsed.error.flatten() });
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const d = parsed.data;
const [row] = await db
.update(heartbeats)
.set({ name: d.name, periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? before.graceMinutes, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? before.enabled })
.where(eq(heartbeats.id, id))
.returning();
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "update", targetType: "heartbeat", targetId: id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
invalidateAlertsCache();
const { token, ...rest } = row;
res.json({ heartbeat: { ...rest, path: pathOf(token) } });
}));
// A new address, when the old one has got out. The old one stops working at once.
heartbeatsRouter.post("/:id/new-address", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.update(heartbeats).set({ token: newToken() }).where(eq(heartbeats.id, id)).returning();
if (!row) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "new_address", targetType: "heartbeat", targetId: id, detail: { name: row.name } });
res.json({ path: pathOf(row.token) });
}));
heartbeatsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(heartbeats).where(eq(heartbeats.id, id));
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "delete", targetType: "heartbeat", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));
+24
View File
@@ -0,0 +1,24 @@
import { Router } from "express";
import { NOT_FOUND, resolveShareLink } from "../services/shareLinks.js";
import { buildAgenda, buildIcs, calendarName, MAX_DAYS } from "../services/agenda.js";
import { env } from "../env.js";
import { rateLimit } from "../utils/rateLimit.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const publicCalendarRouter = Router();
// No sign-in: the secret in the address is the whole of the access, so this reads nothing but what is coming up, and answers a link that doesn't
// work — never made, ended, taken back, or one that opens something other than a calendar — in the same words. The address ends in .ics because
// calendar apps want that.
const limiter = rateLimit({ windowMs: 60_000, max: 60, message: "Too many requests. Wait a minute." });
publicCalendarRouter.get("/:file", limiter, asyncHandler(async (req, res) => {
const token = req.params.file.endsWith(".ics") ? req.params.file.slice(0, -4) : req.params.file;
const link = await resolveShareLink(token);
if (!link || link.kind !== "calendar") return res.status(404).json(NOT_FOUND);
const feed = buildIcs(await buildAgenda(new Date(), MAX_DAYS), { name: calendarName(), detail: link.detail, baseUrl: link.detail === "names" ? env.appBaseUrl : undefined });
res.setHeader("Cache-Control", "private, max-age=300");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
res.setHeader("Content-Disposition", 'inline; filename="homelab-coming-up.ics"');
res.type("text/calendar; charset=utf-8").send(feed);
}));
+2 -2
View File
@@ -20,7 +20,7 @@ const MAX_LINKS = 20;
const createSchema = z.object({
label: z.string().trim().min(1).max(60),
kind: z.enum(shareLinkKinds).default("wall"),
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it.
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it. For a calendar: summary leaves the names out of the events.
detail: z.enum(shareLinkDetails).default("summary"),
title: z.string().trim().max(MAX_TITLE).optional(),
message: z.string().trim().max(MAX_MESSAGE).optional(),
@@ -59,7 +59,7 @@ shareLinksRouter.post("/", asyncHandler(async (req, res) => {
targetId: link.id,
detail: { label: link.label, kind: link.kind, detail: link.detail, expiresAt: link.expiresAt },
});
res.status(201).json({ link: publicShareLink(link), token, path: `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
res.status(201).json({ link: publicShareLink(link), token, path: link.kind === "calendar" ? `/api/public/calendar/${token}.ics` : `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
}));
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);