Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -0,0 +1,15 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { buildAgenda, DEFAULT_DAYS, MAX_DAYS } from "../services/agenda.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const agendaRouter = Router();
|
||||
|
||||
agendaRouter.use(requireAuth);
|
||||
|
||||
// Everything with a date ahead, for everyone signed in: it is only what the pages it comes from already show them. `days` is how far ahead.
|
||||
agendaRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isInteger(asked) && asked >= 1 && asked <= MAX_DAYS ? asked : DEFAULT_DAYS;
|
||||
res.json(await buildAgenda(new Date(), days));
|
||||
}));
|
||||
@@ -7,6 +7,7 @@ import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const endpointsRouter = Router();
|
||||
@@ -84,8 +85,54 @@ async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Checks worth setting up, from what the app already knows: a web port a server is listening on, a secret that reads a certificate. Reads what is
|
||||
// stored, asks nobody. Administrators only, like the checks themselves.
|
||||
endpointsRouter.get("/suggestions", requireRole("admin"), asyncHandler(async (_req, res) => {
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
const keysSchema = z.object({ keys: z.array(z.string().min(1).max(600)).min(1).max(MAX_CHECKS) });
|
||||
|
||||
// Makes the checks for the suggestions named. What each one is comes from the server's own list, not from the request, so a request can only
|
||||
// ever add what was offered.
|
||||
endpointsRouter.post("/suggestions/add", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = keysSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to add." });
|
||||
const offered = new Map((await loadSuggestions()).suggestions.map((s) => [s.key, s]));
|
||||
let have = (await db.select({ id: endpointChecks.id }).from(endpointChecks)).length;
|
||||
const added: { id: number; name: string }[] = [];
|
||||
for (const key of new Set(parsed.data.keys)) {
|
||||
const s = offered.get(key);
|
||||
if (!s) continue;
|
||||
if (have >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.`, added });
|
||||
const [row] = await db
|
||||
.insert(endpointChecks)
|
||||
.values(valuesOf({ name: s.name, kind: s.kind, target: s.target, port: s.port, serverId: s.serverId }))
|
||||
.returning();
|
||||
have++;
|
||||
added.push({ id: row.id, name: row.name });
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target, suggested: true } });
|
||||
}
|
||||
res.status(201).json({ added });
|
||||
}));
|
||||
|
||||
// "Not for me": what is dismissed stops being offered, and can be brought back.
|
||||
endpointsRouter.post("/suggestions/dismiss", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = keysSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to dismiss." });
|
||||
const keys = await loadDismissed();
|
||||
for (const k of parsed.data.keys) keys.add(k);
|
||||
await saveDismissed(keys);
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(async (_req, res) => {
|
||||
await saveDismissed(new Set());
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
// Asking things on the network is an administrator's to set up.
|
||||
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import express, { Router } from "express";
|
||||
import { cleanMessage, recordPing, type PingKind } from "../services/heartbeats.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/**
|
||||
* The address a job calls: /api/heartbeat/<token>, /api/heartbeat/<token>/start, /api/heartbeat/<token>/fail. No session and no sign-in — the
|
||||
* token is the whole of the access, and all it allows is to say "I'm alive". Mounted ahead of the browser-only middleware (a script isn't a
|
||||
* browser and has no cookie), so it answers GET, HEAD, POST and PUT alike: whatever `curl` or `wget` happens to send.
|
||||
*
|
||||
* It answers a token that doesn't exist, and a path that isn't one of the three, in the same two words, and is limited per address.
|
||||
*/
|
||||
export const heartbeatPingRouter = Router();
|
||||
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
|
||||
// Whatever a script sends as the body ("3.2 GB copied") is a note; it is cut down to a line.
|
||||
const body = express.text({ type: () => true, limit: "100kb" });
|
||||
|
||||
const KINDS: Record<string, PingKind> = { "": "ok", ok: "ok", start: "start", fail: "fail" };
|
||||
|
||||
heartbeatPingRouter.all("/:token/:kind?", limiter, body, asyncHandler(async (req, res) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.setHeader("X-Robots-Tag", "noindex, nofollow");
|
||||
if (!["GET", "HEAD", "POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
|
||||
const kind = KINDS[(req.params.kind ?? "").toLowerCase()];
|
||||
if (!kind) return res.status(404).type("text/plain").send("Not found");
|
||||
const fromQuery = typeof req.query.msg === "string" ? req.query.msg : null;
|
||||
const message = cleanMessage(fromQuery ?? (typeof req.body === "string" ? req.body : null));
|
||||
const heartbeat = await recordPing(req.params.token, kind, message);
|
||||
if (!heartbeat) return res.status(404).type("text/plain").send("Not found");
|
||||
res.type("text/plain").send("OK");
|
||||
}));
|
||||
@@ -0,0 +1,111 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { heartbeats, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { dueBy, heartbeatState, listHeartbeats, loadRecent, MAX_GRACE_MINUTES, MAX_PERIOD_MINUTES, newToken } from "../services/heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const heartbeatsRouter = Router();
|
||||
|
||||
heartbeatsRouter.use(requireAuth);
|
||||
|
||||
const MAX_HEARTBEATS = 100;
|
||||
|
||||
const schema = z.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
periodMinutes: z.number().int().min(1).max(MAX_PERIOD_MINUTES),
|
||||
graceMinutes: z.number().int().min(0).max(MAX_GRACE_MINUTES).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
notes: z.string().max(500).nullable().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
const pathOf = (token: string) => `/api/heartbeat/${token}`;
|
||||
|
||||
async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Everyone signed in can see how the jobs are doing, like the Alerts page. The address with its token is shown only to administrators: it
|
||||
// is what lets anyone say "I'm alive" for the job.
|
||||
heartbeatsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const now = new Date();
|
||||
const isAdmin = req.currentUser!.role === "admin";
|
||||
const rows = await listHeartbeats();
|
||||
const recent = await loadRecent(rows.map((r) => r.id));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
baseUrl: isAdmin ? env.appBaseUrl.replace(/\/+$/, "") : null,
|
||||
heartbeats: rows.map(({ token, ...r }) => ({
|
||||
...r,
|
||||
path: isAdmin ? pathOf(token) : null,
|
||||
state: heartbeatState(r, now),
|
||||
dueBy: dueBy(r).toISOString(),
|
||||
server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null,
|
||||
recent: recent.get(r.id) ?? [],
|
||||
})),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
heartbeatsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = schema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: heartbeats.id }).from(heartbeats)).length >= MAX_HEARTBEATS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_HEARTBEATS} heartbeats.` });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const d = parsed.data;
|
||||
const [row] = await db
|
||||
.insert(heartbeats)
|
||||
.values({ name: d.name, token: newToken(), periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? 10, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? true })
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "create", targetType: "heartbeat", targetId: row.id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
|
||||
const { token, ...rest } = row;
|
||||
res.status(201).json({ heartbeat: { ...rest, path: pathOf(token) } });
|
||||
}));
|
||||
|
||||
heartbeatsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = schema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const d = parsed.data;
|
||||
const [row] = await db
|
||||
.update(heartbeats)
|
||||
.set({ name: d.name, periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? before.graceMinutes, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? before.enabled })
|
||||
.where(eq(heartbeats.id, id))
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "update", targetType: "heartbeat", targetId: id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
|
||||
invalidateAlertsCache();
|
||||
const { token, ...rest } = row;
|
||||
res.json({ heartbeat: { ...rest, path: pathOf(token) } });
|
||||
}));
|
||||
|
||||
// A new address, when the old one has got out. The old one stops working at once.
|
||||
heartbeatsRouter.post("/:id/new-address", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.update(heartbeats).set({ token: newToken() }).where(eq(heartbeats.id, id)).returning();
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "new_address", targetType: "heartbeat", targetId: id, detail: { name: row.name } });
|
||||
res.json({ path: pathOf(row.token) });
|
||||
}));
|
||||
|
||||
heartbeatsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(heartbeats).where(eq(heartbeats.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "delete", targetType: "heartbeat", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Router } from "express";
|
||||
import { NOT_FOUND, resolveShareLink } from "../services/shareLinks.js";
|
||||
import { buildAgenda, buildIcs, calendarName, MAX_DAYS } from "../services/agenda.js";
|
||||
import { env } from "../env.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const publicCalendarRouter = Router();
|
||||
|
||||
// No sign-in: the secret in the address is the whole of the access, so this reads nothing but what is coming up, and answers a link that doesn't
|
||||
// work — never made, ended, taken back, or one that opens something other than a calendar — in the same words. The address ends in .ics because
|
||||
// calendar apps want that.
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 60, message: "Too many requests. Wait a minute." });
|
||||
|
||||
publicCalendarRouter.get("/:file", limiter, asyncHandler(async (req, res) => {
|
||||
const token = req.params.file.endsWith(".ics") ? req.params.file.slice(0, -4) : req.params.file;
|
||||
const link = await resolveShareLink(token);
|
||||
if (!link || link.kind !== "calendar") return res.status(404).json(NOT_FOUND);
|
||||
const feed = buildIcs(await buildAgenda(new Date(), MAX_DAYS), { name: calendarName(), detail: link.detail, baseUrl: link.detail === "names" ? env.appBaseUrl : undefined });
|
||||
res.setHeader("Cache-Control", "private, max-age=300");
|
||||
res.setHeader("X-Robots-Tag", "noindex, nofollow");
|
||||
res.setHeader("Content-Disposition", 'inline; filename="homelab-coming-up.ics"');
|
||||
res.type("text/calendar; charset=utf-8").send(feed);
|
||||
}));
|
||||
@@ -20,7 +20,7 @@ const MAX_LINKS = 20;
|
||||
const createSchema = z.object({
|
||||
label: z.string().trim().min(1).max(60),
|
||||
kind: z.enum(shareLinkKinds).default("wall"),
|
||||
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it.
|
||||
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it. For a calendar: summary leaves the names out of the events.
|
||||
detail: z.enum(shareLinkDetails).default("summary"),
|
||||
title: z.string().trim().max(MAX_TITLE).optional(),
|
||||
message: z.string().trim().max(MAX_MESSAGE).optional(),
|
||||
@@ -59,7 +59,7 @@ shareLinksRouter.post("/", asyncHandler(async (req, res) => {
|
||||
targetId: link.id,
|
||||
detail: { label: link.label, kind: link.kind, detail: link.detail, expiresAt: link.expiresAt },
|
||||
});
|
||||
res.status(201).json({ link: publicShareLink(link), token, path: `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
|
||||
res.status(201).json({ link: publicShareLink(link), token, path: link.kind === "calendar" ? `/api/public/calendar/${token}.ics` : `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
|
||||
}));
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
Reference in new issue
Block a user