Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3bc6bbf72
commit
92726568b7
37 files changed
+6918
-21
No files matched your search
@@ -1,6 +1,14 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { desc, eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { restoreTests } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { DEFAULT_STALE_HOURS, getBackupCoverage } from "../services/backupCoverage.js";
|
||||
import { customKey, guestKey, KEY_PATTERN, latestTests, restoreInfo, RESULTS } from "../services/restoreTests.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const backupsRouter = Router();
|
||||
@@ -9,8 +17,68 @@ export const backupsRouter = Router();
|
||||
backupsRouter.use(requireAuth);
|
||||
|
||||
// `staleDays`: how old the newest backup may be before a guest counts as stale (default 2). `refresh=1` asks Proxmox again now.
|
||||
// Each guest also says how its newest restore test went (or that there was none).
|
||||
backupsRouter.get("/coverage", asyncHandler(async (req, res) => {
|
||||
const days = Number(req.query.staleDays);
|
||||
const staleHours = Number.isFinite(days) && days >= 1 && days <= 60 ? Math.round(days * 24) : DEFAULT_STALE_HOURS;
|
||||
res.json(await getBackupCoverage({ staleHours, refresh: req.query.refresh === "1" }));
|
||||
const report = await getBackupCoverage({ staleHours, refresh: req.query.refresh === "1" });
|
||||
const now = new Date();
|
||||
const limit = (await getSettings()).healthChecks.restoreTestDays;
|
||||
const latest = await latestTests();
|
||||
res.json({
|
||||
...report,
|
||||
restoreTestDays: limit,
|
||||
guests: report.guests.map((g) => ({ ...g, restoreKey: guestKey(g.integrationId, g.vmid), restoreTest: restoreInfo(latest.get(guestKey(g.integrationId, g.vmid)), now, limit) })),
|
||||
});
|
||||
}));
|
||||
|
||||
// The history of restore tests, and how each thing that isn't a Proxmox guest (a share, a database) stands.
|
||||
backupsRouter.get("/restore-tests", asyncHandler(async (_req, res) => {
|
||||
const now = new Date();
|
||||
const limit = (await getSettings()).healthChecks.restoreTestDays;
|
||||
const latest = await latestTests();
|
||||
const recent = await db.select().from(restoreTests).orderBy(desc(restoreTests.testedAt), desc(restoreTests.id)).limit(100);
|
||||
res.json({
|
||||
limitDays: limit,
|
||||
tests: recent,
|
||||
custom: [...latest.values()].filter((r) => r.subjectKey.startsWith("custom:")).map((r) => ({ key: r.subjectKey, label: r.label, ...restoreInfo(r, now, limit) })).sort((a, b) => a.label.localeCompare(b.label)),
|
||||
});
|
||||
}));
|
||||
|
||||
const testSchema = z.object({
|
||||
/** guest:<integration>:<vmid>, or left out for something that isn't a guest (then it is told apart by its label). */
|
||||
key: z.string().optional(),
|
||||
label: z.string().trim().min(1).max(120),
|
||||
testedAt: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
|
||||
result: z.enum(RESULTS),
|
||||
note: z.string().trim().max(500).optional(),
|
||||
});
|
||||
|
||||
backupsRouter.post("/restore-tests", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const parsed = testSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Say what was tested, how it went, and on what day (YYYY-MM-DD).", details: parsed.error.flatten() });
|
||||
const d = parsed.data;
|
||||
const key = d.key ?? customKey(d.label);
|
||||
if (!key || !KEY_PATTERN.test(key)) return res.status(400).json({ error: "invalid_key", message: "That can't be told apart from other things; give it a name with some letters or digits." });
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const testedAt = d.testedAt ?? today;
|
||||
if (Number.isNaN(Date.parse(`${testedAt}T00:00:00Z`)) || new Date(`${testedAt}T00:00:00Z`).toISOString().slice(0, 10) !== testedAt) return res.status(400).json({ error: "invalid_date", message: "That isn't a real date." });
|
||||
if (testedAt > today) return res.status(400).json({ error: "future", message: "A restore test can't be in the future." });
|
||||
const actor = req.currentUser!;
|
||||
const [row] = await db.insert(restoreTests).values({ subjectKey: key, label: d.label, testedAt, result: d.result, note: d.note || null, testedBy: actor.name ?? actor.email ?? actor.oidcSub }).returning();
|
||||
await recordAudit({ actor, category: "restore_test", action: "record", targetType: "restore_test", targetId: row.id, detail: { name: row.label, result: row.result, testedAt, note: row.note } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ test: row });
|
||||
}));
|
||||
|
||||
// A mistaken entry can be taken out; that is an administrator's to do, since it is what the alerts are judged on.
|
||||
backupsRouter.delete("/restore-tests/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(restoreTests).where(eq(restoreTests.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(restoreTests).where(eq(restoreTests.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "restore_test", action: "delete", targetType: "restore_test", targetId: id, detail: { name: before.label, testedAt: before.testedAt } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { CHANGE_KINDS, KIND_LABELS, getChangeLog, type ChangeKind } from "../services/changeLog.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const changesRouter = Router();
|
||||
|
||||
// Like the audit log it is made from: operators and administrators.
|
||||
changesRouter.use(requireAuth, requireRole("operator"));
|
||||
|
||||
// ?days=7 &kind=settings &serverId=3 &q=word &before=ISO (the next page) &limit=200
|
||||
changesRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const days = Number(req.query.days);
|
||||
const serverId = typeof req.query.serverId === "string" && /^\d+$/.test(req.query.serverId) ? Number(req.query.serverId) : undefined;
|
||||
const kind = CHANGE_KINDS.find((k) => k === req.query.kind) as ChangeKind | undefined;
|
||||
const before = typeof req.query.before === "string" && !Number.isNaN(Date.parse(req.query.before)) ? req.query.before : undefined;
|
||||
const limit = Number(req.query.limit);
|
||||
const result = await getChangeLog({
|
||||
days: Number.isFinite(days) && days >= 1 ? days : 7,
|
||||
kind,
|
||||
serverId,
|
||||
q: typeof req.query.q === "string" ? req.query.q.slice(0, 100) : undefined,
|
||||
before,
|
||||
limit: Number.isFinite(limit) && limit >= 1 ? limit : 200,
|
||||
});
|
||||
res.json({ ...result, kinds: CHANGE_KINDS.map((k) => ({ id: k, label: KIND_LABELS[k] })) });
|
||||
}));
|
||||
@@ -108,7 +108,7 @@ dependenciesRouter.post("/", requireRole("operator"), asyncHandler(async (req, r
|
||||
return res.status(409).json({ error: "loop", message: `${graph.names.get(to)} already depends on ${graph.names.get(from)}, so this would make them wait for each other.` });
|
||||
}
|
||||
const [row] = await db.insert(dependencies).values(d).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to) } });
|
||||
await recordAudit({ actor: req.currentUser!, category: "dependency", action: "create", targetType: "dependency", targetId: row.id, detail: { from: graph.names.get(from), to: graph.names.get(to), fromKey: from, toKey: to } });
|
||||
invalidateAlertsCache();
|
||||
res.status(201).json({ id: row.id });
|
||||
}));
|
||||
@@ -126,7 +126,7 @@ dependenciesRouter.delete("/:id", requireRole("operator"), asyncHandler(async (r
|
||||
action: "delete",
|
||||
targetType: "dependency",
|
||||
targetId: id,
|
||||
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}` },
|
||||
detail: { from: graph.names.get(nodeKey(before.fromType as "server", before.fromId)) ?? `${before.fromType} #${before.fromId}`, to: graph.names.get(nodeKey(before.toType as "server", before.toId)) ?? `${before.toType} #${before.toId}`, fromKey: `${before.fromType}:${before.fromId}`, toKey: `${before.toType}:${before.toId}` },
|
||||
});
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const endpointsRouter = Router();
|
||||
|
||||
endpointsRouter.use(requireAuth);
|
||||
|
||||
const MAX_CHECKS = 100;
|
||||
|
||||
const checkSchema = z
|
||||
.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
kind: z.enum(CHECK_KINDS),
|
||||
target: z.string().trim().min(1).max(500),
|
||||
port: z.number().int().min(1).max(65535).nullable().optional(),
|
||||
expectStatus: z.string().trim().max(60).nullable().optional(),
|
||||
keyword: z.string().max(200).nullable().optional(),
|
||||
insecure: z.boolean().optional(),
|
||||
intervalMinutes: z.number().int().min(1).max(1440).optional(),
|
||||
timeoutSeconds: z.number().int().min(1).max(60).optional(),
|
||||
failuresBeforeAlert: z.number().int().min(1).max(20).optional(),
|
||||
warnDays: z.number().int().min(1).max(90).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
})
|
||||
.superRefine((v, ctx) => {
|
||||
if (v.kind === "http") {
|
||||
let ok = false;
|
||||
try {
|
||||
const u = new URL(v.target);
|
||||
ok = u.protocol === "http:" || u.protocol === "https:";
|
||||
} catch {
|
||||
ok = false;
|
||||
}
|
||||
if (!ok) ctx.addIssue({ code: "custom", path: ["target"], message: "A web check needs an address that starts with http:// or https://." });
|
||||
if (v.expectStatus && !isStatusSpec(v.expectStatus)) ctx.addIssue({ code: "custom", path: ["expectStatus"], message: "Use status codes like 200, or ranges like 200-299, separated by commas." });
|
||||
} else {
|
||||
if (v.port == null) ctx.addIssue({ code: "custom", path: ["port"], message: v.kind === "tcp" ? "A port check needs a port." : "A certificate check needs a port (usually 443)." });
|
||||
if (/[/\s]/.test(v.target)) ctx.addIssue({ code: "custom", path: ["target"], message: "Give just the host name or address, without http:// or a path." });
|
||||
}
|
||||
});
|
||||
|
||||
/** A blank text field is no value. */
|
||||
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
|
||||
// Everyone signed in can see how things are doing, like the pages the same information appears on.
|
||||
endpointsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const rows = await db.select().from(endpointChecks).orderBy(endpointChecks.name);
|
||||
const stats = await loadStats(rows.map((r) => r.id));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
checks: rows.map((r) => ({ ...r, server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null, stats: stats.get(r.id) })),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
function valuesOf(d: z.infer<typeof checkSchema>) {
|
||||
return {
|
||||
name: d.name,
|
||||
kind: d.kind,
|
||||
target: d.target,
|
||||
port: d.kind === "http" ? null : (d.port ?? null),
|
||||
expectStatus: d.kind === "http" ? blank(d.expectStatus) : null,
|
||||
keyword: d.kind === "http" ? blank(d.keyword) : null,
|
||||
insecure: d.insecure ?? false,
|
||||
intervalMinutes: d.intervalMinutes ?? 5,
|
||||
timeoutSeconds: d.timeoutSeconds ?? 10,
|
||||
failuresBeforeAlert: d.failuresBeforeAlert ?? 2,
|
||||
warnDays: d.warnDays ?? 14,
|
||||
serverId: d.serverId ?? null,
|
||||
enabled: d.enabled ?? true,
|
||||
};
|
||||
}
|
||||
|
||||
async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Asking things on the network is an administrator's to set up.
|
||||
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const [row] = await db.insert(endpointChecks).values(valuesOf(parsed.data)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target } });
|
||||
res.status(201).json({ check: row });
|
||||
}));
|
||||
|
||||
endpointsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.success ? "That isn't a check." : (parsed.error.issues[0]?.message ?? "That isn't a check."), details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const v = valuesOf(parsed.data);
|
||||
// What is asked changed, so what was last found out about it no longer stands.
|
||||
const reset = v.kind !== before.kind || v.target !== before.target || v.port !== before.port ? { lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0 } : {};
|
||||
const [row] = await db.update(endpointChecks).set({ ...v, ...reset }).where(eq(endpointChecks.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "update", targetType: "endpoint", targetId: id, detail: { name: row.name, kind: row.kind, target: row.target } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ check: row });
|
||||
}));
|
||||
|
||||
endpointsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(endpointChecks).where(eq(endpointChecks.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "delete", targetType: "endpoint", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
|
||||
// Ask now, instead of at the next time it is due.
|
||||
endpointsRouter.post("/:id/check", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = /^\d+$/.test(req.params.id) ? Number(req.params.id) : NaN;
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const result = await runCheck(row);
|
||||
invalidateAlertsCache();
|
||||
res.json({ result });
|
||||
}));
|
||||
@@ -143,7 +143,7 @@ const updateSchema = z.object({
|
||||
.partial()
|
||||
.optional(),
|
||||
healthChecks: z
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160) })
|
||||
.object({ serverOfflineMinutes: z.number().int().min(15).max(10080), diskUsagePercent: z.number().int().min(50).max(99), domainWarnDays: z.number().int().min(1).max(365), diskForecastDays: z.number().int().min(0).max(90), backupStaleHours: z.number().int().min(0).max(2160), restoreTestDays: z.number().int().min(0).max(730) })
|
||||
.partial()
|
||||
.optional(),
|
||||
quietHours: z
|
||||
|
||||
Reference in new issue
Block a user