Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE `heartbeat_pings` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`heartbeat_id` integer NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`kind` text NOT NULL,
|
||||
`duration_ms` integer,
|
||||
`message` text,
|
||||
FOREIGN KEY (`heartbeat_id`) REFERENCES `heartbeats`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `heartbeat_pings_idx` ON `heartbeat_pings` (`heartbeat_id`,`at`);--> statement-breakpoint
|
||||
CREATE TABLE `heartbeats` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`token` text NOT NULL,
|
||||
`period_minutes` integer NOT NULL,
|
||||
`grace_minutes` integer DEFAULT 10 NOT NULL,
|
||||
`server_id` integer,
|
||||
`enabled` integer DEFAULT true NOT NULL,
|
||||
`notes` text,
|
||||
`last_ping_at` text,
|
||||
`last_status` text,
|
||||
`last_message` text,
|
||||
`last_duration_ms` integer,
|
||||
`last_start_at` text,
|
||||
`ping_count` integer DEFAULT 0 NOT NULL,
|
||||
`created_at` text DEFAULT (current_timestamp) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `heartbeats_token_unique` ON `heartbeats` (`token`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -295,6 +295,13 @@
|
||||
"when": 1792359507903,
|
||||
"tag": "0041_open_silvermane",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 42,
|
||||
"version": "6",
|
||||
"when": 1792361094159,
|
||||
"tag": "0042_silly_omega_red",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -48,6 +48,10 @@ import { mapRouter } from "./routes/map.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
import { alertTicketsRouter } from "./routes/alertTickets.js";
|
||||
import { avatarsRouter } from "./routes/avatars.js";
|
||||
import { heartbeatsRouter } from "./routes/heartbeats.js";
|
||||
import { agendaRouter } from "./routes/agenda.js";
|
||||
import { publicCalendarRouter } from "./routes/publicCalendar.js";
|
||||
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
|
||||
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
|
||||
import { securityHeaders } from "./utils/securityHeaders.js";
|
||||
import { sameOriginGuard } from "./utils/sameOriginGuard.js";
|
||||
@@ -72,6 +76,8 @@ export function createApp() {
|
||||
// The agent endpoint comes first: it reads its own (larger) body, only after checking the token, and an agent has no use for a
|
||||
// session. It is also the one place a request isn't from a browser, so it isn't subject to the same-origin check below.
|
||||
app.use("/api/agent/report", agentReportRouter);
|
||||
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
|
||||
app.use("/api/heartbeat", heartbeatPingRouter);
|
||||
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
|
||||
app.use(sameOriginGuard);
|
||||
app.use(express.json());
|
||||
@@ -105,6 +111,8 @@ export function createApp() {
|
||||
app.use("/auth", authRouter);
|
||||
app.use("/api/me", meRouter);
|
||||
app.use("/api/avatars", avatarsRouter);
|
||||
app.use("/api/heartbeats", heartbeatsRouter);
|
||||
app.use("/api/agenda", agendaRouter);
|
||||
app.use("/api/uptime", uptimeRouter);
|
||||
app.use("/api/share-links", shareLinksRouter);
|
||||
app.use("/api/runbooks", runbooksRouter);
|
||||
@@ -123,6 +131,7 @@ export function createApp() {
|
||||
app.use("/metrics", metricsRouter);
|
||||
app.use("/api/public/wall", publicWallRouter);
|
||||
app.use("/api/public/status", publicStatusRouter);
|
||||
app.use("/api/public/calendar", publicCalendarRouter);
|
||||
app.use("/api/users", usersRouter);
|
||||
app.use("/api/audit-log", auditLogRouter);
|
||||
app.use("/api/diag-log", diagLogRouter);
|
||||
|
||||
+41
-2
@@ -192,6 +192,44 @@ export const restoreTests = sqliteTable(
|
||||
(t) => [index("restore_tests_subject_idx").on(t.subjectKey, t.testedAt)],
|
||||
);
|
||||
|
||||
// A job that reports in: a cron job or backup script calls its address when it finishes, and the app speaks up when the call doesn't come in time —
|
||||
// see services/heartbeats.ts. The token is part of the address (/api/heartbeat/<token>) and is all the access a ping needs.
|
||||
export const heartbeats = sqliteTable("heartbeats", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
name: text("name").notNull(),
|
||||
token: text("token").notNull().unique(),
|
||||
periodMinutes: integer("period_minutes").notNull(), // how often it should report
|
||||
graceMinutes: integer("grace_minutes").notNull().default(10), // how much later than that is still fine
|
||||
serverId: integer("server_id"), // soft reference: the server it runs on, so its maintenance silences the alerts
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
notes: text("notes"),
|
||||
lastPingAt: text("last_ping_at"), // ISO; the last report of an end (success or failure)
|
||||
lastStatus: text("last_status"), // ok | fail
|
||||
lastMessage: text("last_message"),
|
||||
lastDurationMs: integer("last_duration_ms"), // when it said it had started first
|
||||
lastStartAt: text("last_start_at"), // ISO; the last "started" call
|
||||
pingCount: integer("ping_count").notNull().default(0),
|
||||
createdAt: text("created_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
});
|
||||
|
||||
// The recent calls to each heartbeat, for its history; kept two weeks.
|
||||
export const heartbeatPings = sqliteTable(
|
||||
"heartbeat_pings",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
heartbeatId: integer("heartbeat_id")
|
||||
.notNull()
|
||||
.references(() => heartbeats.id, { onDelete: "cascade" }),
|
||||
at: text("at").notNull(), // ISO
|
||||
kind: text("kind").notNull(), // ok | fail | start
|
||||
durationMs: integer("duration_ms"),
|
||||
message: text("message"),
|
||||
},
|
||||
(t) => [index("heartbeat_pings_idx").on(t.heartbeatId, t.at)],
|
||||
);
|
||||
|
||||
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
|
||||
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
|
||||
export const dependencies = sqliteTable(
|
||||
@@ -628,8 +666,9 @@ export const alertTickets = sqliteTable(
|
||||
export const shareLinkDetails = ["summary", "names"] as const;
|
||||
export type ShareLinkDetail = (typeof shareLinkDetails)[number];
|
||||
|
||||
// What a link opens: the wall display (how things are, for a TV), or a status page (is each of the things I chose working, for people at home).
|
||||
export const shareLinkKinds = ["wall", "status"] as const;
|
||||
// What a link opens: the wall display (how things are, for a TV), a status page (is each of the things I chose working, for people at home), or
|
||||
// a calendar feed (what is coming up, for a phone's calendar to subscribe to).
|
||||
export const shareLinkKinds = ["wall", "status", "calendar"] as const;
|
||||
export type ShareLinkKind = (typeof shareLinkKinds)[number];
|
||||
|
||||
// A link to the wall display that needs no sign-in, for a TV in the hall or someone who only wants to know if things are up. Only a hash
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Router } from "express";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { buildAgenda, DEFAULT_DAYS, MAX_DAYS } from "../services/agenda.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const agendaRouter = Router();
|
||||
|
||||
agendaRouter.use(requireAuth);
|
||||
|
||||
// Everything with a date ahead, for everyone signed in: it is only what the pages it comes from already show them. `days` is how far ahead.
|
||||
agendaRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isInteger(asked) && asked >= 1 && asked <= MAX_DAYS ? asked : DEFAULT_DAYS;
|
||||
res.json(await buildAgenda(new Date(), days));
|
||||
}));
|
||||
@@ -7,6 +7,7 @@ import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const endpointsRouter = Router();
|
||||
@@ -84,8 +85,54 @@ async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Checks worth setting up, from what the app already knows: a web port a server is listening on, a secret that reads a certificate. Reads what is
|
||||
// stored, asks nobody. Administrators only, like the checks themselves.
|
||||
endpointsRouter.get("/suggestions", requireRole("admin"), asyncHandler(async (_req, res) => {
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
const keysSchema = z.object({ keys: z.array(z.string().min(1).max(600)).min(1).max(MAX_CHECKS) });
|
||||
|
||||
// Makes the checks for the suggestions named. What each one is comes from the server's own list, not from the request, so a request can only
|
||||
// ever add what was offered.
|
||||
endpointsRouter.post("/suggestions/add", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = keysSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to add." });
|
||||
const offered = new Map((await loadSuggestions()).suggestions.map((s) => [s.key, s]));
|
||||
let have = (await db.select({ id: endpointChecks.id }).from(endpointChecks)).length;
|
||||
const added: { id: number; name: string }[] = [];
|
||||
for (const key of new Set(parsed.data.keys)) {
|
||||
const s = offered.get(key);
|
||||
if (!s) continue;
|
||||
if (have >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.`, added });
|
||||
const [row] = await db
|
||||
.insert(endpointChecks)
|
||||
.values(valuesOf({ name: s.name, kind: s.kind, target: s.target, port: s.port, serverId: s.serverId }))
|
||||
.returning();
|
||||
have++;
|
||||
added.push({ id: row.id, name: row.name });
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target, suggested: true } });
|
||||
}
|
||||
res.status(201).json({ added });
|
||||
}));
|
||||
|
||||
// "Not for me": what is dismissed stops being offered, and can be brought back.
|
||||
endpointsRouter.post("/suggestions/dismiss", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = keysSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to dismiss." });
|
||||
const keys = await loadDismissed();
|
||||
for (const k of parsed.data.keys) keys.add(k);
|
||||
await saveDismissed(keys);
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(async (_req, res) => {
|
||||
await saveDismissed(new Set());
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
// Asking things on the network is an administrator's to set up.
|
||||
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import express, { Router } from "express";
|
||||
import { cleanMessage, recordPing, type PingKind } from "../services/heartbeats.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
/**
|
||||
* The address a job calls: /api/heartbeat/<token>, /api/heartbeat/<token>/start, /api/heartbeat/<token>/fail. No session and no sign-in — the
|
||||
* token is the whole of the access, and all it allows is to say "I'm alive". Mounted ahead of the browser-only middleware (a script isn't a
|
||||
* browser and has no cookie), so it answers GET, HEAD, POST and PUT alike: whatever `curl` or `wget` happens to send.
|
||||
*
|
||||
* It answers a token that doesn't exist, and a path that isn't one of the three, in the same two words, and is limited per address.
|
||||
*/
|
||||
export const heartbeatPingRouter = Router();
|
||||
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
|
||||
// Whatever a script sends as the body ("3.2 GB copied") is a note; it is cut down to a line.
|
||||
const body = express.text({ type: () => true, limit: "100kb" });
|
||||
|
||||
const KINDS: Record<string, PingKind> = { "": "ok", ok: "ok", start: "start", fail: "fail" };
|
||||
|
||||
heartbeatPingRouter.all("/:token/:kind?", limiter, body, asyncHandler(async (req, res) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.setHeader("X-Robots-Tag", "noindex, nofollow");
|
||||
if (!["GET", "HEAD", "POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
|
||||
const kind = KINDS[(req.params.kind ?? "").toLowerCase()];
|
||||
if (!kind) return res.status(404).type("text/plain").send("Not found");
|
||||
const fromQuery = typeof req.query.msg === "string" ? req.query.msg : null;
|
||||
const message = cleanMessage(fromQuery ?? (typeof req.body === "string" ? req.body : null));
|
||||
const heartbeat = await recordPing(req.params.token, kind, message);
|
||||
if (!heartbeat) return res.status(404).type("text/plain").send("Not found");
|
||||
res.type("text/plain").send("OK");
|
||||
}));
|
||||
@@ -0,0 +1,111 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { z } from "zod";
|
||||
import { db } from "../db/client.js";
|
||||
import { heartbeats, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { dueBy, heartbeatState, listHeartbeats, loadRecent, MAX_GRACE_MINUTES, MAX_PERIOD_MINUTES, newToken } from "../services/heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const heartbeatsRouter = Router();
|
||||
|
||||
heartbeatsRouter.use(requireAuth);
|
||||
|
||||
const MAX_HEARTBEATS = 100;
|
||||
|
||||
const schema = z.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
periodMinutes: z.number().int().min(1).max(MAX_PERIOD_MINUTES),
|
||||
graceMinutes: z.number().int().min(0).max(MAX_GRACE_MINUTES).optional(),
|
||||
serverId: z.number().int().positive().nullable().optional(),
|
||||
notes: z.string().max(500).nullable().optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
const pathOf = (token: string) => `/api/heartbeat/${token}`;
|
||||
|
||||
async function serverExists(id: number | null | undefined) {
|
||||
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
|
||||
}
|
||||
|
||||
// Everyone signed in can see how the jobs are doing, like the Alerts page. The address with its token is shown only to administrators: it
|
||||
// is what lets anyone say "I'm alive" for the job.
|
||||
heartbeatsRouter.get("/", asyncHandler(async (req, res) => {
|
||||
const now = new Date();
|
||||
const isAdmin = req.currentUser!.role === "admin";
|
||||
const rows = await listHeartbeats();
|
||||
const recent = await loadRecent(rows.map((r) => r.id));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
baseUrl: isAdmin ? env.appBaseUrl.replace(/\/+$/, "") : null,
|
||||
heartbeats: rows.map(({ token, ...r }) => ({
|
||||
...r,
|
||||
path: isAdmin ? pathOf(token) : null,
|
||||
state: heartbeatState(r, now),
|
||||
dueBy: dueBy(r).toISOString(),
|
||||
server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null,
|
||||
recent: recent.get(r.id) ?? [],
|
||||
})),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
|
||||
heartbeatsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = schema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.error.flatten() });
|
||||
if ((await db.select({ id: heartbeats.id }).from(heartbeats)).length >= MAX_HEARTBEATS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_HEARTBEATS} heartbeats.` });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const d = parsed.data;
|
||||
const [row] = await db
|
||||
.insert(heartbeats)
|
||||
.values({ name: d.name, token: newToken(), periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? 10, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? true })
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "create", targetType: "heartbeat", targetId: row.id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
|
||||
const { token, ...rest } = row;
|
||||
res.status(201).json({ heartbeat: { ...rest, path: pathOf(token) } });
|
||||
}));
|
||||
|
||||
heartbeatsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
const parsed = schema.safeParse(req.body);
|
||||
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.success ? undefined : parsed.error.flatten() });
|
||||
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const d = parsed.data;
|
||||
const [row] = await db
|
||||
.update(heartbeats)
|
||||
.set({ name: d.name, periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? before.graceMinutes, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? before.enabled })
|
||||
.where(eq(heartbeats.id, id))
|
||||
.returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "update", targetType: "heartbeat", targetId: id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
|
||||
invalidateAlertsCache();
|
||||
const { token, ...rest } = row;
|
||||
res.json({ heartbeat: { ...rest, path: pathOf(token) } });
|
||||
}));
|
||||
|
||||
// A new address, when the old one has got out. The old one stops working at once.
|
||||
heartbeatsRouter.post("/:id/new-address", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.update(heartbeats).set({ token: newToken() }).where(eq(heartbeats.id, id)).returning();
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "new_address", targetType: "heartbeat", targetId: id, detail: { name: row.name } });
|
||||
res.json({ path: pathOf(row.token) });
|
||||
}));
|
||||
|
||||
heartbeatsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const id = idOf(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
await db.delete(heartbeats).where(eq(heartbeats.id, id));
|
||||
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "delete", targetType: "heartbeat", targetId: id, detail: { name: before.name } });
|
||||
invalidateAlertsCache();
|
||||
res.status(204).end();
|
||||
}));
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Router } from "express";
|
||||
import { NOT_FOUND, resolveShareLink } from "../services/shareLinks.js";
|
||||
import { buildAgenda, buildIcs, calendarName, MAX_DAYS } from "../services/agenda.js";
|
||||
import { env } from "../env.js";
|
||||
import { rateLimit } from "../utils/rateLimit.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const publicCalendarRouter = Router();
|
||||
|
||||
// No sign-in: the secret in the address is the whole of the access, so this reads nothing but what is coming up, and answers a link that doesn't
|
||||
// work — never made, ended, taken back, or one that opens something other than a calendar — in the same words. The address ends in .ics because
|
||||
// calendar apps want that.
|
||||
const limiter = rateLimit({ windowMs: 60_000, max: 60, message: "Too many requests. Wait a minute." });
|
||||
|
||||
publicCalendarRouter.get("/:file", limiter, asyncHandler(async (req, res) => {
|
||||
const token = req.params.file.endsWith(".ics") ? req.params.file.slice(0, -4) : req.params.file;
|
||||
const link = await resolveShareLink(token);
|
||||
if (!link || link.kind !== "calendar") return res.status(404).json(NOT_FOUND);
|
||||
const feed = buildIcs(await buildAgenda(new Date(), MAX_DAYS), { name: calendarName(), detail: link.detail, baseUrl: link.detail === "names" ? env.appBaseUrl : undefined });
|
||||
res.setHeader("Cache-Control", "private, max-age=300");
|
||||
res.setHeader("X-Robots-Tag", "noindex, nofollow");
|
||||
res.setHeader("Content-Disposition", 'inline; filename="homelab-coming-up.ics"');
|
||||
res.type("text/calendar; charset=utf-8").send(feed);
|
||||
}));
|
||||
@@ -20,7 +20,7 @@ const MAX_LINKS = 20;
|
||||
const createSchema = z.object({
|
||||
label: z.string().trim().min(1).max(60),
|
||||
kind: z.enum(shareLinkKinds).default("wall"),
|
||||
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it.
|
||||
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it. For a calendar: summary leaves the names out of the events.
|
||||
detail: z.enum(shareLinkDetails).default("summary"),
|
||||
title: z.string().trim().max(MAX_TITLE).optional(),
|
||||
message: z.string().trim().max(MAX_MESSAGE).optional(),
|
||||
@@ -59,7 +59,7 @@ shareLinksRouter.post("/", asyncHandler(async (req, res) => {
|
||||
targetId: link.id,
|
||||
detail: { label: link.label, kind: link.kind, detail: link.detail, expiresAt: link.expiresAt },
|
||||
});
|
||||
res.status(201).json({ link: publicShareLink(link), token, path: `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
|
||||
res.status(201).json({ link: publicShareLink(link), token, path: link.kind === "calendar" ? `/api/public/calendar/${token}.ics` : `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
|
||||
}));
|
||||
|
||||
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
/**
|
||||
* "Coming up": everything with a date ahead, in one list — a secret, domain, certificate or Tailscale key that expires, a warranty or end of support,
|
||||
* planned and recurring maintenance, a restore test that is due, and updates that have been waiting. Each already lives on a page of its own and in the
|
||||
* weekly digest; this puts them side by side, oldest first, so what is about to need attention is one look, not a dozen.
|
||||
*
|
||||
* Built in two parts so it can be tested exactly: `loadAgendaData` reads what is stored (and, for Tailscale keys, asks Tailscale, remembered for ten
|
||||
* minutes), and `agendaFromData` turns that into items. `buildIcs` writes the same items as an iCalendar feed for a phone's calendar to subscribe to.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
|
||||
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
|
||||
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
|
||||
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
|
||||
import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { env } from "../env.js";
|
||||
|
||||
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates"] as const;
|
||||
export type AgendaKind = (typeof AGENDA_KINDS)[number];
|
||||
|
||||
export const KIND_LABELS: Record<AgendaKind, string> = {
|
||||
secret: "Secrets",
|
||||
certificate: "Certificates",
|
||||
domain: "Domains",
|
||||
tailscale: "Tailscale keys",
|
||||
warranty: "Warranties",
|
||||
support: "End of support",
|
||||
maintenance: "Maintenance",
|
||||
restore: "Restore tests",
|
||||
updates: "Updates waiting",
|
||||
};
|
||||
|
||||
export interface AgendaItem {
|
||||
/** Stable, with the date in it: a secret that is renewed is a different entry, and the old one just goes. */
|
||||
id: string;
|
||||
kind: AgendaKind;
|
||||
/** YYYY-MM-DD for something that happens on a day; an ISO time for maintenance, which has a start and an end. */
|
||||
at: string;
|
||||
endsAt: string | null;
|
||||
allDay: boolean;
|
||||
title: string;
|
||||
detail: string | null;
|
||||
/** The page of the app that has more. */
|
||||
link: string;
|
||||
serverId: number | null;
|
||||
/** Whole days from today to the day it falls on, in the time zone set for notifications; negative once it has passed. */
|
||||
daysLeft: number;
|
||||
}
|
||||
|
||||
export interface Agenda {
|
||||
generatedAt: string;
|
||||
today: string;
|
||||
items: AgendaItem[];
|
||||
/** Sources that could not be read, so the agenda is missing something. */
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_DAYS = 90;
|
||||
export const MAX_DAYS = 400;
|
||||
/** What has already happened is shown for this long (a secret that expired last week is still worth seeing). */
|
||||
export const LOOK_BACK_DAYS = 30;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
// ─── Dates ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const isYmd = (v: unknown): v is string => typeof v === "string" && /^\d{4}-\d{2}-\d{2}$/.test(v) && !Number.isNaN(Date.parse(`${v}T00:00:00Z`)) && new Date(`${v}T00:00:00Z`).toISOString().startsWith(v);
|
||||
|
||||
/** The calendar day an instant falls on, as the clock in `tz` sees it. */
|
||||
export function localDate(when: Date, tz: string): string {
|
||||
return new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit" }).format(when);
|
||||
}
|
||||
|
||||
const daysBetween = (fromYmd: string, toYmd: string) => Math.round((Date.parse(`${toYmd}T00:00:00Z`) - Date.parse(`${fromYmd}T00:00:00Z`)) / DAY_MS);
|
||||
|
||||
export const addDays = (ymd: string, n: number) => new Date(Date.parse(`${ymd}T00:00:00Z`) + n * DAY_MS).toISOString().slice(0, 10);
|
||||
|
||||
// ─── What is stored ────────────────────────────────────────────────────────
|
||||
|
||||
export interface AgendaData {
|
||||
secrets: { id: number; name: string; type: string; expiryDate: string | null }[];
|
||||
domains: { id: number; name: string; expiresAt: string | null }[];
|
||||
endpoints: { id: number; name: string; enabled: boolean; certExpiresAt: string | null; serverId: number | null }[];
|
||||
tailscale: { integrationId: number; integrationName: string; deviceLabel: string; daysLeft: number }[];
|
||||
assets: { id: number; name: string; warrantyUntil: string | null; endOfSupport: string | null; serverId: number | null }[];
|
||||
/** Windows that have been set up but not begun, one per target. */
|
||||
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
|
||||
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
|
||||
/** The newest restore test of each subject. */
|
||||
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
|
||||
restoreLimitDays: number;
|
||||
/** Servers with updates waiting. */
|
||||
updates: { name: string; count: number; security: number | null; since: string | null }[];
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
let tailscaleCache: { at: number; items: AgendaData["tailscale"]; failures: string[] } | null = null;
|
||||
const TAILSCALE_CACHE_MS = 10 * 60_000;
|
||||
export const clearAgendaCacheForTests = () => {
|
||||
tailscaleCache = null;
|
||||
};
|
||||
|
||||
async function tailscaleKeys(): Promise<{ items: AgendaData["tailscale"]; failures: string[] }> {
|
||||
if (tailscaleCache && Date.now() - tailscaleCache.at < TAILSCALE_CACHE_MS) return tailscaleCache;
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "tailscale"), eq(integrations.enabled, true)));
|
||||
if (rows.length === 0) return { items: [], failures: [] };
|
||||
const { items, failures } = await collectTailscaleKeyExpiries();
|
||||
tailscaleCache = { at: Date.now(), items, failures: failures.map((f) => `Tailscale (${f.integrationName}): ${f.message}`) };
|
||||
return tailscaleCache;
|
||||
}
|
||||
|
||||
export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData> {
|
||||
const notes: string[] = [];
|
||||
const settings = await getSettings();
|
||||
const [secretRows, domainRows, endpointRows, assetRows, scheduleRows, serverRows, tests] = await Promise.all([
|
||||
db.select({ id: secrets.id, name: secrets.name, type: secrets.type, expiryDate: secrets.expiryDate }).from(secrets),
|
||||
db.select({ id: domains.id, name: domains.name, expiresAt: domains.expiresAt }).from(domains),
|
||||
db.select({ id: endpointChecks.id, name: endpointChecks.name, enabled: endpointChecks.enabled, certExpiresAt: endpointChecks.certExpiresAt, serverId: endpointChecks.serverId }).from(endpointChecks),
|
||||
db.select({ id: assets.id, name: assets.name, warrantyUntil: assets.warrantyUntil, endOfSupport: assets.endOfSupport, serverId: assets.serverId }).from(assets),
|
||||
db.select().from(maintenanceSchedules),
|
||||
db.select({ name: servers.name, pendingUpdates: servers.pendingUpdates, since: servers.updatesPendingSince }).from(servers),
|
||||
latestTests(),
|
||||
]);
|
||||
|
||||
const windows: AgendaData["windows"] = [];
|
||||
for (const w of await listUpcomingWindows(now)) {
|
||||
const target = await describeTarget(w.targetType, w.targetId);
|
||||
if (target) windows.push({ startedAt: w.startedAt, endsAt: w.endsAt, reason: w.reason, source: w.source, targetName: target.name });
|
||||
}
|
||||
|
||||
const schedules: AgendaData["schedules"] = [];
|
||||
for (const s of scheduleRows.filter((r) => r.enabled)) {
|
||||
const names: string[] = [];
|
||||
for (const t of targetsOf(s)) {
|
||||
const d = await describeTarget(t.type, t.id);
|
||||
if (d) names.push(d.name);
|
||||
}
|
||||
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
|
||||
}
|
||||
|
||||
const updates: AgendaData["updates"] = [];
|
||||
for (const s of serverRows) {
|
||||
const u = parsePendingUpdates(s.pendingUpdates);
|
||||
if (u && u.count > 0) updates.push({ name: s.name, count: u.count, security: u.security, since: s.since });
|
||||
}
|
||||
|
||||
let tailscale: AgendaData["tailscale"] = [];
|
||||
try {
|
||||
const t = await tailscaleKeys();
|
||||
tailscale = t.items;
|
||||
notes.push(...t.failures);
|
||||
} catch (err) {
|
||||
notes.push(`Tailscale keys could not be read: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
secrets: secretRows,
|
||||
domains: domainRows,
|
||||
endpoints: endpointRows,
|
||||
tailscale,
|
||||
assets: assetRows,
|
||||
windows,
|
||||
schedules,
|
||||
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
|
||||
restoreLimitDays: settings.healthChecks.restoreTestDays,
|
||||
updates,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Making the list ───────────────────────────────────────────────────────
|
||||
|
||||
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
||||
|
||||
/** Pure: the items, from what was read. `days` is how far ahead; what has just passed is kept for LOOK_BACK_DAYS. */
|
||||
export function agendaFromData(data: AgendaData, now: Date, tz: string, days: number = DEFAULT_DAYS): Agenda {
|
||||
const today = localDate(now, tz);
|
||||
const horizon = Math.min(Math.max(Math.round(days), 1), MAX_DAYS);
|
||||
const items: AgendaItem[] = [];
|
||||
|
||||
const day = (id: string, kind: AgendaKind, ymd: string | null, fn: (past: boolean) => { title: string; detail?: string | null; link: string; serverId?: number | null }) => {
|
||||
if (!isYmd(ymd)) return;
|
||||
const daysLeft = daysBetween(today, ymd);
|
||||
if (daysLeft < -LOOK_BACK_DAYS || daysLeft > horizon) return;
|
||||
const { title, detail = null, link, serverId = null } = fn(daysLeft < 0);
|
||||
items.push({ id: `${id}:${ymd}`, kind, at: ymd, endsAt: null, allDay: true, title, detail, link, serverId, daysLeft });
|
||||
};
|
||||
|
||||
for (const s of data.secrets) {
|
||||
const certificate = s.type === "ssl_certificate";
|
||||
day(`secret:${s.id}`, certificate ? "certificate" : "secret", s.expiryDate, (past) => ({ title: `${certificate ? "Certificate" : "Secret"} “${s.name}” ${past ? "expired" : "expires"}`, detail: certificate ? "kept as a secret" : null, link: "/secrets" }));
|
||||
}
|
||||
for (const d of data.domains) day(`domain:${d.id}`, "domain", d.expiresAt, (past) => ({ title: `Domain ${d.name} ${past ? "expired" : "expires"}`, link: "/domains" }));
|
||||
for (const e of data.endpoints) {
|
||||
if (!e.enabled || !e.certExpiresAt) continue;
|
||||
day(`endpoint-cert:${e.id}`, "certificate", e.certExpiresAt.slice(0, 10), (past) => ({ title: `The certificate for ${e.name} ${past ? "expired" : "expires"}`, detail: "seen by an endpoint check", link: "/endpoints", serverId: e.serverId }));
|
||||
}
|
||||
for (const t of data.tailscale) {
|
||||
day(`tailscale:${t.integrationId}:${t.deviceLabel}`, "tailscale", addDays(today, t.daysLeft), (past) => ({ title: `Tailscale key for ${t.deviceLabel} ${past ? "expired" : "expires"}`, detail: t.integrationName, link: "/tailscale" }));
|
||||
}
|
||||
for (const a of data.assets) {
|
||||
day(`warranty:${a.id}`, "warranty", a.warrantyUntil, (past) => ({ title: `Warranty on ${a.name} ${past ? "ended" : "ends"}`, link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
|
||||
day(`support:${a.id}`, "support", a.endOfSupport, (past) => ({ title: `Support for ${a.name} ${past ? "ended" : "ends"}`, detail: "time to plan its replacement", link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
|
||||
}
|
||||
|
||||
// A restore test is due a set time after the last good one — only if a time limit is set (Settings → Notifications).
|
||||
if (data.restoreLimitDays > 0) {
|
||||
for (const t of data.restoreTests) {
|
||||
if ((t.result !== "ok" && t.result !== "partial") || !isYmd(t.testedAt)) continue;
|
||||
day(`restore:${t.subjectKey}`, "restore", addDays(t.testedAt, data.restoreLimitDays), (past) => ({ title: `Restore test ${past ? "was due" : "due"}: ${t.label}`, detail: `last tested ${t.testedAt}`, link: "/backups" }));
|
||||
}
|
||||
}
|
||||
|
||||
// Maintenance has a start and an end. Windows set up by a recurring schedule are left to the schedule (they are only made a week ahead).
|
||||
const lastMs = now.getTime() + horizon * DAY_MS;
|
||||
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
|
||||
for (const w of data.windows) {
|
||||
if (w.source?.startsWith("schedule:") || Date.parse(w.startedAt) > lastMs) continue;
|
||||
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
|
||||
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
|
||||
g.names.push(w.targetName);
|
||||
groups.set(key, g);
|
||||
}
|
||||
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
|
||||
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
|
||||
const names = (list: string[]) => {
|
||||
const sorted = [...list].sort((a, b) => a.localeCompare(b));
|
||||
return sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
|
||||
};
|
||||
for (const g of groups.values()) timed(`maintenance:${g.start}:${g.names.length}`, g.start, g.end, `Maintenance: ${names(g.names)}`, g.reason);
|
||||
for (const s of data.schedules) {
|
||||
for (const occ of occurrencesBetween(s, now, new Date(lastMs)).slice(0, 60)) {
|
||||
timed(`schedule:${s.id}:${occ.date}`, occ.start.toISOString(), occ.end.toISOString(), `Recurring maintenance: ${s.name}`, [s.targetNames.length > 0 ? names(s.targetNames) : null, s.reason].filter(Boolean).join(" — ") || null);
|
||||
}
|
||||
}
|
||||
|
||||
// Updates that have been waiting: one line for all of them, from the longest wait.
|
||||
if (data.updates.length > 0) {
|
||||
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
|
||||
const security = data.updates.filter((u) => (u.security ?? 0) > 0).length;
|
||||
const waited = Math.max(0, daysBetween(since, today));
|
||||
items.push({
|
||||
id: `updates:${since}`,
|
||||
kind: "updates",
|
||||
at: since,
|
||||
endsAt: null,
|
||||
allDay: true,
|
||||
title: `Updates waiting on ${plural(data.updates.length, "server")}${security > 0 ? ` (${security} with security fixes)` : ""}`,
|
||||
detail: waited > 0 ? `the longest wait is ${plural(waited, "day")}` : "reported today",
|
||||
link: "/updates",
|
||||
serverId: null,
|
||||
daysLeft: -waited,
|
||||
});
|
||||
}
|
||||
|
||||
items.sort((a, b) => a.at.localeCompare(b.at) || a.title.localeCompare(b.title));
|
||||
return { generatedAt: now.toISOString(), today, items, notes: data.notes };
|
||||
}
|
||||
|
||||
export async function buildAgenda(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<Agenda> {
|
||||
const settings = await getSettings();
|
||||
return agendaFromData(await loadAgendaData(now), now, settings.notifications.timezone || "UTC", days);
|
||||
}
|
||||
|
||||
// ─── The calendar feed ─────────────────────────────────────────────────────
|
||||
|
||||
/** Escapes a text value for iCalendar: backslash, semicolon, comma and line breaks. */
|
||||
const escapeText = (s: string) => s.replace(/\\/g, "\\\\").replace(/;/g, "\\;").replace(/,/g, "\\,").replace(/\r?\n/g, "\\n");
|
||||
|
||||
/** Lines are folded at 75 bytes, a space starting each continuation, without splitting a character. */
|
||||
export function foldLine(line: string): string {
|
||||
const out: string[] = [];
|
||||
let current = "";
|
||||
let bytes = 0;
|
||||
for (const ch of line) {
|
||||
const n = Buffer.byteLength(ch, "utf8");
|
||||
if (bytes + n > (out.length === 0 ? 75 : 74)) {
|
||||
out.push(current);
|
||||
current = "";
|
||||
bytes = 0;
|
||||
}
|
||||
current += ch;
|
||||
bytes += n;
|
||||
}
|
||||
out.push(current);
|
||||
return out.join("\r\n ");
|
||||
}
|
||||
|
||||
const icsDate = (ymd: string) => ymd.replace(/-/g, "");
|
||||
const icsTime = (iso: string) => new Date(iso).toISOString().replace(/[-:]/g, "").replace(/\.\d{3}/, "");
|
||||
|
||||
/** What an event is called when the link was made to show no names: only what kind of thing it is. */
|
||||
const GENERIC: Record<AgendaKind, string> = {
|
||||
secret: "A secret expires",
|
||||
certificate: "A certificate expires",
|
||||
domain: "A domain name expires",
|
||||
tailscale: "A Tailscale key expires",
|
||||
warranty: "A warranty ends",
|
||||
support: "Support ends for a device",
|
||||
maintenance: "Planned maintenance",
|
||||
restore: "A restore test is due",
|
||||
updates: "Updates are waiting",
|
||||
};
|
||||
|
||||
/**
|
||||
* The agenda as an iCalendar feed. A day is an all-day event; maintenance is an event from its start to its end. With `detail` "summary" an event only
|
||||
* says what kind of thing it is (no names), so a feed that is shared can be shown without telling anyone what is in the homelab. Updates that
|
||||
* are waiting are not events — they have no day — and are left out.
|
||||
*/
|
||||
export function buildIcs(agenda: Agenda, options: { name: string; detail: "summary" | "names"; baseUrl?: string }): string {
|
||||
const lines = [
|
||||
"BEGIN:VCALENDAR",
|
||||
"VERSION:2.0",
|
||||
"PRODID:-//Homelab Manager//Coming up//EN",
|
||||
"CALSCALE:GREGORIAN",
|
||||
"METHOD:PUBLISH",
|
||||
`X-WR-CALNAME:${escapeText(options.name)}`,
|
||||
"REFRESH-INTERVAL;VALUE=DURATION:PT6H",
|
||||
"X-PUBLISHED-TTL:PT6H",
|
||||
];
|
||||
const stamp = icsTime(agenda.generatedAt);
|
||||
for (const item of agenda.items) {
|
||||
if (item.kind === "updates") continue;
|
||||
if (item.daysLeft < -7) continue; // an old event in a subscribed calendar is just clutter
|
||||
const names = options.detail === "names";
|
||||
lines.push("BEGIN:VEVENT", `UID:${createHash("sha256").update(item.id).digest("hex").slice(0, 24)}@homelab-manager`, `DTSTAMP:${stamp}`);
|
||||
if (item.allDay) lines.push(`DTSTART;VALUE=DATE:${icsDate(item.at)}`, `DTEND;VALUE=DATE:${icsDate(addDays(item.at, 1))}`);
|
||||
else lines.push(`DTSTART:${icsTime(item.at)}`, `DTEND:${icsTime(item.endsAt ?? item.at)}`);
|
||||
lines.push(`SUMMARY:${escapeText(names ? item.title : GENERIC[item.kind])}`);
|
||||
if (names) {
|
||||
const description = [item.detail, options.baseUrl ? `${options.baseUrl.replace(/\/+$/, "")}${item.link}` : null].filter(Boolean).join("\n");
|
||||
if (description) lines.push(`DESCRIPTION:${escapeText(description)}`);
|
||||
}
|
||||
lines.push(`CATEGORIES:${escapeText(KIND_LABELS[item.kind])}`, "TRANSP:TRANSPARENT", "STATUS:CONFIRMED", "END:VEVENT");
|
||||
}
|
||||
lines.push("END:VCALENDAR");
|
||||
return `${lines.map(foldLine).join("\r\n")}\r\n`;
|
||||
}
|
||||
|
||||
/** What the feed calls itself. */
|
||||
export const calendarName = () => `Homelab Manager${env.environment?.name ? ` (${env.environment.name})` : ""} — coming up`;
|
||||
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -35,6 +35,7 @@ import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { heartbeatConditions } from "./heartbeats.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
@@ -65,6 +66,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
|
||||
};
|
||||
|
||||
const SOURCE_TIMEOUT_MS = 20_000;
|
||||
@@ -304,6 +306,22 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Jobs that report in: one that has not done so in time, or said it failed.
|
||||
check("Heartbeats", async () => {
|
||||
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
|
||||
push({
|
||||
category: c.category,
|
||||
severity: c.severity,
|
||||
type: "heartbeat",
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: "/heartbeats",
|
||||
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
||||
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
||||
check("Homelab Manager's own backups", async () => {
|
||||
|
||||
@@ -21,7 +21,7 @@ export const KIND_LABELS: Record<ChangeKind, string> = {
|
||||
dependencies: "Dependencies",
|
||||
inventory: "Inventory",
|
||||
backups: "Backups and restore tests",
|
||||
endpoints: "Endpoint checks",
|
||||
endpoints: "Endpoint checks and heartbeats",
|
||||
other: "Other",
|
||||
};
|
||||
|
||||
@@ -53,6 +53,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
|
||||
db_backup: "backups",
|
||||
restore_test: "backups",
|
||||
endpoint: "endpoints",
|
||||
heartbeat: "endpoints",
|
||||
};
|
||||
|
||||
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
|
||||
@@ -154,6 +155,14 @@ export function describeChange(category: string, action: string, detail: Record<
|
||||
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
|
||||
case "endpoint/delete":
|
||||
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "heartbeat/create":
|
||||
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
|
||||
case "heartbeat/update":
|
||||
return { title: `Heartbeat changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
|
||||
case "heartbeat/new_address":
|
||||
return { title: `Heartbeat given a new address${name ? `: ${name}` : ""}`, tone: "neutral", detail: "the old address no longer works" };
|
||||
case "heartbeat/delete":
|
||||
return { title: `Heartbeat removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
|
||||
case "restore_test/record": {
|
||||
const result = str(detail.result);
|
||||
const tone: ActivityTone = result === "failed" ? "bad" : result === "not_needed" ? "neutral" : "good";
|
||||
@@ -250,7 +259,8 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
|
||||
} else if (a.targetType === "endpoint" && idNum !== null) {
|
||||
targetName = endpointName.get(idNum) ?? targetName;
|
||||
link = "/endpoints";
|
||||
} else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
} else if (a.targetType === "heartbeat") link = "/heartbeats";
|
||||
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
|
||||
else if (a.targetType === "restore_test") link = "/backups";
|
||||
else if (a.targetType === "dependency") link = "/dependencies";
|
||||
else if (a.targetType === "settings") link = "/settings/notifications";
|
||||
@@ -280,7 +290,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
|
||||
.limit(1000);
|
||||
for (const i of incidents) {
|
||||
const serverId = i.targetType === "server" ? i.targetId : null;
|
||||
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/incidents/${i.id}`, serverId };
|
||||
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/alerts/incidents/${i.id}`, serverId };
|
||||
if (i.openedAt >= since.toISOString() && (!until || i.openedAt <= until.toISOString())) {
|
||||
entries.push({ detail: {}, entry: { ...base, id: `incident-open-${i.id}`, at: i.openedAt, tone: i.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised: ${i.message}`, detail: i.causedBy ? `explained by ${i.causedBy}` : null } });
|
||||
}
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats } from "../db/schema.js";
|
||||
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { heartbeatConditions, prunePings } from "./heartbeats.js";
|
||||
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
|
||||
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
@@ -28,7 +29,9 @@ async function loadState(): Promise<ActiveState> {
|
||||
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
|
||||
const asked = await runDueChecks(now, probeFn);
|
||||
const rows = await db.select().from(endpointChecks);
|
||||
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
|
||||
const beats = await db.select().from(heartbeats);
|
||||
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
@@ -48,6 +51,7 @@ export function initEndpointMonitor(): void {
|
||||
if (Date.now() - lastPrune > 24 * 3600_000) {
|
||||
lastPrune = Date.now();
|
||||
await pruneResults();
|
||||
await prunePings();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[endpoints] the pass failed:", err);
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* Suggested endpoint checks: things the app already knows enough about to offer a check for, so setting up monitoring is a click instead of typing.
|
||||
*
|
||||
* - A web port a server's agent sees open (80, 443, 8080, 8443), with a name that points at it in DNS if there is one — "git.example.lan answers on
|
||||
* 443 on nas" — as a web check, linked to that server so its maintenance silences the alerts.
|
||||
* - A secret that already has a host to read the certificate from, as a certificate check: the secret only warns about the expiry, this also hears
|
||||
* when the service stops answering.
|
||||
*
|
||||
* It only reads what is stored (no network), leaves out what a check already covers, and remembers what somebody said no to. Uptime Kuma's monitors are
|
||||
* not offered: they are already being watched, and the adapter only keeps a host and port for them, not the address.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsRecordsCache, endpointChecks, secrets, servers } from "../db/schema.js";
|
||||
import { isLoopback } from "./agentPorts.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export interface Suggestion {
|
||||
/** Stable identity, so a dismissed suggestion stays dismissed and "add" can name it. */
|
||||
key: string;
|
||||
kind: "http" | "tls";
|
||||
name: string;
|
||||
target: string;
|
||||
port: number | null;
|
||||
serverId: number | null;
|
||||
serverName: string | null;
|
||||
/** Why it is offered, in a sentence. */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface SuggestionInput {
|
||||
servers: { id: number; name: string; hostname: string | null; ips: string[]; ports: { protocol?: string; port: number; address?: string }[] }[];
|
||||
dns: { name: string; type: string; content: string }[];
|
||||
secrets: { name: string; checkHost: string | null; checkPort: number | null }[];
|
||||
existing: { kind: string; target: string; port: number | null }[];
|
||||
dismissed: Set<string>;
|
||||
}
|
||||
|
||||
/** The web ports worth offering, and what they speak. */
|
||||
const WEB_PORTS: Record<number, "http" | "https"> = { 80: "http", 443: "https", 8080: "http", 8443: "https" };
|
||||
/** The https twin of a plain port, so a server answering on both gets the one that reads the certificate. */
|
||||
const SECURE_TWIN: Record<number, number> = { 80: 443, 8080: 8443 };
|
||||
const MAX_NAMES_PER_PORT = 3;
|
||||
const MAX_SUGGESTIONS = 200;
|
||||
|
||||
const cleanHost = (h: string) => h.trim().toLowerCase().replace(/\.$/, "");
|
||||
const isIPv4 = (s: string) => /^\d{1,3}(\.\d{1,3}){3}$/.test(s);
|
||||
const usableIPv4 = (s: string) => isIPv4(s) && !s.startsWith("127.") && !s.startsWith("169.254.");
|
||||
|
||||
/** Where a check's address is, in the same words for an existing check and a suggestion: "web:host:port" and "tls:host:port" (a web check on https reads the certificate too). */
|
||||
function coverage(existing: SuggestionInput["existing"]): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const c of existing) {
|
||||
if (c.kind === "http") {
|
||||
try {
|
||||
const u = new URL(c.target);
|
||||
const port = u.port ? Number(u.port) : u.protocol === "https:" ? 443 : 80;
|
||||
const host = cleanHost(u.hostname);
|
||||
out.add(`web:${host}:${port}`);
|
||||
if (u.protocol === "https:") out.add(`tls:${host}:${port}`);
|
||||
} catch {
|
||||
// an address that isn't one covers nothing
|
||||
}
|
||||
} else if (c.kind === "tls" && c.port !== null) out.add(`tls:${cleanHost(c.target)}:${c.port}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the checks worth offering, from what is known. */
|
||||
export function suggestChecks(input: SuggestionInput): { suggestions: Suggestion[]; dismissed: Suggestion[] } {
|
||||
const covered = coverage(input.existing);
|
||||
const all = new Map<string, Suggestion>();
|
||||
|
||||
for (const s of input.servers) {
|
||||
const ips = new Set(s.ips.map((i) => i.trim().toLowerCase()));
|
||||
const hostname = s.hostname ? cleanHost(s.hostname) : null;
|
||||
// Names that point at this server in DNS: an address record with one of its addresses, and an alias to one of those names or to its hostname.
|
||||
const direct = new Set<string>();
|
||||
for (const r of input.dns) {
|
||||
if ((r.type === "A" || r.type === "AAAA") && ips.has(r.content.trim().toLowerCase())) direct.add(cleanHost(r.name));
|
||||
}
|
||||
const names = new Set(direct);
|
||||
for (const r of input.dns) {
|
||||
if (r.type !== "CNAME") continue;
|
||||
const to = cleanHost(r.content);
|
||||
if (direct.has(to) || (hostname !== null && to === hostname)) names.add(cleanHost(r.name));
|
||||
}
|
||||
const dnsNames = [...names].filter((n) => n.includes(".") && !n.startsWith("*") && !n.endsWith(".arpa") && !/\s/.test(n)).sort((a, b) => a.length - b.length || a.localeCompare(b));
|
||||
|
||||
const listening = new Set(s.ports.filter((p) => (p.protocol ?? "tcp") === "tcp" && !(p.address && isLoopback(p.address))).map((p) => p.port));
|
||||
for (const port of Object.keys(WEB_PORTS).map(Number).filter((p) => listening.has(p))) {
|
||||
// Plain and secure on the same server: the secure one is the real service (the plain one is most likely its redirect), and reads the certificate.
|
||||
if (SECURE_TWIN[port] !== undefined && listening.has(SECURE_TWIN[port])) continue;
|
||||
const scheme = WEB_PORTS[port];
|
||||
const fallback = hostname && hostname.includes(".") ? [hostname] : [s.ips.map((i) => i.trim()).find(usableIPv4)].filter((x): x is string => !!x);
|
||||
const hosts = dnsNames.length > 0 ? dnsNames.slice(0, MAX_NAMES_PER_PORT) : fallback;
|
||||
for (const host of hosts) {
|
||||
const defaultPort = scheme === "https" ? 443 : 80;
|
||||
const target = `${scheme}://${host}${port === defaultPort ? "" : `:${port}`}/`;
|
||||
if (covered.has(`web:${host}:${port}`)) continue;
|
||||
const viaDns = dnsNames.includes(host);
|
||||
all.set(`http|${target}`, {
|
||||
key: `http|${target}`,
|
||||
kind: "http",
|
||||
name: `${host}${port === defaultPort ? "" : `:${port}`}`,
|
||||
target,
|
||||
port: null,
|
||||
serverId: s.id,
|
||||
serverName: s.name,
|
||||
reason: `${s.name} is listening on port ${port}${viaDns ? `, and ${host} points at it in DNS` : ""}.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const sec of input.secrets) {
|
||||
if (!sec.checkHost || sec.checkPort === null) continue;
|
||||
const host = cleanHost(sec.checkHost);
|
||||
if (covered.has(`tls:${host}:${sec.checkPort}`)) continue;
|
||||
all.set(`tls|${host}|${sec.checkPort}`, {
|
||||
key: `tls|${host}|${sec.checkPort}`,
|
||||
kind: "tls",
|
||||
name: `Certificate: ${sec.name}`,
|
||||
target: host,
|
||||
port: sec.checkPort,
|
||||
serverId: null,
|
||||
serverName: null,
|
||||
reason: `The secret “${sec.name}” already reads this certificate; a check also tells you when ${host} stops answering.`,
|
||||
});
|
||||
}
|
||||
|
||||
// By server, those about no server last, then by address.
|
||||
const sorted = [...all.values()].sort((a, b) => Number(a.serverName === null) - Number(b.serverName === null) || (a.serverName ?? "").localeCompare(b.serverName ?? "") || a.target.localeCompare(b.target));
|
||||
return {
|
||||
suggestions: sorted.filter((x) => !input.dismissed.has(x.key)).slice(0, MAX_SUGGESTIONS),
|
||||
dismissed: sorted.filter((x) => input.dismissed.has(x.key)),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── What was said no to ───────────────────────────────────────────────────
|
||||
|
||||
const DISMISSED_FLAG = "endpointSuggestionsDismissed";
|
||||
const MAX_DISMISSED = 500;
|
||||
|
||||
export async function loadDismissed(): Promise<Set<string>> {
|
||||
const raw = await getInternalFlag(DISMISSED_FLAG);
|
||||
if (!raw) return new Set();
|
||||
try {
|
||||
const v = JSON.parse(raw);
|
||||
return new Set(Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : []);
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveDismissed(keys: Set<string>): Promise<void> {
|
||||
await setInternalFlag(DISMISSED_FLAG, JSON.stringify([...keys].slice(-MAX_DISMISSED)));
|
||||
}
|
||||
|
||||
// ─── Reading what is stored ────────────────────────────────────────────────
|
||||
|
||||
function jsonArray<T>(stored: string | null): T[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadSuggestions(): Promise<{ suggestions: Suggestion[]; dismissed: Suggestion[] }> {
|
||||
const [serverRows, dnsRows, secretRows, existing, dismissed] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses, listeningPorts: servers.listeningPorts }).from(servers),
|
||||
db.select({ name: dnsRecordsCache.name, type: dnsRecordsCache.type, content: dnsRecordsCache.content }).from(dnsRecordsCache),
|
||||
db.select({ name: secrets.name, checkHost: secrets.checkHost, checkPort: secrets.checkPort }).from(secrets),
|
||||
db.select({ kind: endpointChecks.kind, target: endpointChecks.target, port: endpointChecks.port }).from(endpointChecks),
|
||||
loadDismissed(),
|
||||
]);
|
||||
return suggestChecks({
|
||||
servers: serverRows.map((s) => ({ id: s.id, name: s.name, hostname: s.hostname, ips: jsonArray<string>(s.ipAddresses).filter((i) => typeof i === "string"), ports: jsonArray<{ protocol?: string; port: number; address?: string }>(s.listeningPorts).filter((p) => p && Number.isInteger(p.port)) })),
|
||||
dns: dnsRows,
|
||||
secrets: secretRows,
|
||||
existing,
|
||||
dismissed,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Heartbeats: "did my 3 a.m. job actually run?". A cron job or backup script calls an address when it finishes; the app writes down that it did, and
|
||||
* speaks up — an alert and a notification, like any other problem — when the call doesn't come in time, or when the job says it failed.
|
||||
*
|
||||
* Each heartbeat has an address with a long random token in it (`/api/heartbeat/<token>`) and a period: how often the job should report, plus a grace
|
||||
* period for jobs that run a little late. Three calls: `…/<token>` (it worked), `…/<token>/start` (it began, so the next call can say how long it took)
|
||||
* and `…/<token>/fail` (it ran and went wrong — this is an alert at once, without waiting for the time to pass). Anything after the call, in the
|
||||
* body or in `?msg=`, is kept as a short note ("3.2 GB, 41 s").
|
||||
*
|
||||
* The token is the whole of the access a call needs, and all it allows is to say "I'm alive": it can't read anything. It is kept as it is, not hashed
|
||||
* (unlike a share link), because the address has to be shown again when somebody sets the job up on another machine.
|
||||
*/
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { and, asc, desc, eq, inArray, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { heartbeatPings, heartbeats } from "../db/schema.js";
|
||||
import { invalidateAlertsCache } from "./alerts.js";
|
||||
|
||||
export type HeartbeatRow = typeof heartbeats.$inferSelect;
|
||||
export type PingKind = "ok" | "fail" | "start";
|
||||
export const PING_KINDS: readonly PingKind[] = ["ok", "fail", "start"];
|
||||
|
||||
export const KEEP_PING_DAYS = 14;
|
||||
export const MAX_MESSAGE = 200;
|
||||
/** "hb_" and then random letters and digits: what an address is allowed to carry. */
|
||||
export const TOKEN_PATTERN = /^hb_[A-Za-z0-9_-]{20,64}$/;
|
||||
const MIN_PERIOD = 1;
|
||||
export const MAX_PERIOD_MINUTES = 60 * 24 * 30;
|
||||
export const MAX_GRACE_MINUTES = 60 * 24;
|
||||
/** A "started" call that is older than this is not what a later "finished" call is timing. */
|
||||
const MAX_RUN_MS = 7 * 86_400_000;
|
||||
|
||||
export const newToken = () => `hb_${randomBytes(18).toString("base64url")}`;
|
||||
|
||||
/** SQLite's own timestamps ("2026-10-19 03:10:00") are UTC without a "T"; ours are ISO. Both read as the instant they are. */
|
||||
export function parseStamp(value: string): number {
|
||||
return Date.parse(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value) ? `${value.replace(" ", "T")}Z` : value);
|
||||
}
|
||||
|
||||
/** A note from a job, made fit to store and show: one line, no control characters, not long. */
|
||||
export function cleanMessage(raw: unknown): string | null {
|
||||
if (typeof raw !== "string") return null;
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const text = raw.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
return text === "" ? null : text.slice(0, MAX_MESSAGE);
|
||||
}
|
||||
|
||||
// ─── What a heartbeat's state is ───────────────────────────────────────────
|
||||
|
||||
export type HeartbeatState = "off" | "waiting" | "up" | "running" | "late" | "failed";
|
||||
|
||||
/** When the next call is due by, counting the grace: from the last call, or from the day it was made if it has never called. */
|
||||
export function dueBy(h: Pick<HeartbeatRow, "periodMinutes" | "graceMinutes" | "lastPingAt" | "createdAt">): Date {
|
||||
const base = h.lastPingAt ? parseStamp(h.lastPingAt) : parseStamp(h.createdAt);
|
||||
return new Date(base + (h.periodMinutes + h.graceMinutes) * 60_000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: how a heartbeat is doing. A job that said it failed is failed until it next succeeds; otherwise it is late once the time (period and
|
||||
* grace) has passed without a call. A new one that has not called yet is waiting, and is late if its first call is overdue too.
|
||||
*/
|
||||
export function heartbeatState(h: Pick<HeartbeatRow, "enabled" | "periodMinutes" | "graceMinutes" | "lastPingAt" | "lastStatus" | "lastStartAt" | "createdAt">, now: Date): HeartbeatState {
|
||||
if (!h.enabled) return "off";
|
||||
if (h.lastStatus === "fail") return "failed";
|
||||
if (now.getTime() > dueBy(h).getTime()) return "late";
|
||||
if (!h.lastPingAt) return "waiting";
|
||||
if (h.lastStartAt && parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt)) return "running";
|
||||
return "up";
|
||||
}
|
||||
|
||||
// ─── What is wrong ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface HeartbeatCondition {
|
||||
key: string;
|
||||
source: "heartbeat";
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
category: "automation";
|
||||
heartbeatId: number;
|
||||
serverId: number | null;
|
||||
}
|
||||
|
||||
const stamp = (d: Date) => `${d.toISOString().slice(0, 16).replace("T", " ")} UTC`;
|
||||
|
||||
/**
|
||||
* The problems the heartbeats stand for. A call that didn't come in time is a warning, and says when it was due (a time, not "N minutes ago", so
|
||||
* the message doesn't change on every pass); a job that reported a failure is critical. The key carries the server a heartbeat is linked to, so
|
||||
* maintenance on it silences them.
|
||||
*/
|
||||
export function heartbeatConditions(rows: HeartbeatRow[], now: Date): HeartbeatCondition[] {
|
||||
const out: HeartbeatCondition[] = [];
|
||||
for (const h of rows) {
|
||||
const state = heartbeatState(h, now);
|
||||
const suffix = h.serverId !== null ? `:server:${h.serverId}` : "";
|
||||
if (state === "late") {
|
||||
const heard = h.lastPingAt ? `last heard ${stamp(new Date(parseStamp(h.lastPingAt)))}` : "it has never reported";
|
||||
out.push({ key: `heartbeat:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "warning", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} hasn't reported — it was due by ${stamp(dueBy(h))} (${heard})` });
|
||||
} else if (state === "failed") {
|
||||
out.push({ key: `heartbeat-fail:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "critical", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} reported a failure${h.lastMessage ? ` — ${h.lastMessage}` : ""}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── A call coming in ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Writes down a call. Returns the heartbeat as it is now, or null when there is no heartbeat with that token. A heartbeat that is switched off
|
||||
* still takes calls (so a script isn't broken by pausing it) and notes them, so that switching it back on starts from a fresh clock.
|
||||
*/
|
||||
export async function recordPing(token: string, kind: PingKind, message: string | null, now: Date = new Date()): Promise<HeartbeatRow | null> {
|
||||
if (!TOKEN_PATTERN.test(token)) return null;
|
||||
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.token, token)).limit(1);
|
||||
if (!h) return null;
|
||||
const iso = now.toISOString();
|
||||
let durationMs: number | null = null;
|
||||
let updated: HeartbeatRow;
|
||||
if (kind === "start") {
|
||||
[updated] = await db.update(heartbeats).set({ lastStartAt: iso }).where(eq(heartbeats.id, h.id)).returning();
|
||||
} else {
|
||||
if (h.lastStartAt) {
|
||||
const ran = now.getTime() - parseStamp(h.lastStartAt);
|
||||
const startedAfterLastEnd = !h.lastPingAt || parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt);
|
||||
if (startedAfterLastEnd && ran >= 0 && ran <= MAX_RUN_MS) durationMs = ran;
|
||||
}
|
||||
[updated] = await db
|
||||
.update(heartbeats)
|
||||
.set({ lastPingAt: iso, lastStatus: kind, lastMessage: message, lastDurationMs: durationMs, pingCount: h.pingCount + 1 })
|
||||
.where(eq(heartbeats.id, h.id))
|
||||
.returning();
|
||||
}
|
||||
await db.insert(heartbeatPings).values({ heartbeatId: h.id, at: iso, kind, durationMs, message });
|
||||
invalidateAlertsCache();
|
||||
return updated;
|
||||
}
|
||||
|
||||
// ─── History ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PingSummary {
|
||||
at: string;
|
||||
kind: PingKind;
|
||||
durationMs: number | null;
|
||||
}
|
||||
|
||||
/** The last calls of each heartbeat that ended a run (success or failure), oldest first. */
|
||||
export async function loadRecent(ids: number[], perHeartbeat = 30): Promise<Map<number, PingSummary[]>> {
|
||||
const out = new Map<number, PingSummary[]>();
|
||||
if (ids.length === 0) return out;
|
||||
const rows = await db
|
||||
.select({ heartbeatId: heartbeatPings.heartbeatId, at: heartbeatPings.at, kind: heartbeatPings.kind, durationMs: heartbeatPings.durationMs })
|
||||
.from(heartbeatPings)
|
||||
.where(and(inArray(heartbeatPings.heartbeatId, ids), inArray(heartbeatPings.kind, ["ok", "fail"])))
|
||||
.orderBy(desc(heartbeatPings.at));
|
||||
for (const r of rows) {
|
||||
const list = out.get(r.heartbeatId) ?? [];
|
||||
if (list.length < perHeartbeat) list.push({ at: r.at, kind: r.kind as PingKind, durationMs: r.durationMs });
|
||||
out.set(r.heartbeatId, list);
|
||||
}
|
||||
for (const list of out.values()) list.reverse();
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function prunePings(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(heartbeatPings).where(lt(heartbeatPings.at, new Date(now.getTime() - KEEP_PING_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
export const periodLimits = { min: MIN_PERIOD, max: MAX_PERIOD_MINUTES };
|
||||
|
||||
/** The table order every list uses. */
|
||||
export const listHeartbeats = () => db.select().from(heartbeats).orderBy(asc(heartbeats.name));
|
||||
@@ -25,7 +25,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
|
||||
export function subjectOfConditionKey(key: string): string | null {
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^endpoint(?:-cert)?:\d+:server:(\d+)$/.exec(key);
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
@@ -70,6 +71,18 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Jobs that report in, for /metrics. */
|
||||
async function addHeartbeatMetrics(out: Exposition, now: Date) {
|
||||
for (const h of await db.select().from(heartbeats)) {
|
||||
if (!h.enabled) continue;
|
||||
const labels = { name: h.name };
|
||||
const state = heartbeatState(h, now);
|
||||
out.add("homelab_heartbeat_up", "gauge", "1 if the job has reported in time and not said it failed, else 0.", state === "late" || state === "failed" ? 0 : 1, labels);
|
||||
if (h.lastPingAt) out.add("homelab_heartbeat_last_ping_timestamp_seconds", "gauge", "When the job last reported that it finished (Unix time).", Math.floor(parseStamp(h.lastPingAt) / 1000), labels);
|
||||
if (h.lastDurationMs !== null) out.add("homelab_heartbeat_last_duration_seconds", "gauge", "How long the job said its last run took.", Math.round(h.lastDurationMs / 100) / 10, labels);
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
const started = Date.now();
|
||||
const out = new Exposition();
|
||||
@@ -195,5 +208,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* The name lists behind Operations → Generator's server-name picker.
|
||||
* The name lists behind Administration → Generator's server-name picker.
|
||||
*
|
||||
* The built-in lists below are hand-picked, not taken from any official source — they're a starting point. Admins can
|
||||
* edit every list under Settings → Names, and can replace a Swedish list with real statistics from Skatteverket.
|
||||
|
||||
@@ -109,8 +109,9 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
|
||||
else if (key.startsWith("failed-units:")) category = "services";
|
||||
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
|
||||
else if (key.startsWith("automation:")) category = "automation";
|
||||
else if (key.startsWith("heartbeat")) category = "automation";
|
||||
else if (key.startsWith("endpoint-cert:")) category = "expiry";
|
||||
else if (key.startsWith("endpoint:")) category = "monitoring";
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|endpoint(?:-cert)?:\d+:server:)(\d+)/.exec(key);
|
||||
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
|
||||
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
/**
|
||||
* "Coming up": what the agenda gathers and how it dates it, the calendar feed it can be written as, and the routes — the list for people signed in
|
||||
* and the feed behind a share link.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-agenda-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, secrets, domains, assets, endpointChecks, maintenanceWindows, maintenanceSchedules, restoreTests } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { agendaFromData, buildIcs, foldLine, localDate, addDays, buildAgenda, clearAgendaCacheForTests } = await import("../src/services/agenda.js");
|
||||
const { agendaRouter } = await import("../src/routes/agenda.js");
|
||||
const { publicCalendarRouter } = await import("../src/routes/publicCalendar.js");
|
||||
const { shareLinksRouter } = await import("../src/routes/shareLinks.js");
|
||||
const { createShareLink } = await import("../src/services/shareLinks.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
type Data = Parameters<typeof agendaFromData>[0];
|
||||
const NOW = new Date("2026-10-19T12:00:00Z");
|
||||
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
|
||||
const titles = (data: Data, days?: number, tz = "UTC") => agendaFromData(data, NOW, tz, days).items.map((i) => [i.daysLeft, i.kind, i.title]);
|
||||
|
||||
describe("what is gathered, and how it is dated", () => {
|
||||
test("a day that something expires on, with how many days away it is; a secret that is a certificate is a certificate", () => {
|
||||
const data = empty({
|
||||
secrets: [
|
||||
{ id: 1, name: "API key", type: "api_key", expiryDate: "2026-10-30" },
|
||||
{ id: 2, name: "Wildcard", type: "ssl_certificate", expiryDate: "2026-11-02" },
|
||||
{ id: 3, name: "Never", type: "generic", expiryDate: null },
|
||||
{ id: 4, name: "Garbage", type: "generic", expiryDate: "2026-02-30" },
|
||||
],
|
||||
domains: [{ id: 1, name: "example.se", expiresAt: "2027-01-05" }],
|
||||
});
|
||||
assert.deepEqual(titles(data), [
|
||||
[11, "secret", "Secret “API key” expires"],
|
||||
[14, "certificate", "Certificate “Wildcard” expires"],
|
||||
[78, "domain", "Domain example.se expires"],
|
||||
]);
|
||||
});
|
||||
|
||||
test("what has just passed is shown as passed, for thirty days; what is further off than asked for is not shown", () => {
|
||||
const data = empty({ secrets: [
|
||||
{ id: 1, name: "Yesterday", type: "generic", expiryDate: "2026-10-18" },
|
||||
{ id: 2, name: "Today", type: "generic", expiryDate: "2026-10-19" },
|
||||
{ id: 3, name: "Last month", type: "generic", expiryDate: "2026-09-19" },
|
||||
{ id: 4, name: "Long ago", type: "generic", expiryDate: "2026-09-18" },
|
||||
{ id: 5, name: "In a year", type: "generic", expiryDate: "2027-10-19" },
|
||||
] });
|
||||
assert.deepEqual(titles(data, 90), [[-30, "secret", "Secret “Last month” expired"], [-1, "secret", "Secret “Yesterday” expired"], [0, "secret", "Secret “Today” expires"]]);
|
||||
assert.equal(titles(data, 400).length, 4, "a year ahead, when asked for");
|
||||
});
|
||||
|
||||
test("certificates that endpoint checks have seen, hardware warranties and end of support — each linking to where they are kept", () => {
|
||||
const agenda = agendaFromData(
|
||||
empty({
|
||||
endpoints: [
|
||||
{ id: 1, name: "Gitea", enabled: true, certExpiresAt: "2026-11-01T08:00:00.000Z", serverId: 5 },
|
||||
{ id: 2, name: "Paused", enabled: false, certExpiresAt: "2026-11-01T08:00:00.000Z", serverId: null },
|
||||
{ id: 3, name: "No cert", enabled: true, certExpiresAt: null, serverId: null },
|
||||
],
|
||||
assets: [{ id: 9, name: "Synology DS920+", warrantyUntil: "2026-12-01", endOfSupport: "2026-12-20", serverId: 2 }],
|
||||
}),
|
||||
NOW,
|
||||
"UTC",
|
||||
);
|
||||
assert.deepEqual(agenda.items.map((i) => [i.kind, i.title, i.link, i.serverId]), [
|
||||
["certificate", "The certificate for Gitea expires", "/endpoints", 5],
|
||||
["warranty", "Warranty on Synology DS920+ ends", "/inventory?asset=9", 2],
|
||||
["support", "Support for Synology DS920+ ends", "/inventory?asset=9", 2],
|
||||
]);
|
||||
});
|
||||
|
||||
test("Tailscale keys: only those the integration says are expiring, dated by days left", () => {
|
||||
assert.deepEqual(titles(empty({ tailscale: [{ integrationId: 1, integrationName: "Home", deviceLabel: "laptop", daysLeft: 6 }] })), [[6, "tailscale", "Tailscale key for laptop expires"]]);
|
||||
});
|
||||
|
||||
test("a restore test is due a set time after the last good one — only with a limit set, and never for one that failed", () => {
|
||||
const tests = [
|
||||
{ subjectKey: "guest:1:100", label: "elsa", testedAt: "2026-08-01", result: "ok" },
|
||||
{ subjectKey: "guest:1:101", label: "anna", testedAt: "2026-08-01", result: "failed" },
|
||||
{ subjectKey: "custom:photos", label: "Photos share", testedAt: "2026-07-03", result: "partial" },
|
||||
];
|
||||
assert.deepEqual(titles(empty({ restoreTests: tests, restoreLimitDays: 0 })), []);
|
||||
assert.deepEqual(titles(empty({ restoreTests: tests, restoreLimitDays: 90 })), [[-18, "restore", "Restore test was due: Photos share"], [11, "restore", "Restore test due: elsa"]]);
|
||||
});
|
||||
|
||||
test("updates that are waiting are one line, from the longest wait", () => {
|
||||
const agenda = agendaFromData(empty({ updates: [{ name: "a", count: 5, security: 2, since: "2026-10-10T08:00:00.000Z" }, { name: "b", count: 1, security: 0, since: "2026-10-17T08:00:00.000Z" }, { name: "c", count: 3, security: null, since: null }] }), NOW, "UTC");
|
||||
assert.equal(agenda.items.length, 1);
|
||||
assert.deepEqual([agenda.items[0].title, agenda.items[0].detail, agenda.items[0].daysLeft, agenda.items[0].link], ["Updates waiting on 3 servers (1 with security fixes)", "the longest wait is 9 days", -9, "/updates"]);
|
||||
});
|
||||
|
||||
test("maintenance that is set up is grouped by when it is, the names kept short; windows a schedule made are left to the schedule, which is read forward", () => {
|
||||
const agenda = agendaFromData(
|
||||
empty({
|
||||
windows: [
|
||||
{ startedAt: "2026-10-24T01:00:00.000Z", endsAt: "2026-10-24T03:00:00.000Z", reason: "Firmware", source: null, targetName: "nas" },
|
||||
{ startedAt: "2026-10-24T01:00:00.000Z", endsAt: "2026-10-24T03:00:00.000Z", reason: "Firmware", source: null, targetName: "switch" },
|
||||
{ startedAt: "2026-10-21T02:00:00.000Z", endsAt: "2026-10-21T04:00:00.000Z", reason: "Patch night", source: "schedule:1:2026-10-21", targetName: "nas" },
|
||||
],
|
||||
schedules: [{ id: 1, name: "Patch night", reason: null, days: JSON.stringify([3]), startTime: "02:00", durationMinutes: 120, timezone: "UTC", targetNames: ["nas", "elsa"] }],
|
||||
}),
|
||||
NOW,
|
||||
"UTC",
|
||||
30,
|
||||
);
|
||||
const m = agenda.items.filter((i) => i.kind === "maintenance");
|
||||
assert.deepEqual(m.map((i) => [i.title, i.at.slice(0, 16), i.allDay, i.endsAt?.slice(11, 16)]), [
|
||||
["Recurring maintenance: Patch night", "2026-10-21T02:00", false, "04:00"],
|
||||
["Maintenance: nas, switch", "2026-10-24T01:00", false, "03:00"],
|
||||
["Recurring maintenance: Patch night", "2026-10-28T02:00", false, "04:00"],
|
||||
["Recurring maintenance: Patch night", "2026-11-04T02:00", false, "04:00"],
|
||||
["Recurring maintenance: Patch night", "2026-11-11T02:00", false, "04:00"],
|
||||
["Recurring maintenance: Patch night", "2026-11-18T02:00", false, "04:00"],
|
||||
]);
|
||||
assert.equal(m.filter((i) => i.at.startsWith("2026-10-21")).length, 1, "the window the schedule made and the schedule itself are one entry, not two");
|
||||
assert.equal(m[1].detail, "Firmware");
|
||||
assert.equal(m[0].detail, "elsa, nas");
|
||||
});
|
||||
|
||||
test("a day is the day on the clock set for notifications: a window at 23:30 UTC is already tomorrow in Stockholm", () => {
|
||||
const windows = [{ startedAt: "2026-10-19T23:30:00.000Z", endsAt: "2026-10-20T01:00:00.000Z", reason: null, source: null, targetName: "nas" }];
|
||||
assert.equal(agendaFromData(empty({ windows }), NOW, "UTC").items[0].daysLeft, 0);
|
||||
assert.equal(agendaFromData(empty({ windows }), NOW, "Europe/Stockholm").items[0].daysLeft, 1);
|
||||
assert.equal(localDate(new Date("2026-10-19T23:30:00Z"), "Europe/Stockholm"), "2026-10-20");
|
||||
assert.equal(addDays("2026-10-31", 1), "2026-11-01");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the calendar feed", () => {
|
||||
const data = empty({
|
||||
secrets: [{ id: 1, name: "API key, with; odd\\chars", type: "api_key", expiryDate: "2026-10-30" }],
|
||||
tailscale: [{ integrationId: 1, integrationName: "Home", deviceLabel: "bobs-laptop", daysLeft: 6 }],
|
||||
windows: [{ startedAt: "2026-10-24T01:00:00.000Z", endsAt: "2026-10-24T03:00:00.000Z", reason: "Firmware", source: null, targetName: "nas" }],
|
||||
secretsOld: undefined,
|
||||
updates: [{ name: "a", count: 1, security: 0, since: null }],
|
||||
} as unknown as Partial<Data>);
|
||||
const agenda = agendaFromData(data, NOW, "UTC");
|
||||
const ics = buildIcs(agenda, { name: "Homelab; manager", detail: "names", baseUrl: "https://hm.lan/" });
|
||||
|
||||
test("it is a calendar with CRLF lines: all-day events for days, a start and an end for maintenance, nothing for waiting updates", () => {
|
||||
assert.ok(ics.startsWith("BEGIN:VCALENDAR\r\nVERSION:2.0\r\n") && ics.endsWith("END:VCALENDAR\r\n"));
|
||||
assert.equal(ics.includes("\n") && ics.split("\n").every((l, i, a) => i === a.length - 1 || l.endsWith("\r")), true);
|
||||
assert.equal((ics.match(/BEGIN:VEVENT/g) ?? []).length, 3, "the secret, the key and the window — not the updates");
|
||||
assert.match(ics, /DTSTART;VALUE=DATE:20261030\r\nDTEND;VALUE=DATE:20261031/);
|
||||
assert.match(ics, /DTSTART:20261024T010000Z\r\nDTEND:20261024T030000Z/);
|
||||
assert.match(ics, /X-WR-CALNAME:Homelab\\; manager/);
|
||||
assert.match(ics, /TRANSP:TRANSPARENT/);
|
||||
});
|
||||
|
||||
test("text is escaped, and a line is never more than 75 bytes, without splitting a character", () => {
|
||||
assert.match(ics.replace(/\r\n /g, ""), /SUMMARY:Secret “API key\\, with\\; odd\\\\chars” expires/);
|
||||
const long = foldLine(`SUMMARY:${"å".repeat(100)}`);
|
||||
for (const l of long.split("\r\n")) assert.ok(Buffer.byteLength(l, "utf8") <= 75, `${Buffer.byteLength(l, "utf8")} bytes`);
|
||||
assert.equal(long.replace(/\r\n /g, ""), `SUMMARY:${"å".repeat(100)}`, "and it unfolds to what it was");
|
||||
for (const l of ics.split("\r\n")) assert.ok(Buffer.byteLength(l, "utf8") <= 75);
|
||||
});
|
||||
|
||||
test("with names, an event carries the name and a link back; each has a steady id that does not give the name away", () => {
|
||||
const unfolded = ics.replace(/\r\n /g, "");
|
||||
assert.match(unfolded, /DESCRIPTION:Firmware\\nhttps:\/\/hm\.lan\/maintenance/);
|
||||
const uids = [...ics.matchAll(/UID:(\S+)\r\n/g)].map((m) => m[1]);
|
||||
assert.equal(new Set(uids).size, 3);
|
||||
assert.ok(uids.every((u) => /^[0-9a-f]{24}@homelab-manager$/.test(u)));
|
||||
assert.deepEqual(uids, [...buildIcs(agenda, { name: "x", detail: "names" }).matchAll(/UID:(\S+)\r\n/g)].map((m) => m[1]), "the same on every fetch, or a subscribed calendar would duplicate everything");
|
||||
});
|
||||
|
||||
test("with no names, an event says only what kind of thing it is — and nothing in it gives away a name", () => {
|
||||
const plain = buildIcs(agenda, { name: "Homelab", detail: "summary", baseUrl: "https://hm.lan" });
|
||||
const unfolded = plain.replace(/\r\n /g, "");
|
||||
assert.match(unfolded, /SUMMARY:A secret expires/);
|
||||
assert.match(unfolded, /SUMMARY:A Tailscale key expires/);
|
||||
assert.match(unfolded, /SUMMARY:Planned maintenance/);
|
||||
assert.equal(/API key|bobs-laptop|nas|Firmware|hm\.lan|DESCRIPTION/.test(unfolded), false);
|
||||
});
|
||||
|
||||
test("an event more than a week old is dropped, one from this week is kept", () => {
|
||||
const old = agendaFromData(empty({ secrets: [{ id: 1, name: "Old", type: "generic", expiryDate: "2026-10-10" }, { id: 2, name: "Recent", type: "generic", expiryDate: "2026-10-15" }] }), NOW, "UTC");
|
||||
assert.equal((buildIcs(old, { name: "x", detail: "names" }).match(/BEGIN:VEVENT/g) ?? []).length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("from what is stored, through the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
const day = (n: number) => new Date(Date.now() + n * 86_400_000).toISOString().slice(0, 10);
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
const [s] = await db
|
||||
.insert(servers)
|
||||
.values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", pendingUpdates: JSON.stringify({ manager: "apt", count: 4, security: 1, rebootRequired: false, reportedAt: new Date().toISOString() }), updatesPendingSince: new Date(Date.now() - 3 * 86_400_000).toISOString() })
|
||||
.returning({ id: servers.id });
|
||||
await db.insert(secrets).values({ name: "Deploy key", type: "api_key", expiryDate: day(10) } as never);
|
||||
await db.insert(domains).values({ name: "example.se", expiresAt: day(40) });
|
||||
await db.insert(assets).values({ name: "Switch", kind: "network", warrantyUntil: day(20), createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() } as never);
|
||||
await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/", certExpiresAt: new Date(Date.now() + 5 * 86_400_000).toISOString(), serverId: s.id });
|
||||
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "Disk swap", startedAt: new Date(Date.now() + 2 * 86_400_000).toISOString(), endsAt: new Date(Date.now() + 2 * 86_400_000 + 3600_000).toISOString() });
|
||||
await db.insert(maintenanceSchedules).values({ name: "Patch night", targets: JSON.stringify([{ type: "server", id: s.id }]), days: JSON.stringify([0, 1, 2, 3, 4, 5, 6]), startTime: "03:00", durationMinutes: 60, timezone: "UTC" });
|
||||
await db.insert(restoreTests).values({ subjectKey: "custom:photos", label: "Photos share", testedAt: day(-80), result: "ok" });
|
||||
await updateSettings({ healthChecks: { restoreTestDays: 90 } });
|
||||
clearAgendaCacheForTests();
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use("/api/public/calendar", publicCalendarRouter);
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/agenda", agendaRouter);
|
||||
a.use("/api/share-links", shareLinksRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("everything with a date, from every source, for anyone signed in; the days asked for narrow it", async () => {
|
||||
who = "viewer";
|
||||
const res = await fetch(`${base}/api/agenda`);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
const kinds = new Set(body.items.map((i: { kind: string }) => i.kind));
|
||||
for (const k of ["secret", "domain", "warranty", "certificate", "maintenance", "restore", "updates"]) assert.ok(kinds.has(k), k);
|
||||
assert.ok(body.items.some((i: { title: string }) => i.title === "Restore test due: Photos share"), "tested 80 days ago with a 90-day limit");
|
||||
assert.ok(body.items.filter((i: { title: string }) => i.title.startsWith("Recurring maintenance")).length >= 60 / 1, "a schedule is read forward, not only the week that has windows");
|
||||
const near = await (await fetch(`${base}/api/agenda?days=15`)).json();
|
||||
assert.equal(near.items.some((i: { kind: string }) => i.kind === "domain"), false, "40 days out is beyond 15");
|
||||
assert.equal(near.items.some((i: { kind: string }) => i.kind === "secret"), true);
|
||||
const bad = await (await fetch(`${base}/api/agenda?days=banana`)).json();
|
||||
assert.equal(bad.items.length > 0, true, "an unreadable range is the default one");
|
||||
const direct = await buildAgenda();
|
||||
assert.equal(direct.items.length, body.items.length, "the route and the service agree");
|
||||
});
|
||||
|
||||
test("a calendar link is made like any share link, by an administrator, and its address is the feed", async () => {
|
||||
who = "viewer";
|
||||
assert.equal((await fetch(`${base}/api/share-links`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ label: "Phone", kind: "calendar", detail: "names", expiresInDays: null }) })).status, 403);
|
||||
who = "admin";
|
||||
const res = await fetch(`${base}/api/share-links`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ label: "Phone", kind: "calendar", detail: "names", expiresInDays: null }) });
|
||||
assert.equal(res.status, 201);
|
||||
const made = await res.json();
|
||||
assert.match(made.path, /^\/api\/public\/calendar\/[A-Za-z0-9_-]{20,}\.ics$/);
|
||||
const feed = await fetch(`${base}${made.path}`);
|
||||
assert.equal(feed.status, 200);
|
||||
assert.match(feed.headers.get("content-type") ?? "", /^text\/calendar; charset=utf-8/);
|
||||
const text = (await feed.text()).replace(/\r\n /g, "");
|
||||
assert.ok(text.startsWith("BEGIN:VCALENDAR"));
|
||||
assert.match(text, /SUMMARY:Secret “Deploy key” expires/);
|
||||
assert.match(text, /SUMMARY:Domain example\.se expires/);
|
||||
const bare = made.path.replace(/\.ics$/, "");
|
||||
assert.equal((await fetch(`${base}${bare}`)).status, 200, "the extension is for calendar apps; the address works without it");
|
||||
});
|
||||
|
||||
test("a link for no names gives the feed no names; one for something else, a wrong one and an ended one all say the same thing", async () => {
|
||||
const { token } = await createShareLink({ label: "Private", kind: "calendar", detail: "summary", expiresInDays: null, createdBy: "t" });
|
||||
const text = (await (await fetch(`${base}/api/public/calendar/${token}.ics`)).text()).replace(/\r\n /g, "");
|
||||
assert.match(text, /SUMMARY:A secret expires/);
|
||||
assert.equal(/Deploy key|example\.se|Gitea|nas/.test(text), false);
|
||||
|
||||
const wall = await createShareLink({ label: "Wall", kind: "wall", detail: "summary", expiresInDays: null, createdBy: "t" });
|
||||
const ended = await createShareLink({ label: "Old", kind: "calendar", detail: "names", expiresInDays: 1, createdBy: "t", now: new Date(Date.now() - 3 * 86_400_000) });
|
||||
const answers = await Promise.all([`${wall.token}.ics`, "nonsense.ics", `${"a".repeat(30)}.ics`, `${ended.token}.ics`].map(async (p) => {
|
||||
const r = await fetch(`${base}/api/public/calendar/${p}`);
|
||||
return [r.status, await r.text()];
|
||||
}));
|
||||
assert.deepEqual(new Set(answers.map((a) => JSON.stringify(a))).size, 1, "never made, ended, and a wall's: indistinguishable");
|
||||
assert.equal(answers[0][0], 404);
|
||||
});
|
||||
});
|
||||
@@ -115,7 +115,7 @@ describe("through the routes", () => {
|
||||
assert.ok(!r.entries.some((e: { title: string }) => /login|export/i.test(e.title)));
|
||||
const maint = r.entries[0];
|
||||
assert.deepEqual([maint.kind, maint.actor, maint.link, maint.serverId], ["maintenance", "Ada", `/servers/${nas}`, nas]);
|
||||
assert.equal(r.entries.find((e: { title: string }) => e.title.startsWith("Alert raised")).link.startsWith("/incidents/"), true);
|
||||
assert.equal(r.entries.find((e: { title: string }) => e.title.startsWith("Alert raised")).link.startsWith("/alerts/incidents/"), true);
|
||||
assert.equal(r.kinds.length, 10);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Suggested endpoint checks: what is offered from servers' open web ports, DNS names and secrets, what is left out, and the routes that add or dismiss them.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-suggest-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, secrets, dnsProviders, dnsRecordsCache, endpointChecks } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { suggestChecks } = await import("../src/services/endpointSuggestions.js");
|
||||
const { endpointsRouter } = await import("../src/routes/endpoints.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const nas = { id: 1, name: "nas", hostname: "nas.lan", ips: ["10.0.0.5"], ports: [{ protocol: "tcp", port: 443, address: "0.0.0.0" }, { protocol: "tcp", port: 22, address: "0.0.0.0" }] };
|
||||
const input = (extra: Partial<Parameters<typeof suggestChecks>[0]> = {}) => ({ servers: [nas], dns: [], secrets: [], existing: [], dismissed: new Set<string>(), ...extra });
|
||||
|
||||
describe("what is offered", () => {
|
||||
test("a web port a server listens on, by the name DNS gives it, linked to the server", () => {
|
||||
const { suggestions } = suggestChecks(input({ dns: [{ name: "git.example.lan", type: "A", content: "10.0.0.5" }, { name: "other.example.lan", type: "A", content: "10.0.0.9" }] }));
|
||||
assert.deepEqual(suggestions.map((s) => [s.kind, s.target, s.name, s.serverId, s.serverName]), [["http", "https://git.example.lan/", "git.example.lan", 1, "nas"]]);
|
||||
assert.match(suggestions[0].reason, /nas is listening on port 443, and git\.example\.lan points at it in DNS/);
|
||||
});
|
||||
|
||||
test("an alias to one of those names, or to the server's own hostname, is offered too; wildcards, reverse names and short names are not", () => {
|
||||
const { suggestions } = suggestChecks(
|
||||
input({
|
||||
dns: [
|
||||
{ name: "git.example.lan", type: "A", content: "10.0.0.5" },
|
||||
{ name: "code.example.lan", type: "CNAME", content: "git.example.lan." },
|
||||
{ name: "files.example.lan", type: "CNAME", content: "nas.lan" },
|
||||
{ name: "*.example.lan", type: "A", content: "10.0.0.5" },
|
||||
{ name: "5.0.0.10.in-addr.arpa", type: "A", content: "10.0.0.5" },
|
||||
{ name: "nas", type: "A", content: "10.0.0.5" },
|
||||
{ name: "unrelated.example.lan", type: "CNAME", content: "elsewhere.lan" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(suggestions.map((s) => s.target), ["https://code.example.lan/", "https://files.example.lan/", "https://git.example.lan/"]);
|
||||
});
|
||||
|
||||
test("with no DNS name the server's own hostname or its address is used; with neither, nothing", () => {
|
||||
assert.deepEqual(suggestChecks(input()).suggestions.map((s) => s.target), ["https://nas.lan/"]);
|
||||
const bare = { ...nas, hostname: "nas", ips: ["169.254.1.1", "10.0.0.5"] };
|
||||
assert.deepEqual(suggestChecks(input({ servers: [bare] })).suggestions.map((s) => s.target), ["https://10.0.0.5/"]);
|
||||
assert.deepEqual(suggestChecks(input({ servers: [{ ...bare, ips: [] }] })).suggestions, []);
|
||||
});
|
||||
|
||||
test("only web ports count, only for TCP, and a port that is open on this machine alone does not", () => {
|
||||
const odd = { ...nas, ports: [{ protocol: "udp", port: 443, address: "0.0.0.0" }, { protocol: "tcp", port: 22, address: "0.0.0.0" }, { protocol: "tcp", port: 8443, address: "127.0.0.1" }, { protocol: "tcp", port: 3000, address: "0.0.0.0" }] };
|
||||
assert.deepEqual(suggestChecks(input({ servers: [odd] })).suggestions, []);
|
||||
const mixed = { ...nas, ports: [{ protocol: "tcp", port: 8080, address: "127.0.0.1" }, { protocol: "tcp", port: 8080, address: "10.0.0.5" }] };
|
||||
assert.deepEqual(suggestChecks(input({ servers: [mixed] })).suggestions.map((s) => s.target), ["http://nas.lan:8080/"], "one socket on a real address is enough, and a port that is not the default is in the address");
|
||||
});
|
||||
|
||||
test("plain and secure on the same server: the secure one is offered, since it reads the certificate and the plain one is most likely its redirect", () => {
|
||||
const both = { ...nas, ports: [{ protocol: "tcp", port: 80, address: "::" }, { protocol: "tcp", port: 443, address: "::" }, { protocol: "tcp", port: 8080, address: "::" }] };
|
||||
assert.deepEqual(suggestChecks(input({ servers: [both] })).suggestions.map((s) => s.target), ["http://nas.lan:8080/", "https://nas.lan/"]);
|
||||
});
|
||||
|
||||
test("a secret that already reads a certificate is offered as a certificate check", () => {
|
||||
const { suggestions } = suggestChecks(input({ servers: [], secrets: [{ name: "Wildcard cert", checkHost: "Git.Example.lan.", checkPort: 443 }, { name: "Typed in", checkHost: null, checkPort: null }] }));
|
||||
assert.deepEqual(suggestions.map((s) => [s.kind, s.target, s.port, s.name, s.serverId]), [["tls", "git.example.lan", 443, "Certificate: Wildcard cert", null]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is left out", () => {
|
||||
const dns = [{ name: "git.example.lan", type: "A", content: "10.0.0.5" }];
|
||||
|
||||
test("what a check already covers, however its address is written", () => {
|
||||
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "http", target: "https://Git.example.lan:443/login", port: null }] })).suggestions.length, 0);
|
||||
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "http", target: "https://git.example.lan", port: null }] })).suggestions.length, 0);
|
||||
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "http", target: "http://git.example.lan/", port: null }] })).suggestions.length, 1, "a plain-http check does not cover the secure port");
|
||||
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "tcp", target: "git.example.lan", port: 443 }] })).suggestions.length, 1, "nor does a port check, which doesn't speak to the page");
|
||||
});
|
||||
|
||||
test("a secret's certificate is covered by a certificate check, or by a web check on https that reads it anyway", () => {
|
||||
const sec = [{ name: "Cert", checkHost: "git.example.lan", checkPort: 443 }];
|
||||
assert.equal(suggestChecks(input({ servers: [], secrets: sec, existing: [{ kind: "tls", target: "git.example.lan", port: 443 }] })).suggestions.length, 0);
|
||||
assert.equal(suggestChecks(input({ servers: [], secrets: sec, existing: [{ kind: "http", target: "https://git.example.lan/", port: null }] })).suggestions.length, 0);
|
||||
assert.equal(suggestChecks(input({ servers: [], secrets: sec, existing: [{ kind: "tls", target: "git.example.lan", port: 8443 }] })).suggestions.length, 1);
|
||||
});
|
||||
|
||||
test("what was dismissed is listed apart, not offered", () => {
|
||||
const r = suggestChecks(input({ dns, dismissed: new Set(["http|https://git.example.lan/"]) }));
|
||||
assert.deepEqual([r.suggestions.length, r.dismissed.map((d) => d.target)], [0, ["https://git.example.lan/"]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the routes", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
]);
|
||||
const [s] = await db
|
||||
.insert(servers)
|
||||
.values({ name: "nas", hostname: "nas.lan", apiTokenHash: "n", apiTokenPrefix: "n", ipAddresses: JSON.stringify(["10.0.0.5"]), listeningPorts: JSON.stringify([{ protocol: "tcp", port: 443, address: "0.0.0.0", process: "nginx" }, { protocol: "tcp", port: 8080, address: "0.0.0.0" }]) })
|
||||
.returning({ id: servers.id });
|
||||
const [p] = await db.insert(dnsProviders).values({ providerType: "pihole", name: "Pi-hole" }).returning({ id: dnsProviders.id });
|
||||
await db.insert(dnsRecordsCache).values({ providerId: p.id, zoneId: "z", recordId: "r1", type: "A", name: "git.example.lan", content: "10.0.0.5" });
|
||||
await db.insert(secrets).values({ name: "Mail cert", type: "ssl_certificate", checkHost: "mail.example.lan", checkPort: 443 } as never);
|
||||
void s;
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/endpoints", endpointsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/endpoints`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
test("administrators see what is offered, and nobody else", async () => {
|
||||
who = "operator";
|
||||
assert.equal((await send("GET", "/suggestions")).status, 403);
|
||||
assert.equal((await send("POST", "/suggestions/add", { keys: ["x"] })).status, 403);
|
||||
who = "admin";
|
||||
const body = await (await send("GET", "/suggestions")).json();
|
||||
assert.deepEqual(body.suggestions.map((s: { target: string }) => s.target), ["http://git.example.lan:8080/", "https://git.example.lan/", "mail.example.lan"]);
|
||||
});
|
||||
|
||||
test("adding makes the checks the server itself offered — a key that was not offered adds nothing — linked to their server, and they stop being offered", async () => {
|
||||
const res = await send("POST", "/suggestions/add", { keys: ["https://git.example.lan/", "http|https://git.example.lan/", "http|http://evil.example/", "http|https://git.example.lan/"] });
|
||||
assert.equal(res.status, 201);
|
||||
assert.equal((await res.json()).added.length, 1);
|
||||
const [made] = await db.select().from(endpointChecks).where(eq(endpointChecks.target, "https://git.example.lan/"));
|
||||
assert.deepEqual([made.kind, made.name, made.serverId !== null, made.intervalMinutes, made.enabled], ["http", "git.example.lan", true, 5, true]);
|
||||
const body = await (await send("GET", "/suggestions")).json();
|
||||
assert.equal(body.suggestions.some((s: { target: string }) => s.target === "https://git.example.lan/"), false);
|
||||
assert.equal((await send("POST", "/suggestions/add", { keys: [] })).status, 400);
|
||||
});
|
||||
|
||||
test("dismissing takes one out of the offer, and restoring brings every dismissed one back", async () => {
|
||||
let body = await (await send("POST", "/suggestions/dismiss", { keys: ["tls|mail.example.lan|443"] })).json();
|
||||
assert.equal(body.suggestions.some((s: { kind: string }) => s.kind === "tls"), false);
|
||||
assert.equal(body.dismissed.length, 1);
|
||||
body = await (await send("POST", "/suggestions/restore")).json();
|
||||
assert.equal(body.suggestions.some((s: { kind: string }) => s.kind === "tls"), true);
|
||||
assert.equal(body.dismissed.length, 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Heartbeats: what a job reporting in (or not) amounts to, the address it calls, what the pass announces, and who may set them up.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-heartbeats-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, servers, heartbeats, heartbeatPings } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const { dueBy, heartbeatState, heartbeatConditions, cleanMessage, parseStamp, recordPing, newToken, TOKEN_PATTERN } = await import("../src/services/heartbeats.js");
|
||||
const { heartbeatsRouter } = await import("../src/routes/heartbeats.js");
|
||||
const { heartbeatPingRouter } = await import("../src/routes/heartbeatPing.js");
|
||||
const { runEndpointPass } = await import("../src/services/endpointMonitor.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
|
||||
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { updateSettings } = await import("../src/services/settingsStore.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
type Row = typeof heartbeats.$inferSelect;
|
||||
const NOW = new Date("2026-10-19T03:00:00Z");
|
||||
const row = (extra: Partial<Row> = {}): Row => ({
|
||||
id: 1, name: "Nightly backup", token: "hb_x", periodMinutes: 1440, graceMinutes: 60, serverId: null, enabled: true, notes: null, lastPingAt: null, lastStatus: null, lastMessage: null,
|
||||
lastDurationMs: null, lastStartAt: null, pingCount: 0, createdAt: "2026-10-18 03:00:00", ...extra,
|
||||
});
|
||||
const minutes = (n: number) => new Date(NOW.getTime() + n * 60_000);
|
||||
|
||||
describe("when a job is due, and how it is doing", () => {
|
||||
test("it is due a period and the grace after its last call — or after it was made, if it never called", () => {
|
||||
assert.equal(dueBy(row({ lastPingAt: "2026-10-19T02:00:00.000Z" })).toISOString(), "2026-10-20T03:00:00.000Z");
|
||||
assert.equal(dueBy(row()).toISOString(), "2026-10-19T04:00:00.000Z", "SQLite's own timestamp format is read as UTC");
|
||||
assert.equal(parseStamp("2026-10-18 03:00:00"), Date.parse("2026-10-18T03:00:00Z"));
|
||||
});
|
||||
|
||||
test("new, up, running, late, failed and switched off", () => {
|
||||
const called = { lastPingAt: NOW.toISOString(), lastStatus: "ok" };
|
||||
assert.equal(heartbeatState(row({ createdAt: "2026-10-19 02:00:00" }), NOW), "waiting", "made recently, has not called, not yet due");
|
||||
assert.equal(heartbeatState(row(), NOW), "waiting", "due by 04:00 today");
|
||||
assert.equal(heartbeatState(row(), minutes(61)), "late", "never called, and now it is overdue");
|
||||
assert.equal(heartbeatState(row(called), minutes(60 * 24)), "up");
|
||||
assert.equal(heartbeatState(row(called), minutes(60 * 25)), "up", "inside the grace");
|
||||
assert.equal(heartbeatState(row(called), minutes(60 * 25 + 1)), "late");
|
||||
assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(5).toISOString() }), minutes(10)), "running", "it began after it last finished");
|
||||
assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(-5).toISOString() }), minutes(10)), "up", "a start older than the last finish is not a run in progress");
|
||||
assert.equal(heartbeatState(row({ ...called, lastStatus: "fail", lastMessage: "disk full" }), minutes(10)), "failed", "said so, and does not wait for the time to pass");
|
||||
assert.equal(heartbeatState(row({ enabled: false }), minutes(9999)), "off");
|
||||
});
|
||||
|
||||
test("the problems: late is a warning with the time it was due, a reported failure is critical; each keyed by the server it runs on", () => {
|
||||
const list = heartbeatConditions(
|
||||
[
|
||||
row({ id: 1, serverId: 7 }),
|
||||
row({ id: 2, name: "Photos sync", lastPingAt: NOW.toISOString(), lastStatus: "fail", lastMessage: "exit 3" }),
|
||||
row({ id: 3, name: "Fine", lastPingAt: NOW.toISOString(), lastStatus: "ok" }),
|
||||
row({ id: 4, name: "Paused", enabled: false }),
|
||||
],
|
||||
minutes(120),
|
||||
);
|
||||
assert.deepEqual(list.map((c) => [c.key, c.severity, c.category, c.serverId]), [["heartbeat:1:server:7", "warning", "automation", 7], ["heartbeat-fail:2", "critical", "automation", null]]);
|
||||
assert.match(list[0].message, /Nightly backup hasn't reported — it was due by 2026-10-19 04:00 UTC \(it has never reported\)/);
|
||||
assert.match(list[1].message, /Photos sync reported a failure — exit 3/);
|
||||
assert.deepEqual(metaOfConditionKey("heartbeat:1:server:7", "warning"), { category: "automation", severity: "warning", serverId: 7 }, "tags and categories route it");
|
||||
assert.equal(subjectOfConditionKey("heartbeat-fail:2:server:9"), "server:9", "and maintenance on the server holds it back");
|
||||
});
|
||||
|
||||
test("a note from a job is one short line", () => {
|
||||
assert.equal(cleanMessage("3.2 GB\n\tcopied in 41 s\u0000"), "3.2 GB copied in 41 s");
|
||||
assert.equal(cleanMessage(" "), null);
|
||||
assert.equal(cleanMessage(42), null);
|
||||
assert.equal(cleanMessage("x".repeat(500))?.length, 200);
|
||||
});
|
||||
|
||||
test("an address carries a long random token, and only that shape is looked up", () => {
|
||||
const t = newToken();
|
||||
assert.match(t, TOKEN_PATTERN);
|
||||
assert.notEqual(t, newToken());
|
||||
for (const bad of ["", "hb_", "hb_short", "nope_" + "a".repeat(24), "hb_" + "a/b".repeat(10), "hb_" + "a".repeat(100)]) assert.equal(TOKEN_PATTERN.test(bad), false, bad);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the address, the routes and the pass", () => {
|
||||
let app: Server;
|
||||
let hook: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
let serverId = 0;
|
||||
let id = 0;
|
||||
let path = "";
|
||||
const messages: string[] = [];
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "operator", name: "Olga", role: "operator" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
|
||||
hook = createHttpServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
|
||||
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
|
||||
|
||||
const a = express();
|
||||
// The ping address comes first, as in the real app: no session, no JSON parser, no sign-in.
|
||||
a.use("/api/heartbeat", heartbeatPingRouter);
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/heartbeats", heartbeatsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
app.close();
|
||||
hook.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const admin = (method: string, p: string, body?: unknown) => fetch(`${base}/api/heartbeats${p}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const ping = (p: string, init?: RequestInit) => fetch(`${base}${p}`, init);
|
||||
|
||||
test("an administrator makes one; a wrong one is refused with a reason; nobody else can; everyone can look", async () => {
|
||||
const res = await admin("POST", "/", { name: "Nightly backup", periodMinutes: 1440, graceMinutes: 60, serverId });
|
||||
assert.equal(res.status, 201);
|
||||
const made = (await res.json()).heartbeat;
|
||||
id = made.id;
|
||||
path = made.path;
|
||||
assert.match(path, /^\/api\/heartbeat\/hb_[A-Za-z0-9_-]{20,}$/);
|
||||
for (const bad of [{ periodMinutes: 5 }, { name: "x" }, { name: "x", periodMinutes: 0 }, { name: "x", periodMinutes: 60 * 24 * 31 }, { name: "x", periodMinutes: 5, graceMinutes: 99999 }, { name: "x", periodMinutes: 5, serverId: 99999 }, { name: " ", periodMinutes: 5 }]) {
|
||||
assert.equal((await admin("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
}
|
||||
who = "operator";
|
||||
assert.equal((await admin("POST", "/", { name: "y", periodMinutes: 5 })).status, 403);
|
||||
assert.equal((await admin("DELETE", `/${id}`)).status, 403);
|
||||
const list = await (await admin("GET", "/")).json();
|
||||
assert.equal(list.heartbeats[0].path, null, "the address is for administrators: it is what lets anyone say \"I'm alive\"");
|
||||
assert.equal(list.baseUrl, null);
|
||||
assert.equal(list.heartbeats[0].state, "waiting");
|
||||
assert.equal(list.heartbeats[0].server.name, "nas");
|
||||
who = "admin";
|
||||
const adminList = await (await admin("GET", "/")).json();
|
||||
assert.equal(adminList.heartbeats[0].path, path);
|
||||
assert.ok(adminList.baseUrl);
|
||||
assert.equal("token" in adminList.heartbeats[0], false, "the token is not a field of its own");
|
||||
});
|
||||
|
||||
test("a call is noted: success, a note in the body or the query, and how long a run took when it said it started", async () => {
|
||||
assert.equal((await ping(path)).status, 200);
|
||||
assert.equal(await (await ping(path, { method: "HEAD" })).text(), "");
|
||||
let [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.deepEqual([h.lastStatus, h.pingCount], ["ok", 2]);
|
||||
assert.equal(heartbeatState(h, new Date()), "up");
|
||||
|
||||
assert.equal((await ping(`${path}/start`, { method: "POST" })).status, 200);
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
const done = await ping(path, { method: "POST", body: "3.2 GB\ncopied", headers: { "content-type": "text/plain" } });
|
||||
assert.equal(await done.text(), "OK");
|
||||
[h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal(h.lastMessage, "3.2 GB copied");
|
||||
assert.ok(h.lastDurationMs !== null && h.lastDurationMs >= 20, "timed from the start call");
|
||||
|
||||
await ping(`${path}?msg=from%20the%20query`);
|
||||
[h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal(h.lastMessage, "from the query");
|
||||
assert.equal(h.lastDurationMs, null, "no start before this one, so no duration");
|
||||
const rows = await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id));
|
||||
assert.deepEqual(rows.map((r) => r.kind), ["ok", "ok", "start", "ok", "ok"]);
|
||||
});
|
||||
|
||||
test("a token that does not exist, a path that is not one of the three and a method that is not a call all get no information", async () => {
|
||||
const a = await ping("/api/heartbeat/hb_" + "z".repeat(24));
|
||||
const b = await ping(`${path}/bogus`);
|
||||
const c = await ping("/api/heartbeat/nonsense");
|
||||
assert.deepEqual([a.status, b.status, c.status], [404, 404, 404]);
|
||||
assert.equal(await a.text(), await b.text(), "never made, and a wrong path, read the same");
|
||||
assert.equal((await ping(path, { method: "DELETE" })).status, 405);
|
||||
assert.equal((await ping(path)).headers.get("cache-control"), "no-store");
|
||||
});
|
||||
|
||||
test("a job that says it failed is an alert at once, until it next succeeds; the pass says so once, and once when it is over", async () => {
|
||||
messages.length = 0;
|
||||
await db.update(heartbeats).set({ serverId: null }).where(eq(heartbeats.id, id));
|
||||
assert.equal((await ping(`${path}/fail`, { method: "POST", body: "exit 3: disk full" })).status, 200);
|
||||
invalidateAlertsCache();
|
||||
const alert = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!;
|
||||
assert.match(alert.message, /Nightly backup reported a failure — exit 3: disk full/);
|
||||
assert.deepEqual([alert.severity, alert.category, alert.link], ["critical", "automation", "/heartbeats"]);
|
||||
|
||||
let r = await runEndpointPass(new Date());
|
||||
assert.equal(r.announced, 1);
|
||||
assert.match(messages.join("\n"), /Nightly backup reported a failure/);
|
||||
const sent = messages.length;
|
||||
r = await runEndpointPass(new Date());
|
||||
assert.equal(r.announced + r.recovered, 0);
|
||||
assert.equal(messages.length, sent, "not said again");
|
||||
|
||||
await ping(path);
|
||||
r = await runEndpointPass(new Date());
|
||||
assert.equal(r.recovered, 1);
|
||||
assert.match(messages[messages.length - 1], /Nightly backup reported a failure/, "the recovery names what cleared");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.filter((x) => x.source === "Heartbeat").length, 0);
|
||||
});
|
||||
|
||||
test("a call that does not come in time is a warning once it is overdue, silenced by maintenance on the server, and cleared by the call", async () => {
|
||||
messages.length = 0;
|
||||
await db.update(heartbeats).set({ serverId, periodMinutes: 60, graceMinutes: 10 }).where(eq(heartbeats.id, id));
|
||||
const t = (min: number) => new Date(Date.now() + min * 60_000);
|
||||
let r = await runEndpointPass(t(30));
|
||||
assert.equal(r.announced, 0, "not due yet");
|
||||
r = await runEndpointPass(t(75));
|
||||
assert.equal(r.announced, 1, "an hour and ten minutes with no call");
|
||||
assert.match(messages.join("\n"), /Nightly backup hasn't reported — it was due by/);
|
||||
|
||||
// The Alerts page reads the real clock, so the last call is moved back instead of the clock forward.
|
||||
await db.update(heartbeats).set({ lastPingAt: new Date(Date.now() - 120 * 60_000).toISOString() }).where(eq(heartbeats.id, id));
|
||||
invalidateAlertsCache();
|
||||
const late = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!;
|
||||
assert.equal(late.severity, "warning");
|
||||
assert.deepEqual(late.target && [late.target.type, late.target.id], ["server", serverId]);
|
||||
assert.equal(late.silenced, false);
|
||||
await startOrExtendWindow("server", serverId, 60, "work", "Ada");
|
||||
invalidateAlertsCache();
|
||||
assert.equal((await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!.silenced, true, "maintenance on the server it runs on silences it");
|
||||
|
||||
assert.match(await buildMetrics(t(75)), /homelab_heartbeat_up\{name="Nightly backup"\} 0/);
|
||||
await recordPing((await db.select().from(heartbeats).where(eq(heartbeats.id, id)))[0].token, "ok", null, t(76));
|
||||
r = await runEndpointPass(t(77));
|
||||
assert.equal(r.recovered, 1);
|
||||
assert.match(await buildMetrics(t(77)), /homelab_heartbeat_up\{name="Nightly backup"\} 1/);
|
||||
assert.match(await buildMetrics(t(77)), /homelab_heartbeat_last_ping_timestamp_seconds\{name="Nightly backup"\} \d+/);
|
||||
});
|
||||
|
||||
test("a switched-off heartbeat still takes calls, says nothing, and starts a fresh clock when it is switched on again", async () => {
|
||||
await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: false });
|
||||
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal((await ping(`/api/heartbeat/${h.token}`)).status, 200, "pausing a job's heartbeat does not break its script");
|
||||
assert.equal(heartbeatConditions([{ ...h, lastPingAt: "2020-01-01T00:00:00.000Z", lastStatus: "ok" }], new Date()).length, 0);
|
||||
await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: true });
|
||||
const [after_] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
|
||||
assert.equal(heartbeatState(after_, new Date()), "up");
|
||||
});
|
||||
|
||||
test("a new address replaces the old one at once; removing one takes its history", async () => {
|
||||
const res = await admin("POST", `/${id}/new-address`);
|
||||
assert.equal(res.status, 200);
|
||||
const fresh = (await res.json()).path as string;
|
||||
assert.notEqual(fresh, path);
|
||||
assert.equal((await ping(path)).status, 404, "the old one is dead");
|
||||
assert.equal((await ping(fresh)).status, 200);
|
||||
who = "viewer";
|
||||
assert.equal((await admin("POST", `/${id}/new-address`)).status, 403);
|
||||
who = "admin";
|
||||
assert.equal((await admin("DELETE", `/${id}`)).status, 204);
|
||||
assert.equal((await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id))).length, 0);
|
||||
assert.equal((await ping(fresh)).status, 404);
|
||||
assert.equal((await admin("DELETE", `/${id}`)).status, 404);
|
||||
});
|
||||
});
|
||||
@@ -194,7 +194,9 @@ describe("through the routes, the page, the alerts and the figures", () => {
|
||||
test("metrics say how old each test is and whether it passed", async () => {
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/);
|
||||
assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/);
|
||||
// The test's dates come from a fixed "now" and the metrics from the real clock, so what is expected is worked out the way the metric does.
|
||||
const realAge = Math.floor((Date.now() - Date.parse(`${daysAgo(40)}T12:00:00Z`)) / 86_400_000);
|
||||
assert.match(text, new RegExp(`homelab_restore_test_age_days\\{subject="Photos share"\\} ${realAge}`));
|
||||
assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/);
|
||||
assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured");
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user