Heartbeats, Coming up and suggested checks; Operations menu tidied

Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 21:42:34 +02:00
1 parent c05c847d17
commit 4e0c6927b2
44 files changed
+7092 -75

No files matched your search

+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE `heartbeat_pings` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`heartbeat_id` integer NOT NULL,
`at` text NOT NULL,
`kind` text NOT NULL,
`duration_ms` integer,
`message` text,
FOREIGN KEY (`heartbeat_id`) REFERENCES `heartbeats`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `heartbeat_pings_idx` ON `heartbeat_pings` (`heartbeat_id`,`at`);--> statement-breakpoint
CREATE TABLE `heartbeats` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`token` text NOT NULL,
`period_minutes` integer NOT NULL,
`grace_minutes` integer DEFAULT 10 NOT NULL,
`server_id` integer,
`enabled` integer DEFAULT true NOT NULL,
`notes` text,
`last_ping_at` text,
`last_status` text,
`last_message` text,
`last_duration_ms` integer,
`last_start_at` text,
`ping_count` integer DEFAULT 0 NOT NULL,
`created_at` text DEFAULT (current_timestamp) NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `heartbeats_token_unique` ON `heartbeats` (`token`);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -295,6 +295,13 @@
"when": 1792359507903,
"tag": "0041_open_silvermane",
"breakpoints": true
},
{
"idx": 42,
"version": "6",
"when": 1792361094159,
"tag": "0042_silly_omega_red",
"breakpoints": true
}
]
}
+9
View File
@@ -48,6 +48,10 @@ import { mapRouter } from "./routes/map.js";
import { generatorRouter } from "./routes/generator.js";
import { alertTicketsRouter } from "./routes/alertTickets.js";
import { avatarsRouter } from "./routes/avatars.js";
import { heartbeatsRouter } from "./routes/heartbeats.js";
import { agendaRouter } from "./routes/agenda.js";
import { publicCalendarRouter } from "./routes/publicCalendar.js";
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
import { securityHeaders } from "./utils/securityHeaders.js";
import { sameOriginGuard } from "./utils/sameOriginGuard.js";
@@ -72,6 +76,8 @@ export function createApp() {
// The agent endpoint comes first: it reads its own (larger) body, only after checking the token, and an agent has no use for a
// session. It is also the one place a request isn't from a browser, so it isn't subject to the same-origin check below.
app.use("/api/agent/report", agentReportRouter);
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
app.use("/api/heartbeat", heartbeatPingRouter);
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
app.use(sameOriginGuard);
app.use(express.json());
@@ -105,6 +111,8 @@ export function createApp() {
app.use("/auth", authRouter);
app.use("/api/me", meRouter);
app.use("/api/avatars", avatarsRouter);
app.use("/api/heartbeats", heartbeatsRouter);
app.use("/api/agenda", agendaRouter);
app.use("/api/uptime", uptimeRouter);
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
@@ -123,6 +131,7 @@ export function createApp() {
app.use("/metrics", metricsRouter);
app.use("/api/public/wall", publicWallRouter);
app.use("/api/public/status", publicStatusRouter);
app.use("/api/public/calendar", publicCalendarRouter);
app.use("/api/users", usersRouter);
app.use("/api/audit-log", auditLogRouter);
app.use("/api/diag-log", diagLogRouter);
+41 -2
View File
@@ -192,6 +192,44 @@ export const restoreTests = sqliteTable(
(t) => [index("restore_tests_subject_idx").on(t.subjectKey, t.testedAt)],
);
// A job that reports in: a cron job or backup script calls its address when it finishes, and the app speaks up when the call doesn't come in time —
// see services/heartbeats.ts. The token is part of the address (/api/heartbeat/<token>) and is all the access a ping needs.
export const heartbeats = sqliteTable("heartbeats", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
token: text("token").notNull().unique(),
periodMinutes: integer("period_minutes").notNull(), // how often it should report
graceMinutes: integer("grace_minutes").notNull().default(10), // how much later than that is still fine
serverId: integer("server_id"), // soft reference: the server it runs on, so its maintenance silences the alerts
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
notes: text("notes"),
lastPingAt: text("last_ping_at"), // ISO; the last report of an end (success or failure)
lastStatus: text("last_status"), // ok | fail
lastMessage: text("last_message"),
lastDurationMs: integer("last_duration_ms"), // when it said it had started first
lastStartAt: text("last_start_at"), // ISO; the last "started" call
pingCount: integer("ping_count").notNull().default(0),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
});
// The recent calls to each heartbeat, for its history; kept two weeks.
export const heartbeatPings = sqliteTable(
"heartbeat_pings",
{
id: integer("id").primaryKey({ autoIncrement: true }),
heartbeatId: integer("heartbeat_id")
.notNull()
.references(() => heartbeats.id, { onDelete: "cascade" }),
at: text("at").notNull(), // ISO
kind: text("kind").notNull(), // ok | fail | start
durationMs: integer("duration_ms"),
message: text("message"),
},
(t) => [index("heartbeat_pings_idx").on(t.heartbeatId, t.at)],
);
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
export const dependencies = sqliteTable(
@@ -628,8 +666,9 @@ export const alertTickets = sqliteTable(
export const shareLinkDetails = ["summary", "names"] as const;
export type ShareLinkDetail = (typeof shareLinkDetails)[number];
// What a link opens: the wall display (how things are, for a TV), or a status page (is each of the things I chose working, for people at home).
export const shareLinkKinds = ["wall", "status"] as const;
// What a link opens: the wall display (how things are, for a TV), a status page (is each of the things I chose working, for people at home), or
// a calendar feed (what is coming up, for a phone's calendar to subscribe to).
export const shareLinkKinds = ["wall", "status", "calendar"] as const;
export type ShareLinkKind = (typeof shareLinkKinds)[number];
// A link to the wall display that needs no sign-in, for a TV in the hall or someone who only wants to know if things are up. Only a hash
+15
View File
@@ -0,0 +1,15 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { buildAgenda, DEFAULT_DAYS, MAX_DAYS } from "../services/agenda.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const agendaRouter = Router();
agendaRouter.use(requireAuth);
// Everything with a date ahead, for everyone signed in: it is only what the pages it comes from already show them. `days` is how far ahead.
agendaRouter.get("/", asyncHandler(async (req, res) => {
const asked = Number(req.query.days);
const days = Number.isInteger(asked) && asked >= 1 && asked <= MAX_DAYS ? asked : DEFAULT_DAYS;
res.json(await buildAgenda(new Date(), days));
}));
+48 -1
View File
@@ -7,6 +7,7 @@ import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const endpointsRouter = Router();
@@ -84,8 +85,54 @@ async function serverExists(id: number | null | undefined) {
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
}
// Checks worth setting up, from what the app already knows: a web port a server is listening on, a secret that reads a certificate. Reads what is
// stored, asks nobody. Administrators only, like the checks themselves.
endpointsRouter.get("/suggestions", requireRole("admin"), asyncHandler(async (_req, res) => {
res.json(await loadSuggestions());
}));
const keysSchema = z.object({ keys: z.array(z.string().min(1).max(600)).min(1).max(MAX_CHECKS) });
// Makes the checks for the suggestions named. What each one is comes from the server's own list, not from the request, so a request can only
// ever add what was offered.
endpointsRouter.post("/suggestions/add", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = keysSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to add." });
const offered = new Map((await loadSuggestions()).suggestions.map((s) => [s.key, s]));
let have = (await db.select({ id: endpointChecks.id }).from(endpointChecks)).length;
const added: { id: number; name: string }[] = [];
for (const key of new Set(parsed.data.keys)) {
const s = offered.get(key);
if (!s) continue;
if (have >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.`, added });
const [row] = await db
.insert(endpointChecks)
.values(valuesOf({ name: s.name, kind: s.kind, target: s.target, port: s.port, serverId: s.serverId }))
.returning();
have++;
added.push({ id: row.id, name: row.name });
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target, suggested: true } });
}
res.status(201).json({ added });
}));
// "Not for me": what is dismissed stops being offered, and can be brought back.
endpointsRouter.post("/suggestions/dismiss", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = keysSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to dismiss." });
const keys = await loadDismissed();
for (const k of parsed.data.keys) keys.add(k);
await saveDismissed(keys);
res.json(await loadSuggestions());
}));
endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(async (_req, res) => {
await saveDismissed(new Set());
res.json(await loadSuggestions());
}));
// Asking things on the network is an administrator's to set up.
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = checkSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
+32
View File
@@ -0,0 +1,32 @@
import express, { Router } from "express";
import { cleanMessage, recordPing, type PingKind } from "../services/heartbeats.js";
import { rateLimit } from "../utils/rateLimit.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/**
* The address a job calls: /api/heartbeat/<token>, /api/heartbeat/<token>/start, /api/heartbeat/<token>/fail. No session and no sign-in — the
* token is the whole of the access, and all it allows is to say "I'm alive". Mounted ahead of the browser-only middleware (a script isn't a
* browser and has no cookie), so it answers GET, HEAD, POST and PUT alike: whatever `curl` or `wget` happens to send.
*
* It answers a token that doesn't exist, and a path that isn't one of the three, in the same two words, and is limited per address.
*/
export const heartbeatPingRouter = Router();
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
// Whatever a script sends as the body ("3.2 GB copied") is a note; it is cut down to a line.
const body = express.text({ type: () => true, limit: "100kb" });
const KINDS: Record<string, PingKind> = { "": "ok", ok: "ok", start: "start", fail: "fail" };
heartbeatPingRouter.all("/:token/:kind?", limiter, body, asyncHandler(async (req, res) => {
res.setHeader("Cache-Control", "no-store");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
if (!["GET", "HEAD", "POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
const kind = KINDS[(req.params.kind ?? "").toLowerCase()];
if (!kind) return res.status(404).type("text/plain").send("Not found");
const fromQuery = typeof req.query.msg === "string" ? req.query.msg : null;
const message = cleanMessage(fromQuery ?? (typeof req.body === "string" ? req.body : null));
const heartbeat = await recordPing(req.params.token, kind, message);
if (!heartbeat) return res.status(404).type("text/plain").send("Not found");
res.type("text/plain").send("OK");
}));
+111
View File
@@ -0,0 +1,111 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { heartbeats, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { dueBy, heartbeatState, listHeartbeats, loadRecent, MAX_GRACE_MINUTES, MAX_PERIOD_MINUTES, newToken } from "../services/heartbeats.js";
import { env } from "../env.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const heartbeatsRouter = Router();
heartbeatsRouter.use(requireAuth);
const MAX_HEARTBEATS = 100;
const schema = z.object({
name: z.string().trim().min(1).max(100),
periodMinutes: z.number().int().min(1).max(MAX_PERIOD_MINUTES),
graceMinutes: z.number().int().min(0).max(MAX_GRACE_MINUTES).optional(),
serverId: z.number().int().positive().nullable().optional(),
notes: z.string().max(500).nullable().optional(),
enabled: z.boolean().optional(),
});
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const pathOf = (token: string) => `/api/heartbeat/${token}`;
async function serverExists(id: number | null | undefined) {
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
}
// Everyone signed in can see how the jobs are doing, like the Alerts page. The address with its token is shown only to administrators: it
// is what lets anyone say "I'm alive" for the job.
heartbeatsRouter.get("/", asyncHandler(async (req, res) => {
const now = new Date();
const isAdmin = req.currentUser!.role === "admin";
const rows = await listHeartbeats();
const recent = await loadRecent(rows.map((r) => r.id));
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
res.json({
baseUrl: isAdmin ? env.appBaseUrl.replace(/\/+$/, "") : null,
heartbeats: rows.map(({ token, ...r }) => ({
...r,
path: isAdmin ? pathOf(token) : null,
state: heartbeatState(r, now),
dueBy: dueBy(r).toISOString(),
server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null,
recent: recent.get(r.id) ?? [],
})),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
});
}));
heartbeatsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = schema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.error.flatten() });
if ((await db.select({ id: heartbeats.id }).from(heartbeats)).length >= MAX_HEARTBEATS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_HEARTBEATS} heartbeats.` });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const d = parsed.data;
const [row] = await db
.insert(heartbeats)
.values({ name: d.name, token: newToken(), periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? 10, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? true })
.returning();
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "create", targetType: "heartbeat", targetId: row.id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
const { token, ...rest } = row;
res.status(201).json({ heartbeat: { ...rest, path: pathOf(token) } });
}));
heartbeatsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = schema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.success ? undefined : parsed.error.flatten() });
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const d = parsed.data;
const [row] = await db
.update(heartbeats)
.set({ name: d.name, periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? before.graceMinutes, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? before.enabled })
.where(eq(heartbeats.id, id))
.returning();
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "update", targetType: "heartbeat", targetId: id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
invalidateAlertsCache();
const { token, ...rest } = row;
res.json({ heartbeat: { ...rest, path: pathOf(token) } });
}));
// A new address, when the old one has got out. The old one stops working at once.
heartbeatsRouter.post("/:id/new-address", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.update(heartbeats).set({ token: newToken() }).where(eq(heartbeats.id, id)).returning();
if (!row) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "new_address", targetType: "heartbeat", targetId: id, detail: { name: row.name } });
res.json({ path: pathOf(row.token) });
}));
heartbeatsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(heartbeats).where(eq(heartbeats.id, id));
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "delete", targetType: "heartbeat", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));
+24
View File
@@ -0,0 +1,24 @@
import { Router } from "express";
import { NOT_FOUND, resolveShareLink } from "../services/shareLinks.js";
import { buildAgenda, buildIcs, calendarName, MAX_DAYS } from "../services/agenda.js";
import { env } from "../env.js";
import { rateLimit } from "../utils/rateLimit.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const publicCalendarRouter = Router();
// No sign-in: the secret in the address is the whole of the access, so this reads nothing but what is coming up, and answers a link that doesn't
// work — never made, ended, taken back, or one that opens something other than a calendar — in the same words. The address ends in .ics because
// calendar apps want that.
const limiter = rateLimit({ windowMs: 60_000, max: 60, message: "Too many requests. Wait a minute." });
publicCalendarRouter.get("/:file", limiter, asyncHandler(async (req, res) => {
const token = req.params.file.endsWith(".ics") ? req.params.file.slice(0, -4) : req.params.file;
const link = await resolveShareLink(token);
if (!link || link.kind !== "calendar") return res.status(404).json(NOT_FOUND);
const feed = buildIcs(await buildAgenda(new Date(), MAX_DAYS), { name: calendarName(), detail: link.detail, baseUrl: link.detail === "names" ? env.appBaseUrl : undefined });
res.setHeader("Cache-Control", "private, max-age=300");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
res.setHeader("Content-Disposition", 'inline; filename="homelab-coming-up.ics"');
res.type("text/calendar; charset=utf-8").send(feed);
}));
+2 -2
View File
@@ -20,7 +20,7 @@ const MAX_LINKS = 20;
const createSchema = z.object({
label: z.string().trim().min(1).max(60),
kind: z.enum(shareLinkKinds).default("wall"),
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it.
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it. For a calendar: summary leaves the names out of the events.
detail: z.enum(shareLinkDetails).default("summary"),
title: z.string().trim().max(MAX_TITLE).optional(),
message: z.string().trim().max(MAX_MESSAGE).optional(),
@@ -59,7 +59,7 @@ shareLinksRouter.post("/", asyncHandler(async (req, res) => {
targetId: link.id,
detail: { label: link.label, kind: link.kind, detail: link.detail, expiresAt: link.expiresAt },
});
res.status(201).json({ link: publicShareLink(link), token, path: `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
res.status(201).json({ link: publicShareLink(link), token, path: link.kind === "calendar" ? `/api/public/calendar/${token}.ics` : `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
}));
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
+343
View File
@@ -0,0 +1,343 @@
/**
* "Coming up": everything with a date ahead, in one list — a secret, domain, certificate or Tailscale key that expires, a warranty or end of support,
* planned and recurring maintenance, a restore test that is due, and updates that have been waiting. Each already lives on a page of its own and in the
* weekly digest; this puts them side by side, oldest first, so what is about to need attention is one look, not a dozen.
*
* Built in two parts so it can be tested exactly: `loadAgendaData` reads what is stored (and, for Tailscale keys, asks Tailscale, remembered for ten
* minutes), and `agendaFromData` turns that into items. `buildIcs` writes the same items as an iCalendar feed for a phone's calendar to subscribe to.
*/
import { createHash } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { latestTests } from "./restoreTests.js";
import { getSettings } from "./settingsStore.js";
import { env } from "../env.js";
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates"] as const;
export type AgendaKind = (typeof AGENDA_KINDS)[number];
export const KIND_LABELS: Record<AgendaKind, string> = {
secret: "Secrets",
certificate: "Certificates",
domain: "Domains",
tailscale: "Tailscale keys",
warranty: "Warranties",
support: "End of support",
maintenance: "Maintenance",
restore: "Restore tests",
updates: "Updates waiting",
};
export interface AgendaItem {
/** Stable, with the date in it: a secret that is renewed is a different entry, and the old one just goes. */
id: string;
kind: AgendaKind;
/** YYYY-MM-DD for something that happens on a day; an ISO time for maintenance, which has a start and an end. */
at: string;
endsAt: string | null;
allDay: boolean;
title: string;
detail: string | null;
/** The page of the app that has more. */
link: string;
serverId: number | null;
/** Whole days from today to the day it falls on, in the time zone set for notifications; negative once it has passed. */
daysLeft: number;
}
export interface Agenda {
generatedAt: string;
today: string;
items: AgendaItem[];
/** Sources that could not be read, so the agenda is missing something. */
notes: string[];
}
export const DEFAULT_DAYS = 90;
export const MAX_DAYS = 400;
/** What has already happened is shown for this long (a secret that expired last week is still worth seeing). */
export const LOOK_BACK_DAYS = 30;
const DAY_MS = 86_400_000;
// ─── Dates ─────────────────────────────────────────────────────────────────
const isYmd = (v: unknown): v is string => typeof v === "string" && /^\d{4}-\d{2}-\d{2}$/.test(v) && !Number.isNaN(Date.parse(`${v}T00:00:00Z`)) && new Date(`${v}T00:00:00Z`).toISOString().startsWith(v);
/** The calendar day an instant falls on, as the clock in `tz` sees it. */
export function localDate(when: Date, tz: string): string {
return new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit" }).format(when);
}
const daysBetween = (fromYmd: string, toYmd: string) => Math.round((Date.parse(`${toYmd}T00:00:00Z`) - Date.parse(`${fromYmd}T00:00:00Z`)) / DAY_MS);
export const addDays = (ymd: string, n: number) => new Date(Date.parse(`${ymd}T00:00:00Z`) + n * DAY_MS).toISOString().slice(0, 10);
// ─── What is stored ────────────────────────────────────────────────────────
export interface AgendaData {
secrets: { id: number; name: string; type: string; expiryDate: string | null }[];
domains: { id: number; name: string; expiresAt: string | null }[];
endpoints: { id: number; name: string; enabled: boolean; certExpiresAt: string | null; serverId: number | null }[];
tailscale: { integrationId: number; integrationName: string; deviceLabel: string; daysLeft: number }[];
assets: { id: number; name: string; warrantyUntil: string | null; endOfSupport: string | null; serverId: number | null }[];
/** Windows that have been set up but not begun, one per target. */
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
/** The newest restore test of each subject. */
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
restoreLimitDays: number;
/** Servers with updates waiting. */
updates: { name: string; count: number; security: number | null; since: string | null }[];
notes: string[];
}
let tailscaleCache: { at: number; items: AgendaData["tailscale"]; failures: string[] } | null = null;
const TAILSCALE_CACHE_MS = 10 * 60_000;
export const clearAgendaCacheForTests = () => {
tailscaleCache = null;
};
async function tailscaleKeys(): Promise<{ items: AgendaData["tailscale"]; failures: string[] }> {
if (tailscaleCache && Date.now() - tailscaleCache.at < TAILSCALE_CACHE_MS) return tailscaleCache;
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "tailscale"), eq(integrations.enabled, true)));
if (rows.length === 0) return { items: [], failures: [] };
const { items, failures } = await collectTailscaleKeyExpiries();
tailscaleCache = { at: Date.now(), items, failures: failures.map((f) => `Tailscale (${f.integrationName}): ${f.message}`) };
return tailscaleCache;
}
export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData> {
const notes: string[] = [];
const settings = await getSettings();
const [secretRows, domainRows, endpointRows, assetRows, scheduleRows, serverRows, tests] = await Promise.all([
db.select({ id: secrets.id, name: secrets.name, type: secrets.type, expiryDate: secrets.expiryDate }).from(secrets),
db.select({ id: domains.id, name: domains.name, expiresAt: domains.expiresAt }).from(domains),
db.select({ id: endpointChecks.id, name: endpointChecks.name, enabled: endpointChecks.enabled, certExpiresAt: endpointChecks.certExpiresAt, serverId: endpointChecks.serverId }).from(endpointChecks),
db.select({ id: assets.id, name: assets.name, warrantyUntil: assets.warrantyUntil, endOfSupport: assets.endOfSupport, serverId: assets.serverId }).from(assets),
db.select().from(maintenanceSchedules),
db.select({ name: servers.name, pendingUpdates: servers.pendingUpdates, since: servers.updatesPendingSince }).from(servers),
latestTests(),
]);
const windows: AgendaData["windows"] = [];
for (const w of await listUpcomingWindows(now)) {
const target = await describeTarget(w.targetType, w.targetId);
if (target) windows.push({ startedAt: w.startedAt, endsAt: w.endsAt, reason: w.reason, source: w.source, targetName: target.name });
}
const schedules: AgendaData["schedules"] = [];
for (const s of scheduleRows.filter((r) => r.enabled)) {
const names: string[] = [];
for (const t of targetsOf(s)) {
const d = await describeTarget(t.type, t.id);
if (d) names.push(d.name);
}
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
}
const updates: AgendaData["updates"] = [];
for (const s of serverRows) {
const u = parsePendingUpdates(s.pendingUpdates);
if (u && u.count > 0) updates.push({ name: s.name, count: u.count, security: u.security, since: s.since });
}
let tailscale: AgendaData["tailscale"] = [];
try {
const t = await tailscaleKeys();
tailscale = t.items;
notes.push(...t.failures);
} catch (err) {
notes.push(`Tailscale keys could not be read: ${err instanceof Error ? err.message : String(err)}`);
}
return {
secrets: secretRows,
domains: domainRows,
endpoints: endpointRows,
tailscale,
assets: assetRows,
windows,
schedules,
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
restoreLimitDays: settings.healthChecks.restoreTestDays,
updates,
notes,
};
}
// ─── Making the list ───────────────────────────────────────────────────────
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
/** Pure: the items, from what was read. `days` is how far ahead; what has just passed is kept for LOOK_BACK_DAYS. */
export function agendaFromData(data: AgendaData, now: Date, tz: string, days: number = DEFAULT_DAYS): Agenda {
const today = localDate(now, tz);
const horizon = Math.min(Math.max(Math.round(days), 1), MAX_DAYS);
const items: AgendaItem[] = [];
const day = (id: string, kind: AgendaKind, ymd: string | null, fn: (past: boolean) => { title: string; detail?: string | null; link: string; serverId?: number | null }) => {
if (!isYmd(ymd)) return;
const daysLeft = daysBetween(today, ymd);
if (daysLeft < -LOOK_BACK_DAYS || daysLeft > horizon) return;
const { title, detail = null, link, serverId = null } = fn(daysLeft < 0);
items.push({ id: `${id}:${ymd}`, kind, at: ymd, endsAt: null, allDay: true, title, detail, link, serverId, daysLeft });
};
for (const s of data.secrets) {
const certificate = s.type === "ssl_certificate";
day(`secret:${s.id}`, certificate ? "certificate" : "secret", s.expiryDate, (past) => ({ title: `${certificate ? "Certificate" : "Secret"} “${s.name}” ${past ? "expired" : "expires"}`, detail: certificate ? "kept as a secret" : null, link: "/secrets" }));
}
for (const d of data.domains) day(`domain:${d.id}`, "domain", d.expiresAt, (past) => ({ title: `Domain ${d.name} ${past ? "expired" : "expires"}`, link: "/domains" }));
for (const e of data.endpoints) {
if (!e.enabled || !e.certExpiresAt) continue;
day(`endpoint-cert:${e.id}`, "certificate", e.certExpiresAt.slice(0, 10), (past) => ({ title: `The certificate for ${e.name} ${past ? "expired" : "expires"}`, detail: "seen by an endpoint check", link: "/endpoints", serverId: e.serverId }));
}
for (const t of data.tailscale) {
day(`tailscale:${t.integrationId}:${t.deviceLabel}`, "tailscale", addDays(today, t.daysLeft), (past) => ({ title: `Tailscale key for ${t.deviceLabel} ${past ? "expired" : "expires"}`, detail: t.integrationName, link: "/tailscale" }));
}
for (const a of data.assets) {
day(`warranty:${a.id}`, "warranty", a.warrantyUntil, (past) => ({ title: `Warranty on ${a.name} ${past ? "ended" : "ends"}`, link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
day(`support:${a.id}`, "support", a.endOfSupport, (past) => ({ title: `Support for ${a.name} ${past ? "ended" : "ends"}`, detail: "time to plan its replacement", link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
}
// A restore test is due a set time after the last good one — only if a time limit is set (Settings → Notifications).
if (data.restoreLimitDays > 0) {
for (const t of data.restoreTests) {
if ((t.result !== "ok" && t.result !== "partial") || !isYmd(t.testedAt)) continue;
day(`restore:${t.subjectKey}`, "restore", addDays(t.testedAt, data.restoreLimitDays), (past) => ({ title: `Restore test ${past ? "was due" : "due"}: ${t.label}`, detail: `last tested ${t.testedAt}`, link: "/backups" }));
}
}
// Maintenance has a start and an end. Windows set up by a recurring schedule are left to the schedule (they are only made a week ahead).
const lastMs = now.getTime() + horizon * DAY_MS;
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
for (const w of data.windows) {
if (w.source?.startsWith("schedule:") || Date.parse(w.startedAt) > lastMs) continue;
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
g.names.push(w.targetName);
groups.set(key, g);
}
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
const names = (list: string[]) => {
const sorted = [...list].sort((a, b) => a.localeCompare(b));
return sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
};
for (const g of groups.values()) timed(`maintenance:${g.start}:${g.names.length}`, g.start, g.end, `Maintenance: ${names(g.names)}`, g.reason);
for (const s of data.schedules) {
for (const occ of occurrencesBetween(s, now, new Date(lastMs)).slice(0, 60)) {
timed(`schedule:${s.id}:${occ.date}`, occ.start.toISOString(), occ.end.toISOString(), `Recurring maintenance: ${s.name}`, [s.targetNames.length > 0 ? names(s.targetNames) : null, s.reason].filter(Boolean).join(" — ") || null);
}
}
// Updates that have been waiting: one line for all of them, from the longest wait.
if (data.updates.length > 0) {
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
const security = data.updates.filter((u) => (u.security ?? 0) > 0).length;
const waited = Math.max(0, daysBetween(since, today));
items.push({
id: `updates:${since}`,
kind: "updates",
at: since,
endsAt: null,
allDay: true,
title: `Updates waiting on ${plural(data.updates.length, "server")}${security > 0 ? ` (${security} with security fixes)` : ""}`,
detail: waited > 0 ? `the longest wait is ${plural(waited, "day")}` : "reported today",
link: "/updates",
serverId: null,
daysLeft: -waited,
});
}
items.sort((a, b) => a.at.localeCompare(b.at) || a.title.localeCompare(b.title));
return { generatedAt: now.toISOString(), today, items, notes: data.notes };
}
export async function buildAgenda(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<Agenda> {
const settings = await getSettings();
return agendaFromData(await loadAgendaData(now), now, settings.notifications.timezone || "UTC", days);
}
// ─── The calendar feed ─────────────────────────────────────────────────────
/** Escapes a text value for iCalendar: backslash, semicolon, comma and line breaks. */
const escapeText = (s: string) => s.replace(/\\/g, "\\\\").replace(/;/g, "\\;").replace(/,/g, "\\,").replace(/\r?\n/g, "\\n");
/** Lines are folded at 75 bytes, a space starting each continuation, without splitting a character. */
export function foldLine(line: string): string {
const out: string[] = [];
let current = "";
let bytes = 0;
for (const ch of line) {
const n = Buffer.byteLength(ch, "utf8");
if (bytes + n > (out.length === 0 ? 75 : 74)) {
out.push(current);
current = "";
bytes = 0;
}
current += ch;
bytes += n;
}
out.push(current);
return out.join("\r\n ");
}
const icsDate = (ymd: string) => ymd.replace(/-/g, "");
const icsTime = (iso: string) => new Date(iso).toISOString().replace(/[-:]/g, "").replace(/\.\d{3}/, "");
/** What an event is called when the link was made to show no names: only what kind of thing it is. */
const GENERIC: Record<AgendaKind, string> = {
secret: "A secret expires",
certificate: "A certificate expires",
domain: "A domain name expires",
tailscale: "A Tailscale key expires",
warranty: "A warranty ends",
support: "Support ends for a device",
maintenance: "Planned maintenance",
restore: "A restore test is due",
updates: "Updates are waiting",
};
/**
* The agenda as an iCalendar feed. A day is an all-day event; maintenance is an event from its start to its end. With `detail` "summary" an event only
* says what kind of thing it is (no names), so a feed that is shared can be shown without telling anyone what is in the homelab. Updates that
* are waiting are not events — they have no day — and are left out.
*/
export function buildIcs(agenda: Agenda, options: { name: string; detail: "summary" | "names"; baseUrl?: string }): string {
const lines = [
"BEGIN:VCALENDAR",
"VERSION:2.0",
"PRODID:-//Homelab Manager//Coming up//EN",
"CALSCALE:GREGORIAN",
"METHOD:PUBLISH",
`X-WR-CALNAME:${escapeText(options.name)}`,
"REFRESH-INTERVAL;VALUE=DURATION:PT6H",
"X-PUBLISHED-TTL:PT6H",
];
const stamp = icsTime(agenda.generatedAt);
for (const item of agenda.items) {
if (item.kind === "updates") continue;
if (item.daysLeft < -7) continue; // an old event in a subscribed calendar is just clutter
const names = options.detail === "names";
lines.push("BEGIN:VEVENT", `UID:${createHash("sha256").update(item.id).digest("hex").slice(0, 24)}@homelab-manager`, `DTSTAMP:${stamp}`);
if (item.allDay) lines.push(`DTSTART;VALUE=DATE:${icsDate(item.at)}`, `DTEND;VALUE=DATE:${icsDate(addDays(item.at, 1))}`);
else lines.push(`DTSTART:${icsTime(item.at)}`, `DTEND:${icsTime(item.endsAt ?? item.at)}`);
lines.push(`SUMMARY:${escapeText(names ? item.title : GENERIC[item.kind])}`);
if (names) {
const description = [item.detail, options.baseUrl ? `${options.baseUrl.replace(/\/+$/, "")}${item.link}` : null].filter(Boolean).join("\n");
if (description) lines.push(`DESCRIPTION:${escapeText(description)}`);
}
lines.push(`CATEGORIES:${escapeText(KIND_LABELS[item.kind])}`, "TRANSP:TRANSPARENT", "STATUS:CONFIRMED", "END:VEVENT");
}
lines.push("END:VCALENDAR");
return `${lines.map(foldLine).join("\r\n")}\r\n`;
}
/** What the feed calls itself. */
export const calendarName = () => `Homelab Manager${env.environment?.name ? ` (${env.environment.name})` : ""} — coming up`;
+19 -1
View File
@@ -11,7 +11,7 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
@@ -35,6 +35,7 @@ import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import { endpointConditions } from "./endpointChecks.js";
import { heartbeatConditions } from "./heartbeats.js";
import { latestTests, restoreAlerts } from "./restoreTests.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
@@ -65,6 +66,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
};
const SOURCE_TIMEOUT_MS = 20_000;
@@ -304,6 +306,22 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// Jobs that report in: one that has not done so in time, or said it failed.
check("Heartbeats", async () => {
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
push({
category: c.category,
severity: c.severity,
type: "heartbeat",
message: c.message,
key: c.key,
link: "/heartbeats",
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
});
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
check("Homelab Manager's own backups", async () => {
+13 -3
View File
@@ -21,7 +21,7 @@ export const KIND_LABELS: Record<ChangeKind, string> = {
dependencies: "Dependencies",
inventory: "Inventory",
backups: "Backups and restore tests",
endpoints: "Endpoint checks",
endpoints: "Endpoint checks and heartbeats",
other: "Other",
};
@@ -53,6 +53,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
db_backup: "backups",
restore_test: "backups",
endpoint: "endpoints",
heartbeat: "endpoints",
};
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
@@ -154,6 +155,14 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "heartbeat/create":
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
case "heartbeat/update":
return { title: `Heartbeat changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
case "heartbeat/new_address":
return { title: `Heartbeat given a new address${name ? `: ${name}` : ""}`, tone: "neutral", detail: "the old address no longer works" };
case "heartbeat/delete":
return { title: `Heartbeat removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "restore_test/record": {
const result = str(detail.result);
const tone: ActivityTone = result === "failed" ? "bad" : result === "not_needed" ? "neutral" : "good";
@@ -250,7 +259,8 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
} else if (a.targetType === "endpoint" && idNum !== null) {
targetName = endpointName.get(idNum) ?? targetName;
link = "/endpoints";
} else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
} else if (a.targetType === "heartbeat") link = "/heartbeats";
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
else if (a.targetType === "restore_test") link = "/backups";
else if (a.targetType === "dependency") link = "/dependencies";
else if (a.targetType === "settings") link = "/settings/notifications";
@@ -280,7 +290,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
.limit(1000);
for (const i of incidents) {
const serverId = i.targetType === "server" ? i.targetId : null;
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/incidents/${i.id}`, serverId };
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/alerts/incidents/${i.id}`, serverId };
if (i.openedAt >= since.toISOString() && (!until || i.openedAt <= until.toISOString())) {
entries.push({ detail: {}, entry: { ...base, id: `incident-open-${i.id}`, at: i.openedAt, tone: i.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised: ${i.message}`, detail: i.causedBy ? `explained by ${i.causedBy}` : null } });
}
+6 -2
View File
@@ -5,9 +5,10 @@
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
*/
import { db } from "../db/client.js";
import { endpointChecks } from "../db/schema.js";
import { endpointChecks, heartbeats } from "../db/schema.js";
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
import { activeSubjects } from "./maintenance.js";
import { heartbeatConditions, prunePings } from "./heartbeats.js";
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
@@ -28,7 +29,9 @@ async function loadState(): Promise<ActiveState> {
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
const asked = await runDueChecks(now, probeFn);
const rows = await db.select().from(endpointChecks);
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
const beats = await db.select().from(heartbeats);
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
if (added.length > 0) await notifyHealthIssues(added);
@@ -48,6 +51,7 @@ export function initEndpointMonitor(): void {
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneResults();
await prunePings();
}
} catch (err) {
console.error("[endpoints] the pass failed:", err);
+186
View File
@@ -0,0 +1,186 @@
/**
* Suggested endpoint checks: things the app already knows enough about to offer a check for, so setting up monitoring is a click instead of typing.
*
* - A web port a server's agent sees open (80, 443, 8080, 8443), with a name that points at it in DNS if there is one — "git.example.lan answers on
* 443 on nas" — as a web check, linked to that server so its maintenance silences the alerts.
* - A secret that already has a host to read the certificate from, as a certificate check: the secret only warns about the expiry, this also hears
* when the service stops answering.
*
* It only reads what is stored (no network), leaves out what a check already covers, and remembers what somebody said no to. Uptime Kuma's monitors are
* not offered: they are already being watched, and the adapter only keeps a host and port for them, not the address.
*/
import { db } from "../db/client.js";
import { dnsRecordsCache, endpointChecks, secrets, servers } from "../db/schema.js";
import { isLoopback } from "./agentPorts.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export interface Suggestion {
/** Stable identity, so a dismissed suggestion stays dismissed and "add" can name it. */
key: string;
kind: "http" | "tls";
name: string;
target: string;
port: number | null;
serverId: number | null;
serverName: string | null;
/** Why it is offered, in a sentence. */
reason: string;
}
export interface SuggestionInput {
servers: { id: number; name: string; hostname: string | null; ips: string[]; ports: { protocol?: string; port: number; address?: string }[] }[];
dns: { name: string; type: string; content: string }[];
secrets: { name: string; checkHost: string | null; checkPort: number | null }[];
existing: { kind: string; target: string; port: number | null }[];
dismissed: Set<string>;
}
/** The web ports worth offering, and what they speak. */
const WEB_PORTS: Record<number, "http" | "https"> = { 80: "http", 443: "https", 8080: "http", 8443: "https" };
/** The https twin of a plain port, so a server answering on both gets the one that reads the certificate. */
const SECURE_TWIN: Record<number, number> = { 80: 443, 8080: 8443 };
const MAX_NAMES_PER_PORT = 3;
const MAX_SUGGESTIONS = 200;
const cleanHost = (h: string) => h.trim().toLowerCase().replace(/\.$/, "");
const isIPv4 = (s: string) => /^\d{1,3}(\.\d{1,3}){3}$/.test(s);
const usableIPv4 = (s: string) => isIPv4(s) && !s.startsWith("127.") && !s.startsWith("169.254.");
/** Where a check's address is, in the same words for an existing check and a suggestion: "web:host:port" and "tls:host:port" (a web check on https reads the certificate too). */
function coverage(existing: SuggestionInput["existing"]): Set<string> {
const out = new Set<string>();
for (const c of existing) {
if (c.kind === "http") {
try {
const u = new URL(c.target);
const port = u.port ? Number(u.port) : u.protocol === "https:" ? 443 : 80;
const host = cleanHost(u.hostname);
out.add(`web:${host}:${port}`);
if (u.protocol === "https:") out.add(`tls:${host}:${port}`);
} catch {
// an address that isn't one covers nothing
}
} else if (c.kind === "tls" && c.port !== null) out.add(`tls:${cleanHost(c.target)}:${c.port}`);
}
return out;
}
/** Pure: the checks worth offering, from what is known. */
export function suggestChecks(input: SuggestionInput): { suggestions: Suggestion[]; dismissed: Suggestion[] } {
const covered = coverage(input.existing);
const all = new Map<string, Suggestion>();
for (const s of input.servers) {
const ips = new Set(s.ips.map((i) => i.trim().toLowerCase()));
const hostname = s.hostname ? cleanHost(s.hostname) : null;
// Names that point at this server in DNS: an address record with one of its addresses, and an alias to one of those names or to its hostname.
const direct = new Set<string>();
for (const r of input.dns) {
if ((r.type === "A" || r.type === "AAAA") && ips.has(r.content.trim().toLowerCase())) direct.add(cleanHost(r.name));
}
const names = new Set(direct);
for (const r of input.dns) {
if (r.type !== "CNAME") continue;
const to = cleanHost(r.content);
if (direct.has(to) || (hostname !== null && to === hostname)) names.add(cleanHost(r.name));
}
const dnsNames = [...names].filter((n) => n.includes(".") && !n.startsWith("*") && !n.endsWith(".arpa") && !/\s/.test(n)).sort((a, b) => a.length - b.length || a.localeCompare(b));
const listening = new Set(s.ports.filter((p) => (p.protocol ?? "tcp") === "tcp" && !(p.address && isLoopback(p.address))).map((p) => p.port));
for (const port of Object.keys(WEB_PORTS).map(Number).filter((p) => listening.has(p))) {
// Plain and secure on the same server: the secure one is the real service (the plain one is most likely its redirect), and reads the certificate.
if (SECURE_TWIN[port] !== undefined && listening.has(SECURE_TWIN[port])) continue;
const scheme = WEB_PORTS[port];
const fallback = hostname && hostname.includes(".") ? [hostname] : [s.ips.map((i) => i.trim()).find(usableIPv4)].filter((x): x is string => !!x);
const hosts = dnsNames.length > 0 ? dnsNames.slice(0, MAX_NAMES_PER_PORT) : fallback;
for (const host of hosts) {
const defaultPort = scheme === "https" ? 443 : 80;
const target = `${scheme}://${host}${port === defaultPort ? "" : `:${port}`}/`;
if (covered.has(`web:${host}:${port}`)) continue;
const viaDns = dnsNames.includes(host);
all.set(`http|${target}`, {
key: `http|${target}`,
kind: "http",
name: `${host}${port === defaultPort ? "" : `:${port}`}`,
target,
port: null,
serverId: s.id,
serverName: s.name,
reason: `${s.name} is listening on port ${port}${viaDns ? `, and ${host} points at it in DNS` : ""}.`,
});
}
}
}
for (const sec of input.secrets) {
if (!sec.checkHost || sec.checkPort === null) continue;
const host = cleanHost(sec.checkHost);
if (covered.has(`tls:${host}:${sec.checkPort}`)) continue;
all.set(`tls|${host}|${sec.checkPort}`, {
key: `tls|${host}|${sec.checkPort}`,
kind: "tls",
name: `Certificate: ${sec.name}`,
target: host,
port: sec.checkPort,
serverId: null,
serverName: null,
reason: `The secret “${sec.name}” already reads this certificate; a check also tells you when ${host} stops answering.`,
});
}
// By server, those about no server last, then by address.
const sorted = [...all.values()].sort((a, b) => Number(a.serverName === null) - Number(b.serverName === null) || (a.serverName ?? "").localeCompare(b.serverName ?? "") || a.target.localeCompare(b.target));
return {
suggestions: sorted.filter((x) => !input.dismissed.has(x.key)).slice(0, MAX_SUGGESTIONS),
dismissed: sorted.filter((x) => input.dismissed.has(x.key)),
};
}
// ─── What was said no to ───────────────────────────────────────────────────
const DISMISSED_FLAG = "endpointSuggestionsDismissed";
const MAX_DISMISSED = 500;
export async function loadDismissed(): Promise<Set<string>> {
const raw = await getInternalFlag(DISMISSED_FLAG);
if (!raw) return new Set();
try {
const v = JSON.parse(raw);
return new Set(Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : []);
} catch {
return new Set();
}
}
export async function saveDismissed(keys: Set<string>): Promise<void> {
await setInternalFlag(DISMISSED_FLAG, JSON.stringify([...keys].slice(-MAX_DISMISSED)));
}
// ─── Reading what is stored ────────────────────────────────────────────────
function jsonArray<T>(stored: string | null): T[] {
if (!stored) return [];
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? (v as T[]) : [];
} catch {
return [];
}
}
export async function loadSuggestions(): Promise<{ suggestions: Suggestion[]; dismissed: Suggestion[] }> {
const [serverRows, dnsRows, secretRows, existing, dismissed] = await Promise.all([
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses, listeningPorts: servers.listeningPorts }).from(servers),
db.select({ name: dnsRecordsCache.name, type: dnsRecordsCache.type, content: dnsRecordsCache.content }).from(dnsRecordsCache),
db.select({ name: secrets.name, checkHost: secrets.checkHost, checkPort: secrets.checkPort }).from(secrets),
db.select({ kind: endpointChecks.kind, target: endpointChecks.target, port: endpointChecks.port }).from(endpointChecks),
loadDismissed(),
]);
return suggestChecks({
servers: serverRows.map((s) => ({ id: s.id, name: s.name, hostname: s.hostname, ips: jsonArray<string>(s.ipAddresses).filter((i) => typeof i === "string"), ports: jsonArray<{ protocol?: string; port: number; address?: string }>(s.listeningPorts).filter((p) => p && Number.isInteger(p.port)) })),
dns: dnsRows,
secrets: secretRows,
existing,
dismissed,
});
}
+170
View File
@@ -0,0 +1,170 @@
/**
* Heartbeats: "did my 3 a.m. job actually run?". A cron job or backup script calls an address when it finishes; the app writes down that it did, and
* speaks up — an alert and a notification, like any other problem — when the call doesn't come in time, or when the job says it failed.
*
* Each heartbeat has an address with a long random token in it (`/api/heartbeat/<token>`) and a period: how often the job should report, plus a grace
* period for jobs that run a little late. Three calls: `…/<token>` (it worked), `…/<token>/start` (it began, so the next call can say how long it took)
* and `…/<token>/fail` (it ran and went wrong — this is an alert at once, without waiting for the time to pass). Anything after the call, in the
* body or in `?msg=`, is kept as a short note ("3.2 GB, 41 s").
*
* The token is the whole of the access a call needs, and all it allows is to say "I'm alive": it can't read anything. It is kept as it is, not hashed
* (unlike a share link), because the address has to be shown again when somebody sets the job up on another machine.
*/
import { randomBytes } from "node:crypto";
import { and, asc, desc, eq, inArray, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { heartbeatPings, heartbeats } from "../db/schema.js";
import { invalidateAlertsCache } from "./alerts.js";
export type HeartbeatRow = typeof heartbeats.$inferSelect;
export type PingKind = "ok" | "fail" | "start";
export const PING_KINDS: readonly PingKind[] = ["ok", "fail", "start"];
export const KEEP_PING_DAYS = 14;
export const MAX_MESSAGE = 200;
/** "hb_" and then random letters and digits: what an address is allowed to carry. */
export const TOKEN_PATTERN = /^hb_[A-Za-z0-9_-]{20,64}$/;
const MIN_PERIOD = 1;
export const MAX_PERIOD_MINUTES = 60 * 24 * 30;
export const MAX_GRACE_MINUTES = 60 * 24;
/** A "started" call that is older than this is not what a later "finished" call is timing. */
const MAX_RUN_MS = 7 * 86_400_000;
export const newToken = () => `hb_${randomBytes(18).toString("base64url")}`;
/** SQLite's own timestamps ("2026-10-19 03:10:00") are UTC without a "T"; ours are ISO. Both read as the instant they are. */
export function parseStamp(value: string): number {
return Date.parse(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value) ? `${value.replace(" ", "T")}Z` : value);
}
/** A note from a job, made fit to store and show: one line, no control characters, not long. */
export function cleanMessage(raw: unknown): string | null {
if (typeof raw !== "string") return null;
// eslint-disable-next-line no-control-regex
const text = raw.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
return text === "" ? null : text.slice(0, MAX_MESSAGE);
}
// ─── What a heartbeat's state is ───────────────────────────────────────────
export type HeartbeatState = "off" | "waiting" | "up" | "running" | "late" | "failed";
/** When the next call is due by, counting the grace: from the last call, or from the day it was made if it has never called. */
export function dueBy(h: Pick<HeartbeatRow, "periodMinutes" | "graceMinutes" | "lastPingAt" | "createdAt">): Date {
const base = h.lastPingAt ? parseStamp(h.lastPingAt) : parseStamp(h.createdAt);
return new Date(base + (h.periodMinutes + h.graceMinutes) * 60_000);
}
/**
* Pure: how a heartbeat is doing. A job that said it failed is failed until it next succeeds; otherwise it is late once the time (period and
* grace) has passed without a call. A new one that has not called yet is waiting, and is late if its first call is overdue too.
*/
export function heartbeatState(h: Pick<HeartbeatRow, "enabled" | "periodMinutes" | "graceMinutes" | "lastPingAt" | "lastStatus" | "lastStartAt" | "createdAt">, now: Date): HeartbeatState {
if (!h.enabled) return "off";
if (h.lastStatus === "fail") return "failed";
if (now.getTime() > dueBy(h).getTime()) return "late";
if (!h.lastPingAt) return "waiting";
if (h.lastStartAt && parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt)) return "running";
return "up";
}
// ─── What is wrong ─────────────────────────────────────────────────────────
export interface HeartbeatCondition {
key: string;
source: "heartbeat";
message: string;
severity: "critical" | "warning";
category: "automation";
heartbeatId: number;
serverId: number | null;
}
const stamp = (d: Date) => `${d.toISOString().slice(0, 16).replace("T", " ")} UTC`;
/**
* The problems the heartbeats stand for. A call that didn't come in time is a warning, and says when it was due (a time, not "N minutes ago", so
* the message doesn't change on every pass); a job that reported a failure is critical. The key carries the server a heartbeat is linked to, so
* maintenance on it silences them.
*/
export function heartbeatConditions(rows: HeartbeatRow[], now: Date): HeartbeatCondition[] {
const out: HeartbeatCondition[] = [];
for (const h of rows) {
const state = heartbeatState(h, now);
const suffix = h.serverId !== null ? `:server:${h.serverId}` : "";
if (state === "late") {
const heard = h.lastPingAt ? `last heard ${stamp(new Date(parseStamp(h.lastPingAt)))}` : "it has never reported";
out.push({ key: `heartbeat:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "warning", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} hasn't reported — it was due by ${stamp(dueBy(h))} (${heard})` });
} else if (state === "failed") {
out.push({ key: `heartbeat-fail:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "critical", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} reported a failure${h.lastMessage ? ` — ${h.lastMessage}` : ""}` });
}
}
return out;
}
// ─── A call coming in ──────────────────────────────────────────────────────
/**
* Writes down a call. Returns the heartbeat as it is now, or null when there is no heartbeat with that token. A heartbeat that is switched off
* still takes calls (so a script isn't broken by pausing it) and notes them, so that switching it back on starts from a fresh clock.
*/
export async function recordPing(token: string, kind: PingKind, message: string | null, now: Date = new Date()): Promise<HeartbeatRow | null> {
if (!TOKEN_PATTERN.test(token)) return null;
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.token, token)).limit(1);
if (!h) return null;
const iso = now.toISOString();
let durationMs: number | null = null;
let updated: HeartbeatRow;
if (kind === "start") {
[updated] = await db.update(heartbeats).set({ lastStartAt: iso }).where(eq(heartbeats.id, h.id)).returning();
} else {
if (h.lastStartAt) {
const ran = now.getTime() - parseStamp(h.lastStartAt);
const startedAfterLastEnd = !h.lastPingAt || parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt);
if (startedAfterLastEnd && ran >= 0 && ran <= MAX_RUN_MS) durationMs = ran;
}
[updated] = await db
.update(heartbeats)
.set({ lastPingAt: iso, lastStatus: kind, lastMessage: message, lastDurationMs: durationMs, pingCount: h.pingCount + 1 })
.where(eq(heartbeats.id, h.id))
.returning();
}
await db.insert(heartbeatPings).values({ heartbeatId: h.id, at: iso, kind, durationMs, message });
invalidateAlertsCache();
return updated;
}
// ─── History ───────────────────────────────────────────────────────────────
export interface PingSummary {
at: string;
kind: PingKind;
durationMs: number | null;
}
/** The last calls of each heartbeat that ended a run (success or failure), oldest first. */
export async function loadRecent(ids: number[], perHeartbeat = 30): Promise<Map<number, PingSummary[]>> {
const out = new Map<number, PingSummary[]>();
if (ids.length === 0) return out;
const rows = await db
.select({ heartbeatId: heartbeatPings.heartbeatId, at: heartbeatPings.at, kind: heartbeatPings.kind, durationMs: heartbeatPings.durationMs })
.from(heartbeatPings)
.where(and(inArray(heartbeatPings.heartbeatId, ids), inArray(heartbeatPings.kind, ["ok", "fail"])))
.orderBy(desc(heartbeatPings.at));
for (const r of rows) {
const list = out.get(r.heartbeatId) ?? [];
if (list.length < perHeartbeat) list.push({ at: r.at, kind: r.kind as PingKind, durationMs: r.durationMs });
out.set(r.heartbeatId, list);
}
for (const list of out.values()) list.reverse();
return out;
}
export async function prunePings(now: Date = new Date()): Promise<void> {
await db.delete(heartbeatPings).where(lt(heartbeatPings.at, new Date(now.getTime() - KEEP_PING_DAYS * 86_400_000).toISOString()));
}
export const periodLimits = { min: MIN_PERIOD, max: MAX_PERIOD_MINUTES };
/** The table order every list uses. */
export const listHeartbeats = () => db.select().from(heartbeats).orderBy(asc(heartbeats.name));
+1 -1
View File
@@ -25,7 +25,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^endpoint(?:-cert)?:\d+:server:(\d+)$/.exec(key);
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
+15 -1
View File
@@ -4,8 +4,9 @@
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { db } from "../db/client.js";
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
@@ -70,6 +71,18 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
}
}
/** Jobs that report in, for /metrics. */
async function addHeartbeatMetrics(out: Exposition, now: Date) {
for (const h of await db.select().from(heartbeats)) {
if (!h.enabled) continue;
const labels = { name: h.name };
const state = heartbeatState(h, now);
out.add("homelab_heartbeat_up", "gauge", "1 if the job has reported in time and not said it failed, else 0.", state === "late" || state === "failed" ? 0 : 1, labels);
if (h.lastPingAt) out.add("homelab_heartbeat_last_ping_timestamp_seconds", "gauge", "When the job last reported that it finished (Unix time).", Math.floor(parseStamp(h.lastPingAt) / 1000), labels);
if (h.lastDurationMs !== null) out.add("homelab_heartbeat_last_duration_seconds", "gauge", "How long the job said its last run took.", Math.round(h.lastDurationMs / 100) / 10, labels);
}
}
export async function buildMetrics(now: Date = new Date()): Promise<string> {
const started = Date.now();
const out = new Exposition();
@@ -195,5 +208,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
await addEndpointMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+1 -1
View File
@@ -1,5 +1,5 @@
/**
* The name lists behind Operations → Generator's server-name picker.
* The name lists behind Administration → Generator's server-name picker.
*
* The built-in lists below are hand-picked, not taken from any official source — they're a starting point. Admins can
* edit every list under Settings → Names, and can replace a Swedish list with real statistics from Skatteverket.
+2 -1
View File
@@ -109,8 +109,9 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
else if (key.startsWith("failed-units:")) category = "services";
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
else if (key.startsWith("automation:")) category = "automation";
else if (key.startsWith("heartbeat")) category = "automation";
else if (key.startsWith("endpoint-cert:")) category = "expiry";
else if (key.startsWith("endpoint:")) category = "monitoring";
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|endpoint(?:-cert)?:\d+:server:)(\d+)/.exec(key);
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
}
+300
View File
@@ -0,0 +1,300 @@
/**
* "Coming up": what the agenda gathers and how it dates it, the calendar feed it can be written as, and the routes — the list for people signed in
* and the feed behind a share link.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-agenda-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, secrets, domains, assets, endpointChecks, maintenanceWindows, maintenanceSchedules, restoreTests } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { agendaFromData, buildIcs, foldLine, localDate, addDays, buildAgenda, clearAgendaCacheForTests } = await import("../src/services/agenda.js");
const { agendaRouter } = await import("../src/routes/agenda.js");
const { publicCalendarRouter } = await import("../src/routes/publicCalendar.js");
const { shareLinksRouter } = await import("../src/routes/shareLinks.js");
const { createShareLink } = await import("../src/services/shareLinks.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
await runMigrations();
type Data = Parameters<typeof agendaFromData>[0];
const NOW = new Date("2026-10-19T12:00:00Z");
const empty = (extra: Partial<Data> = {}): Data => ({ secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], restoreTests: [], restoreLimitDays: 0, updates: [], notes: [], ...extra });
const titles = (data: Data, days?: number, tz = "UTC") => agendaFromData(data, NOW, tz, days).items.map((i) => [i.daysLeft, i.kind, i.title]);
describe("what is gathered, and how it is dated", () => {
test("a day that something expires on, with how many days away it is; a secret that is a certificate is a certificate", () => {
const data = empty({
secrets: [
{ id: 1, name: "API key", type: "api_key", expiryDate: "2026-10-30" },
{ id: 2, name: "Wildcard", type: "ssl_certificate", expiryDate: "2026-11-02" },
{ id: 3, name: "Never", type: "generic", expiryDate: null },
{ id: 4, name: "Garbage", type: "generic", expiryDate: "2026-02-30" },
],
domains: [{ id: 1, name: "example.se", expiresAt: "2027-01-05" }],
});
assert.deepEqual(titles(data), [
[11, "secret", "Secret “API key” expires"],
[14, "certificate", "Certificate “Wildcard” expires"],
[78, "domain", "Domain example.se expires"],
]);
});
test("what has just passed is shown as passed, for thirty days; what is further off than asked for is not shown", () => {
const data = empty({ secrets: [
{ id: 1, name: "Yesterday", type: "generic", expiryDate: "2026-10-18" },
{ id: 2, name: "Today", type: "generic", expiryDate: "2026-10-19" },
{ id: 3, name: "Last month", type: "generic", expiryDate: "2026-09-19" },
{ id: 4, name: "Long ago", type: "generic", expiryDate: "2026-09-18" },
{ id: 5, name: "In a year", type: "generic", expiryDate: "2027-10-19" },
] });
assert.deepEqual(titles(data, 90), [[-30, "secret", "Secret “Last month” expired"], [-1, "secret", "Secret “Yesterday” expired"], [0, "secret", "Secret “Today” expires"]]);
assert.equal(titles(data, 400).length, 4, "a year ahead, when asked for");
});
test("certificates that endpoint checks have seen, hardware warranties and end of support — each linking to where they are kept", () => {
const agenda = agendaFromData(
empty({
endpoints: [
{ id: 1, name: "Gitea", enabled: true, certExpiresAt: "2026-11-01T08:00:00.000Z", serverId: 5 },
{ id: 2, name: "Paused", enabled: false, certExpiresAt: "2026-11-01T08:00:00.000Z", serverId: null },
{ id: 3, name: "No cert", enabled: true, certExpiresAt: null, serverId: null },
],
assets: [{ id: 9, name: "Synology DS920+", warrantyUntil: "2026-12-01", endOfSupport: "2026-12-20", serverId: 2 }],
}),
NOW,
"UTC",
);
assert.deepEqual(agenda.items.map((i) => [i.kind, i.title, i.link, i.serverId]), [
["certificate", "The certificate for Gitea expires", "/endpoints", 5],
["warranty", "Warranty on Synology DS920+ ends", "/inventory?asset=9", 2],
["support", "Support for Synology DS920+ ends", "/inventory?asset=9", 2],
]);
});
test("Tailscale keys: only those the integration says are expiring, dated by days left", () => {
assert.deepEqual(titles(empty({ tailscale: [{ integrationId: 1, integrationName: "Home", deviceLabel: "laptop", daysLeft: 6 }] })), [[6, "tailscale", "Tailscale key for laptop expires"]]);
});
test("a restore test is due a set time after the last good one — only with a limit set, and never for one that failed", () => {
const tests = [
{ subjectKey: "guest:1:100", label: "elsa", testedAt: "2026-08-01", result: "ok" },
{ subjectKey: "guest:1:101", label: "anna", testedAt: "2026-08-01", result: "failed" },
{ subjectKey: "custom:photos", label: "Photos share", testedAt: "2026-07-03", result: "partial" },
];
assert.deepEqual(titles(empty({ restoreTests: tests, restoreLimitDays: 0 })), []);
assert.deepEqual(titles(empty({ restoreTests: tests, restoreLimitDays: 90 })), [[-18, "restore", "Restore test was due: Photos share"], [11, "restore", "Restore test due: elsa"]]);
});
test("updates that are waiting are one line, from the longest wait", () => {
const agenda = agendaFromData(empty({ updates: [{ name: "a", count: 5, security: 2, since: "2026-10-10T08:00:00.000Z" }, { name: "b", count: 1, security: 0, since: "2026-10-17T08:00:00.000Z" }, { name: "c", count: 3, security: null, since: null }] }), NOW, "UTC");
assert.equal(agenda.items.length, 1);
assert.deepEqual([agenda.items[0].title, agenda.items[0].detail, agenda.items[0].daysLeft, agenda.items[0].link], ["Updates waiting on 3 servers (1 with security fixes)", "the longest wait is 9 days", -9, "/updates"]);
});
test("maintenance that is set up is grouped by when it is, the names kept short; windows a schedule made are left to the schedule, which is read forward", () => {
const agenda = agendaFromData(
empty({
windows: [
{ startedAt: "2026-10-24T01:00:00.000Z", endsAt: "2026-10-24T03:00:00.000Z", reason: "Firmware", source: null, targetName: "nas" },
{ startedAt: "2026-10-24T01:00:00.000Z", endsAt: "2026-10-24T03:00:00.000Z", reason: "Firmware", source: null, targetName: "switch" },
{ startedAt: "2026-10-21T02:00:00.000Z", endsAt: "2026-10-21T04:00:00.000Z", reason: "Patch night", source: "schedule:1:2026-10-21", targetName: "nas" },
],
schedules: [{ id: 1, name: "Patch night", reason: null, days: JSON.stringify([3]), startTime: "02:00", durationMinutes: 120, timezone: "UTC", targetNames: ["nas", "elsa"] }],
}),
NOW,
"UTC",
30,
);
const m = agenda.items.filter((i) => i.kind === "maintenance");
assert.deepEqual(m.map((i) => [i.title, i.at.slice(0, 16), i.allDay, i.endsAt?.slice(11, 16)]), [
["Recurring maintenance: Patch night", "2026-10-21T02:00", false, "04:00"],
["Maintenance: nas, switch", "2026-10-24T01:00", false, "03:00"],
["Recurring maintenance: Patch night", "2026-10-28T02:00", false, "04:00"],
["Recurring maintenance: Patch night", "2026-11-04T02:00", false, "04:00"],
["Recurring maintenance: Patch night", "2026-11-11T02:00", false, "04:00"],
["Recurring maintenance: Patch night", "2026-11-18T02:00", false, "04:00"],
]);
assert.equal(m.filter((i) => i.at.startsWith("2026-10-21")).length, 1, "the window the schedule made and the schedule itself are one entry, not two");
assert.equal(m[1].detail, "Firmware");
assert.equal(m[0].detail, "elsa, nas");
});
test("a day is the day on the clock set for notifications: a window at 23:30 UTC is already tomorrow in Stockholm", () => {
const windows = [{ startedAt: "2026-10-19T23:30:00.000Z", endsAt: "2026-10-20T01:00:00.000Z", reason: null, source: null, targetName: "nas" }];
assert.equal(agendaFromData(empty({ windows }), NOW, "UTC").items[0].daysLeft, 0);
assert.equal(agendaFromData(empty({ windows }), NOW, "Europe/Stockholm").items[0].daysLeft, 1);
assert.equal(localDate(new Date("2026-10-19T23:30:00Z"), "Europe/Stockholm"), "2026-10-20");
assert.equal(addDays("2026-10-31", 1), "2026-11-01");
});
});
describe("the calendar feed", () => {
const data = empty({
secrets: [{ id: 1, name: "API key, with; odd\\chars", type: "api_key", expiryDate: "2026-10-30" }],
tailscale: [{ integrationId: 1, integrationName: "Home", deviceLabel: "bobs-laptop", daysLeft: 6 }],
windows: [{ startedAt: "2026-10-24T01:00:00.000Z", endsAt: "2026-10-24T03:00:00.000Z", reason: "Firmware", source: null, targetName: "nas" }],
secretsOld: undefined,
updates: [{ name: "a", count: 1, security: 0, since: null }],
} as unknown as Partial<Data>);
const agenda = agendaFromData(data, NOW, "UTC");
const ics = buildIcs(agenda, { name: "Homelab; manager", detail: "names", baseUrl: "https://hm.lan/" });
test("it is a calendar with CRLF lines: all-day events for days, a start and an end for maintenance, nothing for waiting updates", () => {
assert.ok(ics.startsWith("BEGIN:VCALENDAR\r\nVERSION:2.0\r\n") && ics.endsWith("END:VCALENDAR\r\n"));
assert.equal(ics.includes("\n") && ics.split("\n").every((l, i, a) => i === a.length - 1 || l.endsWith("\r")), true);
assert.equal((ics.match(/BEGIN:VEVENT/g) ?? []).length, 3, "the secret, the key and the window — not the updates");
assert.match(ics, /DTSTART;VALUE=DATE:20261030\r\nDTEND;VALUE=DATE:20261031/);
assert.match(ics, /DTSTART:20261024T010000Z\r\nDTEND:20261024T030000Z/);
assert.match(ics, /X-WR-CALNAME:Homelab\\; manager/);
assert.match(ics, /TRANSP:TRANSPARENT/);
});
test("text is escaped, and a line is never more than 75 bytes, without splitting a character", () => {
assert.match(ics.replace(/\r\n /g, ""), /SUMMARY:Secret “API key\\, with\\; odd\\\\chars” expires/);
const long = foldLine(`SUMMARY:${"å".repeat(100)}`);
for (const l of long.split("\r\n")) assert.ok(Buffer.byteLength(l, "utf8") <= 75, `${Buffer.byteLength(l, "utf8")} bytes`);
assert.equal(long.replace(/\r\n /g, ""), `SUMMARY:${"å".repeat(100)}`, "and it unfolds to what it was");
for (const l of ics.split("\r\n")) assert.ok(Buffer.byteLength(l, "utf8") <= 75);
});
test("with names, an event carries the name and a link back; each has a steady id that does not give the name away", () => {
const unfolded = ics.replace(/\r\n /g, "");
assert.match(unfolded, /DESCRIPTION:Firmware\\nhttps:\/\/hm\.lan\/maintenance/);
const uids = [...ics.matchAll(/UID:(\S+)\r\n/g)].map((m) => m[1]);
assert.equal(new Set(uids).size, 3);
assert.ok(uids.every((u) => /^[0-9a-f]{24}@homelab-manager$/.test(u)));
assert.deepEqual(uids, [...buildIcs(agenda, { name: "x", detail: "names" }).matchAll(/UID:(\S+)\r\n/g)].map((m) => m[1]), "the same on every fetch, or a subscribed calendar would duplicate everything");
});
test("with no names, an event says only what kind of thing it is — and nothing in it gives away a name", () => {
const plain = buildIcs(agenda, { name: "Homelab", detail: "summary", baseUrl: "https://hm.lan" });
const unfolded = plain.replace(/\r\n /g, "");
assert.match(unfolded, /SUMMARY:A secret expires/);
assert.match(unfolded, /SUMMARY:A Tailscale key expires/);
assert.match(unfolded, /SUMMARY:Planned maintenance/);
assert.equal(/API key|bobs-laptop|nas|Firmware|hm\.lan|DESCRIPTION/.test(unfolded), false);
});
test("an event more than a week old is dropped, one from this week is kept", () => {
const old = agendaFromData(empty({ secrets: [{ id: 1, name: "Old", type: "generic", expiryDate: "2026-10-10" }, { id: 2, name: "Recent", type: "generic", expiryDate: "2026-10-15" }] }), NOW, "UTC");
assert.equal((buildIcs(old, { name: "x", detail: "names" }).match(/BEGIN:VEVENT/g) ?? []).length, 1);
});
});
describe("from what is stored, through the routes", () => {
let app: Server;
let base = "";
let who = "admin";
const day = (n: number) => new Date(Date.now() + n * 86_400_000).toISOString().slice(0, 10);
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const [s] = await db
.insert(servers)
.values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", pendingUpdates: JSON.stringify({ manager: "apt", count: 4, security: 1, rebootRequired: false, reportedAt: new Date().toISOString() }), updatesPendingSince: new Date(Date.now() - 3 * 86_400_000).toISOString() })
.returning({ id: servers.id });
await db.insert(secrets).values({ name: "Deploy key", type: "api_key", expiryDate: day(10) } as never);
await db.insert(domains).values({ name: "example.se", expiresAt: day(40) });
await db.insert(assets).values({ name: "Switch", kind: "network", warrantyUntil: day(20), createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() } as never);
await db.insert(endpointChecks).values({ name: "Gitea", kind: "http", target: "https://git.lab/", certExpiresAt: new Date(Date.now() + 5 * 86_400_000).toISOString(), serverId: s.id });
await db.insert(maintenanceWindows).values({ targetType: "server", targetId: s.id, reason: "Disk swap", startedAt: new Date(Date.now() + 2 * 86_400_000).toISOString(), endsAt: new Date(Date.now() + 2 * 86_400_000 + 3600_000).toISOString() });
await db.insert(maintenanceSchedules).values({ name: "Patch night", targets: JSON.stringify([{ type: "server", id: s.id }]), days: JSON.stringify([0, 1, 2, 3, 4, 5, 6]), startTime: "03:00", durationMinutes: 60, timezone: "UTC" });
await db.insert(restoreTests).values({ subjectKey: "custom:photos", label: "Photos share", testedAt: day(-80), result: "ok" });
await updateSettings({ healthChecks: { restoreTestDays: 90 } });
clearAgendaCacheForTests();
const a = express();
a.use(express.json());
a.use("/api/public/calendar", publicCalendarRouter);
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/agenda", agendaRouter);
a.use("/api/share-links", shareLinksRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("everything with a date, from every source, for anyone signed in; the days asked for narrow it", async () => {
who = "viewer";
const res = await fetch(`${base}/api/agenda`);
assert.equal(res.status, 200);
const body = await res.json();
const kinds = new Set(body.items.map((i: { kind: string }) => i.kind));
for (const k of ["secret", "domain", "warranty", "certificate", "maintenance", "restore", "updates"]) assert.ok(kinds.has(k), k);
assert.ok(body.items.some((i: { title: string }) => i.title === "Restore test due: Photos share"), "tested 80 days ago with a 90-day limit");
assert.ok(body.items.filter((i: { title: string }) => i.title.startsWith("Recurring maintenance")).length >= 60 / 1, "a schedule is read forward, not only the week that has windows");
const near = await (await fetch(`${base}/api/agenda?days=15`)).json();
assert.equal(near.items.some((i: { kind: string }) => i.kind === "domain"), false, "40 days out is beyond 15");
assert.equal(near.items.some((i: { kind: string }) => i.kind === "secret"), true);
const bad = await (await fetch(`${base}/api/agenda?days=banana`)).json();
assert.equal(bad.items.length > 0, true, "an unreadable range is the default one");
const direct = await buildAgenda();
assert.equal(direct.items.length, body.items.length, "the route and the service agree");
});
test("a calendar link is made like any share link, by an administrator, and its address is the feed", async () => {
who = "viewer";
assert.equal((await fetch(`${base}/api/share-links`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ label: "Phone", kind: "calendar", detail: "names", expiresInDays: null }) })).status, 403);
who = "admin";
const res = await fetch(`${base}/api/share-links`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ label: "Phone", kind: "calendar", detail: "names", expiresInDays: null }) });
assert.equal(res.status, 201);
const made = await res.json();
assert.match(made.path, /^\/api\/public\/calendar\/[A-Za-z0-9_-]{20,}\.ics$/);
const feed = await fetch(`${base}${made.path}`);
assert.equal(feed.status, 200);
assert.match(feed.headers.get("content-type") ?? "", /^text\/calendar; charset=utf-8/);
const text = (await feed.text()).replace(/\r\n /g, "");
assert.ok(text.startsWith("BEGIN:VCALENDAR"));
assert.match(text, /SUMMARY:Secret “Deploy key” expires/);
assert.match(text, /SUMMARY:Domain example\.se expires/);
const bare = made.path.replace(/\.ics$/, "");
assert.equal((await fetch(`${base}${bare}`)).status, 200, "the extension is for calendar apps; the address works without it");
});
test("a link for no names gives the feed no names; one for something else, a wrong one and an ended one all say the same thing", async () => {
const { token } = await createShareLink({ label: "Private", kind: "calendar", detail: "summary", expiresInDays: null, createdBy: "t" });
const text = (await (await fetch(`${base}/api/public/calendar/${token}.ics`)).text()).replace(/\r\n /g, "");
assert.match(text, /SUMMARY:A secret expires/);
assert.equal(/Deploy key|example\.se|Gitea|nas/.test(text), false);
const wall = await createShareLink({ label: "Wall", kind: "wall", detail: "summary", expiresInDays: null, createdBy: "t" });
const ended = await createShareLink({ label: "Old", kind: "calendar", detail: "names", expiresInDays: 1, createdBy: "t", now: new Date(Date.now() - 3 * 86_400_000) });
const answers = await Promise.all([`${wall.token}.ics`, "nonsense.ics", `${"a".repeat(30)}.ics`, `${ended.token}.ics`].map(async (p) => {
const r = await fetch(`${base}/api/public/calendar/${p}`);
return [r.status, await r.text()];
}));
assert.deepEqual(new Set(answers.map((a) => JSON.stringify(a))).size, 1, "never made, ended, and a wall's: indistinguishable");
assert.equal(answers[0][0], 404);
});
});
+1 -1
View File
@@ -115,7 +115,7 @@ describe("through the routes", () => {
assert.ok(!r.entries.some((e: { title: string }) => /login|export/i.test(e.title)));
const maint = r.entries[0];
assert.deepEqual([maint.kind, maint.actor, maint.link, maint.serverId], ["maintenance", "Ada", `/servers/${nas}`, nas]);
assert.equal(r.entries.find((e: { title: string }) => e.title.startsWith("Alert raised")).link.startsWith("/incidents/"), true);
assert.equal(r.entries.find((e: { title: string }) => e.title.startsWith("Alert raised")).link.startsWith("/alerts/incidents/"), true);
assert.equal(r.kinds.length, 10);
});
+176
View File
@@ -0,0 +1,176 @@
/**
* Suggested endpoint checks: what is offered from servers' open web ports, DNS names and secrets, what is left out, and the routes that add or dismiss them.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-suggest-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, secrets, dnsProviders, dnsRecordsCache, endpointChecks } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { suggestChecks } = await import("../src/services/endpointSuggestions.js");
const { endpointsRouter } = await import("../src/routes/endpoints.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const nas = { id: 1, name: "nas", hostname: "nas.lan", ips: ["10.0.0.5"], ports: [{ protocol: "tcp", port: 443, address: "0.0.0.0" }, { protocol: "tcp", port: 22, address: "0.0.0.0" }] };
const input = (extra: Partial<Parameters<typeof suggestChecks>[0]> = {}) => ({ servers: [nas], dns: [], secrets: [], existing: [], dismissed: new Set<string>(), ...extra });
describe("what is offered", () => {
test("a web port a server listens on, by the name DNS gives it, linked to the server", () => {
const { suggestions } = suggestChecks(input({ dns: [{ name: "git.example.lan", type: "A", content: "10.0.0.5" }, { name: "other.example.lan", type: "A", content: "10.0.0.9" }] }));
assert.deepEqual(suggestions.map((s) => [s.kind, s.target, s.name, s.serverId, s.serverName]), [["http", "https://git.example.lan/", "git.example.lan", 1, "nas"]]);
assert.match(suggestions[0].reason, /nas is listening on port 443, and git\.example\.lan points at it in DNS/);
});
test("an alias to one of those names, or to the server's own hostname, is offered too; wildcards, reverse names and short names are not", () => {
const { suggestions } = suggestChecks(
input({
dns: [
{ name: "git.example.lan", type: "A", content: "10.0.0.5" },
{ name: "code.example.lan", type: "CNAME", content: "git.example.lan." },
{ name: "files.example.lan", type: "CNAME", content: "nas.lan" },
{ name: "*.example.lan", type: "A", content: "10.0.0.5" },
{ name: "5.0.0.10.in-addr.arpa", type: "A", content: "10.0.0.5" },
{ name: "nas", type: "A", content: "10.0.0.5" },
{ name: "unrelated.example.lan", type: "CNAME", content: "elsewhere.lan" },
],
}),
);
assert.deepEqual(suggestions.map((s) => s.target), ["https://code.example.lan/", "https://files.example.lan/", "https://git.example.lan/"]);
});
test("with no DNS name the server's own hostname or its address is used; with neither, nothing", () => {
assert.deepEqual(suggestChecks(input()).suggestions.map((s) => s.target), ["https://nas.lan/"]);
const bare = { ...nas, hostname: "nas", ips: ["169.254.1.1", "10.0.0.5"] };
assert.deepEqual(suggestChecks(input({ servers: [bare] })).suggestions.map((s) => s.target), ["https://10.0.0.5/"]);
assert.deepEqual(suggestChecks(input({ servers: [{ ...bare, ips: [] }] })).suggestions, []);
});
test("only web ports count, only for TCP, and a port that is open on this machine alone does not", () => {
const odd = { ...nas, ports: [{ protocol: "udp", port: 443, address: "0.0.0.0" }, { protocol: "tcp", port: 22, address: "0.0.0.0" }, { protocol: "tcp", port: 8443, address: "127.0.0.1" }, { protocol: "tcp", port: 3000, address: "0.0.0.0" }] };
assert.deepEqual(suggestChecks(input({ servers: [odd] })).suggestions, []);
const mixed = { ...nas, ports: [{ protocol: "tcp", port: 8080, address: "127.0.0.1" }, { protocol: "tcp", port: 8080, address: "10.0.0.5" }] };
assert.deepEqual(suggestChecks(input({ servers: [mixed] })).suggestions.map((s) => s.target), ["http://nas.lan:8080/"], "one socket on a real address is enough, and a port that is not the default is in the address");
});
test("plain and secure on the same server: the secure one is offered, since it reads the certificate and the plain one is most likely its redirect", () => {
const both = { ...nas, ports: [{ protocol: "tcp", port: 80, address: "::" }, { protocol: "tcp", port: 443, address: "::" }, { protocol: "tcp", port: 8080, address: "::" }] };
assert.deepEqual(suggestChecks(input({ servers: [both] })).suggestions.map((s) => s.target), ["http://nas.lan:8080/", "https://nas.lan/"]);
});
test("a secret that already reads a certificate is offered as a certificate check", () => {
const { suggestions } = suggestChecks(input({ servers: [], secrets: [{ name: "Wildcard cert", checkHost: "Git.Example.lan.", checkPort: 443 }, { name: "Typed in", checkHost: null, checkPort: null }] }));
assert.deepEqual(suggestions.map((s) => [s.kind, s.target, s.port, s.name, s.serverId]), [["tls", "git.example.lan", 443, "Certificate: Wildcard cert", null]]);
});
});
describe("what is left out", () => {
const dns = [{ name: "git.example.lan", type: "A", content: "10.0.0.5" }];
test("what a check already covers, however its address is written", () => {
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "http", target: "https://Git.example.lan:443/login", port: null }] })).suggestions.length, 0);
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "http", target: "https://git.example.lan", port: null }] })).suggestions.length, 0);
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "http", target: "http://git.example.lan/", port: null }] })).suggestions.length, 1, "a plain-http check does not cover the secure port");
assert.equal(suggestChecks(input({ dns, existing: [{ kind: "tcp", target: "git.example.lan", port: 443 }] })).suggestions.length, 1, "nor does a port check, which doesn't speak to the page");
});
test("a secret's certificate is covered by a certificate check, or by a web check on https that reads it anyway", () => {
const sec = [{ name: "Cert", checkHost: "git.example.lan", checkPort: 443 }];
assert.equal(suggestChecks(input({ servers: [], secrets: sec, existing: [{ kind: "tls", target: "git.example.lan", port: 443 }] })).suggestions.length, 0);
assert.equal(suggestChecks(input({ servers: [], secrets: sec, existing: [{ kind: "http", target: "https://git.example.lan/", port: null }] })).suggestions.length, 0);
assert.equal(suggestChecks(input({ servers: [], secrets: sec, existing: [{ kind: "tls", target: "git.example.lan", port: 8443 }] })).suggestions.length, 1);
});
test("what was dismissed is listed apart, not offered", () => {
const r = suggestChecks(input({ dns, dismissed: new Set(["http|https://git.example.lan/"]) }));
assert.deepEqual([r.suggestions.length, r.dismissed.map((d) => d.target)], [0, ["https://git.example.lan/"]]);
});
});
describe("the routes", () => {
let app: Server;
let base = "";
let who = "admin";
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
]);
const [s] = await db
.insert(servers)
.values({ name: "nas", hostname: "nas.lan", apiTokenHash: "n", apiTokenPrefix: "n", ipAddresses: JSON.stringify(["10.0.0.5"]), listeningPorts: JSON.stringify([{ protocol: "tcp", port: 443, address: "0.0.0.0", process: "nginx" }, { protocol: "tcp", port: 8080, address: "0.0.0.0" }]) })
.returning({ id: servers.id });
const [p] = await db.insert(dnsProviders).values({ providerType: "pihole", name: "Pi-hole" }).returning({ id: dnsProviders.id });
await db.insert(dnsRecordsCache).values({ providerId: p.id, zoneId: "z", recordId: "r1", type: "A", name: "git.example.lan", content: "10.0.0.5" });
await db.insert(secrets).values({ name: "Mail cert", type: "ssl_certificate", checkHost: "mail.example.lan", checkPort: 443 } as never);
void s;
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/endpoints", endpointsRouter);
app = await new Promise<Server>((resolve) => {
const x = a.listen(0, "127.0.0.1", () => resolve(x));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/endpoints`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
test("administrators see what is offered, and nobody else", async () => {
who = "operator";
assert.equal((await send("GET", "/suggestions")).status, 403);
assert.equal((await send("POST", "/suggestions/add", { keys: ["x"] })).status, 403);
who = "admin";
const body = await (await send("GET", "/suggestions")).json();
assert.deepEqual(body.suggestions.map((s: { target: string }) => s.target), ["http://git.example.lan:8080/", "https://git.example.lan/", "mail.example.lan"]);
});
test("adding makes the checks the server itself offered — a key that was not offered adds nothing — linked to their server, and they stop being offered", async () => {
const res = await send("POST", "/suggestions/add", { keys: ["https://git.example.lan/", "http|https://git.example.lan/", "http|http://evil.example/", "http|https://git.example.lan/"] });
assert.equal(res.status, 201);
assert.equal((await res.json()).added.length, 1);
const [made] = await db.select().from(endpointChecks).where(eq(endpointChecks.target, "https://git.example.lan/"));
assert.deepEqual([made.kind, made.name, made.serverId !== null, made.intervalMinutes, made.enabled], ["http", "git.example.lan", true, 5, true]);
const body = await (await send("GET", "/suggestions")).json();
assert.equal(body.suggestions.some((s: { target: string }) => s.target === "https://git.example.lan/"), false);
assert.equal((await send("POST", "/suggestions/add", { keys: [] })).status, 400);
});
test("dismissing takes one out of the offer, and restoring brings every dismissed one back", async () => {
let body = await (await send("POST", "/suggestions/dismiss", { keys: ["tls|mail.example.lan|443"] })).json();
assert.equal(body.suggestions.some((s: { kind: string }) => s.kind === "tls"), false);
assert.equal(body.dismissed.length, 1);
body = await (await send("POST", "/suggestions/restore")).json();
assert.equal(body.suggestions.some((s: { kind: string }) => s.kind === "tls"), true);
assert.equal(body.dismissed.length, 0);
});
});
+292
View File
@@ -0,0 +1,292 @@
/**
* Heartbeats: what a job reporting in (or not) amounts to, the address it calls, what the pass announces, and who may set them up.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-heartbeats-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, servers, heartbeats, heartbeatPings } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const { dueBy, heartbeatState, heartbeatConditions, cleanMessage, parseStamp, recordPing, newToken, TOKEN_PATTERN } = await import("../src/services/heartbeats.js");
const { heartbeatsRouter } = await import("../src/routes/heartbeats.js");
const { heartbeatPingRouter } = await import("../src/routes/heartbeatPing.js");
const { runEndpointPass } = await import("../src/services/endpointMonitor.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { startOrExtendWindow } = await import("../src/services/maintenance.js");
const { metaOfConditionKey } = await import("../src/services/notifyRouting.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { updateSettings } = await import("../src/services/settingsStore.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
type Row = typeof heartbeats.$inferSelect;
const NOW = new Date("2026-10-19T03:00:00Z");
const row = (extra: Partial<Row> = {}): Row => ({
id: 1, name: "Nightly backup", token: "hb_x", periodMinutes: 1440, graceMinutes: 60, serverId: null, enabled: true, notes: null, lastPingAt: null, lastStatus: null, lastMessage: null,
lastDurationMs: null, lastStartAt: null, pingCount: 0, createdAt: "2026-10-18 03:00:00", ...extra,
});
const minutes = (n: number) => new Date(NOW.getTime() + n * 60_000);
describe("when a job is due, and how it is doing", () => {
test("it is due a period and the grace after its last call — or after it was made, if it never called", () => {
assert.equal(dueBy(row({ lastPingAt: "2026-10-19T02:00:00.000Z" })).toISOString(), "2026-10-20T03:00:00.000Z");
assert.equal(dueBy(row()).toISOString(), "2026-10-19T04:00:00.000Z", "SQLite's own timestamp format is read as UTC");
assert.equal(parseStamp("2026-10-18 03:00:00"), Date.parse("2026-10-18T03:00:00Z"));
});
test("new, up, running, late, failed and switched off", () => {
const called = { lastPingAt: NOW.toISOString(), lastStatus: "ok" };
assert.equal(heartbeatState(row({ createdAt: "2026-10-19 02:00:00" }), NOW), "waiting", "made recently, has not called, not yet due");
assert.equal(heartbeatState(row(), NOW), "waiting", "due by 04:00 today");
assert.equal(heartbeatState(row(), minutes(61)), "late", "never called, and now it is overdue");
assert.equal(heartbeatState(row(called), minutes(60 * 24)), "up");
assert.equal(heartbeatState(row(called), minutes(60 * 25)), "up", "inside the grace");
assert.equal(heartbeatState(row(called), minutes(60 * 25 + 1)), "late");
assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(5).toISOString() }), minutes(10)), "running", "it began after it last finished");
assert.equal(heartbeatState(row({ ...called, lastStartAt: minutes(-5).toISOString() }), minutes(10)), "up", "a start older than the last finish is not a run in progress");
assert.equal(heartbeatState(row({ ...called, lastStatus: "fail", lastMessage: "disk full" }), minutes(10)), "failed", "said so, and does not wait for the time to pass");
assert.equal(heartbeatState(row({ enabled: false }), minutes(9999)), "off");
});
test("the problems: late is a warning with the time it was due, a reported failure is critical; each keyed by the server it runs on", () => {
const list = heartbeatConditions(
[
row({ id: 1, serverId: 7 }),
row({ id: 2, name: "Photos sync", lastPingAt: NOW.toISOString(), lastStatus: "fail", lastMessage: "exit 3" }),
row({ id: 3, name: "Fine", lastPingAt: NOW.toISOString(), lastStatus: "ok" }),
row({ id: 4, name: "Paused", enabled: false }),
],
minutes(120),
);
assert.deepEqual(list.map((c) => [c.key, c.severity, c.category, c.serverId]), [["heartbeat:1:server:7", "warning", "automation", 7], ["heartbeat-fail:2", "critical", "automation", null]]);
assert.match(list[0].message, /Nightly backup hasn't reported — it was due by 2026-10-19 04:00 UTC \(it has never reported\)/);
assert.match(list[1].message, /Photos sync reported a failure — exit 3/);
assert.deepEqual(metaOfConditionKey("heartbeat:1:server:7", "warning"), { category: "automation", severity: "warning", serverId: 7 }, "tags and categories route it");
assert.equal(subjectOfConditionKey("heartbeat-fail:2:server:9"), "server:9", "and maintenance on the server holds it back");
});
test("a note from a job is one short line", () => {
assert.equal(cleanMessage("3.2 GB\n\tcopied in 41 s\u0000"), "3.2 GB copied in 41 s");
assert.equal(cleanMessage(" "), null);
assert.equal(cleanMessage(42), null);
assert.equal(cleanMessage("x".repeat(500))?.length, 200);
});
test("an address carries a long random token, and only that shape is looked up", () => {
const t = newToken();
assert.match(t, TOKEN_PATTERN);
assert.notEqual(t, newToken());
for (const bad of ["", "hb_", "hb_short", "nope_" + "a".repeat(24), "hb_" + "a/b".repeat(10), "hb_" + "a".repeat(100)]) assert.equal(TOKEN_PATTERN.test(bad), false, bad);
});
});
describe("the address, the routes and the pass", () => {
let app: Server;
let hook: Server;
let base = "";
let who = "admin";
let serverId = 0;
let id = 0;
let path = "";
const messages: string[] = [];
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id;
hook = createHttpServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content));
res.end("{}");
});
});
await new Promise<void>((r) => hook.listen(0, "127.0.0.1", r));
await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } });
const a = express();
// The ping address comes first, as in the real app: no session, no JSON parser, no sign-in.
a.use("/api/heartbeat", heartbeatPingRouter);
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/heartbeats", heartbeatsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}`;
});
after(() => {
app.close();
hook.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
const admin = (method: string, p: string, body?: unknown) => fetch(`${base}/api/heartbeats${p}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
const ping = (p: string, init?: RequestInit) => fetch(`${base}${p}`, init);
test("an administrator makes one; a wrong one is refused with a reason; nobody else can; everyone can look", async () => {
const res = await admin("POST", "/", { name: "Nightly backup", periodMinutes: 1440, graceMinutes: 60, serverId });
assert.equal(res.status, 201);
const made = (await res.json()).heartbeat;
id = made.id;
path = made.path;
assert.match(path, /^\/api\/heartbeat\/hb_[A-Za-z0-9_-]{20,}$/);
for (const bad of [{ periodMinutes: 5 }, { name: "x" }, { name: "x", periodMinutes: 0 }, { name: "x", periodMinutes: 60 * 24 * 31 }, { name: "x", periodMinutes: 5, graceMinutes: 99999 }, { name: "x", periodMinutes: 5, serverId: 99999 }, { name: " ", periodMinutes: 5 }]) {
assert.equal((await admin("POST", "/", bad)).status, 400, JSON.stringify(bad));
}
who = "operator";
assert.equal((await admin("POST", "/", { name: "y", periodMinutes: 5 })).status, 403);
assert.equal((await admin("DELETE", `/${id}`)).status, 403);
const list = await (await admin("GET", "/")).json();
assert.equal(list.heartbeats[0].path, null, "the address is for administrators: it is what lets anyone say \"I'm alive\"");
assert.equal(list.baseUrl, null);
assert.equal(list.heartbeats[0].state, "waiting");
assert.equal(list.heartbeats[0].server.name, "nas");
who = "admin";
const adminList = await (await admin("GET", "/")).json();
assert.equal(adminList.heartbeats[0].path, path);
assert.ok(adminList.baseUrl);
assert.equal("token" in adminList.heartbeats[0], false, "the token is not a field of its own");
});
test("a call is noted: success, a note in the body or the query, and how long a run took when it said it started", async () => {
assert.equal((await ping(path)).status, 200);
assert.equal(await (await ping(path, { method: "HEAD" })).text(), "");
let [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
assert.deepEqual([h.lastStatus, h.pingCount], ["ok", 2]);
assert.equal(heartbeatState(h, new Date()), "up");
assert.equal((await ping(`${path}/start`, { method: "POST" })).status, 200);
await new Promise((r) => setTimeout(r, 30));
const done = await ping(path, { method: "POST", body: "3.2 GB\ncopied", headers: { "content-type": "text/plain" } });
assert.equal(await done.text(), "OK");
[h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
assert.equal(h.lastMessage, "3.2 GB copied");
assert.ok(h.lastDurationMs !== null && h.lastDurationMs >= 20, "timed from the start call");
await ping(`${path}?msg=from%20the%20query`);
[h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
assert.equal(h.lastMessage, "from the query");
assert.equal(h.lastDurationMs, null, "no start before this one, so no duration");
const rows = await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id));
assert.deepEqual(rows.map((r) => r.kind), ["ok", "ok", "start", "ok", "ok"]);
});
test("a token that does not exist, a path that is not one of the three and a method that is not a call all get no information", async () => {
const a = await ping("/api/heartbeat/hb_" + "z".repeat(24));
const b = await ping(`${path}/bogus`);
const c = await ping("/api/heartbeat/nonsense");
assert.deepEqual([a.status, b.status, c.status], [404, 404, 404]);
assert.equal(await a.text(), await b.text(), "never made, and a wrong path, read the same");
assert.equal((await ping(path, { method: "DELETE" })).status, 405);
assert.equal((await ping(path)).headers.get("cache-control"), "no-store");
});
test("a job that says it failed is an alert at once, until it next succeeds; the pass says so once, and once when it is over", async () => {
messages.length = 0;
await db.update(heartbeats).set({ serverId: null }).where(eq(heartbeats.id, id));
assert.equal((await ping(`${path}/fail`, { method: "POST", body: "exit 3: disk full" })).status, 200);
invalidateAlertsCache();
const alert = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!;
assert.match(alert.message, /Nightly backup reported a failure — exit 3: disk full/);
assert.deepEqual([alert.severity, alert.category, alert.link], ["critical", "automation", "/heartbeats"]);
let r = await runEndpointPass(new Date());
assert.equal(r.announced, 1);
assert.match(messages.join("\n"), /Nightly backup reported a failure/);
const sent = messages.length;
r = await runEndpointPass(new Date());
assert.equal(r.announced + r.recovered, 0);
assert.equal(messages.length, sent, "not said again");
await ping(path);
r = await runEndpointPass(new Date());
assert.equal(r.recovered, 1);
assert.match(messages[messages.length - 1], /Nightly backup reported a failure/, "the recovery names what cleared");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.filter((x) => x.source === "Heartbeat").length, 0);
});
test("a call that does not come in time is a warning once it is overdue, silenced by maintenance on the server, and cleared by the call", async () => {
messages.length = 0;
await db.update(heartbeats).set({ serverId, periodMinutes: 60, graceMinutes: 10 }).where(eq(heartbeats.id, id));
const t = (min: number) => new Date(Date.now() + min * 60_000);
let r = await runEndpointPass(t(30));
assert.equal(r.announced, 0, "not due yet");
r = await runEndpointPass(t(75));
assert.equal(r.announced, 1, "an hour and ten minutes with no call");
assert.match(messages.join("\n"), /Nightly backup hasn't reported — it was due by/);
// The Alerts page reads the real clock, so the last call is moved back instead of the clock forward.
await db.update(heartbeats).set({ lastPingAt: new Date(Date.now() - 120 * 60_000).toISOString() }).where(eq(heartbeats.id, id));
invalidateAlertsCache();
const late = (await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!;
assert.equal(late.severity, "warning");
assert.deepEqual(late.target && [late.target.type, late.target.id], ["server", serverId]);
assert.equal(late.silenced, false);
await startOrExtendWindow("server", serverId, 60, "work", "Ada");
invalidateAlertsCache();
assert.equal((await getAlerts(true)).alerts.find((x) => x.source === "Heartbeat")!.silenced, true, "maintenance on the server it runs on silences it");
assert.match(await buildMetrics(t(75)), /homelab_heartbeat_up\{name="Nightly backup"\} 0/);
await recordPing((await db.select().from(heartbeats).where(eq(heartbeats.id, id)))[0].token, "ok", null, t(76));
r = await runEndpointPass(t(77));
assert.equal(r.recovered, 1);
assert.match(await buildMetrics(t(77)), /homelab_heartbeat_up\{name="Nightly backup"\} 1/);
assert.match(await buildMetrics(t(77)), /homelab_heartbeat_last_ping_timestamp_seconds\{name="Nightly backup"\} \d+/);
});
test("a switched-off heartbeat still takes calls, says nothing, and starts a fresh clock when it is switched on again", async () => {
await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: false });
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
assert.equal((await ping(`/api/heartbeat/${h.token}`)).status, 200, "pausing a job's heartbeat does not break its script");
assert.equal(heartbeatConditions([{ ...h, lastPingAt: "2020-01-01T00:00:00.000Z", lastStatus: "ok" }], new Date()).length, 0);
await admin("PUT", `/${id}`, { name: "Nightly backup", periodMinutes: 60, graceMinutes: 10, serverId: null, enabled: true });
const [after_] = await db.select().from(heartbeats).where(eq(heartbeats.id, id));
assert.equal(heartbeatState(after_, new Date()), "up");
});
test("a new address replaces the old one at once; removing one takes its history", async () => {
const res = await admin("POST", `/${id}/new-address`);
assert.equal(res.status, 200);
const fresh = (await res.json()).path as string;
assert.notEqual(fresh, path);
assert.equal((await ping(path)).status, 404, "the old one is dead");
assert.equal((await ping(fresh)).status, 200);
who = "viewer";
assert.equal((await admin("POST", `/${id}/new-address`)).status, 403);
who = "admin";
assert.equal((await admin("DELETE", `/${id}`)).status, 204);
assert.equal((await db.select().from(heartbeatPings).where(eq(heartbeatPings.heartbeatId, id))).length, 0);
assert.equal((await ping(fresh)).status, 404);
assert.equal((await admin("DELETE", `/${id}`)).status, 404);
});
});
+3 -1
View File
@@ -194,7 +194,9 @@ describe("through the routes, the page, the alerts and the figures", () => {
test("metrics say how old each test is and whether it passed", async () => {
const text = await buildMetrics();
assert.match(text, /homelab_restore_test_age_days\{subject="elsa"\} 0/);
assert.match(text, /homelab_restore_test_age_days\{subject="Photos share"\} 40/);
// The test's dates come from a fixed "now" and the metrics from the real clock, so what is expected is worked out the way the metric does.
const realAge = Math.floor((Date.now() - Date.parse(`${daysAgo(40)}T12:00:00Z`)) / 86_400_000);
assert.match(text, new RegExp(`homelab_restore_test_age_days\\{subject="Photos share"\\} ${realAge}`));
assert.match(text, /homelab_restore_test_passed\{subject="elsa"\} 1/);
assert.doesNotMatch(text, /subject="anna"/, "'not needed' isn't measured");
});