Heartbeats, Coming up and suggested checks; Operations menu tidied

Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 21:42:34 +02:00
1 parent c05c847d17
commit 4e0c6927b2
44 files changed
+7092 -75

No files matched your search

+9
View File
@@ -48,6 +48,10 @@ import { mapRouter } from "./routes/map.js";
import { generatorRouter } from "./routes/generator.js";
import { alertTicketsRouter } from "./routes/alertTickets.js";
import { avatarsRouter } from "./routes/avatars.js";
import { heartbeatsRouter } from "./routes/heartbeats.js";
import { agendaRouter } from "./routes/agenda.js";
import { publicCalendarRouter } from "./routes/publicCalendar.js";
import { heartbeatPingRouter } from "./routes/heartbeatPing.js";
import { apiErrorHandler } from "./utils/apiErrorHandler.js";
import { securityHeaders } from "./utils/securityHeaders.js";
import { sameOriginGuard } from "./utils/sameOriginGuard.js";
@@ -72,6 +76,8 @@ export function createApp() {
// The agent endpoint comes first: it reads its own (larger) body, only after checking the token, and an agent has no use for a
// session. It is also the one place a request isn't from a browser, so it isn't subject to the same-origin check below.
app.use("/api/agent/report", agentReportRouter);
// A job reporting that it ran: a token in the address, no session, and no browser to be checked for where it came from.
app.use("/api/heartbeat", heartbeatPingRouter);
// Everything after this is reached with a session cookie, so a request that came from another origin is refused here.
app.use(sameOriginGuard);
app.use(express.json());
@@ -105,6 +111,8 @@ export function createApp() {
app.use("/auth", authRouter);
app.use("/api/me", meRouter);
app.use("/api/avatars", avatarsRouter);
app.use("/api/heartbeats", heartbeatsRouter);
app.use("/api/agenda", agendaRouter);
app.use("/api/uptime", uptimeRouter);
app.use("/api/share-links", shareLinksRouter);
app.use("/api/runbooks", runbooksRouter);
@@ -123,6 +131,7 @@ export function createApp() {
app.use("/metrics", metricsRouter);
app.use("/api/public/wall", publicWallRouter);
app.use("/api/public/status", publicStatusRouter);
app.use("/api/public/calendar", publicCalendarRouter);
app.use("/api/users", usersRouter);
app.use("/api/audit-log", auditLogRouter);
app.use("/api/diag-log", diagLogRouter);
+41 -2
View File
@@ -192,6 +192,44 @@ export const restoreTests = sqliteTable(
(t) => [index("restore_tests_subject_idx").on(t.subjectKey, t.testedAt)],
);
// A job that reports in: a cron job or backup script calls its address when it finishes, and the app speaks up when the call doesn't come in time —
// see services/heartbeats.ts. The token is part of the address (/api/heartbeat/<token>) and is all the access a ping needs.
export const heartbeats = sqliteTable("heartbeats", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
token: text("token").notNull().unique(),
periodMinutes: integer("period_minutes").notNull(), // how often it should report
graceMinutes: integer("grace_minutes").notNull().default(10), // how much later than that is still fine
serverId: integer("server_id"), // soft reference: the server it runs on, so its maintenance silences the alerts
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
notes: text("notes"),
lastPingAt: text("last_ping_at"), // ISO; the last report of an end (success or failure)
lastStatus: text("last_status"), // ok | fail
lastMessage: text("last_message"),
lastDurationMs: integer("last_duration_ms"), // when it said it had started first
lastStartAt: text("last_start_at"), // ISO; the last "started" call
pingCount: integer("ping_count").notNull().default(0),
createdAt: text("created_at")
.notNull()
.default(sql`(current_timestamp)`),
});
// The recent calls to each heartbeat, for its history; kept two weeks.
export const heartbeatPings = sqliteTable(
"heartbeat_pings",
{
id: integer("id").primaryKey({ autoIncrement: true }),
heartbeatId: integer("heartbeat_id")
.notNull()
.references(() => heartbeats.id, { onDelete: "cascade" }),
at: text("at").notNull(), // ISO
kind: text("kind").notNull(), // ok | fail | start
durationMs: integer("duration_ms"),
message: text("message"),
},
(t) => [index("heartbeat_pings_idx").on(t.heartbeatId, t.at)],
);
// "A needs B", for things that can be down: a server or an integration each. Polymorphic like maintenance_windows (no foreign key); an edge whose
// end was deleted is ignored. When B is down, A's alert is folded under B's — see services/dependencies.ts.
export const dependencies = sqliteTable(
@@ -628,8 +666,9 @@ export const alertTickets = sqliteTable(
export const shareLinkDetails = ["summary", "names"] as const;
export type ShareLinkDetail = (typeof shareLinkDetails)[number];
// What a link opens: the wall display (how things are, for a TV), or a status page (is each of the things I chose working, for people at home).
export const shareLinkKinds = ["wall", "status"] as const;
// What a link opens: the wall display (how things are, for a TV), a status page (is each of the things I chose working, for people at home), or
// a calendar feed (what is coming up, for a phone's calendar to subscribe to).
export const shareLinkKinds = ["wall", "status", "calendar"] as const;
export type ShareLinkKind = (typeof shareLinkKinds)[number];
// A link to the wall display that needs no sign-in, for a TV in the hall or someone who only wants to know if things are up. Only a hash
+15
View File
@@ -0,0 +1,15 @@
import { Router } from "express";
import { requireAuth } from "../auth/middleware.js";
import { buildAgenda, DEFAULT_DAYS, MAX_DAYS } from "../services/agenda.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const agendaRouter = Router();
agendaRouter.use(requireAuth);
// Everything with a date ahead, for everyone signed in: it is only what the pages it comes from already show them. `days` is how far ahead.
agendaRouter.get("/", asyncHandler(async (req, res) => {
const asked = Number(req.query.days);
const days = Number.isInteger(asked) && asked >= 1 && asked <= MAX_DAYS ? asked : DEFAULT_DAYS;
res.json(await buildAgenda(new Date(), days));
}));
+48 -1
View File
@@ -7,6 +7,7 @@ import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { CHECK_KINDS, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const endpointsRouter = Router();
@@ -84,8 +85,54 @@ async function serverExists(id: number | null | undefined) {
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
}
// Checks worth setting up, from what the app already knows: a web port a server is listening on, a secret that reads a certificate. Reads what is
// stored, asks nobody. Administrators only, like the checks themselves.
endpointsRouter.get("/suggestions", requireRole("admin"), asyncHandler(async (_req, res) => {
res.json(await loadSuggestions());
}));
const keysSchema = z.object({ keys: z.array(z.string().min(1).max(600)).min(1).max(MAX_CHECKS) });
// Makes the checks for the suggestions named. What each one is comes from the server's own list, not from the request, so a request can only
// ever add what was offered.
endpointsRouter.post("/suggestions/add", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = keysSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to add." });
const offered = new Map((await loadSuggestions()).suggestions.map((s) => [s.key, s]));
let have = (await db.select({ id: endpointChecks.id }).from(endpointChecks)).length;
const added: { id: number; name: string }[] = [];
for (const key of new Set(parsed.data.keys)) {
const s = offered.get(key);
if (!s) continue;
if (have >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.`, added });
const [row] = await db
.insert(endpointChecks)
.values(valuesOf({ name: s.name, kind: s.kind, target: s.target, port: s.port, serverId: s.serverId }))
.returning();
have++;
added.push({ id: row.id, name: row.name });
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target, suggested: true } });
}
res.status(201).json({ added });
}));
// "Not for me": what is dismissed stops being offered, and can be brought back.
endpointsRouter.post("/suggestions/dismiss", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = keysSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Name the suggestions to dismiss." });
const keys = await loadDismissed();
for (const k of parsed.data.keys) keys.add(k);
await saveDismissed(keys);
res.json(await loadSuggestions());
}));
endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(async (_req, res) => {
await saveDismissed(new Set());
res.json(await loadSuggestions());
}));
// Asking things on the network is an administrator's to set up.
endpointsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = checkSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
+32
View File
@@ -0,0 +1,32 @@
import express, { Router } from "express";
import { cleanMessage, recordPing, type PingKind } from "../services/heartbeats.js";
import { rateLimit } from "../utils/rateLimit.js";
import { asyncHandler } from "../utils/asyncHandler.js";
/**
* The address a job calls: /api/heartbeat/<token>, /api/heartbeat/<token>/start, /api/heartbeat/<token>/fail. No session and no sign-in — the
* token is the whole of the access, and all it allows is to say "I'm alive". Mounted ahead of the browser-only middleware (a script isn't a
* browser and has no cookie), so it answers GET, HEAD, POST and PUT alike: whatever `curl` or `wget` happens to send.
*
* It answers a token that doesn't exist, and a path that isn't one of the three, in the same two words, and is limited per address.
*/
export const heartbeatPingRouter = Router();
const limiter = rateLimit({ windowMs: 60_000, max: 120, message: "Too many requests. Wait a minute." });
// Whatever a script sends as the body ("3.2 GB copied") is a note; it is cut down to a line.
const body = express.text({ type: () => true, limit: "100kb" });
const KINDS: Record<string, PingKind> = { "": "ok", ok: "ok", start: "start", fail: "fail" };
heartbeatPingRouter.all("/:token/:kind?", limiter, body, asyncHandler(async (req, res) => {
res.setHeader("Cache-Control", "no-store");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
if (!["GET", "HEAD", "POST", "PUT"].includes(req.method)) return res.status(405).type("text/plain").send("Method not allowed");
const kind = KINDS[(req.params.kind ?? "").toLowerCase()];
if (!kind) return res.status(404).type("text/plain").send("Not found");
const fromQuery = typeof req.query.msg === "string" ? req.query.msg : null;
const message = cleanMessage(fromQuery ?? (typeof req.body === "string" ? req.body : null));
const heartbeat = await recordPing(req.params.token, kind, message);
if (!heartbeat) return res.status(404).type("text/plain").send("Not found");
res.type("text/plain").send("OK");
}));
+111
View File
@@ -0,0 +1,111 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { heartbeats, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { dueBy, heartbeatState, listHeartbeats, loadRecent, MAX_GRACE_MINUTES, MAX_PERIOD_MINUTES, newToken } from "../services/heartbeats.js";
import { env } from "../env.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const heartbeatsRouter = Router();
heartbeatsRouter.use(requireAuth);
const MAX_HEARTBEATS = 100;
const schema = z.object({
name: z.string().trim().min(1).max(100),
periodMinutes: z.number().int().min(1).max(MAX_PERIOD_MINUTES),
graceMinutes: z.number().int().min(0).max(MAX_GRACE_MINUTES).optional(),
serverId: z.number().int().positive().nullable().optional(),
notes: z.string().max(500).nullable().optional(),
enabled: z.boolean().optional(),
});
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
const pathOf = (token: string) => `/api/heartbeat/${token}`;
async function serverExists(id: number | null | undefined) {
return id == null || (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id))).length > 0;
}
// Everyone signed in can see how the jobs are doing, like the Alerts page. The address with its token is shown only to administrators: it
// is what lets anyone say "I'm alive" for the job.
heartbeatsRouter.get("/", asyncHandler(async (req, res) => {
const now = new Date();
const isAdmin = req.currentUser!.role === "admin";
const rows = await listHeartbeats();
const recent = await loadRecent(rows.map((r) => r.id));
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
res.json({
baseUrl: isAdmin ? env.appBaseUrl.replace(/\/+$/, "") : null,
heartbeats: rows.map(({ token, ...r }) => ({
...r,
path: isAdmin ? pathOf(token) : null,
state: heartbeatState(r, now),
dueBy: dueBy(r).toISOString(),
server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null,
recent: recent.get(r.id) ?? [],
})),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
});
}));
heartbeatsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = schema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.error.flatten() });
if ((await db.select({ id: heartbeats.id }).from(heartbeats)).length >= MAX_HEARTBEATS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_HEARTBEATS} heartbeats.` });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const d = parsed.data;
const [row] = await db
.insert(heartbeats)
.values({ name: d.name, token: newToken(), periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? 10, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? true })
.returning();
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "create", targetType: "heartbeat", targetId: row.id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
const { token, ...rest } = row;
res.status(201).json({ heartbeat: { ...rest, path: pathOf(token) } });
}));
heartbeatsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
const parsed = schema.safeParse(req.body);
if (!Number.isInteger(id) || !parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name and say how often it should report (1 minute to 30 days).", details: parsed.success ? undefined : parsed.error.flatten() });
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const d = parsed.data;
const [row] = await db
.update(heartbeats)
.set({ name: d.name, periodMinutes: d.periodMinutes, graceMinutes: d.graceMinutes ?? before.graceMinutes, serverId: d.serverId ?? null, notes: blank(d.notes), enabled: d.enabled ?? before.enabled })
.where(eq(heartbeats.id, id))
.returning();
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "update", targetType: "heartbeat", targetId: id, detail: { name: row.name, periodMinutes: row.periodMinutes } });
invalidateAlertsCache();
const { token, ...rest } = row;
res.json({ heartbeat: { ...rest, path: pathOf(token) } });
}));
// A new address, when the old one has got out. The old one stops working at once.
heartbeatsRouter.post("/:id/new-address", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.update(heartbeats).set({ token: newToken() }).where(eq(heartbeats.id, id)).returning();
if (!row) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "new_address", targetType: "heartbeat", targetId: id, detail: { name: row.name } });
res.json({ path: pathOf(row.token) });
}));
heartbeatsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = idOf(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [before] = await db.select().from(heartbeats).where(eq(heartbeats.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
await db.delete(heartbeats).where(eq(heartbeats.id, id));
await recordAudit({ actor: req.currentUser!, category: "heartbeat", action: "delete", targetType: "heartbeat", targetId: id, detail: { name: before.name } });
invalidateAlertsCache();
res.status(204).end();
}));
+24
View File
@@ -0,0 +1,24 @@
import { Router } from "express";
import { NOT_FOUND, resolveShareLink } from "../services/shareLinks.js";
import { buildAgenda, buildIcs, calendarName, MAX_DAYS } from "../services/agenda.js";
import { env } from "../env.js";
import { rateLimit } from "../utils/rateLimit.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const publicCalendarRouter = Router();
// No sign-in: the secret in the address is the whole of the access, so this reads nothing but what is coming up, and answers a link that doesn't
// work — never made, ended, taken back, or one that opens something other than a calendar — in the same words. The address ends in .ics because
// calendar apps want that.
const limiter = rateLimit({ windowMs: 60_000, max: 60, message: "Too many requests. Wait a minute." });
publicCalendarRouter.get("/:file", limiter, asyncHandler(async (req, res) => {
const token = req.params.file.endsWith(".ics") ? req.params.file.slice(0, -4) : req.params.file;
const link = await resolveShareLink(token);
if (!link || link.kind !== "calendar") return res.status(404).json(NOT_FOUND);
const feed = buildIcs(await buildAgenda(new Date(), MAX_DAYS), { name: calendarName(), detail: link.detail, baseUrl: link.detail === "names" ? env.appBaseUrl : undefined });
res.setHeader("Cache-Control", "private, max-age=300");
res.setHeader("X-Robots-Tag", "noindex, nofollow");
res.setHeader("Content-Disposition", 'inline; filename="homelab-coming-up.ics"');
res.type("text/calendar; charset=utf-8").send(feed);
}));
+2 -2
View File
@@ -20,7 +20,7 @@ const MAX_LINKS = 20;
const createSchema = z.object({
label: z.string().trim().min(1).max(60),
kind: z.enum(shareLinkKinds).default("wall"),
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it.
// For a wall: how much it shows. A status page always shows only names and states, so this is ignored for it. For a calendar: summary leaves the names out of the events.
detail: z.enum(shareLinkDetails).default("summary"),
title: z.string().trim().max(MAX_TITLE).optional(),
message: z.string().trim().max(MAX_MESSAGE).optional(),
@@ -59,7 +59,7 @@ shareLinksRouter.post("/", asyncHandler(async (req, res) => {
targetId: link.id,
detail: { label: link.label, kind: link.kind, detail: link.detail, expiresAt: link.expiresAt },
});
res.status(201).json({ link: publicShareLink(link), token, path: `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
res.status(201).json({ link: publicShareLink(link), token, path: link.kind === "calendar" ? `/api/public/calendar/${token}.ics` : `/share/${link.kind === "status" ? "status" : "wall"}/${token}` });
}));
const idOf = (v: string) => (/^\d+$/.test(v) ? Number(v) : NaN);
+343
View File
@@ -0,0 +1,343 @@
/**
* "Coming up": everything with a date ahead, in one list — a secret, domain, certificate or Tailscale key that expires, a warranty or end of support,
* planned and recurring maintenance, a restore test that is due, and updates that have been waiting. Each already lives on a page of its own and in the
* weekly digest; this puts them side by side, oldest first, so what is about to need attention is one look, not a dozen.
*
* Built in two parts so it can be tested exactly: `loadAgendaData` reads what is stored (and, for Tailscale keys, asks Tailscale, remembered for ten
* minutes), and `agendaFromData` turns that into items. `buildIcs` writes the same items as an iCalendar feed for a phone's calendar to subscribe to.
*/
import { createHash } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, domains, endpointChecks, integrations, maintenanceSchedules, secrets, servers } from "../db/schema.js";
import { collectTailscaleKeyExpiries } from "./tailscaleKeyExpiryScheduler.js";
import { describeTarget, listUpcomingWindows } from "./maintenance.js";
import { occurrencesBetween, targetsOf } from "./maintenanceSchedules.js";
import { parsePendingUpdates } from "./pendingUpdates.js";
import { latestTests } from "./restoreTests.js";
import { getSettings } from "./settingsStore.js";
import { env } from "../env.js";
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates"] as const;
export type AgendaKind = (typeof AGENDA_KINDS)[number];
export const KIND_LABELS: Record<AgendaKind, string> = {
secret: "Secrets",
certificate: "Certificates",
domain: "Domains",
tailscale: "Tailscale keys",
warranty: "Warranties",
support: "End of support",
maintenance: "Maintenance",
restore: "Restore tests",
updates: "Updates waiting",
};
export interface AgendaItem {
/** Stable, with the date in it: a secret that is renewed is a different entry, and the old one just goes. */
id: string;
kind: AgendaKind;
/** YYYY-MM-DD for something that happens on a day; an ISO time for maintenance, which has a start and an end. */
at: string;
endsAt: string | null;
allDay: boolean;
title: string;
detail: string | null;
/** The page of the app that has more. */
link: string;
serverId: number | null;
/** Whole days from today to the day it falls on, in the time zone set for notifications; negative once it has passed. */
daysLeft: number;
}
export interface Agenda {
generatedAt: string;
today: string;
items: AgendaItem[];
/** Sources that could not be read, so the agenda is missing something. */
notes: string[];
}
export const DEFAULT_DAYS = 90;
export const MAX_DAYS = 400;
/** What has already happened is shown for this long (a secret that expired last week is still worth seeing). */
export const LOOK_BACK_DAYS = 30;
const DAY_MS = 86_400_000;
// ─── Dates ─────────────────────────────────────────────────────────────────
const isYmd = (v: unknown): v is string => typeof v === "string" && /^\d{4}-\d{2}-\d{2}$/.test(v) && !Number.isNaN(Date.parse(`${v}T00:00:00Z`)) && new Date(`${v}T00:00:00Z`).toISOString().startsWith(v);
/** The calendar day an instant falls on, as the clock in `tz` sees it. */
export function localDate(when: Date, tz: string): string {
return new Intl.DateTimeFormat("en-CA", { timeZone: tz, year: "numeric", month: "2-digit", day: "2-digit" }).format(when);
}
const daysBetween = (fromYmd: string, toYmd: string) => Math.round((Date.parse(`${toYmd}T00:00:00Z`) - Date.parse(`${fromYmd}T00:00:00Z`)) / DAY_MS);
export const addDays = (ymd: string, n: number) => new Date(Date.parse(`${ymd}T00:00:00Z`) + n * DAY_MS).toISOString().slice(0, 10);
// ─── What is stored ────────────────────────────────────────────────────────
export interface AgendaData {
secrets: { id: number; name: string; type: string; expiryDate: string | null }[];
domains: { id: number; name: string; expiresAt: string | null }[];
endpoints: { id: number; name: string; enabled: boolean; certExpiresAt: string | null; serverId: number | null }[];
tailscale: { integrationId: number; integrationName: string; deviceLabel: string; daysLeft: number }[];
assets: { id: number; name: string; warrantyUntil: string | null; endOfSupport: string | null; serverId: number | null }[];
/** Windows that have been set up but not begun, one per target. */
windows: { startedAt: string; endsAt: string; reason: string | null; source: string | null; targetName: string }[];
schedules: { id: number; name: string; reason: string | null; days: string; startTime: string; durationMinutes: number; timezone: string; targetNames: string[] }[];
/** The newest restore test of each subject. */
restoreTests: { subjectKey: string; label: string; testedAt: string; result: string }[];
restoreLimitDays: number;
/** Servers with updates waiting. */
updates: { name: string; count: number; security: number | null; since: string | null }[];
notes: string[];
}
let tailscaleCache: { at: number; items: AgendaData["tailscale"]; failures: string[] } | null = null;
const TAILSCALE_CACHE_MS = 10 * 60_000;
export const clearAgendaCacheForTests = () => {
tailscaleCache = null;
};
async function tailscaleKeys(): Promise<{ items: AgendaData["tailscale"]; failures: string[] }> {
if (tailscaleCache && Date.now() - tailscaleCache.at < TAILSCALE_CACHE_MS) return tailscaleCache;
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "tailscale"), eq(integrations.enabled, true)));
if (rows.length === 0) return { items: [], failures: [] };
const { items, failures } = await collectTailscaleKeyExpiries();
tailscaleCache = { at: Date.now(), items, failures: failures.map((f) => `Tailscale (${f.integrationName}): ${f.message}`) };
return tailscaleCache;
}
export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData> {
const notes: string[] = [];
const settings = await getSettings();
const [secretRows, domainRows, endpointRows, assetRows, scheduleRows, serverRows, tests] = await Promise.all([
db.select({ id: secrets.id, name: secrets.name, type: secrets.type, expiryDate: secrets.expiryDate }).from(secrets),
db.select({ id: domains.id, name: domains.name, expiresAt: domains.expiresAt }).from(domains),
db.select({ id: endpointChecks.id, name: endpointChecks.name, enabled: endpointChecks.enabled, certExpiresAt: endpointChecks.certExpiresAt, serverId: endpointChecks.serverId }).from(endpointChecks),
db.select({ id: assets.id, name: assets.name, warrantyUntil: assets.warrantyUntil, endOfSupport: assets.endOfSupport, serverId: assets.serverId }).from(assets),
db.select().from(maintenanceSchedules),
db.select({ name: servers.name, pendingUpdates: servers.pendingUpdates, since: servers.updatesPendingSince }).from(servers),
latestTests(),
]);
const windows: AgendaData["windows"] = [];
for (const w of await listUpcomingWindows(now)) {
const target = await describeTarget(w.targetType, w.targetId);
if (target) windows.push({ startedAt: w.startedAt, endsAt: w.endsAt, reason: w.reason, source: w.source, targetName: target.name });
}
const schedules: AgendaData["schedules"] = [];
for (const s of scheduleRows.filter((r) => r.enabled)) {
const names: string[] = [];
for (const t of targetsOf(s)) {
const d = await describeTarget(t.type, t.id);
if (d) names.push(d.name);
}
schedules.push({ id: s.id, name: s.name, reason: s.reason, days: s.days, startTime: s.startTime, durationMinutes: s.durationMinutes, timezone: s.timezone, targetNames: names });
}
const updates: AgendaData["updates"] = [];
for (const s of serverRows) {
const u = parsePendingUpdates(s.pendingUpdates);
if (u && u.count > 0) updates.push({ name: s.name, count: u.count, security: u.security, since: s.since });
}
let tailscale: AgendaData["tailscale"] = [];
try {
const t = await tailscaleKeys();
tailscale = t.items;
notes.push(...t.failures);
} catch (err) {
notes.push(`Tailscale keys could not be read: ${err instanceof Error ? err.message : String(err)}`);
}
return {
secrets: secretRows,
domains: domainRows,
endpoints: endpointRows,
tailscale,
assets: assetRows,
windows,
schedules,
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
restoreLimitDays: settings.healthChecks.restoreTestDays,
updates,
notes,
};
}
// ─── Making the list ───────────────────────────────────────────────────────
const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
/** Pure: the items, from what was read. `days` is how far ahead; what has just passed is kept for LOOK_BACK_DAYS. */
export function agendaFromData(data: AgendaData, now: Date, tz: string, days: number = DEFAULT_DAYS): Agenda {
const today = localDate(now, tz);
const horizon = Math.min(Math.max(Math.round(days), 1), MAX_DAYS);
const items: AgendaItem[] = [];
const day = (id: string, kind: AgendaKind, ymd: string | null, fn: (past: boolean) => { title: string; detail?: string | null; link: string; serverId?: number | null }) => {
if (!isYmd(ymd)) return;
const daysLeft = daysBetween(today, ymd);
if (daysLeft < -LOOK_BACK_DAYS || daysLeft > horizon) return;
const { title, detail = null, link, serverId = null } = fn(daysLeft < 0);
items.push({ id: `${id}:${ymd}`, kind, at: ymd, endsAt: null, allDay: true, title, detail, link, serverId, daysLeft });
};
for (const s of data.secrets) {
const certificate = s.type === "ssl_certificate";
day(`secret:${s.id}`, certificate ? "certificate" : "secret", s.expiryDate, (past) => ({ title: `${certificate ? "Certificate" : "Secret"} “${s.name}” ${past ? "expired" : "expires"}`, detail: certificate ? "kept as a secret" : null, link: "/secrets" }));
}
for (const d of data.domains) day(`domain:${d.id}`, "domain", d.expiresAt, (past) => ({ title: `Domain ${d.name} ${past ? "expired" : "expires"}`, link: "/domains" }));
for (const e of data.endpoints) {
if (!e.enabled || !e.certExpiresAt) continue;
day(`endpoint-cert:${e.id}`, "certificate", e.certExpiresAt.slice(0, 10), (past) => ({ title: `The certificate for ${e.name} ${past ? "expired" : "expires"}`, detail: "seen by an endpoint check", link: "/endpoints", serverId: e.serverId }));
}
for (const t of data.tailscale) {
day(`tailscale:${t.integrationId}:${t.deviceLabel}`, "tailscale", addDays(today, t.daysLeft), (past) => ({ title: `Tailscale key for ${t.deviceLabel} ${past ? "expired" : "expires"}`, detail: t.integrationName, link: "/tailscale" }));
}
for (const a of data.assets) {
day(`warranty:${a.id}`, "warranty", a.warrantyUntil, (past) => ({ title: `Warranty on ${a.name} ${past ? "ended" : "ends"}`, link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
day(`support:${a.id}`, "support", a.endOfSupport, (past) => ({ title: `Support for ${a.name} ${past ? "ended" : "ends"}`, detail: "time to plan its replacement", link: `/inventory?asset=${a.id}`, serverId: a.serverId }));
}
// A restore test is due a set time after the last good one — only if a time limit is set (Settings → Notifications).
if (data.restoreLimitDays > 0) {
for (const t of data.restoreTests) {
if ((t.result !== "ok" && t.result !== "partial") || !isYmd(t.testedAt)) continue;
day(`restore:${t.subjectKey}`, "restore", addDays(t.testedAt, data.restoreLimitDays), (past) => ({ title: `Restore test ${past ? "was due" : "due"}: ${t.label}`, detail: `last tested ${t.testedAt}`, link: "/backups" }));
}
}
// Maintenance has a start and an end. Windows set up by a recurring schedule are left to the schedule (they are only made a week ahead).
const lastMs = now.getTime() + horizon * DAY_MS;
const groups = new Map<string, { start: string; end: string; reason: string | null; names: string[] }>();
for (const w of data.windows) {
if (w.source?.startsWith("schedule:") || Date.parse(w.startedAt) > lastMs) continue;
const key = `${w.startedAt}|${w.endsAt}|${w.reason ?? ""}`;
const g = groups.get(key) ?? { start: w.startedAt, end: w.endsAt, reason: w.reason, names: [] };
g.names.push(w.targetName);
groups.set(key, g);
}
const timed = (id: string, start: string, end: string, title: string, detail: string | null) =>
items.push({ id, kind: "maintenance", at: start, endsAt: end, allDay: false, title, detail, link: "/maintenance", serverId: null, daysLeft: daysBetween(today, localDate(new Date(start), tz)) });
const names = (list: string[]) => {
const sorted = [...list].sort((a, b) => a.localeCompare(b));
return sorted.length <= 3 ? sorted.join(", ") : `${sorted.slice(0, 3).join(", ")} and ${sorted.length - 3} more`;
};
for (const g of groups.values()) timed(`maintenance:${g.start}:${g.names.length}`, g.start, g.end, `Maintenance: ${names(g.names)}`, g.reason);
for (const s of data.schedules) {
for (const occ of occurrencesBetween(s, now, new Date(lastMs)).slice(0, 60)) {
timed(`schedule:${s.id}:${occ.date}`, occ.start.toISOString(), occ.end.toISOString(), `Recurring maintenance: ${s.name}`, [s.targetNames.length > 0 ? names(s.targetNames) : null, s.reason].filter(Boolean).join(" — ") || null);
}
}
// Updates that have been waiting: one line for all of them, from the longest wait.
if (data.updates.length > 0) {
const since = data.updates.map((u) => u.since).filter((s): s is string => !!s).map((s) => localDate(new Date(s), tz)).sort()[0] ?? today;
const security = data.updates.filter((u) => (u.security ?? 0) > 0).length;
const waited = Math.max(0, daysBetween(since, today));
items.push({
id: `updates:${since}`,
kind: "updates",
at: since,
endsAt: null,
allDay: true,
title: `Updates waiting on ${plural(data.updates.length, "server")}${security > 0 ? ` (${security} with security fixes)` : ""}`,
detail: waited > 0 ? `the longest wait is ${plural(waited, "day")}` : "reported today",
link: "/updates",
serverId: null,
daysLeft: -waited,
});
}
items.sort((a, b) => a.at.localeCompare(b.at) || a.title.localeCompare(b.title));
return { generatedAt: now.toISOString(), today, items, notes: data.notes };
}
export async function buildAgenda(now: Date = new Date(), days: number = DEFAULT_DAYS): Promise<Agenda> {
const settings = await getSettings();
return agendaFromData(await loadAgendaData(now), now, settings.notifications.timezone || "UTC", days);
}
// ─── The calendar feed ─────────────────────────────────────────────────────
/** Escapes a text value for iCalendar: backslash, semicolon, comma and line breaks. */
const escapeText = (s: string) => s.replace(/\\/g, "\\\\").replace(/;/g, "\\;").replace(/,/g, "\\,").replace(/\r?\n/g, "\\n");
/** Lines are folded at 75 bytes, a space starting each continuation, without splitting a character. */
export function foldLine(line: string): string {
const out: string[] = [];
let current = "";
let bytes = 0;
for (const ch of line) {
const n = Buffer.byteLength(ch, "utf8");
if (bytes + n > (out.length === 0 ? 75 : 74)) {
out.push(current);
current = "";
bytes = 0;
}
current += ch;
bytes += n;
}
out.push(current);
return out.join("\r\n ");
}
const icsDate = (ymd: string) => ymd.replace(/-/g, "");
const icsTime = (iso: string) => new Date(iso).toISOString().replace(/[-:]/g, "").replace(/\.\d{3}/, "");
/** What an event is called when the link was made to show no names: only what kind of thing it is. */
const GENERIC: Record<AgendaKind, string> = {
secret: "A secret expires",
certificate: "A certificate expires",
domain: "A domain name expires",
tailscale: "A Tailscale key expires",
warranty: "A warranty ends",
support: "Support ends for a device",
maintenance: "Planned maintenance",
restore: "A restore test is due",
updates: "Updates are waiting",
};
/**
* The agenda as an iCalendar feed. A day is an all-day event; maintenance is an event from its start to its end. With `detail` "summary" an event only
* says what kind of thing it is (no names), so a feed that is shared can be shown without telling anyone what is in the homelab. Updates that
* are waiting are not events — they have no day — and are left out.
*/
export function buildIcs(agenda: Agenda, options: { name: string; detail: "summary" | "names"; baseUrl?: string }): string {
const lines = [
"BEGIN:VCALENDAR",
"VERSION:2.0",
"PRODID:-//Homelab Manager//Coming up//EN",
"CALSCALE:GREGORIAN",
"METHOD:PUBLISH",
`X-WR-CALNAME:${escapeText(options.name)}`,
"REFRESH-INTERVAL;VALUE=DURATION:PT6H",
"X-PUBLISHED-TTL:PT6H",
];
const stamp = icsTime(agenda.generatedAt);
for (const item of agenda.items) {
if (item.kind === "updates") continue;
if (item.daysLeft < -7) continue; // an old event in a subscribed calendar is just clutter
const names = options.detail === "names";
lines.push("BEGIN:VEVENT", `UID:${createHash("sha256").update(item.id).digest("hex").slice(0, 24)}@homelab-manager`, `DTSTAMP:${stamp}`);
if (item.allDay) lines.push(`DTSTART;VALUE=DATE:${icsDate(item.at)}`, `DTEND;VALUE=DATE:${icsDate(addDays(item.at, 1))}`);
else lines.push(`DTSTART:${icsTime(item.at)}`, `DTEND:${icsTime(item.endsAt ?? item.at)}`);
lines.push(`SUMMARY:${escapeText(names ? item.title : GENERIC[item.kind])}`);
if (names) {
const description = [item.detail, options.baseUrl ? `${options.baseUrl.replace(/\/+$/, "")}${item.link}` : null].filter(Boolean).join("\n");
if (description) lines.push(`DESCRIPTION:${escapeText(description)}`);
}
lines.push(`CATEGORIES:${escapeText(KIND_LABELS[item.kind])}`, "TRANSP:TRANSPARENT", "STATUS:CONFIRMED", "END:VEVENT");
}
lines.push("END:VCALENDAR");
return `${lines.map(foldLine).join("\r\n")}\r\n`;
}
/** What the feed calls itself. */
export const calendarName = () => `Homelab Manager${env.environment?.name ? ` (${env.environment.name})` : ""} — coming up`;
+19 -1
View File
@@ -11,7 +11,7 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
@@ -35,6 +35,7 @@ import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import { endpointConditions } from "./endpointChecks.js";
import { heartbeatConditions } from "./heartbeats.js";
import { latestTests, restoreAlerts } from "./restoreTests.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
@@ -65,6 +66,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
};
const SOURCE_TIMEOUT_MS = 20_000;
@@ -304,6 +306,22 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// Jobs that report in: one that has not done so in time, or said it failed.
check("Heartbeats", async () => {
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
push({
category: c.category,
severity: c.severity,
type: "heartbeat",
message: c.message,
key: c.key,
link: "/heartbeats",
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
});
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
check("Homelab Manager's own backups", async () => {
+13 -3
View File
@@ -21,7 +21,7 @@ export const KIND_LABELS: Record<ChangeKind, string> = {
dependencies: "Dependencies",
inventory: "Inventory",
backups: "Backups and restore tests",
endpoints: "Endpoint checks",
endpoints: "Endpoint checks and heartbeats",
other: "Other",
};
@@ -53,6 +53,7 @@ const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
db_backup: "backups",
restore_test: "backups",
endpoint: "endpoints",
heartbeat: "endpoints",
};
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
@@ -154,6 +155,14 @@ export function describeChange(category: string, action: string, detail: Record<
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "heartbeat/create":
return { title: `Heartbeat added${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
case "heartbeat/update":
return { title: `Heartbeat changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: typeof detail.periodMinutes === "number" ? `expected every ${detail.periodMinutes} min` : null };
case "heartbeat/new_address":
return { title: `Heartbeat given a new address${name ? `: ${name}` : ""}`, tone: "neutral", detail: "the old address no longer works" };
case "heartbeat/delete":
return { title: `Heartbeat removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "restore_test/record": {
const result = str(detail.result);
const tone: ActivityTone = result === "failed" ? "bad" : result === "not_needed" ? "neutral" : "good";
@@ -250,7 +259,8 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
} else if (a.targetType === "endpoint" && idNum !== null) {
targetName = endpointName.get(idNum) ?? targetName;
link = "/endpoints";
} else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
} else if (a.targetType === "heartbeat") link = "/heartbeats";
else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
else if (a.targetType === "restore_test") link = "/backups";
else if (a.targetType === "dependency") link = "/dependencies";
else if (a.targetType === "settings") link = "/settings/notifications";
@@ -280,7 +290,7 @@ export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries:
.limit(1000);
for (const i of incidents) {
const serverId = i.targetType === "server" ? i.targetId : null;
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/incidents/${i.id}`, serverId };
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/alerts/incidents/${i.id}`, serverId };
if (i.openedAt >= since.toISOString() && (!until || i.openedAt <= until.toISOString())) {
entries.push({ detail: {}, entry: { ...base, id: `incident-open-${i.id}`, at: i.openedAt, tone: i.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised: ${i.message}`, detail: i.causedBy ? `explained by ${i.causedBy}` : null } });
}
+6 -2
View File
@@ -5,9 +5,10 @@
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
*/
import { db } from "../db/client.js";
import { endpointChecks } from "../db/schema.js";
import { endpointChecks, heartbeats } from "../db/schema.js";
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
import { activeSubjects } from "./maintenance.js";
import { heartbeatConditions, prunePings } from "./heartbeats.js";
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
@@ -28,7 +29,9 @@ async function loadState(): Promise<ActiveState> {
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
const asked = await runDueChecks(now, probeFn);
const rows = await db.select().from(endpointChecks);
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
const beats = await db.select().from(heartbeats);
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
if (added.length > 0) await notifyHealthIssues(added);
@@ -48,6 +51,7 @@ export function initEndpointMonitor(): void {
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneResults();
await prunePings();
}
} catch (err) {
console.error("[endpoints] the pass failed:", err);
+186
View File
@@ -0,0 +1,186 @@
/**
* Suggested endpoint checks: things the app already knows enough about to offer a check for, so setting up monitoring is a click instead of typing.
*
* - A web port a server's agent sees open (80, 443, 8080, 8443), with a name that points at it in DNS if there is one — "git.example.lan answers on
* 443 on nas" — as a web check, linked to that server so its maintenance silences the alerts.
* - A secret that already has a host to read the certificate from, as a certificate check: the secret only warns about the expiry, this also hears
* when the service stops answering.
*
* It only reads what is stored (no network), leaves out what a check already covers, and remembers what somebody said no to. Uptime Kuma's monitors are
* not offered: they are already being watched, and the adapter only keeps a host and port for them, not the address.
*/
import { db } from "../db/client.js";
import { dnsRecordsCache, endpointChecks, secrets, servers } from "../db/schema.js";
import { isLoopback } from "./agentPorts.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export interface Suggestion {
/** Stable identity, so a dismissed suggestion stays dismissed and "add" can name it. */
key: string;
kind: "http" | "tls";
name: string;
target: string;
port: number | null;
serverId: number | null;
serverName: string | null;
/** Why it is offered, in a sentence. */
reason: string;
}
export interface SuggestionInput {
servers: { id: number; name: string; hostname: string | null; ips: string[]; ports: { protocol?: string; port: number; address?: string }[] }[];
dns: { name: string; type: string; content: string }[];
secrets: { name: string; checkHost: string | null; checkPort: number | null }[];
existing: { kind: string; target: string; port: number | null }[];
dismissed: Set<string>;
}
/** The web ports worth offering, and what they speak. */
const WEB_PORTS: Record<number, "http" | "https"> = { 80: "http", 443: "https", 8080: "http", 8443: "https" };
/** The https twin of a plain port, so a server answering on both gets the one that reads the certificate. */
const SECURE_TWIN: Record<number, number> = { 80: 443, 8080: 8443 };
const MAX_NAMES_PER_PORT = 3;
const MAX_SUGGESTIONS = 200;
const cleanHost = (h: string) => h.trim().toLowerCase().replace(/\.$/, "");
const isIPv4 = (s: string) => /^\d{1,3}(\.\d{1,3}){3}$/.test(s);
const usableIPv4 = (s: string) => isIPv4(s) && !s.startsWith("127.") && !s.startsWith("169.254.");
/** Where a check's address is, in the same words for an existing check and a suggestion: "web:host:port" and "tls:host:port" (a web check on https reads the certificate too). */
function coverage(existing: SuggestionInput["existing"]): Set<string> {
const out = new Set<string>();
for (const c of existing) {
if (c.kind === "http") {
try {
const u = new URL(c.target);
const port = u.port ? Number(u.port) : u.protocol === "https:" ? 443 : 80;
const host = cleanHost(u.hostname);
out.add(`web:${host}:${port}`);
if (u.protocol === "https:") out.add(`tls:${host}:${port}`);
} catch {
// an address that isn't one covers nothing
}
} else if (c.kind === "tls" && c.port !== null) out.add(`tls:${cleanHost(c.target)}:${c.port}`);
}
return out;
}
/** Pure: the checks worth offering, from what is known. */
export function suggestChecks(input: SuggestionInput): { suggestions: Suggestion[]; dismissed: Suggestion[] } {
const covered = coverage(input.existing);
const all = new Map<string, Suggestion>();
for (const s of input.servers) {
const ips = new Set(s.ips.map((i) => i.trim().toLowerCase()));
const hostname = s.hostname ? cleanHost(s.hostname) : null;
// Names that point at this server in DNS: an address record with one of its addresses, and an alias to one of those names or to its hostname.
const direct = new Set<string>();
for (const r of input.dns) {
if ((r.type === "A" || r.type === "AAAA") && ips.has(r.content.trim().toLowerCase())) direct.add(cleanHost(r.name));
}
const names = new Set(direct);
for (const r of input.dns) {
if (r.type !== "CNAME") continue;
const to = cleanHost(r.content);
if (direct.has(to) || (hostname !== null && to === hostname)) names.add(cleanHost(r.name));
}
const dnsNames = [...names].filter((n) => n.includes(".") && !n.startsWith("*") && !n.endsWith(".arpa") && !/\s/.test(n)).sort((a, b) => a.length - b.length || a.localeCompare(b));
const listening = new Set(s.ports.filter((p) => (p.protocol ?? "tcp") === "tcp" && !(p.address && isLoopback(p.address))).map((p) => p.port));
for (const port of Object.keys(WEB_PORTS).map(Number).filter((p) => listening.has(p))) {
// Plain and secure on the same server: the secure one is the real service (the plain one is most likely its redirect), and reads the certificate.
if (SECURE_TWIN[port] !== undefined && listening.has(SECURE_TWIN[port])) continue;
const scheme = WEB_PORTS[port];
const fallback = hostname && hostname.includes(".") ? [hostname] : [s.ips.map((i) => i.trim()).find(usableIPv4)].filter((x): x is string => !!x);
const hosts = dnsNames.length > 0 ? dnsNames.slice(0, MAX_NAMES_PER_PORT) : fallback;
for (const host of hosts) {
const defaultPort = scheme === "https" ? 443 : 80;
const target = `${scheme}://${host}${port === defaultPort ? "" : `:${port}`}/`;
if (covered.has(`web:${host}:${port}`)) continue;
const viaDns = dnsNames.includes(host);
all.set(`http|${target}`, {
key: `http|${target}`,
kind: "http",
name: `${host}${port === defaultPort ? "" : `:${port}`}`,
target,
port: null,
serverId: s.id,
serverName: s.name,
reason: `${s.name} is listening on port ${port}${viaDns ? `, and ${host} points at it in DNS` : ""}.`,
});
}
}
}
for (const sec of input.secrets) {
if (!sec.checkHost || sec.checkPort === null) continue;
const host = cleanHost(sec.checkHost);
if (covered.has(`tls:${host}:${sec.checkPort}`)) continue;
all.set(`tls|${host}|${sec.checkPort}`, {
key: `tls|${host}|${sec.checkPort}`,
kind: "tls",
name: `Certificate: ${sec.name}`,
target: host,
port: sec.checkPort,
serverId: null,
serverName: null,
reason: `The secret “${sec.name}” already reads this certificate; a check also tells you when ${host} stops answering.`,
});
}
// By server, those about no server last, then by address.
const sorted = [...all.values()].sort((a, b) => Number(a.serverName === null) - Number(b.serverName === null) || (a.serverName ?? "").localeCompare(b.serverName ?? "") || a.target.localeCompare(b.target));
return {
suggestions: sorted.filter((x) => !input.dismissed.has(x.key)).slice(0, MAX_SUGGESTIONS),
dismissed: sorted.filter((x) => input.dismissed.has(x.key)),
};
}
// ─── What was said no to ───────────────────────────────────────────────────
const DISMISSED_FLAG = "endpointSuggestionsDismissed";
const MAX_DISMISSED = 500;
export async function loadDismissed(): Promise<Set<string>> {
const raw = await getInternalFlag(DISMISSED_FLAG);
if (!raw) return new Set();
try {
const v = JSON.parse(raw);
return new Set(Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : []);
} catch {
return new Set();
}
}
export async function saveDismissed(keys: Set<string>): Promise<void> {
await setInternalFlag(DISMISSED_FLAG, JSON.stringify([...keys].slice(-MAX_DISMISSED)));
}
// ─── Reading what is stored ────────────────────────────────────────────────
function jsonArray<T>(stored: string | null): T[] {
if (!stored) return [];
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? (v as T[]) : [];
} catch {
return [];
}
}
export async function loadSuggestions(): Promise<{ suggestions: Suggestion[]; dismissed: Suggestion[] }> {
const [serverRows, dnsRows, secretRows, existing, dismissed] = await Promise.all([
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses, listeningPorts: servers.listeningPorts }).from(servers),
db.select({ name: dnsRecordsCache.name, type: dnsRecordsCache.type, content: dnsRecordsCache.content }).from(dnsRecordsCache),
db.select({ name: secrets.name, checkHost: secrets.checkHost, checkPort: secrets.checkPort }).from(secrets),
db.select({ kind: endpointChecks.kind, target: endpointChecks.target, port: endpointChecks.port }).from(endpointChecks),
loadDismissed(),
]);
return suggestChecks({
servers: serverRows.map((s) => ({ id: s.id, name: s.name, hostname: s.hostname, ips: jsonArray<string>(s.ipAddresses).filter((i) => typeof i === "string"), ports: jsonArray<{ protocol?: string; port: number; address?: string }>(s.listeningPorts).filter((p) => p && Number.isInteger(p.port)) })),
dns: dnsRows,
secrets: secretRows,
existing,
dismissed,
});
}
+170
View File
@@ -0,0 +1,170 @@
/**
* Heartbeats: "did my 3 a.m. job actually run?". A cron job or backup script calls an address when it finishes; the app writes down that it did, and
* speaks up — an alert and a notification, like any other problem — when the call doesn't come in time, or when the job says it failed.
*
* Each heartbeat has an address with a long random token in it (`/api/heartbeat/<token>`) and a period: how often the job should report, plus a grace
* period for jobs that run a little late. Three calls: `…/<token>` (it worked), `…/<token>/start` (it began, so the next call can say how long it took)
* and `…/<token>/fail` (it ran and went wrong — this is an alert at once, without waiting for the time to pass). Anything after the call, in the
* body or in `?msg=`, is kept as a short note ("3.2 GB, 41 s").
*
* The token is the whole of the access a call needs, and all it allows is to say "I'm alive": it can't read anything. It is kept as it is, not hashed
* (unlike a share link), because the address has to be shown again when somebody sets the job up on another machine.
*/
import { randomBytes } from "node:crypto";
import { and, asc, desc, eq, inArray, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { heartbeatPings, heartbeats } from "../db/schema.js";
import { invalidateAlertsCache } from "./alerts.js";
export type HeartbeatRow = typeof heartbeats.$inferSelect;
export type PingKind = "ok" | "fail" | "start";
export const PING_KINDS: readonly PingKind[] = ["ok", "fail", "start"];
export const KEEP_PING_DAYS = 14;
export const MAX_MESSAGE = 200;
/** "hb_" and then random letters and digits: what an address is allowed to carry. */
export const TOKEN_PATTERN = /^hb_[A-Za-z0-9_-]{20,64}$/;
const MIN_PERIOD = 1;
export const MAX_PERIOD_MINUTES = 60 * 24 * 30;
export const MAX_GRACE_MINUTES = 60 * 24;
/** A "started" call that is older than this is not what a later "finished" call is timing. */
const MAX_RUN_MS = 7 * 86_400_000;
export const newToken = () => `hb_${randomBytes(18).toString("base64url")}`;
/** SQLite's own timestamps ("2026-10-19 03:10:00") are UTC without a "T"; ours are ISO. Both read as the instant they are. */
export function parseStamp(value: string): number {
return Date.parse(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value) ? `${value.replace(" ", "T")}Z` : value);
}
/** A note from a job, made fit to store and show: one line, no control characters, not long. */
export function cleanMessage(raw: unknown): string | null {
if (typeof raw !== "string") return null;
// eslint-disable-next-line no-control-regex
const text = raw.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
return text === "" ? null : text.slice(0, MAX_MESSAGE);
}
// ─── What a heartbeat's state is ───────────────────────────────────────────
export type HeartbeatState = "off" | "waiting" | "up" | "running" | "late" | "failed";
/** When the next call is due by, counting the grace: from the last call, or from the day it was made if it has never called. */
export function dueBy(h: Pick<HeartbeatRow, "periodMinutes" | "graceMinutes" | "lastPingAt" | "createdAt">): Date {
const base = h.lastPingAt ? parseStamp(h.lastPingAt) : parseStamp(h.createdAt);
return new Date(base + (h.periodMinutes + h.graceMinutes) * 60_000);
}
/**
* Pure: how a heartbeat is doing. A job that said it failed is failed until it next succeeds; otherwise it is late once the time (period and
* grace) has passed without a call. A new one that has not called yet is waiting, and is late if its first call is overdue too.
*/
export function heartbeatState(h: Pick<HeartbeatRow, "enabled" | "periodMinutes" | "graceMinutes" | "lastPingAt" | "lastStatus" | "lastStartAt" | "createdAt">, now: Date): HeartbeatState {
if (!h.enabled) return "off";
if (h.lastStatus === "fail") return "failed";
if (now.getTime() > dueBy(h).getTime()) return "late";
if (!h.lastPingAt) return "waiting";
if (h.lastStartAt && parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt)) return "running";
return "up";
}
// ─── What is wrong ─────────────────────────────────────────────────────────
export interface HeartbeatCondition {
key: string;
source: "heartbeat";
message: string;
severity: "critical" | "warning";
category: "automation";
heartbeatId: number;
serverId: number | null;
}
const stamp = (d: Date) => `${d.toISOString().slice(0, 16).replace("T", " ")} UTC`;
/**
* The problems the heartbeats stand for. A call that didn't come in time is a warning, and says when it was due (a time, not "N minutes ago", so
* the message doesn't change on every pass); a job that reported a failure is critical. The key carries the server a heartbeat is linked to, so
* maintenance on it silences them.
*/
export function heartbeatConditions(rows: HeartbeatRow[], now: Date): HeartbeatCondition[] {
const out: HeartbeatCondition[] = [];
for (const h of rows) {
const state = heartbeatState(h, now);
const suffix = h.serverId !== null ? `:server:${h.serverId}` : "";
if (state === "late") {
const heard = h.lastPingAt ? `last heard ${stamp(new Date(parseStamp(h.lastPingAt)))}` : "it has never reported";
out.push({ key: `heartbeat:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "warning", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} hasn't reported — it was due by ${stamp(dueBy(h))} (${heard})` });
} else if (state === "failed") {
out.push({ key: `heartbeat-fail:${h.id}${suffix}`, source: "heartbeat", category: "automation", severity: "critical", heartbeatId: h.id, serverId: h.serverId, message: `${h.name} reported a failure${h.lastMessage ? ` — ${h.lastMessage}` : ""}` });
}
}
return out;
}
// ─── A call coming in ──────────────────────────────────────────────────────
/**
* Writes down a call. Returns the heartbeat as it is now, or null when there is no heartbeat with that token. A heartbeat that is switched off
* still takes calls (so a script isn't broken by pausing it) and notes them, so that switching it back on starts from a fresh clock.
*/
export async function recordPing(token: string, kind: PingKind, message: string | null, now: Date = new Date()): Promise<HeartbeatRow | null> {
if (!TOKEN_PATTERN.test(token)) return null;
const [h] = await db.select().from(heartbeats).where(eq(heartbeats.token, token)).limit(1);
if (!h) return null;
const iso = now.toISOString();
let durationMs: number | null = null;
let updated: HeartbeatRow;
if (kind === "start") {
[updated] = await db.update(heartbeats).set({ lastStartAt: iso }).where(eq(heartbeats.id, h.id)).returning();
} else {
if (h.lastStartAt) {
const ran = now.getTime() - parseStamp(h.lastStartAt);
const startedAfterLastEnd = !h.lastPingAt || parseStamp(h.lastStartAt) > parseStamp(h.lastPingAt);
if (startedAfterLastEnd && ran >= 0 && ran <= MAX_RUN_MS) durationMs = ran;
}
[updated] = await db
.update(heartbeats)
.set({ lastPingAt: iso, lastStatus: kind, lastMessage: message, lastDurationMs: durationMs, pingCount: h.pingCount + 1 })
.where(eq(heartbeats.id, h.id))
.returning();
}
await db.insert(heartbeatPings).values({ heartbeatId: h.id, at: iso, kind, durationMs, message });
invalidateAlertsCache();
return updated;
}
// ─── History ───────────────────────────────────────────────────────────────
export interface PingSummary {
at: string;
kind: PingKind;
durationMs: number | null;
}
/** The last calls of each heartbeat that ended a run (success or failure), oldest first. */
export async function loadRecent(ids: number[], perHeartbeat = 30): Promise<Map<number, PingSummary[]>> {
const out = new Map<number, PingSummary[]>();
if (ids.length === 0) return out;
const rows = await db
.select({ heartbeatId: heartbeatPings.heartbeatId, at: heartbeatPings.at, kind: heartbeatPings.kind, durationMs: heartbeatPings.durationMs })
.from(heartbeatPings)
.where(and(inArray(heartbeatPings.heartbeatId, ids), inArray(heartbeatPings.kind, ["ok", "fail"])))
.orderBy(desc(heartbeatPings.at));
for (const r of rows) {
const list = out.get(r.heartbeatId) ?? [];
if (list.length < perHeartbeat) list.push({ at: r.at, kind: r.kind as PingKind, durationMs: r.durationMs });
out.set(r.heartbeatId, list);
}
for (const list of out.values()) list.reverse();
return out;
}
export async function prunePings(now: Date = new Date()): Promise<void> {
await db.delete(heartbeatPings).where(lt(heartbeatPings.at, new Date(now.getTime() - KEEP_PING_DAYS * 86_400_000).toISOString()));
}
export const periodLimits = { min: MIN_PERIOD, max: MAX_PERIOD_MINUTES };
/** The table order every list uses. */
export const listHeartbeats = () => db.select().from(heartbeats).orderBy(asc(heartbeats.name));
+1 -1
View File
@@ -25,7 +25,7 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^endpoint(?:-cert)?:\d+:server:(\d+)$/.exec(key);
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
+15 -1
View File
@@ -4,8 +4,9 @@
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { db } from "../db/client.js";
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
@@ -70,6 +71,18 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
}
}
/** Jobs that report in, for /metrics. */
async function addHeartbeatMetrics(out: Exposition, now: Date) {
for (const h of await db.select().from(heartbeats)) {
if (!h.enabled) continue;
const labels = { name: h.name };
const state = heartbeatState(h, now);
out.add("homelab_heartbeat_up", "gauge", "1 if the job has reported in time and not said it failed, else 0.", state === "late" || state === "failed" ? 0 : 1, labels);
if (h.lastPingAt) out.add("homelab_heartbeat_last_ping_timestamp_seconds", "gauge", "When the job last reported that it finished (Unix time).", Math.floor(parseStamp(h.lastPingAt) / 1000), labels);
if (h.lastDurationMs !== null) out.add("homelab_heartbeat_last_duration_seconds", "gauge", "How long the job said its last run took.", Math.round(h.lastDurationMs / 100) / 10, labels);
}
}
export async function buildMetrics(now: Date = new Date()): Promise<string> {
const started = Date.now();
const out = new Exposition();
@@ -195,5 +208,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
await addEndpointMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+1 -1
View File
@@ -1,5 +1,5 @@
/**
* The name lists behind Operations → Generator's server-name picker.
* The name lists behind Administration → Generator's server-name picker.
*
* The built-in lists below are hand-picked, not taken from any official source — they're a starting point. Admins can
* edit every list under Settings → Names, and can replace a Swedish list with real statistics from Skatteverket.
+2 -1
View File
@@ -109,8 +109,9 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
else if (key.startsWith("failed-units:")) category = "services";
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
else if (key.startsWith("automation:")) category = "automation";
else if (key.startsWith("heartbeat")) category = "automation";
else if (key.startsWith("endpoint-cert:")) category = "expiry";
else if (key.startsWith("endpoint:")) category = "monitoring";
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|endpoint(?:-cert)?:\d+:server:)(\d+)/.exec(key);
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:)(\d+)/.exec(key);
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
}