Heartbeats, Coming up and suggested checks; Operations menu tidied

Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 21:42:34 +02:00
1 parent c05c847d17
commit 4e0c6927b2
44 files changed
+7092 -75

No files matched your search

+186
View File
@@ -0,0 +1,186 @@
/**
* Suggested endpoint checks: things the app already knows enough about to offer a check for, so setting up monitoring is a click instead of typing.
*
* - A web port a server's agent sees open (80, 443, 8080, 8443), with a name that points at it in DNS if there is one — "git.example.lan answers on
* 443 on nas" — as a web check, linked to that server so its maintenance silences the alerts.
* - A secret that already has a host to read the certificate from, as a certificate check: the secret only warns about the expiry, this also hears
* when the service stops answering.
*
* It only reads what is stored (no network), leaves out what a check already covers, and remembers what somebody said no to. Uptime Kuma's monitors are
* not offered: they are already being watched, and the adapter only keeps a host and port for them, not the address.
*/
import { db } from "../db/client.js";
import { dnsRecordsCache, endpointChecks, secrets, servers } from "../db/schema.js";
import { isLoopback } from "./agentPorts.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export interface Suggestion {
/** Stable identity, so a dismissed suggestion stays dismissed and "add" can name it. */
key: string;
kind: "http" | "tls";
name: string;
target: string;
port: number | null;
serverId: number | null;
serverName: string | null;
/** Why it is offered, in a sentence. */
reason: string;
}
export interface SuggestionInput {
servers: { id: number; name: string; hostname: string | null; ips: string[]; ports: { protocol?: string; port: number; address?: string }[] }[];
dns: { name: string; type: string; content: string }[];
secrets: { name: string; checkHost: string | null; checkPort: number | null }[];
existing: { kind: string; target: string; port: number | null }[];
dismissed: Set<string>;
}
/** The web ports worth offering, and what they speak. */
const WEB_PORTS: Record<number, "http" | "https"> = { 80: "http", 443: "https", 8080: "http", 8443: "https" };
/** The https twin of a plain port, so a server answering on both gets the one that reads the certificate. */
const SECURE_TWIN: Record<number, number> = { 80: 443, 8080: 8443 };
const MAX_NAMES_PER_PORT = 3;
const MAX_SUGGESTIONS = 200;
const cleanHost = (h: string) => h.trim().toLowerCase().replace(/\.$/, "");
const isIPv4 = (s: string) => /^\d{1,3}(\.\d{1,3}){3}$/.test(s);
const usableIPv4 = (s: string) => isIPv4(s) && !s.startsWith("127.") && !s.startsWith("169.254.");
/** Where a check's address is, in the same words for an existing check and a suggestion: "web:host:port" and "tls:host:port" (a web check on https reads the certificate too). */
function coverage(existing: SuggestionInput["existing"]): Set<string> {
const out = new Set<string>();
for (const c of existing) {
if (c.kind === "http") {
try {
const u = new URL(c.target);
const port = u.port ? Number(u.port) : u.protocol === "https:" ? 443 : 80;
const host = cleanHost(u.hostname);
out.add(`web:${host}:${port}`);
if (u.protocol === "https:") out.add(`tls:${host}:${port}`);
} catch {
// an address that isn't one covers nothing
}
} else if (c.kind === "tls" && c.port !== null) out.add(`tls:${cleanHost(c.target)}:${c.port}`);
}
return out;
}
/** Pure: the checks worth offering, from what is known. */
export function suggestChecks(input: SuggestionInput): { suggestions: Suggestion[]; dismissed: Suggestion[] } {
const covered = coverage(input.existing);
const all = new Map<string, Suggestion>();
for (const s of input.servers) {
const ips = new Set(s.ips.map((i) => i.trim().toLowerCase()));
const hostname = s.hostname ? cleanHost(s.hostname) : null;
// Names that point at this server in DNS: an address record with one of its addresses, and an alias to one of those names or to its hostname.
const direct = new Set<string>();
for (const r of input.dns) {
if ((r.type === "A" || r.type === "AAAA") && ips.has(r.content.trim().toLowerCase())) direct.add(cleanHost(r.name));
}
const names = new Set(direct);
for (const r of input.dns) {
if (r.type !== "CNAME") continue;
const to = cleanHost(r.content);
if (direct.has(to) || (hostname !== null && to === hostname)) names.add(cleanHost(r.name));
}
const dnsNames = [...names].filter((n) => n.includes(".") && !n.startsWith("*") && !n.endsWith(".arpa") && !/\s/.test(n)).sort((a, b) => a.length - b.length || a.localeCompare(b));
const listening = new Set(s.ports.filter((p) => (p.protocol ?? "tcp") === "tcp" && !(p.address && isLoopback(p.address))).map((p) => p.port));
for (const port of Object.keys(WEB_PORTS).map(Number).filter((p) => listening.has(p))) {
// Plain and secure on the same server: the secure one is the real service (the plain one is most likely its redirect), and reads the certificate.
if (SECURE_TWIN[port] !== undefined && listening.has(SECURE_TWIN[port])) continue;
const scheme = WEB_PORTS[port];
const fallback = hostname && hostname.includes(".") ? [hostname] : [s.ips.map((i) => i.trim()).find(usableIPv4)].filter((x): x is string => !!x);
const hosts = dnsNames.length > 0 ? dnsNames.slice(0, MAX_NAMES_PER_PORT) : fallback;
for (const host of hosts) {
const defaultPort = scheme === "https" ? 443 : 80;
const target = `${scheme}://${host}${port === defaultPort ? "" : `:${port}`}/`;
if (covered.has(`web:${host}:${port}`)) continue;
const viaDns = dnsNames.includes(host);
all.set(`http|${target}`, {
key: `http|${target}`,
kind: "http",
name: `${host}${port === defaultPort ? "" : `:${port}`}`,
target,
port: null,
serverId: s.id,
serverName: s.name,
reason: `${s.name} is listening on port ${port}${viaDns ? `, and ${host} points at it in DNS` : ""}.`,
});
}
}
}
for (const sec of input.secrets) {
if (!sec.checkHost || sec.checkPort === null) continue;
const host = cleanHost(sec.checkHost);
if (covered.has(`tls:${host}:${sec.checkPort}`)) continue;
all.set(`tls|${host}|${sec.checkPort}`, {
key: `tls|${host}|${sec.checkPort}`,
kind: "tls",
name: `Certificate: ${sec.name}`,
target: host,
port: sec.checkPort,
serverId: null,
serverName: null,
reason: `The secret “${sec.name}” already reads this certificate; a check also tells you when ${host} stops answering.`,
});
}
// By server, those about no server last, then by address.
const sorted = [...all.values()].sort((a, b) => Number(a.serverName === null) - Number(b.serverName === null) || (a.serverName ?? "").localeCompare(b.serverName ?? "") || a.target.localeCompare(b.target));
return {
suggestions: sorted.filter((x) => !input.dismissed.has(x.key)).slice(0, MAX_SUGGESTIONS),
dismissed: sorted.filter((x) => input.dismissed.has(x.key)),
};
}
// ─── What was said no to ───────────────────────────────────────────────────
const DISMISSED_FLAG = "endpointSuggestionsDismissed";
const MAX_DISMISSED = 500;
export async function loadDismissed(): Promise<Set<string>> {
const raw = await getInternalFlag(DISMISSED_FLAG);
if (!raw) return new Set();
try {
const v = JSON.parse(raw);
return new Set(Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : []);
} catch {
return new Set();
}
}
export async function saveDismissed(keys: Set<string>): Promise<void> {
await setInternalFlag(DISMISSED_FLAG, JSON.stringify([...keys].slice(-MAX_DISMISSED)));
}
// ─── Reading what is stored ────────────────────────────────────────────────
function jsonArray<T>(stored: string | null): T[] {
if (!stored) return [];
try {
const v = JSON.parse(stored);
return Array.isArray(v) ? (v as T[]) : [];
} catch {
return [];
}
}
export async function loadSuggestions(): Promise<{ suggestions: Suggestion[]; dismissed: Suggestion[] }> {
const [serverRows, dnsRows, secretRows, existing, dismissed] = await Promise.all([
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses, listeningPorts: servers.listeningPorts }).from(servers),
db.select({ name: dnsRecordsCache.name, type: dnsRecordsCache.type, content: dnsRecordsCache.content }).from(dnsRecordsCache),
db.select({ name: secrets.name, checkHost: secrets.checkHost, checkPort: secrets.checkPort }).from(secrets),
db.select({ kind: endpointChecks.kind, target: endpointChecks.target, port: endpointChecks.port }).from(endpointChecks),
loadDismissed(),
]);
return suggestChecks({
servers: serverRows.map((s) => ({ id: s.id, name: s.name, hostname: s.hostname, ips: jsonArray<string>(s.ipAddresses).filter((i) => typeof i === "string"), ports: jsonArray<{ protocol?: string; port: number; address?: string }>(s.listeningPorts).filter((p) => p && Number.isInteger(p.port)) })),
dns: dnsRows,
secrets: secretRows,
existing,
dismissed,
});
}