/** * Endpoint checks: the app asks a web address, a TCP port or a TLS certificate whether it is fine, every few minutes, and remembers how it * went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a * certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own. * * What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses * are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts). * LAN and public hosts are fine; that is what these are for. */ import http from "node:http"; import https from "node:https"; import net from "node:net"; import tls from "node:tls"; import type { TLSSocket } from "node:tls"; import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm"; import { db } from "../db/client.js"; import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js"; import { noteRenewal } from "./renewals.js"; import { resolveTarget } from "../utils/networkGuard.js"; export const CHECK_KINDS = ["http", "tcp", "tls"] as const; export type CheckKind = (typeof CHECK_KINDS)[number]; export type CheckRow = typeof endpointChecks.$inferSelect; export interface ProbeResult { ok: boolean; /** How long it took to answer, when it did. */ ms: number | null; message: string; /** ISO — when the certificate it showed runs out (https and tls checks). */ certExpiresAt?: string | null; } export const KEEP_RESULT_DAYS = 14; const MAX_BODY_BYTES = 64 * 1024; const MAX_REDIRECTS = 5; /** "200-299, 301" → a test for a status code. Anything unreadable falls back to the usual 200–399. */ export function statusAccepter(spec: string | null | undefined): (code: number) => boolean { const ranges: [number, number][] = []; for (const part of (spec ?? "").split(",")) { const m = /^\s*(\d{3})(?:\s*-\s*(\d{3}))?\s*$/.exec(part); if (m) ranges.push([Number(m[1]), Number(m[2] ?? m[1])]); } if (ranges.length === 0) ranges.push([200, 399]); return (code) => ranges.some(([lo, hi]) => code >= lo && code <= hi); } /** Whether a status spec is one statusAccepter would read (for refusing a typo when it is saved). */ export const isStatusSpec = (spec: string) => spec.split(",").every((p) => /^\s*\d{3}(\s*-\s*\d{3})?\s*$/.test(p)); const certExpiry = (socket: unknown): string | null => { try { const cert = (socket as TLSSocket).getPeerCertificate?.(); if (!cert || !cert.valid_to) return null; const t = new Date(cert.valid_to); return Number.isNaN(t.getTime()) ? null : t.toISOString(); } catch { return null; } }; /** What to tell a person when the certificate can't be trusted: the usual reasons, said plainly. */ function trustProblem(code: string | undefined, text: string): string { const known: Record = { DEPTH_ZERO_SELF_SIGNED_CERT: "it is self-signed", SELF_SIGNED_CERT_IN_CHAIN: "it is signed by a certificate authority this server doesn't know", UNABLE_TO_VERIFY_LEAF_SIGNATURE: "it is signed by a certificate authority this server doesn't know", UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "it is signed by a certificate authority this server doesn't know", CERT_HAS_EXPIRED: "it has expired", ERR_TLS_CERT_ALTNAME_INVALID: "it isn't for this name", }; return `The certificate can't be trusted: ${(code && known[code]) || text}. If that is expected for this service, tick "Accept a certificate that isn't trusted".`; } interface OnceResult { status: number; location: string | null; body: string; cert: string | null; } function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise { const family = net.isIP(address) === 6 ? 6 : 4; return new Promise((resolve, reject) => { const isHttps = url.protocol === "https:"; const bare = url.hostname.replace(/^\[|\]$/g, ""); const req = (isHttps ? https : http).request( { host: bare, port: url.port ? Number(url.port) : isHttps ? 443 : 80, method: "GET", path: `${url.pathname}${url.search}`, headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host }, // Connect to the address that passed the check, not to the name again. lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never, timeout: timeoutMs, ...(isHttps ? { rejectUnauthorized: !insecure, servername: net.isIP(bare) === 0 ? bare : undefined } : {}), }, (res) => { const cert = isHttps ? certExpiry(res.socket) : null; const chunks: Buffer[] = []; let size = 0; const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert }); if (!wantBody) { res.resume(); res.on("end", finish); res.on("close", finish); return; } res.on("data", (c: Buffer) => { size += c.length; chunks.push(c); if (size >= MAX_BODY_BYTES) { res.destroy(); finish(); } }); res.on("end", finish); res.on("close", finish); }, ); req.on("timeout", () => req.destroy(new Error(`no answer within ${Math.round(timeoutMs / 1000)} s`))); req.on("error", reject); req.end(); }); } async function probeHttp(c: CheckRow): Promise { const started = performance.now(); const timeoutMs = c.timeoutSeconds * 1000; const accepts = statusAccepter(c.expectStatus); let url: URL; try { url = new URL(c.target); if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("not http"); } catch { return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." }; } let cert: string | null = null; for (let hops = 0; ; hops++) { const left = timeoutMs - (performance.now() - started); if (left <= 0) return { ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }; let once: OnceResult; try { const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true }); once = await requestOnce(url, address, c.insecure, left, !!c.keyword); } catch (err) { const e = err as NodeJS.ErrnoException; const text = err instanceof Error ? err.message : String(err); const trust = url.protocol === "https:" && /^(DEPTH_ZERO|SELF_SIGNED|UNABLE_TO|CERT_|ERR_TLS_CERT)/.test(e.code ?? ""); return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert }; } cert = once.cert ?? cert; // Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer. if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) { url = new URL(once.location, url); continue; } const ms = Math.round(performance.now() - started); if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert }; if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert }; return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert }; } } async function probeTcp(c: CheckRow): Promise { const started = performance.now(); let address: string; try { address = await resolveTarget(c.target, { allowPrivate: true }); } catch (err) { return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) }; } return new Promise((resolve) => { const socket = net.connect({ host: address, port: c.port ?? 0 }); const done = (r: ProbeResult) => { socket.destroy(); resolve(r); }; socket.setTimeout(c.timeoutSeconds * 1000, () => done({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` })); socket.on("connect", () => { const ms = Math.round(performance.now() - started); done({ ok: true, ms, message: `port ${c.port} answered in ${ms} ms` }); }); socket.on("error", (err: NodeJS.ErrnoException) => done({ ok: false, ms: null, message: err.code === "ECONNREFUSED" ? `port ${c.port} refused the connection` : err.message })); }); } async function probeTls(c: CheckRow): Promise { const started = performance.now(); let address: string; try { address = await resolveTarget(c.target, { allowPrivate: true }); } catch (err) { return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) }; } return new Promise((resolve) => { const socket = tls.connect({ host: address, port: c.port ?? 443, servername: net.isIP(c.target) === 0 ? c.target : undefined, rejectUnauthorized: false }, () => { const ms = Math.round(performance.now() - started); const expires = certExpiry(socket); const authorized = socket.authorized; const reason = socket.authorizationError ? String(socket.authorizationError) : ""; socket.end(); if (!expires) return resolve({ ok: false, ms, message: "The server showed no certificate." }); const days = Math.floor((Date.parse(expires) - Date.now()) / 86_400_000); if (days < 0) return resolve({ ok: false, ms, message: `The certificate expired on ${expires.slice(0, 10)}.`, certExpiresAt: expires }); if (!authorized && !c.insecure) return resolve({ ok: false, ms, message: trustProblem(reason, reason || "it didn't verify"), certExpiresAt: expires }); resolve({ ok: true, ms, message: `The certificate is good until ${expires.slice(0, 10)} (${days} days).`, certExpiresAt: expires }); }); socket.setTimeout(c.timeoutSeconds * 1000, () => { socket.destroy(); resolve({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }); }); socket.on("error", (err) => { resolve({ ok: false, ms: null, message: /wrong version number|packet length too long/i.test(err.message) ? `${c.target}:${c.port ?? 443} isn't speaking TLS — is that the right port?` : err.message }); }); }); } export function probe(c: CheckRow): Promise { return c.kind === "tcp" ? probeTcp(c) : c.kind === "tls" ? probeTls(c) : probeHttp(c); } /** Asks once, and writes down how it went: the check's current state, and one more point in its history. */ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c: CheckRow) => Promise = probe): Promise { let result: ProbeResult; try { result = await probeFn(c); } catch (err) { result = { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) }; } const iso = now.toISOString(); const flipped = c.lastOk === null || c.lastOk !== result.ok; await db .update(endpointChecks) .set({ lastCheckedAt: iso, lastOk: result.ok, lastMessage: result.message.slice(0, 300), lastResponseMs: result.ms, certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt, consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1, lastChangeAt: flipped ? iso : c.lastChangeAt, }) .where(eq(endpointChecks.id, c.id)); await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) }); await addToDaily(c.id, iso, result); // A certificate that is now good for much longer than it was has been renewed. if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now); return result; } /** How long the daily figures are kept: years, where the individual results are kept two weeks. */ export const KEEP_DAILY_DAYS = 800; /** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */ async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise { const timed = result.ok && result.ms !== null; const ok = result.ok ? 1 : 0; const ms = timed ? (result.ms as number) : 0; try { await db .insert(endpointDaily) .values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 }) .onConflictDoUpdate({ target: [endpointDaily.checkId, endpointDaily.day], set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` }, }); } catch (err) { console.error("[endpoints] couldn't add to the daily figures:", err); } } /** Runs every enabled check that is due, a few at a time. Returns how many it asked. */ export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise): Promise { const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id)); const due = rows.filter((c) => !c.lastCheckedAt || now.getTime() - Date.parse(c.lastCheckedAt) >= c.intervalMinutes * 60_000 - 5_000); let next = 0; const worker = async () => { while (next < due.length) { const c = due[next++]; await runCheck(c, now, probeFn); } }; await Promise.all(Array.from({ length: Math.min(5, due.length) }, worker)); return due.length; } export async function pruneResults(now: Date = new Date()): Promise { await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString())); await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10))); } // ─── What the history says ───────────────────────────────────────────────── export interface CheckStats { uptime24h: number | null; uptime7d: number | null; avgMs24h: number | null; /** The last results, oldest first: how the recent checks went. */ recent: { at: string; ok: boolean; ms: number | null }[]; } /** Pure: uptime, average answer time and the recent run of results, from a check's stored results. */ export function summariseResults(results: { at: string; ok: boolean; ms: number | null }[], now: Date): CheckStats { const sorted = [...results].sort((a, b) => a.at.localeCompare(b.at)); const since = (ms: number) => sorted.filter((r) => now.getTime() - Date.parse(r.at) <= ms); const pct = (list: typeof sorted) => (list.length === 0 ? null : Math.round((list.filter((r) => r.ok).length / list.length) * 1000) / 10); const day = since(86_400_000); const timed = day.filter((r) => r.ok && r.ms !== null); return { uptime24h: pct(day), uptime7d: pct(since(7 * 86_400_000)), avgMs24h: timed.length === 0 ? null : Math.round(timed.reduce((n, r) => n + (r.ms ?? 0), 0) / timed.length), recent: sorted.slice(-30), }; } export async function loadStats(ids: number[], now: Date = new Date()): Promise> { const out = new Map(); if (ids.length === 0) return out; const rows = await db .select({ checkId: endpointResults.checkId, at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms }) .from(endpointResults) .where(and(inArray(endpointResults.checkId, ids), gte(endpointResults.at, new Date(now.getTime() - 7 * 86_400_000).toISOString()))); const by = new Map(); for (const r of rows) by.set(r.checkId, [...(by.get(r.checkId) ?? []), r]); for (const id of ids) out.set(id, summariseResults(by.get(id) ?? [], now)); return out; } // ─── What is wrong ───────────────────────────────────────────────────────── export interface EndpointCondition { key: string; source: "endpoint"; message: string; severity: "critical" | "warning"; /** "monitoring" for a check that fails, "expiry" for a certificate running out. */ category: "monitoring" | "expiry"; checkId: number; serverId: number | null; } const daysLeft = (iso: string, now: Date) => Math.floor((Date.parse(iso) - now.getTime()) / 86_400_000); /** * The problems the checks stand for. A check must have failed several times in a row before it is one (so a blip isn't), and a certificate is * one from its warning time on — critical in the last three days. The key carries the server a check is linked to, so maintenance on it silences them. */ export function endpointConditions(rows: CheckRow[], now: Date): EndpointCondition[] { const out: EndpointCondition[] = []; for (const c of rows) { if (!c.enabled) continue; const suffix = c.serverId !== null ? `:server:${c.serverId}` : ""; const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null; const certGone = left !== null && left < 0; if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) { out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` }); } if (c.certExpiresAt && left !== null && left <= c.warnDays) { out.push({ key: `endpoint-cert:${c.id}${suffix}`, source: "endpoint", category: "expiry", severity: left <= 3 ? "critical" : "warning", checkId: c.id, serverId: c.serverId, message: `The certificate for ${c.name} ${left < 0 ? `expired ${-left} day${-left === 1 ? "" : "s"} ago` : left === 0 ? "expires today" : `expires in ${left} day${left === 1 ? "" : "s"}`} (${c.certExpiresAt.slice(0, 10)})`, }); } } return out; }