Files
Homelab-manager/server/src/services/endpointChecks.ts
T
bobbanandClaude Sonnet 5.5 be6c5552bd Patch plans, a monthly report and new-port alerts
Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an
ordinary update run, stopping at the first failure by default, silencing what depends on a
server while it is updated, with one message at the end.

Monthly report (Operations -> Monthly report, and optionally a message on a day of the month):
availability, incidents, updates, restore tests, backups and renewals for a calendar month,
printable. Backed by daily endpoint figures and a record of renewals.

New ports: the first report with ports is the baseline; a TCP port that begins listening later and
is not noted on the server's Ports card is a warning, cleared by Mark as expected.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-19 23:16:14 +02:00

377 lines
18 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Endpoint checks: the app asks a web address, a TCP port or a TLS certificate whether it is fine, every few minutes, and remembers how it
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
*
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
* LAN and public hosts are fine; that is what these are for.
*/
import http from "node:http";
import https from "node:https";
import net from "node:net";
import tls from "node:tls";
import type { TLSSocket } from "node:tls";
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
import { noteRenewal } from "./renewals.js";
import { resolveTarget } from "../utils/networkGuard.js";
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
export type CheckKind = (typeof CHECK_KINDS)[number];
export type CheckRow = typeof endpointChecks.$inferSelect;
export interface ProbeResult {
ok: boolean;
/** How long it took to answer, when it did. */
ms: number | null;
message: string;
/** ISO — when the certificate it showed runs out (https and tls checks). */
certExpiresAt?: string | null;
}
export const KEEP_RESULT_DAYS = 14;
const MAX_BODY_BYTES = 64 * 1024;
const MAX_REDIRECTS = 5;
/** "200-299, 301" → a test for a status code. Anything unreadable falls back to the usual 200–399. */
export function statusAccepter(spec: string | null | undefined): (code: number) => boolean {
const ranges: [number, number][] = [];
for (const part of (spec ?? "").split(",")) {
const m = /^\s*(\d{3})(?:\s*-\s*(\d{3}))?\s*$/.exec(part);
if (m) ranges.push([Number(m[1]), Number(m[2] ?? m[1])]);
}
if (ranges.length === 0) ranges.push([200, 399]);
return (code) => ranges.some(([lo, hi]) => code >= lo && code <= hi);
}
/** Whether a status spec is one statusAccepter would read (for refusing a typo when it is saved). */
export const isStatusSpec = (spec: string) => spec.split(",").every((p) => /^\s*\d{3}(\s*-\s*\d{3})?\s*$/.test(p));
const certExpiry = (socket: unknown): string | null => {
try {
const cert = (socket as TLSSocket).getPeerCertificate?.();
if (!cert || !cert.valid_to) return null;
const t = new Date(cert.valid_to);
return Number.isNaN(t.getTime()) ? null : t.toISOString();
} catch {
return null;
}
};
/** What to tell a person when the certificate can't be trusted: the usual reasons, said plainly. */
function trustProblem(code: string | undefined, text: string): string {
const known: Record<string, string> = {
DEPTH_ZERO_SELF_SIGNED_CERT: "it is self-signed",
SELF_SIGNED_CERT_IN_CHAIN: "it is signed by a certificate authority this server doesn't know",
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "it is signed by a certificate authority this server doesn't know",
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "it is signed by a certificate authority this server doesn't know",
CERT_HAS_EXPIRED: "it has expired",
ERR_TLS_CERT_ALTNAME_INVALID: "it isn't for this name",
};
return `The certificate can't be trusted: ${(code && known[code]) || text}. If that is expected for this service, tick "Accept a certificate that isn't trusted".`;
}
interface OnceResult {
status: number;
location: string | null;
body: string;
cert: string | null;
}
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
const family = net.isIP(address) === 6 ? 6 : 4;
return new Promise((resolve, reject) => {
const isHttps = url.protocol === "https:";
const bare = url.hostname.replace(/^\[|\]$/g, "");
const req = (isHttps ? https : http).request(
{
host: bare,
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
method: "GET",
path: `${url.pathname}${url.search}`,
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
// Connect to the address that passed the check, not to the name again.
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
timeout: timeoutMs,
...(isHttps ? { rejectUnauthorized: !insecure, servername: net.isIP(bare) === 0 ? bare : undefined } : {}),
},
(res) => {
const cert = isHttps ? certExpiry(res.socket) : null;
const chunks: Buffer[] = [];
let size = 0;
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
if (!wantBody) {
res.resume();
res.on("end", finish);
res.on("close", finish);
return;
}
res.on("data", (c: Buffer) => {
size += c.length;
chunks.push(c);
if (size >= MAX_BODY_BYTES) {
res.destroy();
finish();
}
});
res.on("end", finish);
res.on("close", finish);
},
);
req.on("timeout", () => req.destroy(new Error(`no answer within ${Math.round(timeoutMs / 1000)} s`)));
req.on("error", reject);
req.end();
});
}
async function probeHttp(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
const timeoutMs = c.timeoutSeconds * 1000;
const accepts = statusAccepter(c.expectStatus);
let url: URL;
try {
url = new URL(c.target);
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("not http");
} catch {
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
}
let cert: string | null = null;
for (let hops = 0; ; hops++) {
const left = timeoutMs - (performance.now() - started);
if (left <= 0) return { ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` };
let once: OnceResult;
try {
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
} catch (err) {
const e = err as NodeJS.ErrnoException;
const text = err instanceof Error ? err.message : String(err);
const trust = url.protocol === "https:" && /^(DEPTH_ZERO|SELF_SIGNED|UNABLE_TO|CERT_|ERR_TLS_CERT)/.test(e.code ?? "");
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
}
cert = once.cert ?? cert;
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
url = new URL(once.location, url);
continue;
}
const ms = Math.round(performance.now() - started);
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
}
}
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
let address: string;
try {
address = await resolveTarget(c.target, { allowPrivate: true });
} catch (err) {
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
}
return new Promise((resolve) => {
const socket = net.connect({ host: address, port: c.port ?? 0 });
const done = (r: ProbeResult) => {
socket.destroy();
resolve(r);
};
socket.setTimeout(c.timeoutSeconds * 1000, () => done({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }));
socket.on("connect", () => {
const ms = Math.round(performance.now() - started);
done({ ok: true, ms, message: `port ${c.port} answered in ${ms} ms` });
});
socket.on("error", (err: NodeJS.ErrnoException) => done({ ok: false, ms: null, message: err.code === "ECONNREFUSED" ? `port ${c.port} refused the connection` : err.message }));
});
}
async function probeTls(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
let address: string;
try {
address = await resolveTarget(c.target, { allowPrivate: true });
} catch (err) {
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
}
return new Promise((resolve) => {
const socket = tls.connect({ host: address, port: c.port ?? 443, servername: net.isIP(c.target) === 0 ? c.target : undefined, rejectUnauthorized: false }, () => {
const ms = Math.round(performance.now() - started);
const expires = certExpiry(socket);
const authorized = socket.authorized;
const reason = socket.authorizationError ? String(socket.authorizationError) : "";
socket.end();
if (!expires) return resolve({ ok: false, ms, message: "The server showed no certificate." });
const days = Math.floor((Date.parse(expires) - Date.now()) / 86_400_000);
if (days < 0) return resolve({ ok: false, ms, message: `The certificate expired on ${expires.slice(0, 10)}.`, certExpiresAt: expires });
if (!authorized && !c.insecure) return resolve({ ok: false, ms, message: trustProblem(reason, reason || "it didn't verify"), certExpiresAt: expires });
resolve({ ok: true, ms, message: `The certificate is good until ${expires.slice(0, 10)} (${days} days).`, certExpiresAt: expires });
});
socket.setTimeout(c.timeoutSeconds * 1000, () => {
socket.destroy();
resolve({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` });
});
socket.on("error", (err) => {
resolve({ ok: false, ms: null, message: /wrong version number|packet length too long/i.test(err.message) ? `${c.target}:${c.port ?? 443} isn't speaking TLS — is that the right port?` : err.message });
});
});
}
export function probe(c: CheckRow): Promise<ProbeResult> {
return c.kind === "tcp" ? probeTcp(c) : c.kind === "tls" ? probeTls(c) : probeHttp(c);
}
/** Asks once, and writes down how it went: the check's current state, and one more point in its history. */
export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c: CheckRow) => Promise<ProbeResult> = probe): Promise<ProbeResult> {
let result: ProbeResult;
try {
result = await probeFn(c);
} catch (err) {
result = { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
}
const iso = now.toISOString();
const flipped = c.lastOk === null || c.lastOk !== result.ok;
await db
.update(endpointChecks)
.set({
lastCheckedAt: iso,
lastOk: result.ok,
lastMessage: result.message.slice(0, 300),
lastResponseMs: result.ms,
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
lastChangeAt: flipped ? iso : c.lastChangeAt,
})
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
await addToDaily(c.id, iso, result);
// A certificate that is now good for much longer than it was has been renewed.
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
return result;
}
/** How long the daily figures are kept: years, where the individual results are kept two weeks. */
export const KEEP_DAILY_DAYS = 800;
/** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */
async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
const timed = result.ok && result.ms !== null;
const ok = result.ok ? 1 : 0;
const ms = timed ? (result.ms as number) : 0;
try {
await db
.insert(endpointDaily)
.values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 })
.onConflictDoUpdate({
target: [endpointDaily.checkId, endpointDaily.day],
set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` },
});
} catch (err) {
console.error("[endpoints] couldn't add to the daily figures:", err);
}
}
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
const due = rows.filter((c) => !c.lastCheckedAt || now.getTime() - Date.parse(c.lastCheckedAt) >= c.intervalMinutes * 60_000 - 5_000);
let next = 0;
const worker = async () => {
while (next < due.length) {
const c = due[next++];
await runCheck(c, now, probeFn);
}
};
await Promise.all(Array.from({ length: Math.min(5, due.length) }, worker));
return due.length;
}
export async function pruneResults(now: Date = new Date()): Promise<void> {
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
}
// ─── What the history says ─────────────────────────────────────────────────
export interface CheckStats {
uptime24h: number | null;
uptime7d: number | null;
avgMs24h: number | null;
/** The last results, oldest first: how the recent checks went. */
recent: { at: string; ok: boolean; ms: number | null }[];
}
/** Pure: uptime, average answer time and the recent run of results, from a check's stored results. */
export function summariseResults(results: { at: string; ok: boolean; ms: number | null }[], now: Date): CheckStats {
const sorted = [...results].sort((a, b) => a.at.localeCompare(b.at));
const since = (ms: number) => sorted.filter((r) => now.getTime() - Date.parse(r.at) <= ms);
const pct = (list: typeof sorted) => (list.length === 0 ? null : Math.round((list.filter((r) => r.ok).length / list.length) * 1000) / 10);
const day = since(86_400_000);
const timed = day.filter((r) => r.ok && r.ms !== null);
return {
uptime24h: pct(day),
uptime7d: pct(since(7 * 86_400_000)),
avgMs24h: timed.length === 0 ? null : Math.round(timed.reduce((n, r) => n + (r.ms ?? 0), 0) / timed.length),
recent: sorted.slice(-30),
};
}
export async function loadStats(ids: number[], now: Date = new Date()): Promise<Map<number, CheckStats>> {
const out = new Map<number, CheckStats>();
if (ids.length === 0) return out;
const rows = await db
.select({ checkId: endpointResults.checkId, at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
.from(endpointResults)
.where(and(inArray(endpointResults.checkId, ids), gte(endpointResults.at, new Date(now.getTime() - 7 * 86_400_000).toISOString())));
const by = new Map<number, { at: string; ok: boolean; ms: number | null }[]>();
for (const r of rows) by.set(r.checkId, [...(by.get(r.checkId) ?? []), r]);
for (const id of ids) out.set(id, summariseResults(by.get(id) ?? [], now));
return out;
}
// ─── What is wrong ─────────────────────────────────────────────────────────
export interface EndpointCondition {
key: string;
source: "endpoint";
message: string;
severity: "critical" | "warning";
/** "monitoring" for a check that fails, "expiry" for a certificate running out. */
category: "monitoring" | "expiry";
checkId: number;
serverId: number | null;
}
const daysLeft = (iso: string, now: Date) => Math.floor((Date.parse(iso) - now.getTime()) / 86_400_000);
/**
* The problems the checks stand for. A check must have failed several times in a row before it is one (so a blip isn't), and a certificate is
* one from its warning time on — critical in the last three days. The key carries the server a check is linked to, so maintenance on it silences them.
*/
export function endpointConditions(rows: CheckRow[], now: Date): EndpointCondition[] {
const out: EndpointCondition[] = [];
for (const c of rows) {
if (!c.enabled) continue;
const suffix = c.serverId !== null ? `:server:${c.serverId}` : "";
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
const certGone = left !== null && left < 0;
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
}
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
out.push({
key: `endpoint-cert:${c.id}${suffix}`,
source: "endpoint",
category: "expiry",
severity: left <= 3 ? "critical" : "warning",
checkId: c.id,
serverId: c.serverId,
message: `The certificate for ${c.name} ${left < 0 ? `expired ${-left} day${-left === 1 ? "" : "s"} ago` : left === 0 ? "expires today" : `expires in ${left} day${left === 1 ? "" : "s"}`} (${c.certExpiresAt.slice(0, 10)})`,
});
}
}
return out;
}