Patch plans (Updates -> Patch plans): updates on a schedule, one server at a time, each by an ordinary update run, stopping at the first failure by default, silencing what depends on a server while it is updated, with one message at the end. Monthly report (Operations -> Monthly report, and optionally a message on a day of the month): availability, incidents, updates, restore tests, backups and renewals for a calendar month, printable. Backed by daily endpoint figures and a record of renewals. New ports: the first report with ports is the baseline; a TCP port that begins listening later and is not noted on the server's Ports card is a warning, cleared by Mark as expected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
377 lines
18 KiB
TypeScript
377 lines
18 KiB
TypeScript
/**
|
||
* Endpoint checks: the app asks a web address, a TCP port or a TLS certificate whether it is fine, every few minutes, and remembers how it
|
||
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
|
||
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
|
||
*
|
||
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
|
||
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
|
||
* LAN and public hosts are fine; that is what these are for.
|
||
*/
|
||
import http from "node:http";
|
||
import https from "node:https";
|
||
import net from "node:net";
|
||
import tls from "node:tls";
|
||
import type { TLSSocket } from "node:tls";
|
||
import { and, asc, eq, gte, inArray, lt, sql } from "drizzle-orm";
|
||
import { db } from "../db/client.js";
|
||
import { endpointChecks, endpointDaily, endpointResults } from "../db/schema.js";
|
||
import { noteRenewal } from "./renewals.js";
|
||
import { resolveTarget } from "../utils/networkGuard.js";
|
||
|
||
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
|
||
export type CheckKind = (typeof CHECK_KINDS)[number];
|
||
export type CheckRow = typeof endpointChecks.$inferSelect;
|
||
|
||
export interface ProbeResult {
|
||
ok: boolean;
|
||
/** How long it took to answer, when it did. */
|
||
ms: number | null;
|
||
message: string;
|
||
/** ISO — when the certificate it showed runs out (https and tls checks). */
|
||
certExpiresAt?: string | null;
|
||
}
|
||
|
||
export const KEEP_RESULT_DAYS = 14;
|
||
const MAX_BODY_BYTES = 64 * 1024;
|
||
const MAX_REDIRECTS = 5;
|
||
|
||
/** "200-299, 301" → a test for a status code. Anything unreadable falls back to the usual 200–399. */
|
||
export function statusAccepter(spec: string | null | undefined): (code: number) => boolean {
|
||
const ranges: [number, number][] = [];
|
||
for (const part of (spec ?? "").split(",")) {
|
||
const m = /^\s*(\d{3})(?:\s*-\s*(\d{3}))?\s*$/.exec(part);
|
||
if (m) ranges.push([Number(m[1]), Number(m[2] ?? m[1])]);
|
||
}
|
||
if (ranges.length === 0) ranges.push([200, 399]);
|
||
return (code) => ranges.some(([lo, hi]) => code >= lo && code <= hi);
|
||
}
|
||
|
||
/** Whether a status spec is one statusAccepter would read (for refusing a typo when it is saved). */
|
||
export const isStatusSpec = (spec: string) => spec.split(",").every((p) => /^\s*\d{3}(\s*-\s*\d{3})?\s*$/.test(p));
|
||
|
||
const certExpiry = (socket: unknown): string | null => {
|
||
try {
|
||
const cert = (socket as TLSSocket).getPeerCertificate?.();
|
||
if (!cert || !cert.valid_to) return null;
|
||
const t = new Date(cert.valid_to);
|
||
return Number.isNaN(t.getTime()) ? null : t.toISOString();
|
||
} catch {
|
||
return null;
|
||
}
|
||
};
|
||
|
||
/** What to tell a person when the certificate can't be trusted: the usual reasons, said plainly. */
|
||
function trustProblem(code: string | undefined, text: string): string {
|
||
const known: Record<string, string> = {
|
||
DEPTH_ZERO_SELF_SIGNED_CERT: "it is self-signed",
|
||
SELF_SIGNED_CERT_IN_CHAIN: "it is signed by a certificate authority this server doesn't know",
|
||
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "it is signed by a certificate authority this server doesn't know",
|
||
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "it is signed by a certificate authority this server doesn't know",
|
||
CERT_HAS_EXPIRED: "it has expired",
|
||
ERR_TLS_CERT_ALTNAME_INVALID: "it isn't for this name",
|
||
};
|
||
return `The certificate can't be trusted: ${(code && known[code]) || text}. If that is expected for this service, tick "Accept a certificate that isn't trusted".`;
|
||
}
|
||
|
||
interface OnceResult {
|
||
status: number;
|
||
location: string | null;
|
||
body: string;
|
||
cert: string | null;
|
||
}
|
||
|
||
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
|
||
const family = net.isIP(address) === 6 ? 6 : 4;
|
||
return new Promise((resolve, reject) => {
|
||
const isHttps = url.protocol === "https:";
|
||
const bare = url.hostname.replace(/^\[|\]$/g, "");
|
||
const req = (isHttps ? https : http).request(
|
||
{
|
||
host: bare,
|
||
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
|
||
method: "GET",
|
||
path: `${url.pathname}${url.search}`,
|
||
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
|
||
// Connect to the address that passed the check, not to the name again.
|
||
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
|
||
timeout: timeoutMs,
|
||
...(isHttps ? { rejectUnauthorized: !insecure, servername: net.isIP(bare) === 0 ? bare : undefined } : {}),
|
||
},
|
||
(res) => {
|
||
const cert = isHttps ? certExpiry(res.socket) : null;
|
||
const chunks: Buffer[] = [];
|
||
let size = 0;
|
||
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
|
||
if (!wantBody) {
|
||
res.resume();
|
||
res.on("end", finish);
|
||
res.on("close", finish);
|
||
return;
|
||
}
|
||
res.on("data", (c: Buffer) => {
|
||
size += c.length;
|
||
chunks.push(c);
|
||
if (size >= MAX_BODY_BYTES) {
|
||
res.destroy();
|
||
finish();
|
||
}
|
||
});
|
||
res.on("end", finish);
|
||
res.on("close", finish);
|
||
},
|
||
);
|
||
req.on("timeout", () => req.destroy(new Error(`no answer within ${Math.round(timeoutMs / 1000)} s`)));
|
||
req.on("error", reject);
|
||
req.end();
|
||
});
|
||
}
|
||
|
||
async function probeHttp(c: CheckRow): Promise<ProbeResult> {
|
||
const started = performance.now();
|
||
const timeoutMs = c.timeoutSeconds * 1000;
|
||
const accepts = statusAccepter(c.expectStatus);
|
||
let url: URL;
|
||
try {
|
||
url = new URL(c.target);
|
||
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("not http");
|
||
} catch {
|
||
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
|
||
}
|
||
let cert: string | null = null;
|
||
for (let hops = 0; ; hops++) {
|
||
const left = timeoutMs - (performance.now() - started);
|
||
if (left <= 0) return { ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` };
|
||
let once: OnceResult;
|
||
try {
|
||
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
|
||
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
|
||
} catch (err) {
|
||
const e = err as NodeJS.ErrnoException;
|
||
const text = err instanceof Error ? err.message : String(err);
|
||
const trust = url.protocol === "https:" && /^(DEPTH_ZERO|SELF_SIGNED|UNABLE_TO|CERT_|ERR_TLS_CERT)/.test(e.code ?? "");
|
||
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
|
||
}
|
||
cert = once.cert ?? cert;
|
||
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
|
||
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
|
||
url = new URL(once.location, url);
|
||
continue;
|
||
}
|
||
const ms = Math.round(performance.now() - started);
|
||
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
|
||
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
|
||
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
|
||
}
|
||
}
|
||
|
||
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
|
||
const started = performance.now();
|
||
let address: string;
|
||
try {
|
||
address = await resolveTarget(c.target, { allowPrivate: true });
|
||
} catch (err) {
|
||
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||
}
|
||
return new Promise((resolve) => {
|
||
const socket = net.connect({ host: address, port: c.port ?? 0 });
|
||
const done = (r: ProbeResult) => {
|
||
socket.destroy();
|
||
resolve(r);
|
||
};
|
||
socket.setTimeout(c.timeoutSeconds * 1000, () => done({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }));
|
||
socket.on("connect", () => {
|
||
const ms = Math.round(performance.now() - started);
|
||
done({ ok: true, ms, message: `port ${c.port} answered in ${ms} ms` });
|
||
});
|
||
socket.on("error", (err: NodeJS.ErrnoException) => done({ ok: false, ms: null, message: err.code === "ECONNREFUSED" ? `port ${c.port} refused the connection` : err.message }));
|
||
});
|
||
}
|
||
|
||
async function probeTls(c: CheckRow): Promise<ProbeResult> {
|
||
const started = performance.now();
|
||
let address: string;
|
||
try {
|
||
address = await resolveTarget(c.target, { allowPrivate: true });
|
||
} catch (err) {
|
||
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||
}
|
||
return new Promise((resolve) => {
|
||
const socket = tls.connect({ host: address, port: c.port ?? 443, servername: net.isIP(c.target) === 0 ? c.target : undefined, rejectUnauthorized: false }, () => {
|
||
const ms = Math.round(performance.now() - started);
|
||
const expires = certExpiry(socket);
|
||
const authorized = socket.authorized;
|
||
const reason = socket.authorizationError ? String(socket.authorizationError) : "";
|
||
socket.end();
|
||
if (!expires) return resolve({ ok: false, ms, message: "The server showed no certificate." });
|
||
const days = Math.floor((Date.parse(expires) - Date.now()) / 86_400_000);
|
||
if (days < 0) return resolve({ ok: false, ms, message: `The certificate expired on ${expires.slice(0, 10)}.`, certExpiresAt: expires });
|
||
if (!authorized && !c.insecure) return resolve({ ok: false, ms, message: trustProblem(reason, reason || "it didn't verify"), certExpiresAt: expires });
|
||
resolve({ ok: true, ms, message: `The certificate is good until ${expires.slice(0, 10)} (${days} days).`, certExpiresAt: expires });
|
||
});
|
||
socket.setTimeout(c.timeoutSeconds * 1000, () => {
|
||
socket.destroy();
|
||
resolve({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` });
|
||
});
|
||
socket.on("error", (err) => {
|
||
resolve({ ok: false, ms: null, message: /wrong version number|packet length too long/i.test(err.message) ? `${c.target}:${c.port ?? 443} isn't speaking TLS — is that the right port?` : err.message });
|
||
});
|
||
});
|
||
}
|
||
|
||
export function probe(c: CheckRow): Promise<ProbeResult> {
|
||
return c.kind === "tcp" ? probeTcp(c) : c.kind === "tls" ? probeTls(c) : probeHttp(c);
|
||
}
|
||
|
||
/** Asks once, and writes down how it went: the check's current state, and one more point in its history. */
|
||
export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c: CheckRow) => Promise<ProbeResult> = probe): Promise<ProbeResult> {
|
||
let result: ProbeResult;
|
||
try {
|
||
result = await probeFn(c);
|
||
} catch (err) {
|
||
result = { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||
}
|
||
const iso = now.toISOString();
|
||
const flipped = c.lastOk === null || c.lastOk !== result.ok;
|
||
await db
|
||
.update(endpointChecks)
|
||
.set({
|
||
lastCheckedAt: iso,
|
||
lastOk: result.ok,
|
||
lastMessage: result.message.slice(0, 300),
|
||
lastResponseMs: result.ms,
|
||
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
|
||
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
|
||
lastChangeAt: flipped ? iso : c.lastChangeAt,
|
||
})
|
||
.where(eq(endpointChecks.id, c.id));
|
||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||
await addToDaily(c.id, iso, result);
|
||
// A certificate that is now good for much longer than it was has been renewed.
|
||
if (result.certExpiresAt) await noteRenewal("certificate", c.name, c.certExpiresAt, result.certExpiresAt, now);
|
||
return result;
|
||
}
|
||
|
||
/** How long the daily figures are kept: years, where the individual results are kept two weeks. */
|
||
export const KEEP_DAILY_DAYS = 800;
|
||
|
||
/** Adds one result to its day: the monthly report's availability is worked from these, long after the results themselves are gone. */
|
||
async function addToDaily(checkId: number, atIso: string, result: ProbeResult): Promise<void> {
|
||
const timed = result.ok && result.ms !== null;
|
||
const ok = result.ok ? 1 : 0;
|
||
const ms = timed ? (result.ms as number) : 0;
|
||
try {
|
||
await db
|
||
.insert(endpointDaily)
|
||
.values({ checkId, day: atIso.slice(0, 10), total: 1, ok, msSum: ms, msCount: timed ? 1 : 0 })
|
||
.onConflictDoUpdate({
|
||
target: [endpointDaily.checkId, endpointDaily.day],
|
||
set: { total: sql`${endpointDaily.total} + 1`, ok: sql`${endpointDaily.ok} + ${ok}`, msSum: sql`${endpointDaily.msSum} + ${ms}`, msCount: sql`${endpointDaily.msCount} + ${timed ? 1 : 0}` },
|
||
});
|
||
} catch (err) {
|
||
console.error("[endpoints] couldn't add to the daily figures:", err);
|
||
}
|
||
}
|
||
|
||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||
const due = rows.filter((c) => !c.lastCheckedAt || now.getTime() - Date.parse(c.lastCheckedAt) >= c.intervalMinutes * 60_000 - 5_000);
|
||
let next = 0;
|
||
const worker = async () => {
|
||
while (next < due.length) {
|
||
const c = due[next++];
|
||
await runCheck(c, now, probeFn);
|
||
}
|
||
};
|
||
await Promise.all(Array.from({ length: Math.min(5, due.length) }, worker));
|
||
return due.length;
|
||
}
|
||
|
||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||
await db.delete(endpointDaily).where(lt(endpointDaily.day, new Date(now.getTime() - KEEP_DAILY_DAYS * 86_400_000).toISOString().slice(0, 10)));
|
||
}
|
||
|
||
// ─── What the history says ─────────────────────────────────────────────────
|
||
|
||
export interface CheckStats {
|
||
uptime24h: number | null;
|
||
uptime7d: number | null;
|
||
avgMs24h: number | null;
|
||
/** The last results, oldest first: how the recent checks went. */
|
||
recent: { at: string; ok: boolean; ms: number | null }[];
|
||
}
|
||
|
||
/** Pure: uptime, average answer time and the recent run of results, from a check's stored results. */
|
||
export function summariseResults(results: { at: string; ok: boolean; ms: number | null }[], now: Date): CheckStats {
|
||
const sorted = [...results].sort((a, b) => a.at.localeCompare(b.at));
|
||
const since = (ms: number) => sorted.filter((r) => now.getTime() - Date.parse(r.at) <= ms);
|
||
const pct = (list: typeof sorted) => (list.length === 0 ? null : Math.round((list.filter((r) => r.ok).length / list.length) * 1000) / 10);
|
||
const day = since(86_400_000);
|
||
const timed = day.filter((r) => r.ok && r.ms !== null);
|
||
return {
|
||
uptime24h: pct(day),
|
||
uptime7d: pct(since(7 * 86_400_000)),
|
||
avgMs24h: timed.length === 0 ? null : Math.round(timed.reduce((n, r) => n + (r.ms ?? 0), 0) / timed.length),
|
||
recent: sorted.slice(-30),
|
||
};
|
||
}
|
||
|
||
export async function loadStats(ids: number[], now: Date = new Date()): Promise<Map<number, CheckStats>> {
|
||
const out = new Map<number, CheckStats>();
|
||
if (ids.length === 0) return out;
|
||
const rows = await db
|
||
.select({ checkId: endpointResults.checkId, at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
|
||
.from(endpointResults)
|
||
.where(and(inArray(endpointResults.checkId, ids), gte(endpointResults.at, new Date(now.getTime() - 7 * 86_400_000).toISOString())));
|
||
const by = new Map<number, { at: string; ok: boolean; ms: number | null }[]>();
|
||
for (const r of rows) by.set(r.checkId, [...(by.get(r.checkId) ?? []), r]);
|
||
for (const id of ids) out.set(id, summariseResults(by.get(id) ?? [], now));
|
||
return out;
|
||
}
|
||
|
||
// ─── What is wrong ─────────────────────────────────────────────────────────
|
||
|
||
export interface EndpointCondition {
|
||
key: string;
|
||
source: "endpoint";
|
||
message: string;
|
||
severity: "critical" | "warning";
|
||
/** "monitoring" for a check that fails, "expiry" for a certificate running out. */
|
||
category: "monitoring" | "expiry";
|
||
checkId: number;
|
||
serverId: number | null;
|
||
}
|
||
|
||
const daysLeft = (iso: string, now: Date) => Math.floor((Date.parse(iso) - now.getTime()) / 86_400_000);
|
||
|
||
/**
|
||
* The problems the checks stand for. A check must have failed several times in a row before it is one (so a blip isn't), and a certificate is
|
||
* one from its warning time on — critical in the last three days. The key carries the server a check is linked to, so maintenance on it silences them.
|
||
*/
|
||
export function endpointConditions(rows: CheckRow[], now: Date): EndpointCondition[] {
|
||
const out: EndpointCondition[] = [];
|
||
for (const c of rows) {
|
||
if (!c.enabled) continue;
|
||
const suffix = c.serverId !== null ? `:server:${c.serverId}` : "";
|
||
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
|
||
const certGone = left !== null && left < 0;
|
||
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
|
||
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
|
||
}
|
||
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
|
||
out.push({
|
||
key: `endpoint-cert:${c.id}${suffix}`,
|
||
source: "endpoint",
|
||
category: "expiry",
|
||
severity: left <= 3 ? "critical" : "warning",
|
||
checkId: c.id,
|
||
serverId: c.serverId,
|
||
message: `The certificate for ${c.name} ${left < 0 ? `expired ${-left} day${-left === 1 ? "" : "s"} ago` : left === 0 ? "expires today" : `expires in ${left} day${left === 1 ? "" : "s"}`} (${c.certExpiresAt.slice(0, 10)})`,
|
||
});
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|