Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3bc6bbf72
commit
92726568b7
37 files changed
+6918
-21
No files matched your search
@@ -0,0 +1,350 @@
|
||||
/**
|
||||
* Endpoint checks: the app asks a web address, a TCP port or a TLS certificate whether it is fine, every few minutes, and remembers how it
|
||||
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
|
||||
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
|
||||
*
|
||||
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
|
||||
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
|
||||
* LAN and public hosts are fine; that is what these are for.
|
||||
*/
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import net from "node:net";
|
||||
import tls from "node:tls";
|
||||
import type { TLSSocket } from "node:tls";
|
||||
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, endpointResults } from "../db/schema.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
|
||||
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
|
||||
export type CheckKind = (typeof CHECK_KINDS)[number];
|
||||
export type CheckRow = typeof endpointChecks.$inferSelect;
|
||||
|
||||
export interface ProbeResult {
|
||||
ok: boolean;
|
||||
/** How long it took to answer, when it did. */
|
||||
ms: number | null;
|
||||
message: string;
|
||||
/** ISO — when the certificate it showed runs out (https and tls checks). */
|
||||
certExpiresAt?: string | null;
|
||||
}
|
||||
|
||||
export const KEEP_RESULT_DAYS = 14;
|
||||
const MAX_BODY_BYTES = 64 * 1024;
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
/** "200-299, 301" → a test for a status code. Anything unreadable falls back to the usual 200–399. */
|
||||
export function statusAccepter(spec: string | null | undefined): (code: number) => boolean {
|
||||
const ranges: [number, number][] = [];
|
||||
for (const part of (spec ?? "").split(",")) {
|
||||
const m = /^\s*(\d{3})(?:\s*-\s*(\d{3}))?\s*$/.exec(part);
|
||||
if (m) ranges.push([Number(m[1]), Number(m[2] ?? m[1])]);
|
||||
}
|
||||
if (ranges.length === 0) ranges.push([200, 399]);
|
||||
return (code) => ranges.some(([lo, hi]) => code >= lo && code <= hi);
|
||||
}
|
||||
|
||||
/** Whether a status spec is one statusAccepter would read (for refusing a typo when it is saved). */
|
||||
export const isStatusSpec = (spec: string) => spec.split(",").every((p) => /^\s*\d{3}(\s*-\s*\d{3})?\s*$/.test(p));
|
||||
|
||||
const certExpiry = (socket: unknown): string | null => {
|
||||
try {
|
||||
const cert = (socket as TLSSocket).getPeerCertificate?.();
|
||||
if (!cert || !cert.valid_to) return null;
|
||||
const t = new Date(cert.valid_to);
|
||||
return Number.isNaN(t.getTime()) ? null : t.toISOString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** What to tell a person when the certificate can't be trusted: the usual reasons, said plainly. */
|
||||
function trustProblem(code: string | undefined, text: string): string {
|
||||
const known: Record<string, string> = {
|
||||
DEPTH_ZERO_SELF_SIGNED_CERT: "it is self-signed",
|
||||
SELF_SIGNED_CERT_IN_CHAIN: "it is signed by a certificate authority this server doesn't know",
|
||||
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "it is signed by a certificate authority this server doesn't know",
|
||||
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "it is signed by a certificate authority this server doesn't know",
|
||||
CERT_HAS_EXPIRED: "it has expired",
|
||||
ERR_TLS_CERT_ALTNAME_INVALID: "it isn't for this name",
|
||||
};
|
||||
return `The certificate can't be trusted: ${(code && known[code]) || text}. If that is expected for this service, tick "Accept a certificate that isn't trusted".`;
|
||||
}
|
||||
|
||||
interface OnceResult {
|
||||
status: number;
|
||||
location: string | null;
|
||||
body: string;
|
||||
cert: string | null;
|
||||
}
|
||||
|
||||
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
|
||||
const family = net.isIP(address) === 6 ? 6 : 4;
|
||||
return new Promise((resolve, reject) => {
|
||||
const isHttps = url.protocol === "https:";
|
||||
const bare = url.hostname.replace(/^\[|\]$/g, "");
|
||||
const req = (isHttps ? https : http).request(
|
||||
{
|
||||
host: bare,
|
||||
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
|
||||
method: "GET",
|
||||
path: `${url.pathname}${url.search}`,
|
||||
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
|
||||
// Connect to the address that passed the check, not to the name again.
|
||||
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
|
||||
timeout: timeoutMs,
|
||||
...(isHttps ? { rejectUnauthorized: !insecure, servername: net.isIP(bare) === 0 ? bare : undefined } : {}),
|
||||
},
|
||||
(res) => {
|
||||
const cert = isHttps ? certExpiry(res.socket) : null;
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
|
||||
if (!wantBody) {
|
||||
res.resume();
|
||||
res.on("end", finish);
|
||||
res.on("close", finish);
|
||||
return;
|
||||
}
|
||||
res.on("data", (c: Buffer) => {
|
||||
size += c.length;
|
||||
chunks.push(c);
|
||||
if (size >= MAX_BODY_BYTES) {
|
||||
res.destroy();
|
||||
finish();
|
||||
}
|
||||
});
|
||||
res.on("end", finish);
|
||||
res.on("close", finish);
|
||||
},
|
||||
);
|
||||
req.on("timeout", () => req.destroy(new Error(`no answer within ${Math.round(timeoutMs / 1000)} s`)));
|
||||
req.on("error", reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function probeHttp(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
const timeoutMs = c.timeoutSeconds * 1000;
|
||||
const accepts = statusAccepter(c.expectStatus);
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(c.target);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("not http");
|
||||
} catch {
|
||||
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
|
||||
}
|
||||
let cert: string | null = null;
|
||||
for (let hops = 0; ; hops++) {
|
||||
const left = timeoutMs - (performance.now() - started);
|
||||
if (left <= 0) return { ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` };
|
||||
let once: OnceResult;
|
||||
try {
|
||||
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
|
||||
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
|
||||
} catch (err) {
|
||||
const e = err as NodeJS.ErrnoException;
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
const trust = url.protocol === "https:" && /^(DEPTH_ZERO|SELF_SIGNED|UNABLE_TO|CERT_|ERR_TLS_CERT)/.test(e.code ?? "");
|
||||
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
|
||||
}
|
||||
cert = once.cert ?? cert;
|
||||
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
|
||||
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
|
||||
url = new URL(once.location, url);
|
||||
continue;
|
||||
}
|
||||
const ms = Math.round(performance.now() - started);
|
||||
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
|
||||
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
|
||||
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
|
||||
}
|
||||
}
|
||||
|
||||
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
let address: string;
|
||||
try {
|
||||
address = await resolveTarget(c.target, { allowPrivate: true });
|
||||
} catch (err) {
|
||||
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const socket = net.connect({ host: address, port: c.port ?? 0 });
|
||||
const done = (r: ProbeResult) => {
|
||||
socket.destroy();
|
||||
resolve(r);
|
||||
};
|
||||
socket.setTimeout(c.timeoutSeconds * 1000, () => done({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }));
|
||||
socket.on("connect", () => {
|
||||
const ms = Math.round(performance.now() - started);
|
||||
done({ ok: true, ms, message: `port ${c.port} answered in ${ms} ms` });
|
||||
});
|
||||
socket.on("error", (err: NodeJS.ErrnoException) => done({ ok: false, ms: null, message: err.code === "ECONNREFUSED" ? `port ${c.port} refused the connection` : err.message }));
|
||||
});
|
||||
}
|
||||
|
||||
async function probeTls(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
let address: string;
|
||||
try {
|
||||
address = await resolveTarget(c.target, { allowPrivate: true });
|
||||
} catch (err) {
|
||||
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const socket = tls.connect({ host: address, port: c.port ?? 443, servername: net.isIP(c.target) === 0 ? c.target : undefined, rejectUnauthorized: false }, () => {
|
||||
const ms = Math.round(performance.now() - started);
|
||||
const expires = certExpiry(socket);
|
||||
const authorized = socket.authorized;
|
||||
const reason = socket.authorizationError ? String(socket.authorizationError) : "";
|
||||
socket.end();
|
||||
if (!expires) return resolve({ ok: false, ms, message: "The server showed no certificate." });
|
||||
const days = Math.floor((Date.parse(expires) - Date.now()) / 86_400_000);
|
||||
if (days < 0) return resolve({ ok: false, ms, message: `The certificate expired on ${expires.slice(0, 10)}.`, certExpiresAt: expires });
|
||||
if (!authorized && !c.insecure) return resolve({ ok: false, ms, message: trustProblem(reason, reason || "it didn't verify"), certExpiresAt: expires });
|
||||
resolve({ ok: true, ms, message: `The certificate is good until ${expires.slice(0, 10)} (${days} days).`, certExpiresAt: expires });
|
||||
});
|
||||
socket.setTimeout(c.timeoutSeconds * 1000, () => {
|
||||
socket.destroy();
|
||||
resolve({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` });
|
||||
});
|
||||
socket.on("error", (err) => {
|
||||
resolve({ ok: false, ms: null, message: /wrong version number|packet length too long/i.test(err.message) ? `${c.target}:${c.port ?? 443} isn't speaking TLS — is that the right port?` : err.message });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function probe(c: CheckRow): Promise<ProbeResult> {
|
||||
return c.kind === "tcp" ? probeTcp(c) : c.kind === "tls" ? probeTls(c) : probeHttp(c);
|
||||
}
|
||||
|
||||
/** Asks once, and writes down how it went: the check's current state, and one more point in its history. */
|
||||
export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c: CheckRow) => Promise<ProbeResult> = probe): Promise<ProbeResult> {
|
||||
let result: ProbeResult;
|
||||
try {
|
||||
result = await probeFn(c);
|
||||
} catch (err) {
|
||||
result = { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
const iso = now.toISOString();
|
||||
const flipped = c.lastOk === null || c.lastOk !== result.ok;
|
||||
await db
|
||||
.update(endpointChecks)
|
||||
.set({
|
||||
lastCheckedAt: iso,
|
||||
lastOk: result.ok,
|
||||
lastMessage: result.message.slice(0, 300),
|
||||
lastResponseMs: result.ms,
|
||||
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
|
||||
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
|
||||
lastChangeAt: flipped ? iso : c.lastChangeAt,
|
||||
})
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
|
||||
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
|
||||
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
|
||||
const due = rows.filter((c) => !c.lastCheckedAt || now.getTime() - Date.parse(c.lastCheckedAt) >= c.intervalMinutes * 60_000 - 5_000);
|
||||
let next = 0;
|
||||
const worker = async () => {
|
||||
while (next < due.length) {
|
||||
const c = due[next++];
|
||||
await runCheck(c, now, probeFn);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(5, due.length) }, worker));
|
||||
return due.length;
|
||||
}
|
||||
|
||||
export async function pruneResults(now: Date = new Date()): Promise<void> {
|
||||
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
|
||||
}
|
||||
|
||||
// ─── What the history says ─────────────────────────────────────────────────
|
||||
|
||||
export interface CheckStats {
|
||||
uptime24h: number | null;
|
||||
uptime7d: number | null;
|
||||
avgMs24h: number | null;
|
||||
/** The last results, oldest first: how the recent checks went. */
|
||||
recent: { at: string; ok: boolean; ms: number | null }[];
|
||||
}
|
||||
|
||||
/** Pure: uptime, average answer time and the recent run of results, from a check's stored results. */
|
||||
export function summariseResults(results: { at: string; ok: boolean; ms: number | null }[], now: Date): CheckStats {
|
||||
const sorted = [...results].sort((a, b) => a.at.localeCompare(b.at));
|
||||
const since = (ms: number) => sorted.filter((r) => now.getTime() - Date.parse(r.at) <= ms);
|
||||
const pct = (list: typeof sorted) => (list.length === 0 ? null : Math.round((list.filter((r) => r.ok).length / list.length) * 1000) / 10);
|
||||
const day = since(86_400_000);
|
||||
const timed = day.filter((r) => r.ok && r.ms !== null);
|
||||
return {
|
||||
uptime24h: pct(day),
|
||||
uptime7d: pct(since(7 * 86_400_000)),
|
||||
avgMs24h: timed.length === 0 ? null : Math.round(timed.reduce((n, r) => n + (r.ms ?? 0), 0) / timed.length),
|
||||
recent: sorted.slice(-30),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadStats(ids: number[], now: Date = new Date()): Promise<Map<number, CheckStats>> {
|
||||
const out = new Map<number, CheckStats>();
|
||||
if (ids.length === 0) return out;
|
||||
const rows = await db
|
||||
.select({ checkId: endpointResults.checkId, at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
|
||||
.from(endpointResults)
|
||||
.where(and(inArray(endpointResults.checkId, ids), gte(endpointResults.at, new Date(now.getTime() - 7 * 86_400_000).toISOString())));
|
||||
const by = new Map<number, { at: string; ok: boolean; ms: number | null }[]>();
|
||||
for (const r of rows) by.set(r.checkId, [...(by.get(r.checkId) ?? []), r]);
|
||||
for (const id of ids) out.set(id, summariseResults(by.get(id) ?? [], now));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── What is wrong ─────────────────────────────────────────────────────────
|
||||
|
||||
export interface EndpointCondition {
|
||||
key: string;
|
||||
source: "endpoint";
|
||||
message: string;
|
||||
severity: "critical" | "warning";
|
||||
/** "monitoring" for a check that fails, "expiry" for a certificate running out. */
|
||||
category: "monitoring" | "expiry";
|
||||
checkId: number;
|
||||
serverId: number | null;
|
||||
}
|
||||
|
||||
const daysLeft = (iso: string, now: Date) => Math.floor((Date.parse(iso) - now.getTime()) / 86_400_000);
|
||||
|
||||
/**
|
||||
* The problems the checks stand for. A check must have failed several times in a row before it is one (so a blip isn't), and a certificate is
|
||||
* one from its warning time on — critical in the last three days. The key carries the server a check is linked to, so maintenance on it silences them.
|
||||
*/
|
||||
export function endpointConditions(rows: CheckRow[], now: Date): EndpointCondition[] {
|
||||
const out: EndpointCondition[] = [];
|
||||
for (const c of rows) {
|
||||
if (!c.enabled) continue;
|
||||
const suffix = c.serverId !== null ? `:server:${c.serverId}` : "";
|
||||
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
|
||||
const certGone = left !== null && left < 0;
|
||||
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
|
||||
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
|
||||
}
|
||||
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
|
||||
out.push({
|
||||
key: `endpoint-cert:${c.id}${suffix}`,
|
||||
source: "endpoint",
|
||||
category: "expiry",
|
||||
severity: left <= 3 ? "critical" : "warning",
|
||||
checkId: c.id,
|
||||
serverId: c.serverId,
|
||||
message: `The certificate for ${c.name} ${left < 0 ? `expired ${-left} day${-left === 1 ? "" : "s"} ago` : left === 0 ? "expires today" : `expires in ${left} day${left === 1 ? "" : "s"}`} (${c.certExpiresAt.slice(0, 10)})`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user