/** * Endpoint checks: what counts as an answer, the probes themselves (against servers this machine runs), what a run records, the problems they * stand for, and what the routes, the alerts and the notifications do with them. * * Run with: npm test --workspace server */ import { test, describe, before, after } from "node:test"; import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { createServer as createHttpServer, type Server } from "node:http"; import { createServer as createHttpsServer } from "node:https"; import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { networkInterfaces, tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; const dir = mkdtempSync(join(tmpdir(), "hm-endpoints-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, servers, endpointChecks, endpointResults } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const { statusAccepter, isStatusSpec, summariseResults, endpointConditions, probe, runCheck, runDueChecks } = await import("../src/services/endpointChecks.js"); const { runEndpointPass } = await import("../src/services/endpointMonitor.js"); const { endpointsRouter } = await import("../src/routes/endpoints.js"); const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js"); const { startOrExtendWindow } = await import("../src/services/maintenance.js"); const { updateSettings } = await import("../src/services/settingsStore.js"); const { buildMetrics } = await import("../src/services/metrics.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); type Row = typeof endpointChecks.$inferSelect; const NOW = new Date("2026-10-18T12:00:00Z"); const row = (extra: Partial = {}): Row => ({ id: 1, name: "Gitea", kind: "http", target: "https://git.lab/", port: null, expectStatus: null, keyword: null, insecure: false, intervalMinutes: 5, timeoutSeconds: 10, failuresBeforeAlert: 2, warnDays: 14, serverId: null, enabled: true, lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0, lastChangeAt: null, createdAt: "2026-10-01 00:00:00", ...extra, }); describe("what counts as an answer", () => { test("status codes, ranges and the default of 200 to 399", () => { const usual = statusAccepter(null); assert.deepEqual([199, 200, 301, 399, 400, 500].map(usual), [false, true, true, true, false, false]); const custom = statusAccepter("200-204, 401"); assert.deepEqual([200, 204, 205, 401, 403].map(custom), [true, true, false, true, false]); assert.equal(statusAccepter("rubbish")(200), true, "unreadable falls back to the usual"); assert.equal(isStatusSpec("200, 301-302"), true); assert.equal(isStatusSpec("2xx"), false); }); test("uptime, the average answer time and the recent run come from the stored results", () => { const at = (min: number) => new Date(NOW.getTime() - min * 60_000).toISOString(); const stats = summariseResults( [ { at: at(10), ok: true, ms: 100 }, { at: at(20), ok: true, ms: 300 }, { at: at(30), ok: false, ms: null }, { at: at(60 * 30), ok: false, ms: null }, // yesterday and a bit { at: at(60 * 24 * 6), ok: true, ms: 50 }, ], NOW, ); assert.equal(stats.uptime24h, 66.7, "2 of 3 in the last day"); assert.equal(stats.uptime7d, 60, "3 of 5 in the last week"); assert.equal(stats.avgMs24h, 200, "only answers that came count"); assert.equal(stats.recent[0].at, at(60 * 24 * 6), "oldest first"); assert.deepEqual(summariseResults([], NOW), { uptime24h: null, uptime7d: null, avgMs24h: null, recent: [] }); }); }); describe("what is a problem", () => { test("a check has to fail several times in a row; a blip isn't", () => { assert.deepEqual(endpointConditions([row({ lastOk: false, consecutiveFailures: 1, lastMessage: "answered 500" })], NOW), []); const [c] = endpointConditions([row({ lastOk: false, consecutiveFailures: 2, lastMessage: "answered 500" })], NOW); assert.equal(c.key, "endpoint:1"); assert.equal(c.category, "monitoring"); assert.equal(c.severity, "critical"); assert.match(c.message, /Gitea isn't answering — answered 500/); assert.deepEqual(endpointConditions([row({ lastOk: false, consecutiveFailures: 9, enabled: false })], NOW), [], "a check that is switched off says nothing"); assert.deepEqual(endpointConditions([row({ lastOk: true, consecutiveFailures: 0 })], NOW), []); }); test("a certificate is a problem from its warning time on, critical in the last three days, and the key carries the server it is linked to", () => { const days = (n: number) => new Date(NOW.getTime() + n * 86_400_000).toISOString(); assert.deepEqual(endpointConditions([row({ certExpiresAt: days(20) })], NOW), []); const [warn] = endpointConditions([row({ certExpiresAt: days(10), serverId: 4 })], NOW); assert.equal(warn.key, "endpoint-cert:1:server:4"); assert.deepEqual([warn.category, warn.severity], ["expiry", "warning"]); assert.match(warn.message, /expires in 10 days/); assert.equal(endpointConditions([row({ certExpiresAt: days(2) })], NOW)[0].severity, "critical"); assert.match(endpointConditions([row({ certExpiresAt: days(-3) })], NOW)[0].message, /expired 3 days ago/); }); test("a certificate check whose certificate has expired says so once, not also that it isn't answering", () => { const expired = new Date(NOW.getTime() - 86_400_000).toISOString(); const list = endpointConditions([row({ kind: "tls", target: "git.lab", port: 443, lastOk: false, consecutiveFailures: 5, lastMessage: "The certificate expired", certExpiresAt: expired })], NOW); assert.deepEqual(list.map((c) => c.category), ["expiry"]); }); }); /** An address of this machine that isn't loopback (the checks refuse that), if it has one. */ function lanAddress(): string | null { for (const list of Object.values(networkInterfaces())) { for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address; } return null; } const LAN = lanAddress(); describe("asking", { skip: LAN === null && "this machine has no address other than loopback" }, () => { let web: Server; let port = 0; let sleeping: Server; let sleepingPort = 0; const hits: string[] = []; before(async () => { web = createHttpServer((req, res) => { hits.push(req.url ?? ""); if (req.url === "/ok") return void res.end("

Welcome to the lab

"); if (req.url === "/broken") { res.statusCode = 500; return void res.end("oops"); } if (req.url === "/moved") { res.statusCode = 301; res.setHeader("location", "/ok"); return void res.end(); } res.statusCode = 404; res.end("no"); }); await new Promise((r) => web.listen(0, "0.0.0.0", r)); port = (web.address() as AddressInfo).port; // Accepts the connection and never answers. sleeping = createHttpServer(() => {}); await new Promise((r) => sleeping.listen(0, "0.0.0.0", r)); sleepingPort = (sleeping.address() as AddressInfo).port; }); after(() => { web.close(); sleeping.close(); }); const http = (path: string, extra: Partial = {}) => row({ kind: "http", target: `http://${LAN}:${port}${path}`, timeoutSeconds: 3, ...extra }); test("a web address that answers is fine, with how long it took", async () => { const r = await probe(http("/ok")); assert.equal(r.ok, true); assert.match(r.message, /answered 200 in \d+ ms/); assert.ok(r.ms !== null && r.ms >= 0); }); test("a wrong status fails, unless that is what was expected; a redirect is an answer", async () => { assert.match((await probe(http("/broken"))).message, /answered 500/); assert.equal((await probe(http("/broken", { expectStatus: "500" }))).ok, true); assert.equal((await probe(http("/missing"))).ok, false); assert.equal((await probe(http("/moved"))).ok, true, "301 is within 200 to 399"); }); test("a word the page must contain: found after following a redirect, or missing", async () => { assert.equal((await probe(http("/ok", { keyword: "Welcome" }))).ok, true); const missing = await probe(http("/ok", { keyword: "Goodbye" })); assert.equal(missing.ok, false); assert.match(missing.message, /doesn't contain “Goodbye”/); assert.equal((await probe(http("/moved", { keyword: "Welcome" }))).ok, true, "with a word to find, the redirect is followed to the page"); }); test("no answer in time, and a refused connection, say so", async () => { const slow = await probe(row({ kind: "http", target: `http://${LAN}:${sleepingPort}/`, timeoutSeconds: 1 })); assert.equal(slow.ok, false); assert.match(slow.message, /no answer within 1 s/); const refused = await probe(row({ kind: "tcp", target: LAN!, port: 1, timeoutSeconds: 2 })); assert.equal(refused.ok, false); const open = await probe(row({ kind: "tcp", target: LAN!, port, timeoutSeconds: 2 })); assert.equal(open.ok, true); assert.match(open.message, new RegExp(`port ${port} answered in \\d+ ms`)); }); test("it will not be pointed at this machine itself or at the cloud's metadata address", async () => { for (const target of ["http://127.0.0.1:1/", "http://169.254.169.254/latest/meta-data/"]) { const r = await probe(row({ kind: "http", target })); assert.equal(r.ok, false); assert.match(r.message, /can't be used|isn't allowed/); } assert.equal((await probe(row({ kind: "tcp", target: "127.0.0.1", port: 22 }))).ok, false); assert.equal((await probe(row({ kind: "http", target: "ftp://example.com/" }))).ok, false); }); test("a certificate that can't be trusted fails unless it is accepted, and either way its expiry is read", async () => { const work = mkdtempSync(join(tmpdir(), "hm-cert-")); const made = spawnSync("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "30", "-subj", "/CN=lab.example", "-keyout", join(work, "k.pem"), "-out", join(work, "c.pem")], { encoding: "utf8" }); if (made.status !== 0) { rmSync(work, { recursive: true, force: true }); return; // no openssl here: the rest is covered by what the pure parts say } const secure = createHttpsServer({ key: readFileSync(join(work, "k.pem")), cert: readFileSync(join(work, "c.pem")) }, (_req, res) => res.end("secure")); await new Promise((r) => secure.listen(0, "0.0.0.0", r)); const sp = (secure.address() as AddressInfo).port; try { const strict = await probe(row({ kind: "http", target: `https://${LAN}:${sp}/`, timeoutSeconds: 5 })); assert.equal(strict.ok, false); assert.match(strict.message, /certificate can't be trusted: it is self-signed/); const lenient = await probe(row({ kind: "http", target: `https://${LAN}:${sp}/`, insecure: true, timeoutSeconds: 5 })); assert.equal(lenient.ok, true); assert.ok(lenient.certExpiresAt && Date.parse(lenient.certExpiresAt) > Date.now() + 25 * 86_400_000, "about 30 days left"); const tlsStrict = await probe(row({ kind: "tls", target: LAN!, port: sp, timeoutSeconds: 5 })); assert.equal(tlsStrict.ok, false); assert.match(tlsStrict.message, /can't be trusted/); const tlsLenient = await probe(row({ kind: "tls", target: LAN!, port: sp, insecure: true, timeoutSeconds: 5 })); assert.equal(tlsLenient.ok, true); assert.match(tlsLenient.message, /good until \d{4}-\d{2}-\d{2}/); const plain = await probe(row({ kind: "tls", target: LAN!, port, timeoutSeconds: 5 })); assert.equal(plain.ok, false); assert.match(plain.message, /isn't speaking TLS/); } finally { secure.close(); rmSync(work, { recursive: true, force: true }); } }); }); describe("a run, a pass, the routes, the alerts and the notifications", () => { let app: Server; let base = ""; let who = "admin"; let serverId = 0; const messages: string[] = []; let hook: Server; let answer: { ok: boolean; ms: number | null; message: string; certExpiresAt?: string | null } = { ok: true, ms: 12, message: "answered 200 in 12 ms" }; before(async () => { await db.insert(users).values([ { oidcSub: "admin", name: "Ada", role: "admin" }, { oidcSub: "operator", name: "Olga", role: "operator" }, { oidcSub: "viewer", name: "Vic", role: "viewer" }, ]); serverId = (await db.insert(servers).values({ name: "nas", apiTokenHash: "n", apiTokenPrefix: "n", lastSeenAt: new Date().toISOString() }).returning({ id: servers.id }))[0].id; hook = createHttpServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c) => chunks.push(c)); req.on("end", () => { messages.push(String(JSON.parse(Buffer.concat(chunks).toString()).content)); res.end("{}"); }); }); await new Promise((r) => hook.listen(0, "127.0.0.1", r)); await updateSettings({ webhook: { enabled: true, url: `http://127.0.0.1:${(hook.address() as AddressInfo).port}/` } }); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: who } }; next(); }); a.use("/api/endpoints", endpointsRouter); app = await new Promise((resolve) => { const s = a.listen(0, "127.0.0.1", () => resolve(s)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/endpoints`; }); after(() => { app.close(); hook.close(); try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); const fake = async () => answer; let id = 0; test("an administrator adds a check; a wrong one is refused with a reason; nobody else can", async () => { const res = await send("POST", "/", { name: "Gitea", kind: "http", target: "https://git.lab/", serverId }); assert.equal(res.status, 201); id = (await res.json()).check.id; for (const bad of [ { name: "x", kind: "http", target: "git.lab" }, { name: "x", kind: "http", target: "https://a/", expectStatus: "2xx" }, { name: "x", kind: "tcp", target: "host" }, { name: "x", kind: "tcp", target: "http://host", port: 80 }, { name: "x", kind: "tls", target: "host", port: 70000 }, { name: "x", kind: "ping", target: "host" }, { name: "x", kind: "http", target: "https://a/", serverId: 99999 }, ]) assert.equal((await send("POST", "/", bad)).status, 400, JSON.stringify(bad)); who = "operator"; assert.equal((await send("POST", "/", { name: "y", kind: "tcp", target: "h", port: 22 })).status, 403); who = "viewer"; assert.equal((await send("GET", "/")).status, 200, "everyone can look"); who = "admin"; }); test("a run records the answer on the check and as a point in its history", async () => { const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)); answer = { ok: true, ms: 40, message: "answered 200 in 40 ms", certExpiresAt: new Date(NOW.getTime() + 3 * 86_400_000).toISOString() }; await runCheck(c, new Date("2026-10-18T12:00:00Z"), fake); const [after1] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)); assert.deepEqual([after1.lastOk, after1.lastResponseMs, after1.consecutiveFailures, after1.lastChangeAt], [true, 40, 0, "2026-10-18T12:00:00.000Z"]); assert.ok(after1.certExpiresAt); assert.equal((await db.select().from(endpointResults).where(eq(endpointResults.checkId, id))).length, 1); answer = { ok: false, ms: null, message: "answered 500" }; await runCheck(after1, new Date("2026-10-18T12:05:00Z"), fake); const [failed] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)); assert.deepEqual([failed.lastOk, failed.consecutiveFailures, failed.lastChangeAt, failed.lastMessage], [false, 1, "2026-10-18T12:05:00.000Z", "answered 500"]); await runCheck(failed, new Date("2026-10-18T12:10:00Z"), fake); const [again] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)); assert.deepEqual([again.consecutiveFailures, again.lastChangeAt], [2, "2026-10-18T12:05:00.000Z"], "the time of the change is when it began failing, not each failure"); }); test("only the checks that are due are asked", async () => { await db.update(endpointChecks).set({ lastCheckedAt: "2026-10-18T12:10:00.000Z" }).where(eq(endpointChecks.id, id)); let asked = 0; const counting = async () => { asked++; return answer; }; assert.equal(await runDueChecks(new Date("2026-10-18T12:12:00Z"), counting), 0, "two minutes after a five-minute check"); assert.equal(await runDueChecks(new Date("2026-10-18T12:15:30Z"), counting), 1); assert.equal(asked, 1); await db.update(endpointChecks).set({ enabled: false }).where(eq(endpointChecks.id, id)); assert.equal(await runDueChecks(new Date("2026-10-18T13:00:00Z"), counting), 0, "a check that is switched off is never asked"); await db.update(endpointChecks).set({ enabled: true }).where(eq(endpointChecks.id, id)); }); test("a check that keeps failing is an alert, silenced by maintenance on its server, and a certificate running out is another", async () => { await db.update(endpointChecks).set({ lastOk: false, consecutiveFailures: 3, lastMessage: "answered 500", certExpiresAt: new Date(Date.now() + 2 * 86_400_000).toISOString() }).where(eq(endpointChecks.id, id)); invalidateAlertsCache(); let alerts = (await getAlerts(true)).alerts.filter((a) => a.source === "Endpoint"); const down = alerts.find((a) => a.category === "monitoring")!; const cert = alerts.find((a) => a.category === "expiry")!; assert.match(down.message, /Gitea isn't answering — answered 500/); assert.equal(down.severity, "critical"); assert.equal(down.link, "/endpoints"); assert.deepEqual(down.target && [down.target.type, down.target.id], ["server", serverId]); assert.match(cert.message, /expires in/); assert.equal(cert.severity, "critical", "two days left"); assert.equal(down.silenced, false); await startOrExtendWindow("server", serverId, 60, "work", "Ada"); invalidateAlertsCache(); alerts = (await getAlerts(true)).alerts.filter((a) => a.source === "Endpoint"); assert.ok(alerts.length === 2 && alerts.every((a) => a.silenced), "both are under the server's maintenance"); }); test("the pass tells you once when a check starts failing, and once when it is fine again", async () => { const { listActiveWindows } = await import("../src/services/maintenance.js"); const { maintenanceWindows } = await import("../src/db/schema.js"); for (const w of await listActiveWindows()) await db.delete(maintenanceWindows).where(eq(maintenanceWindows.id, w.id)); await db.update(endpointChecks).set({ lastOk: true, consecutiveFailures: 0, lastCheckedAt: null, certExpiresAt: null }).where(eq(endpointChecks.id, id)); messages.length = 0; answer = { ok: false, ms: null, message: "answered 503" }; const t = (min: number) => new Date(Date.now() + min * 60_000); let r = await runEndpointPass(t(0), fake); assert.equal(r.announced, 0, "the first failure is a blip"); r = await runEndpointPass(t(6), fake); assert.equal(r.announced, 1, "the second is a problem"); assert.match(messages.join("\n"), /Gitea isn't answering — answered 503/); const sent = messages.length; await runEndpointPass(t(12), fake); assert.equal(messages.length, sent, "and it isn't said again"); answer = { ok: true, ms: 9, message: "answered 200 in 9 ms" }; r = await runEndpointPass(t(18), fake); assert.equal(r.recovered, 1); assert.match(messages[messages.length - 1], /Gitea isn't answering/, "the recovery names what cleared"); }); test("check now is for operators and answers at once; the list carries the figures; metrics have them", async () => { who = "viewer"; assert.equal((await send("POST", `/${id}/check`)).status, 403); who = "operator"; // the real probe, aimed at a name nothing answers to (and refused if it were loopback): it says what happened await db.update(endpointChecks).set({ target: "http://127.0.0.1:1/" }).where(eq(endpointChecks.id, id)); const res = await send("POST", `/${id}/check`); assert.equal(res.status, 200); assert.equal((await res.json()).result.ok, false); who = "viewer"; const list = await (await send("GET", "/")).json(); const c = list.checks.find((x: { id: number }) => x.id === id); assert.equal(c.server.name, "nas"); assert.ok(c.stats.recent.length >= 1); assert.equal(typeof c.stats.uptime24h, "number"); assert.match(await buildMetrics(), /homelab_endpoint_up\{name="Gitea",kind="http"\} 0/); }); test("changing what is asked forgets what was found; removing it takes its history", async () => { who = "admin"; const res = await send("PUT", `/${id}`, { name: "Gitea", kind: "http", target: "https://git2.lab/", serverId }); assert.equal(res.status, 200); const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)); assert.deepEqual([c.lastOk, c.consecutiveFailures, c.lastMessage], [null, 0, null]); assert.equal((await send("DELETE", `/${id}`)).status, 204); assert.equal((await db.select().from(endpointResults).where(eq(endpointResults.checkId, id))).length, 0); assert.equal((await send("DELETE", `/${id}`)).status, 404); }); });