Check a value, not just "is it up": JSON value checks on web endpoints

A web check can read a value out of a JSON answer (power.now, state,
data.result[0].value[1]) and judge it: below, above, equal to, contains, is there.
The value read is shown on the Endpoints page, charted under History, exported as
homelab_endpoint_value and in the read API. A check can also send one header (a
token), stored encrypted, never returned, and not forwarded to another host on a
redirect.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-21 21:13:44 +02:00
1 parent e114567d64
commit 3cb3e4988f
21 files changed
+6804 -47

No files matched your search

+8
View File
@@ -141,6 +141,13 @@ export const endpointChecks = sqliteTable("endpoint_checks", {
expectStatus: text("expect_status"), // http: "200-299, 301"; null = 200-399
keyword: text("keyword"), // http: text the page must contain
insecure: integer("insecure", { mode: "boolean" }).notNull().default(false), // accept a certificate that isn't trusted (self-signed, a private CA)
valuePath: text("value_path"), // http: where in a JSON answer the value to judge is ("data.result[0].value[1]")
valueOp: text("value_op"), // http: "<" "<=" ">" ">=" "=" "!=" "contains" "exists"
valueExpected: text("value_expected"), // http: what it is compared with (a number, or text for "=", "!=" and "contains")
valueUnit: text("value_unit"), // http: shown after the value ("W", "%", "°C")
headerName: text("header_name"), // http: a header to send with the request ("Authorization")
headerValue: text("header_value"), // its value, encrypted at rest like the notification channels' secrets; never returned by the API
lastValue: text("last_value"), // http: what was read the last time, as text
intervalMinutes: integer("interval_minutes").notNull().default(5),
timeoutSeconds: integer("timeout_seconds").notNull().default(10),
failuresBeforeAlert: integer("failures_before_alert").notNull().default(2),
@@ -170,6 +177,7 @@ export const endpointResults = sqliteTable(
ok: integer("ok", { mode: "boolean" }).notNull(),
ms: integer("ms"),
message: text("message"),
value: real("value"), // the number read from the answer, for a check that judges a value
},
(t) => [index("endpoint_results_check_idx").on(t.checkId, t.at)],
);
+51 -9
View File
@@ -6,7 +6,9 @@ import { endpointChecks, servers } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck, type CheckRow } from "../services/endpointChecks.js";
import { sealSecret } from "../services/sealed.js";
import { VALUE_OPS, valueProblem } from "../services/valueCheck.js";
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -16,6 +18,17 @@ endpointsRouter.use(requireAuth);
const MAX_CHECKS = 100;
/** A blank text field is no value. */
function blank(v: string | null | undefined): string | null {
return v === undefined || v === null || v.trim() === "" ? null : v.trim();
}
/** A check as the API shows it: the header's value is never sent back, only that there is one. */
function publicCheck(row: CheckRow) {
const { headerValue, ...rest } = row;
return { ...rest, hasHeader: !!row.headerName && !!headerValue };
}
const checkSchema = z
.object({
name: z.string().trim().min(1).max(100),
@@ -24,6 +37,13 @@ const checkSchema = z
port: z.number().int().min(1).max(65535).nullable().optional(),
expectStatus: z.string().trim().max(60).nullable().optional(),
keyword: z.string().max(200).nullable().optional(),
valuePath: z.string().trim().max(200).nullable().optional(),
valueOp: z.enum(VALUE_OPS).nullable().optional(),
valueExpected: z.string().max(200).nullable().optional(),
valueUnit: z.string().trim().max(12).nullable().optional(),
headerName: z.string().trim().max(100).nullable().optional(),
/** Left out or blank on an edit keeps the one already saved. */
headerValue: z.string().max(2000).nullable().optional(),
insecure: z.boolean().optional(),
intervalMinutes: z.number().int().min(1).max(1440).optional(),
timeoutSeconds: z.number().int().min(1).max(60).optional(),
@@ -43,14 +63,17 @@ const checkSchema = z
}
if (!ok) ctx.addIssue({ code: "custom", path: ["target"], message: "A web check needs an address that starts with http:// or https://." });
if (v.expectStatus && !isStatusSpec(v.expectStatus)) ctx.addIssue({ code: "custom", path: ["expectStatus"], message: "Use status codes like 200, or ranges like 200-299, separated by commas." });
const problem = valueProblem(blank(v.valuePath), v.valueOp ?? null, blank(v.valueExpected));
if (problem) ctx.addIssue({ code: "custom", path: ["valuePath"], message: problem });
const name = blank(v.headerName);
if (name && !/^[A-Za-z0-9!#$%&'*+.^_`|~-]+$/.test(name)) ctx.addIssue({ code: "custom", path: ["headerName"], message: "A header name is letters, digits and dashes, like Authorization." });
if (name && /^(host|content-length|transfer-encoding|connection)$/i.test(name)) ctx.addIssue({ code: "custom", path: ["headerName"], message: `${name} is set by the app and can't be changed.` });
} else {
if (v.port == null) ctx.addIssue({ code: "custom", path: ["port"], message: v.kind === "tcp" ? "A port check needs a port." : "A certificate check needs a port (usually 443)." });
if (/[/\s]/.test(v.target)) ctx.addIssue({ code: "custom", path: ["target"], message: "Give just the host name or address, without http:// or a path." });
}
});
/** A blank text field is no value. */
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
// Everyone signed in can see how things are doing, like the pages the same information appears on.
endpointsRouter.get("/", asyncHandler(async (_req, res) => {
@@ -58,7 +81,7 @@ endpointsRouter.get("/", asyncHandler(async (_req, res) => {
const stats = await loadStats(rows.map((r) => r.id));
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
res.json({
checks: rows.map((r) => ({ ...r, server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null, stats: stats.get(r.id) })),
checks: rows.map((r) => ({ ...publicCheck(r), server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null, stats: stats.get(r.id) })),
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
});
}));
@@ -74,8 +97,19 @@ endpointsRouter.get("/:id/history", asyncHandler(async (req, res) => {
res.json({ check, ...(await getEndpointHistory(id, days)) });
}));
function valuesOf(d: z.infer<typeof checkSchema>) {
function valuesOf(d: z.infer<typeof checkSchema>, before?: CheckRow) {
const http = d.kind === "http";
const hasValue = http && !!blank(d.valuePath);
const headerName = http ? blank(d.headerName) : null;
// What is already saved stays unless a new value is given; clearing the name removes the header.
const headerValue = !headerName ? null : blank(d.headerValue) !== null ? sealSecret(d.headerValue!.trim()) : (before?.headerValue ?? null);
return {
valuePath: hasValue ? blank(d.valuePath) : null,
valueOp: hasValue ? (d.valueOp ?? null) : null,
valueExpected: hasValue && d.valueOp !== "exists" ? blank(d.valueExpected) : null,
valueUnit: hasValue ? blank(d.valueUnit) : null,
headerName,
headerValue,
name: d.name,
kind: d.kind,
target: d.target,
@@ -142,15 +176,21 @@ endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(
res.json(await loadSuggestions());
}));
/** A header named with nothing to send, and nothing saved to send either. */
function v0HeaderMissing(d: z.infer<typeof checkSchema>, before?: CheckRow): boolean {
return d.kind === "http" && !!blank(d.headerName) && blank(d.headerValue) === null && !before?.headerValue;
}
// Asking things on the network is an administrator's to set up.
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = checkSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
if (v0HeaderMissing(parsed.data)) return res.status(400).json({ error: "invalid_body", message: "Give the header a value." });
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const [row] = await db.insert(endpointChecks).values(valuesOf(parsed.data)).returning();
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target } });
res.status(201).json({ check: row });
res.status(201).json({ check: publicCheck(row) });
}));
endpointsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
@@ -160,13 +200,15 @@ endpointsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res)
const [before] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
if (!before) return res.status(404).json({ error: "not_found" });
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
const v = valuesOf(parsed.data);
if (v0HeaderMissing(parsed.data, before)) return res.status(400).json({ error: "invalid_body", message: "Give the header a value." });
const v = valuesOf(parsed.data, before);
// What is asked changed, so what was last found out about it no longer stands.
const reset = v.kind !== before.kind || v.target !== before.target || v.port !== before.port ? { lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0 } : {};
const changed = v.kind !== before.kind || v.target !== before.target || v.port !== before.port || v.valuePath !== before.valuePath || v.valueOp !== before.valueOp || v.valueExpected !== before.valueExpected || v.headerName !== before.headerName;
const reset = changed ? { lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, lastValue: null, certExpiresAt: null, consecutiveFailures: 0 } : {};
const [row] = await db.update(endpointChecks).set({ ...v, ...reset }).where(eq(endpointChecks.id, id)).returning();
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "update", targetType: "endpoint", targetId: id, detail: { name: row.name, kind: row.kind, target: row.target } });
invalidateAlertsCache();
res.json({ check: row });
res.json({ check: publicCheck(row) });
}));
endpointsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
+96 -8
View File
@@ -3,6 +3,9 @@
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
*
* A web check can also judge a value in a JSON answer (services/valueCheck.ts): the power is under 5 000 W, the player's state is "playing", the query
* returned a number. The value read is remembered with each result, so it can be charted.
*
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
* LAN and public hosts are fine; that is what these are for.
@@ -18,6 +21,8 @@ import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
import { noteRenewal } from "./renewals.js";
import { resolveTarget } from "../utils/networkGuard.js";
import { openSecret } from "./sealed.js";
import { isValueOp, judgeValue, readPath, type ValueOp } from "./valueCheck.js";
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
export type CheckKind = (typeof CHECK_KINDS)[number];
@@ -30,6 +35,9 @@ export interface ProbeResult {
message: string;
/** ISO — when the certificate it showed runs out (https and tls checks). */
certExpiresAt?: string | null;
/** What was read from the answer, as text, for a check that judges a value; the number, when it is one. */
value?: string | null;
valueNumber?: number | null;
}
export const KEEP_RESULT_DAYS = 14;
@@ -78,10 +86,12 @@ interface OnceResult {
status: number;
location: string | null;
body: string;
/** The answer reached the most that is read, so what is here may be only the start of it. */
truncated: boolean;
cert: string | null;
}
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean, extraHeaders: Record<string, string> = {}): Promise<OnceResult> {
const family = net.isIP(address) === 6 ? 6 : 4;
return new Promise((resolve, reject) => {
const isHttps = url.protocol === "https:";
@@ -92,7 +102,7 @@ function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: nu
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
method: "GET",
path: `${url.pathname}${url.search}`,
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", ...extraHeaders, host: url.host },
// Connect to the address that passed the check, not to the name again.
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
timeout: timeoutMs,
@@ -102,7 +112,7 @@ function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: nu
const cert = isHttps ? certExpiry(res.socket) : null;
const chunks: Buffer[] = [];
let size = 0;
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), truncated: size >= MAX_BODY_BYTES, cert });
if (!wantBody) {
res.resume();
res.on("end", finish);
@@ -138,6 +148,15 @@ async function probeHttp(c: CheckRow): Promise<ProbeResult> {
} catch {
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
}
// A header the person set (a token for an API) goes to the address they gave, and to nowhere a redirect leads to on another host.
let header: Record<string, string> = {};
if (c.headerName) {
const value = openSecret(c.headerValue);
if (value === null) return { ok: false, ms: null, message: `The value of the ${c.headerName} header can't be read — enter it again.` };
header = { [c.headerName]: value };
}
const origin = url.origin;
const wantBody = !!c.keyword || !!c.valuePath;
let cert: string | null = null;
for (let hops = 0; ; hops++) {
const left = timeoutMs - (performance.now() - started);
@@ -145,7 +164,7 @@ async function probeHttp(c: CheckRow): Promise<ProbeResult> {
let once: OnceResult;
try {
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
once = await requestOnce(url, address, c.insecure, left, wantBody, url.origin === origin ? header : {});
} catch (err) {
const e = err as NodeJS.ErrnoException;
const text = err instanceof Error ? err.message : String(err);
@@ -153,18 +172,32 @@ async function probeHttp(c: CheckRow): Promise<ProbeResult> {
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
}
cert = once.cert ?? cert;
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
// Redirects are only followed when there is something to look for in the page they lead to; otherwise a redirect is an answer.
if (wantBody && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
url = new URL(once.location, url);
continue;
}
const ms = Math.round(performance.now() - started);
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
if (c.valuePath && c.valueOp && isValueOp(c.valueOp)) return judgeAnswer(c, c.valuePath, c.valueOp, once.body, once.truncated, ms, cert);
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
}
}
/** The answer, read as JSON, and the value in it judged. */
function judgeAnswer(c: CheckRow, path: string, op: ValueOp, body: string, truncated: boolean, ms: number, cert: string | null): ProbeResult {
if (truncated) return { ok: false, ms, message: `the answer is more than ${MAX_BODY_BYTES / 1024} KB, which is more than is read`, certExpiresAt: cert };
let json: unknown;
try {
json = JSON.parse(body);
} catch {
return { ok: false, ms, message: "the answer isn't JSON", certExpiresAt: cert };
}
const v = judgeValue(readPath(json, path), path, op, c.valueExpected, c.valueUnit);
return { ok: v.ok, ms, message: v.message, certExpiresAt: cert, value: v.shown === "" ? null : v.shown, valueNumber: v.number };
}
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
let address: string;
@@ -240,12 +273,14 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
lastOk: result.ok,
lastMessage: result.message.slice(0, 300),
lastResponseMs: result.ms,
// A check that judges a value remembers what it read; one that couldn't reach the answer keeps the last it did read.
lastValue: c.valuePath ? (result.value === undefined ? c.lastValue : result.value) : null,
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
lastChangeAt: flipped ? iso : c.lastChangeAt,
})
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300), value: result.valueNumber ?? null });
await addToDaily(c.id, iso, result);
await addToHourly(c.id, iso, result);
// A certificate that is now good for much longer than it was has been renewed.
@@ -387,8 +422,39 @@ export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxP
}));
}
/** A stretch of the values a check read: the average, with the lowest and highest in it. */
export interface ValuePoint {
/** ISO. */
t: string;
v: number;
lo: number;
hi: number;
}
/** Pure: readings into at most `maxPoints` equal slices of [from, to], each as its average with its lowest and highest. */
export function bucketValues(rows: { t: number; v: number }[], from: number, to: number, maxPoints: number): ValuePoint[] {
if (rows.length === 0 || to <= from) return [];
const size = Math.max((to - from) / maxPoints, 1);
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number }>();
for (const r of rows) {
if (r.t < from || r.t > to) continue;
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: r.v, hi: r.v, tSum: 0 };
b.sum += r.v;
b.n += 1;
b.lo = Math.min(b.lo, r.v);
b.hi = Math.max(b.hi, r.v);
b.tSum += r.t;
buckets.set(key, b);
}
const round = (n: number) => Math.round(n * 1000) / 1000;
return [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round(b.sum / b.n), lo: round(b.lo), hi: round(b.hi) }));
}
export interface EndpointHistory {
days: number;
/** The numbers a value check read, as far back as the results are kept (14 days); null for a check that reads none. */
values: { unit: string | null; days: number; points: ValuePoint[]; latest: number | null; lowest: number | null; highest: number | null } | null;
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
source: "results" | "hourly" | "daily";
points: HistoryPoint[];
@@ -428,8 +494,28 @@ export async function getEndpointHistory(checkId: number, days: number, now: Dat
const checks = rows.reduce((n, r) => n + r.total, 0);
const okCount = rows.reduce((n, r) => n + r.ok, 0);
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
const valueFrom = Math.max(from, to - KEEP_RESULT_DAYS * 86_400_000);
const read = await db
.select({ at: endpointResults.at, value: endpointResults.value })
.from(endpointResults)
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(valueFrom).toISOString()), sql`${endpointResults.value} IS NOT NULL`))
.orderBy(asc(endpointResults.at));
let values: EndpointHistory["values"] = null;
if (read.length > 0) {
const [c] = await db.select({ unit: endpointChecks.valueUnit }).from(endpointChecks).where(eq(endpointChecks.id, checkId)).limit(1);
const nums = read.map((r) => r.value as number);
values = {
unit: c?.unit ?? null,
days: Math.min(days, KEEP_RESULT_DAYS),
points: bucketValues(read.map((r) => ({ t: Date.parse(r.at), v: r.value as number })), valueFrom, to, 200),
latest: nums[nums.length - 1],
lowest: Math.min(...nums),
highest: Math.max(...nums),
};
}
return {
days,
values,
source,
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
@@ -503,7 +589,9 @@ export function endpointConditions(rows: CheckRow[], now: Date): EndpointConditi
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
const certGone = left !== null && left < 0;
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
// A check of a value says what is wrong with it; "isn't answering" would be untrue of one that answered, but not as it should.
const message = c.valuePath ? `${c.name} — ${c.lastMessage ?? "no answer"}` : `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}`;
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message });
}
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
out.push({
+3 -18
View File
@@ -14,30 +14,15 @@ import { createHmac, randomUUID } from "node:crypto";
import { and, desc, eq, lt, notInArray } from "drizzle-orm";
import { db } from "../db/client.js";
import { eventWebhooks, webhookDeliveries, webhookEvents, type WebhookEvent } from "../db/schema.js";
import { decryptSecret, encryptSecret } from "../crypto.js";
import { env } from "../env.js";
import { openSecret, sealSecret } from "./sealed.js";
export { sealSecret };
export type EventWebhookRow = typeof eventWebhooks.$inferSelect;
const ENCRYPTED_PREFIX = "enc:v1:";
const TIMEOUT_MS = 5000;
const RETRY_AFTER_MS = 2000;
/** The stored form of a secret: encrypted when an encryption key is set, as the notification channels' secrets are. */
export function sealSecret(plain: string): string {
return env.credentialsEncryptionEnabled ? ENCRYPTED_PREFIX + encryptSecret(plain) : plain;
}
function openSecret(stored: string | null): string | null {
if (!stored) return null;
if (!stored.startsWith(ENCRYPTED_PREFIX)) return stored;
try {
return decryptSecret(stored.slice(ENCRYPTED_PREFIX.length));
} catch {
return null;
}
}
export function parseEvents(stored: string): WebhookEvent[] {
try {
const v = JSON.parse(stored);
+1
View File
@@ -62,6 +62,7 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
if (!c.enabled || c.lastOk === null) continue;
const labels = { name: c.name, kind: c.kind };
out.add("homelab_endpoint_up", "gauge", "1 if the endpoint check's last answer was fine, else 0.", c.lastOk ? 1 : 0, labels);
if (c.valuePath && c.lastValue !== null && c.lastValue.trim() !== "" && Number.isFinite(Number(c.lastValue))) out.add("homelab_endpoint_value", "gauge", "The number a value check last read from the endpoint's answer.", Number(c.lastValue), labels);
if (c.lastResponseMs !== null) out.add("homelab_endpoint_response_milliseconds", "gauge", "How long the endpoint took to answer the last time.", c.lastResponseMs, labels);
if (c.certExpiresAt) out.add("homelab_endpoint_certificate_days_left", "gauge", "Days until the certificate the endpoint shows expires (negative once it has).", Math.floor((Date.parse(c.certExpiresAt) - now.getTime()) / 86_400_000), labels);
}
+1
View File
@@ -159,6 +159,7 @@ export async function endpointsV1(now: Date = new Date()) {
lastCheckedAt: r.lastCheckedAt,
lastResponseMs: r.lastResponseMs,
lastMessage: r.lastOk === false ? r.lastMessage : null,
value: r.valuePath ? { path: r.valuePath, operator: r.valueOp, expected: r.valueExpected, unit: r.valueUnit, last: r.lastValue } : null,
certificateExpiresAt: r.certExpiresAt,
uptime24hPercent: st?.uptime24h ?? null,
uptime7dPercent: st?.uptime7d ?? null,
+24
View File
@@ -0,0 +1,24 @@
/**
* A secret kept in the database: encrypted when an encryption key is set (as the notification channels' secrets are), plain otherwise. The webhooks' signing
* secrets and the header an endpoint check sends use it.
*/
import { decryptSecret, encryptSecret } from "../crypto.js";
import { env } from "../env.js";
const ENCRYPTED_PREFIX = "enc:v1:";
/** The stored form of a secret. */
export function sealSecret(plain: string): string {
return env.credentialsEncryptionEnabled ? ENCRYPTED_PREFIX + encryptSecret(plain) : plain;
}
/** What was stored, as it was given; null when there is none or it can no longer be decrypted. */
export function openSecret(stored: string | null): string | null {
if (!stored) return null;
if (!stored.startsWith(ENCRYPTED_PREFIX)) return stored;
try {
return decryptSecret(stored.slice(ENCRYPTED_PREFIX.length));
} catch {
return null;
}
}
+140
View File
@@ -0,0 +1,140 @@
/**
* Judging a value in an answer: "the power is under 5 000 W", "the status is playing", "the Prometheus query returned something". The answer is JSON; a path says
* where the value is in it, and one comparison says whether it is fine. Everything here is pure — the asking is in endpointChecks.ts.
*
* A path is the dots and brackets people already write: `power.now`, `data.result[0].value[1]`, `items[2].name`, and `["a.b"]` for a name with a dot in it.
* A leading `$.` is allowed and means nothing.
*/
export const VALUE_OPS = ["<", "<=", ">", ">=", "=", "!=", "contains", "exists"] as const;
export type ValueOp = (typeof VALUE_OPS)[number];
const NUMERIC_OPS: readonly ValueOp[] = ["<", "<=", ">", ">="];
export const isValueOp = (op: string): op is ValueOp => (VALUE_OPS as readonly string[]).includes(op);
type Step = string | number;
/** A path into its steps, or null when it can't be read. */
export function parsePath(path: string): Step[] | null {
let p = path.trim();
if (p.startsWith("$")) p = p.slice(1);
if (p.startsWith(".")) p = p.slice(1);
const steps: Step[] = [];
let i = 0;
while (i < p.length) {
const c = p[i];
if (c === ".") {
i++;
if (i >= p.length || p[i] === "." || p[i] === "[") return null;
} else if (c === "[") {
const close = p.indexOf("]", i);
if (close < 0) return null;
const inner = p.slice(i + 1, close).trim();
i = close + 1;
if (/^\d+$/.test(inner)) steps.push(Number(inner));
else if (/^(".*"|'.*')$/.test(inner) && inner.length >= 2) steps.push(inner.slice(1, -1));
else return null;
} else {
const m = /^[^.[\]]+/.exec(p.slice(i));
if (!m) return null;
steps.push(m[0]);
i += m[0].length;
}
}
return steps.length > 0 ? steps : null;
}
export const isValuePath = (path: string) => parsePath(path) !== null;
export type Read = { found: true; value: unknown } | { found: false; at: string };
/** What is at a path in a parsed answer. `at` says how far it got when it isn't there. */
export function readPath(json: unknown, path: string): Read {
const steps = parsePath(path);
if (!steps) return { found: false, at: path };
let here: unknown = json;
const walked: string[] = [];
for (const step of steps) {
walked.push(typeof step === "number" ? `[${step}]` : walked.length === 0 ? step : `.${step}`);
if (typeof step === "number") {
if (!Array.isArray(here) || step >= here.length) return { found: false, at: walked.join("") };
here = here[step];
} else {
if (here === null || typeof here !== "object" || Array.isArray(here) || !Object.prototype.hasOwnProperty.call(here, step)) return { found: false, at: walked.join("") };
here = (here as Record<string, unknown>)[step];
}
}
return { found: true, value: here };
}
/** A value as a number when it is one, or text that is one ("12.5": Prometheus gives its numbers as text); otherwise null. */
export function asNumber(value: unknown): number | null {
if (typeof value === "number") return Number.isFinite(value) ? value : null;
if (typeof value === "string" && value.trim() !== "" && Number.isFinite(Number(value))) return Number(value);
return null;
}
/** A value as text, short enough to keep and show. */
export function showValue(value: unknown): string {
const text = typeof value === "string" ? value : typeof value === "number" || typeof value === "boolean" || value === null ? String(value) : JSON.stringify(value);
return text.length > 120 ? `${text.slice(0, 117)}…` : text;
}
export interface Verdict {
ok: boolean;
/** What was read, as text, to show and to remember. */
shown: string;
/** The number, when it is one. */
number: number | null;
message: string;
}
const WORDS: Record<ValueOp, string> = { "<": "below", "<=": "at most", ">": "above", ">=": "at least", "=": "equal to", "!=": "anything but", contains: "containing", exists: "there" };
/** How a limit reads: "below 5000 W". */
export function describeLimit(op: ValueOp, expected: string | null, unit: string | null): string {
if (op === "exists") return "there";
const n = asNumber(expected);
const quoted = NUMERIC_OPS.includes(op) || n !== null ? String(expected) : `“${expected}”`;
return `${WORDS[op]} ${quoted}${unit && (NUMERIC_OPS.includes(op) || n !== null) ? ` ${unit}` : ""}`;
}
/** Whether what was read meets the limit, and what to say about it. `read` is what readPath found. */
export function judgeValue(read: Read, path: string, op: ValueOp, expected: string | null, unit: string | null): Verdict {
if (!read.found) {
// "exists" is satisfied by the value being there, so its absence is the thing to say; the others cannot be judged without it.
return { ok: false, shown: "", number: null, message: `there is no “${path}” in the answer (it stops at ${read.at})` };
}
const number = asNumber(read.value);
const shown = showValue(read.value);
const withUnit = number !== null && unit ? `${shown} ${unit}` : shown;
const limit = describeLimit(op, expected, unit);
if (op === "exists") return { ok: true, shown, number, message: `${path} is ${withUnit}` };
let ok: boolean;
if (NUMERIC_OPS.includes(op)) {
const want = asNumber(expected);
if (number === null) return { ok: false, shown, number, message: `${path} is “${shown}”, which isn't a number, so it can't be compared with ${expected}` };
if (want === null) return { ok: false, shown, number, message: `the limit “${expected}” isn't a number` };
ok = op === "<" ? number < want : op === "<=" ? number <= want : op === ">" ? number > want : number >= want;
} else if (op === "contains") {
ok = shown.includes(expected ?? "");
} else {
// Numbers are equal as numbers ("5" and 5.0), anything else as the same text.
const want = asNumber(expected);
const same = number !== null && want !== null ? number === want : shown === (expected ?? "");
ok = op === "=" ? same : !same;
}
return { ok, shown, number, message: ok ? `${path} is ${withUnit} (should be ${limit})` : `${path} is ${withUnit}, but should be ${limit}` };
}
/** Whether a saved check's value settings make sense together: the reason why not, or null. */
export function valueProblem(path: string | null, op: string | null, expected: string | null): string | null {
if (!path && !op && !expected) return null;
if (!path) return "Say where in the answer the value is, e.g. data.result[0].value[1].";
if (!isValuePath(path)) return "That isn't a path I can read. Use names separated by dots, and [0] for a place in a list, like data.result[0].value[1].";
if (!op || !isValueOp(op)) return "Say how the value should compare.";
if (op !== "exists" && (expected === null || expected.trim() === "")) return "Say what to compare it with.";
if (NUMERIC_OPS.includes(op) && asNumber(expected) === null) return "A limit to be above or below has to be a number.";
return null;
}