Check a value, not just "is it up": JSON value checks on web endpoints
A web check can read a value out of a JSON answer (power.now, state, data.result[0].value[1]) and judge it: below, above, equal to, contains, is there. The value read is shown on the Endpoints page, charted under History, exported as homelab_endpoint_value and in the read API. A check can also send one header (a token), stored encrypted, never returned, and not forwarded to another host on a redirect. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e114567d64
commit
3cb3e4988f
21 files changed
+6804
-47
No files matched your search
@@ -0,0 +1,8 @@
|
||||
ALTER TABLE `endpoint_checks` ADD `value_path` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_checks` ADD `value_op` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_checks` ADD `value_expected` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_checks` ADD `value_unit` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_checks` ADD `header_name` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_checks` ADD `header_value` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_checks` ADD `last_value` text;--> statement-breakpoint
|
||||
ALTER TABLE `endpoint_results` ADD `value` real;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -372,6 +372,13 @@
|
||||
"when": 1792608282223,
|
||||
"tag": "0052_brainy_steve_rogers",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 53,
|
||||
"version": "6",
|
||||
"when": 1792609540216,
|
||||
"tag": "0053_dashing_stark_industries",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -141,6 +141,13 @@ export const endpointChecks = sqliteTable("endpoint_checks", {
|
||||
expectStatus: text("expect_status"), // http: "200-299, 301"; null = 200-399
|
||||
keyword: text("keyword"), // http: text the page must contain
|
||||
insecure: integer("insecure", { mode: "boolean" }).notNull().default(false), // accept a certificate that isn't trusted (self-signed, a private CA)
|
||||
valuePath: text("value_path"), // http: where in a JSON answer the value to judge is ("data.result[0].value[1]")
|
||||
valueOp: text("value_op"), // http: "<" "<=" ">" ">=" "=" "!=" "contains" "exists"
|
||||
valueExpected: text("value_expected"), // http: what it is compared with (a number, or text for "=", "!=" and "contains")
|
||||
valueUnit: text("value_unit"), // http: shown after the value ("W", "%", "°C")
|
||||
headerName: text("header_name"), // http: a header to send with the request ("Authorization")
|
||||
headerValue: text("header_value"), // its value, encrypted at rest like the notification channels' secrets; never returned by the API
|
||||
lastValue: text("last_value"), // http: what was read the last time, as text
|
||||
intervalMinutes: integer("interval_minutes").notNull().default(5),
|
||||
timeoutSeconds: integer("timeout_seconds").notNull().default(10),
|
||||
failuresBeforeAlert: integer("failures_before_alert").notNull().default(2),
|
||||
@@ -170,6 +177,7 @@ export const endpointResults = sqliteTable(
|
||||
ok: integer("ok", { mode: "boolean" }).notNull(),
|
||||
ms: integer("ms"),
|
||||
message: text("message"),
|
||||
value: real("value"), // the number read from the answer, for a check that judges a value
|
||||
},
|
||||
(t) => [index("endpoint_results_check_idx").on(t.checkId, t.at)],
|
||||
);
|
||||
|
||||
@@ -6,7 +6,9 @@ import { endpointChecks, servers } from "../db/schema.js";
|
||||
import { requireAuth, requireRole } from "../auth/middleware.js";
|
||||
import { recordAudit } from "../services/audit.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck } from "../services/endpointChecks.js";
|
||||
import { CHECK_KINDS, getEndpointHistory, isStatusSpec, loadStats, runCheck, type CheckRow } from "../services/endpointChecks.js";
|
||||
import { sealSecret } from "../services/sealed.js";
|
||||
import { VALUE_OPS, valueProblem } from "../services/valueCheck.js";
|
||||
import { loadDismissed, loadSuggestions, saveDismissed } from "../services/endpointSuggestions.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
@@ -16,6 +18,17 @@ endpointsRouter.use(requireAuth);
|
||||
|
||||
const MAX_CHECKS = 100;
|
||||
|
||||
/** A blank text field is no value. */
|
||||
function blank(v: string | null | undefined): string | null {
|
||||
return v === undefined || v === null || v.trim() === "" ? null : v.trim();
|
||||
}
|
||||
|
||||
/** A check as the API shows it: the header's value is never sent back, only that there is one. */
|
||||
function publicCheck(row: CheckRow) {
|
||||
const { headerValue, ...rest } = row;
|
||||
return { ...rest, hasHeader: !!row.headerName && !!headerValue };
|
||||
}
|
||||
|
||||
const checkSchema = z
|
||||
.object({
|
||||
name: z.string().trim().min(1).max(100),
|
||||
@@ -24,6 +37,13 @@ const checkSchema = z
|
||||
port: z.number().int().min(1).max(65535).nullable().optional(),
|
||||
expectStatus: z.string().trim().max(60).nullable().optional(),
|
||||
keyword: z.string().max(200).nullable().optional(),
|
||||
valuePath: z.string().trim().max(200).nullable().optional(),
|
||||
valueOp: z.enum(VALUE_OPS).nullable().optional(),
|
||||
valueExpected: z.string().max(200).nullable().optional(),
|
||||
valueUnit: z.string().trim().max(12).nullable().optional(),
|
||||
headerName: z.string().trim().max(100).nullable().optional(),
|
||||
/** Left out or blank on an edit keeps the one already saved. */
|
||||
headerValue: z.string().max(2000).nullable().optional(),
|
||||
insecure: z.boolean().optional(),
|
||||
intervalMinutes: z.number().int().min(1).max(1440).optional(),
|
||||
timeoutSeconds: z.number().int().min(1).max(60).optional(),
|
||||
@@ -43,14 +63,17 @@ const checkSchema = z
|
||||
}
|
||||
if (!ok) ctx.addIssue({ code: "custom", path: ["target"], message: "A web check needs an address that starts with http:// or https://." });
|
||||
if (v.expectStatus && !isStatusSpec(v.expectStatus)) ctx.addIssue({ code: "custom", path: ["expectStatus"], message: "Use status codes like 200, or ranges like 200-299, separated by commas." });
|
||||
const problem = valueProblem(blank(v.valuePath), v.valueOp ?? null, blank(v.valueExpected));
|
||||
if (problem) ctx.addIssue({ code: "custom", path: ["valuePath"], message: problem });
|
||||
const name = blank(v.headerName);
|
||||
if (name && !/^[A-Za-z0-9!#$%&'*+.^_`|~-]+$/.test(name)) ctx.addIssue({ code: "custom", path: ["headerName"], message: "A header name is letters, digits and dashes, like Authorization." });
|
||||
if (name && /^(host|content-length|transfer-encoding|connection)$/i.test(name)) ctx.addIssue({ code: "custom", path: ["headerName"], message: `${name} is set by the app and can't be changed.` });
|
||||
} else {
|
||||
if (v.port == null) ctx.addIssue({ code: "custom", path: ["port"], message: v.kind === "tcp" ? "A port check needs a port." : "A certificate check needs a port (usually 443)." });
|
||||
if (/[/\s]/.test(v.target)) ctx.addIssue({ code: "custom", path: ["target"], message: "Give just the host name or address, without http:// or a path." });
|
||||
}
|
||||
});
|
||||
|
||||
/** A blank text field is no value. */
|
||||
const blank = (v: string | null | undefined) => (v === undefined || v === null || v.trim() === "" ? null : v.trim());
|
||||
|
||||
// Everyone signed in can see how things are doing, like the pages the same information appears on.
|
||||
endpointsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
@@ -58,7 +81,7 @@ endpointsRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const stats = await loadStats(rows.map((r) => r.id));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((s) => [s.id, s.name]));
|
||||
res.json({
|
||||
checks: rows.map((r) => ({ ...r, server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null, stats: stats.get(r.id) })),
|
||||
checks: rows.map((r) => ({ ...publicCheck(r), server: r.serverId !== null && serverNames.has(r.serverId) ? { id: r.serverId, name: serverNames.get(r.serverId)! } : null, stats: stats.get(r.id) })),
|
||||
servers: [...serverNames].map(([id, name]) => ({ id, name })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
});
|
||||
}));
|
||||
@@ -74,8 +97,19 @@ endpointsRouter.get("/:id/history", asyncHandler(async (req, res) => {
|
||||
res.json({ check, ...(await getEndpointHistory(id, days)) });
|
||||
}));
|
||||
|
||||
function valuesOf(d: z.infer<typeof checkSchema>) {
|
||||
function valuesOf(d: z.infer<typeof checkSchema>, before?: CheckRow) {
|
||||
const http = d.kind === "http";
|
||||
const hasValue = http && !!blank(d.valuePath);
|
||||
const headerName = http ? blank(d.headerName) : null;
|
||||
// What is already saved stays unless a new value is given; clearing the name removes the header.
|
||||
const headerValue = !headerName ? null : blank(d.headerValue) !== null ? sealSecret(d.headerValue!.trim()) : (before?.headerValue ?? null);
|
||||
return {
|
||||
valuePath: hasValue ? blank(d.valuePath) : null,
|
||||
valueOp: hasValue ? (d.valueOp ?? null) : null,
|
||||
valueExpected: hasValue && d.valueOp !== "exists" ? blank(d.valueExpected) : null,
|
||||
valueUnit: hasValue ? blank(d.valueUnit) : null,
|
||||
headerName,
|
||||
headerValue,
|
||||
name: d.name,
|
||||
kind: d.kind,
|
||||
target: d.target,
|
||||
@@ -142,15 +176,21 @@ endpointsRouter.post("/suggestions/restore", requireRole("admin"), asyncHandler(
|
||||
res.json(await loadSuggestions());
|
||||
}));
|
||||
|
||||
/** A header named with nothing to send, and nothing saved to send either. */
|
||||
function v0HeaderMissing(d: z.infer<typeof checkSchema>, before?: CheckRow): boolean {
|
||||
return d.kind === "http" && !!blank(d.headerName) && blank(d.headerValue) === null && !before?.headerValue;
|
||||
}
|
||||
|
||||
// Asking things on the network is an administrator's to set up.
|
||||
endpointsRouter.post("/",requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
const parsed = checkSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: parsed.error.issues[0]?.message ?? "That isn't a check.", details: parsed.error.flatten() });
|
||||
if (v0HeaderMissing(parsed.data)) return res.status(400).json({ error: "invalid_body", message: "Give the header a value." });
|
||||
if ((await db.select({ id: endpointChecks.id }).from(endpointChecks)).length >= MAX_CHECKS) return res.status(409).json({ error: "too_many", message: `At most ${MAX_CHECKS} checks.` });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const [row] = await db.insert(endpointChecks).values(valuesOf(parsed.data)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "create", targetType: "endpoint", targetId: row.id, detail: { name: row.name, kind: row.kind, target: row.target } });
|
||||
res.status(201).json({ check: row });
|
||||
res.status(201).json({ check: publicCheck(row) });
|
||||
}));
|
||||
|
||||
endpointsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
@@ -160,13 +200,15 @@ endpointsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res)
|
||||
const [before] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, id)).limit(1);
|
||||
if (!before) return res.status(404).json({ error: "not_found" });
|
||||
if (!(await serverExists(parsed.data.serverId))) return res.status(400).json({ error: "unknown_server", message: "That server doesn't exist." });
|
||||
const v = valuesOf(parsed.data);
|
||||
if (v0HeaderMissing(parsed.data, before)) return res.status(400).json({ error: "invalid_body", message: "Give the header a value." });
|
||||
const v = valuesOf(parsed.data, before);
|
||||
// What is asked changed, so what was last found out about it no longer stands.
|
||||
const reset = v.kind !== before.kind || v.target !== before.target || v.port !== before.port ? { lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0 } : {};
|
||||
const changed = v.kind !== before.kind || v.target !== before.target || v.port !== before.port || v.valuePath !== before.valuePath || v.valueOp !== before.valueOp || v.valueExpected !== before.valueExpected || v.headerName !== before.headerName;
|
||||
const reset = changed ? { lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, lastValue: null, certExpiresAt: null, consecutiveFailures: 0 } : {};
|
||||
const [row] = await db.update(endpointChecks).set({ ...v, ...reset }).where(eq(endpointChecks.id, id)).returning();
|
||||
await recordAudit({ actor: req.currentUser!, category: "endpoint", action: "update", targetType: "endpoint", targetId: id, detail: { name: row.name, kind: row.kind, target: row.target } });
|
||||
invalidateAlertsCache();
|
||||
res.json({ check: row });
|
||||
res.json({ check: publicCheck(row) });
|
||||
}));
|
||||
|
||||
endpointsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
|
||||
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
|
||||
*
|
||||
* A web check can also judge a value in a JSON answer (services/valueCheck.ts): the power is under 5 000 W, the player's state is "playing", the query
|
||||
* returned a number. The value read is remembered with each result, so it can be charted.
|
||||
*
|
||||
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
|
||||
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
|
||||
* LAN and public hosts are fine; that is what these are for.
|
||||
@@ -18,6 +21,8 @@ import { endpointChecks, endpointDaily, endpointHourly, endpointResults } from "
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
import { noteRenewal } from "./renewals.js";
|
||||
import { resolveTarget } from "../utils/networkGuard.js";
|
||||
import { openSecret } from "./sealed.js";
|
||||
import { isValueOp, judgeValue, readPath, type ValueOp } from "./valueCheck.js";
|
||||
|
||||
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
|
||||
export type CheckKind = (typeof CHECK_KINDS)[number];
|
||||
@@ -30,6 +35,9 @@ export interface ProbeResult {
|
||||
message: string;
|
||||
/** ISO — when the certificate it showed runs out (https and tls checks). */
|
||||
certExpiresAt?: string | null;
|
||||
/** What was read from the answer, as text, for a check that judges a value; the number, when it is one. */
|
||||
value?: string | null;
|
||||
valueNumber?: number | null;
|
||||
}
|
||||
|
||||
export const KEEP_RESULT_DAYS = 14;
|
||||
@@ -78,10 +86,12 @@ interface OnceResult {
|
||||
status: number;
|
||||
location: string | null;
|
||||
body: string;
|
||||
/** The answer reached the most that is read, so what is here may be only the start of it. */
|
||||
truncated: boolean;
|
||||
cert: string | null;
|
||||
}
|
||||
|
||||
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
|
||||
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean, extraHeaders: Record<string, string> = {}): Promise<OnceResult> {
|
||||
const family = net.isIP(address) === 6 ? 6 : 4;
|
||||
return new Promise((resolve, reject) => {
|
||||
const isHttps = url.protocol === "https:";
|
||||
@@ -92,7 +102,7 @@ function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: nu
|
||||
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
|
||||
method: "GET",
|
||||
path: `${url.pathname}${url.search}`,
|
||||
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
|
||||
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", ...extraHeaders, host: url.host },
|
||||
// Connect to the address that passed the check, not to the name again.
|
||||
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
|
||||
timeout: timeoutMs,
|
||||
@@ -102,7 +112,7 @@ function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: nu
|
||||
const cert = isHttps ? certExpiry(res.socket) : null;
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
|
||||
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), truncated: size >= MAX_BODY_BYTES, cert });
|
||||
if (!wantBody) {
|
||||
res.resume();
|
||||
res.on("end", finish);
|
||||
@@ -138,6 +148,15 @@ async function probeHttp(c: CheckRow): Promise<ProbeResult> {
|
||||
} catch {
|
||||
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
|
||||
}
|
||||
// A header the person set (a token for an API) goes to the address they gave, and to nowhere a redirect leads to on another host.
|
||||
let header: Record<string, string> = {};
|
||||
if (c.headerName) {
|
||||
const value = openSecret(c.headerValue);
|
||||
if (value === null) return { ok: false, ms: null, message: `The value of the ${c.headerName} header can't be read — enter it again.` };
|
||||
header = { [c.headerName]: value };
|
||||
}
|
||||
const origin = url.origin;
|
||||
const wantBody = !!c.keyword || !!c.valuePath;
|
||||
let cert: string | null = null;
|
||||
for (let hops = 0; ; hops++) {
|
||||
const left = timeoutMs - (performance.now() - started);
|
||||
@@ -145,7 +164,7 @@ async function probeHttp(c: CheckRow): Promise<ProbeResult> {
|
||||
let once: OnceResult;
|
||||
try {
|
||||
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
|
||||
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
|
||||
once = await requestOnce(url, address, c.insecure, left, wantBody, url.origin === origin ? header : {});
|
||||
} catch (err) {
|
||||
const e = err as NodeJS.ErrnoException;
|
||||
const text = err instanceof Error ? err.message : String(err);
|
||||
@@ -153,18 +172,32 @@ async function probeHttp(c: CheckRow): Promise<ProbeResult> {
|
||||
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
|
||||
}
|
||||
cert = once.cert ?? cert;
|
||||
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
|
||||
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
|
||||
// Redirects are only followed when there is something to look for in the page they lead to; otherwise a redirect is an answer.
|
||||
if (wantBody && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
|
||||
url = new URL(once.location, url);
|
||||
continue;
|
||||
}
|
||||
const ms = Math.round(performance.now() - started);
|
||||
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
|
||||
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
|
||||
if (c.valuePath && c.valueOp && isValueOp(c.valueOp)) return judgeAnswer(c, c.valuePath, c.valueOp, once.body, once.truncated, ms, cert);
|
||||
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
|
||||
}
|
||||
}
|
||||
|
||||
/** The answer, read as JSON, and the value in it judged. */
|
||||
function judgeAnswer(c: CheckRow, path: string, op: ValueOp, body: string, truncated: boolean, ms: number, cert: string | null): ProbeResult {
|
||||
if (truncated) return { ok: false, ms, message: `the answer is more than ${MAX_BODY_BYTES / 1024} KB, which is more than is read`, certExpiresAt: cert };
|
||||
let json: unknown;
|
||||
try {
|
||||
json = JSON.parse(body);
|
||||
} catch {
|
||||
return { ok: false, ms, message: "the answer isn't JSON", certExpiresAt: cert };
|
||||
}
|
||||
const v = judgeValue(readPath(json, path), path, op, c.valueExpected, c.valueUnit);
|
||||
return { ok: v.ok, ms, message: v.message, certExpiresAt: cert, value: v.shown === "" ? null : v.shown, valueNumber: v.number };
|
||||
}
|
||||
|
||||
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
|
||||
const started = performance.now();
|
||||
let address: string;
|
||||
@@ -240,12 +273,14 @@ export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c:
|
||||
lastOk: result.ok,
|
||||
lastMessage: result.message.slice(0, 300),
|
||||
lastResponseMs: result.ms,
|
||||
// A check that judges a value remembers what it read; one that couldn't reach the answer keeps the last it did read.
|
||||
lastValue: c.valuePath ? (result.value === undefined ? c.lastValue : result.value) : null,
|
||||
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
|
||||
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
|
||||
lastChangeAt: flipped ? iso : c.lastChangeAt,
|
||||
})
|
||||
.where(eq(endpointChecks.id, c.id));
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
|
||||
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300), value: result.valueNumber ?? null });
|
||||
await addToDaily(c.id, iso, result);
|
||||
await addToHourly(c.id, iso, result);
|
||||
// A certificate that is now good for much longer than it was has been renewed.
|
||||
@@ -387,8 +422,39 @@ export function bucketHistory(rows: HistoryRow[], from: number, to: number, maxP
|
||||
}));
|
||||
}
|
||||
|
||||
/** A stretch of the values a check read: the average, with the lowest and highest in it. */
|
||||
export interface ValuePoint {
|
||||
/** ISO. */
|
||||
t: string;
|
||||
v: number;
|
||||
lo: number;
|
||||
hi: number;
|
||||
}
|
||||
|
||||
/** Pure: readings into at most `maxPoints` equal slices of [from, to], each as its average with its lowest and highest. */
|
||||
export function bucketValues(rows: { t: number; v: number }[], from: number, to: number, maxPoints: number): ValuePoint[] {
|
||||
if (rows.length === 0 || to <= from) return [];
|
||||
const size = Math.max((to - from) / maxPoints, 1);
|
||||
const buckets = new Map<number, { sum: number; n: number; lo: number; hi: number; tSum: number }>();
|
||||
for (const r of rows) {
|
||||
if (r.t < from || r.t > to) continue;
|
||||
const key = Math.min(maxPoints - 1, Math.floor((r.t - from) / size));
|
||||
const b = buckets.get(key) ?? { sum: 0, n: 0, lo: r.v, hi: r.v, tSum: 0 };
|
||||
b.sum += r.v;
|
||||
b.n += 1;
|
||||
b.lo = Math.min(b.lo, r.v);
|
||||
b.hi = Math.max(b.hi, r.v);
|
||||
b.tSum += r.t;
|
||||
buckets.set(key, b);
|
||||
}
|
||||
const round = (n: number) => Math.round(n * 1000) / 1000;
|
||||
return [...buckets.entries()].sort((a, b) => a[0] - b[0]).map(([, b]) => ({ t: new Date(b.tSum / b.n).toISOString(), v: round(b.sum / b.n), lo: round(b.lo), hi: round(b.hi) }));
|
||||
}
|
||||
|
||||
export interface EndpointHistory {
|
||||
days: number;
|
||||
/** The numbers a value check read, as far back as the results are kept (14 days); null for a check that reads none. */
|
||||
values: { unit: string | null; days: number; points: ValuePoint[]; latest: number | null; lowest: number | null; highest: number | null } | null;
|
||||
/** Which figures it was drawn from: the results themselves (a day or two), the hourly figures, or the daily ones. */
|
||||
source: "results" | "hourly" | "daily";
|
||||
points: HistoryPoint[];
|
||||
@@ -428,8 +494,28 @@ export async function getEndpointHistory(checkId: number, days: number, now: Dat
|
||||
const checks = rows.reduce((n, r) => n + r.total, 0);
|
||||
const okCount = rows.reduce((n, r) => n + r.ok, 0);
|
||||
const msCount = rows.reduce((n, r) => n + r.msCount, 0);
|
||||
const valueFrom = Math.max(from, to - KEEP_RESULT_DAYS * 86_400_000);
|
||||
const read = await db
|
||||
.select({ at: endpointResults.at, value: endpointResults.value })
|
||||
.from(endpointResults)
|
||||
.where(and(eq(endpointResults.checkId, checkId), gte(endpointResults.at, new Date(valueFrom).toISOString()), sql`${endpointResults.value} IS NOT NULL`))
|
||||
.orderBy(asc(endpointResults.at));
|
||||
let values: EndpointHistory["values"] = null;
|
||||
if (read.length > 0) {
|
||||
const [c] = await db.select({ unit: endpointChecks.valueUnit }).from(endpointChecks).where(eq(endpointChecks.id, checkId)).limit(1);
|
||||
const nums = read.map((r) => r.value as number);
|
||||
values = {
|
||||
unit: c?.unit ?? null,
|
||||
days: Math.min(days, KEEP_RESULT_DAYS),
|
||||
points: bucketValues(read.map((r) => ({ t: Date.parse(r.at), v: r.value as number })), valueFrom, to, 200),
|
||||
latest: nums[nums.length - 1],
|
||||
lowest: Math.min(...nums),
|
||||
highest: Math.max(...nums),
|
||||
};
|
||||
}
|
||||
return {
|
||||
days,
|
||||
values,
|
||||
source,
|
||||
points: bucketHistory(rows, from, to, days <= 2 ? 96 : days <= KEEP_HOURLY_DAYS ? 200 : 370),
|
||||
availability: checks === 0 ? null : Math.round((okCount / checks) * 1000) / 10,
|
||||
@@ -503,7 +589,9 @@ export function endpointConditions(rows: CheckRow[], now: Date): EndpointConditi
|
||||
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
|
||||
const certGone = left !== null && left < 0;
|
||||
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
|
||||
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
|
||||
// A check of a value says what is wrong with it; "isn't answering" would be untrue of one that answered, but not as it should.
|
||||
const message = c.valuePath ? `${c.name} — ${c.lastMessage ?? "no answer"}` : `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}`;
|
||||
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message });
|
||||
}
|
||||
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
|
||||
out.push({
|
||||
|
||||
@@ -14,30 +14,15 @@ import { createHmac, randomUUID } from "node:crypto";
|
||||
import { and, desc, eq, lt, notInArray } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { eventWebhooks, webhookDeliveries, webhookEvents, type WebhookEvent } from "../db/schema.js";
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { env } from "../env.js";
|
||||
import { openSecret, sealSecret } from "./sealed.js";
|
||||
|
||||
export { sealSecret };
|
||||
|
||||
export type EventWebhookRow = typeof eventWebhooks.$inferSelect;
|
||||
|
||||
const ENCRYPTED_PREFIX = "enc:v1:";
|
||||
const TIMEOUT_MS = 5000;
|
||||
const RETRY_AFTER_MS = 2000;
|
||||
|
||||
/** The stored form of a secret: encrypted when an encryption key is set, as the notification channels' secrets are. */
|
||||
export function sealSecret(plain: string): string {
|
||||
return env.credentialsEncryptionEnabled ? ENCRYPTED_PREFIX + encryptSecret(plain) : plain;
|
||||
}
|
||||
|
||||
function openSecret(stored: string | null): string | null {
|
||||
if (!stored) return null;
|
||||
if (!stored.startsWith(ENCRYPTED_PREFIX)) return stored;
|
||||
try {
|
||||
return decryptSecret(stored.slice(ENCRYPTED_PREFIX.length));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseEvents(stored: string): WebhookEvent[] {
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
|
||||
@@ -62,6 +62,7 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
if (!c.enabled || c.lastOk === null) continue;
|
||||
const labels = { name: c.name, kind: c.kind };
|
||||
out.add("homelab_endpoint_up", "gauge", "1 if the endpoint check's last answer was fine, else 0.", c.lastOk ? 1 : 0, labels);
|
||||
if (c.valuePath && c.lastValue !== null && c.lastValue.trim() !== "" && Number.isFinite(Number(c.lastValue))) out.add("homelab_endpoint_value", "gauge", "The number a value check last read from the endpoint's answer.", Number(c.lastValue), labels);
|
||||
if (c.lastResponseMs !== null) out.add("homelab_endpoint_response_milliseconds", "gauge", "How long the endpoint took to answer the last time.", c.lastResponseMs, labels);
|
||||
if (c.certExpiresAt) out.add("homelab_endpoint_certificate_days_left", "gauge", "Days until the certificate the endpoint shows expires (negative once it has).", Math.floor((Date.parse(c.certExpiresAt) - now.getTime()) / 86_400_000), labels);
|
||||
}
|
||||
|
||||
@@ -159,6 +159,7 @@ export async function endpointsV1(now: Date = new Date()) {
|
||||
lastCheckedAt: r.lastCheckedAt,
|
||||
lastResponseMs: r.lastResponseMs,
|
||||
lastMessage: r.lastOk === false ? r.lastMessage : null,
|
||||
value: r.valuePath ? { path: r.valuePath, operator: r.valueOp, expected: r.valueExpected, unit: r.valueUnit, last: r.lastValue } : null,
|
||||
certificateExpiresAt: r.certExpiresAt,
|
||||
uptime24hPercent: st?.uptime24h ?? null,
|
||||
uptime7dPercent: st?.uptime7d ?? null,
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* A secret kept in the database: encrypted when an encryption key is set (as the notification channels' secrets are), plain otherwise. The webhooks' signing
|
||||
* secrets and the header an endpoint check sends use it.
|
||||
*/
|
||||
import { decryptSecret, encryptSecret } from "../crypto.js";
|
||||
import { env } from "../env.js";
|
||||
|
||||
const ENCRYPTED_PREFIX = "enc:v1:";
|
||||
|
||||
/** The stored form of a secret. */
|
||||
export function sealSecret(plain: string): string {
|
||||
return env.credentialsEncryptionEnabled ? ENCRYPTED_PREFIX + encryptSecret(plain) : plain;
|
||||
}
|
||||
|
||||
/** What was stored, as it was given; null when there is none or it can no longer be decrypted. */
|
||||
export function openSecret(stored: string | null): string | null {
|
||||
if (!stored) return null;
|
||||
if (!stored.startsWith(ENCRYPTED_PREFIX)) return stored;
|
||||
try {
|
||||
return decryptSecret(stored.slice(ENCRYPTED_PREFIX.length));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Judging a value in an answer: "the power is under 5 000 W", "the status is playing", "the Prometheus query returned something". The answer is JSON; a path says
|
||||
* where the value is in it, and one comparison says whether it is fine. Everything here is pure — the asking is in endpointChecks.ts.
|
||||
*
|
||||
* A path is the dots and brackets people already write: `power.now`, `data.result[0].value[1]`, `items[2].name`, and `["a.b"]` for a name with a dot in it.
|
||||
* A leading `$.` is allowed and means nothing.
|
||||
*/
|
||||
|
||||
export const VALUE_OPS = ["<", "<=", ">", ">=", "=", "!=", "contains", "exists"] as const;
|
||||
export type ValueOp = (typeof VALUE_OPS)[number];
|
||||
|
||||
const NUMERIC_OPS: readonly ValueOp[] = ["<", "<=", ">", ">="];
|
||||
|
||||
export const isValueOp = (op: string): op is ValueOp => (VALUE_OPS as readonly string[]).includes(op);
|
||||
|
||||
type Step = string | number;
|
||||
|
||||
/** A path into its steps, or null when it can't be read. */
|
||||
export function parsePath(path: string): Step[] | null {
|
||||
let p = path.trim();
|
||||
if (p.startsWith("$")) p = p.slice(1);
|
||||
if (p.startsWith(".")) p = p.slice(1);
|
||||
const steps: Step[] = [];
|
||||
let i = 0;
|
||||
while (i < p.length) {
|
||||
const c = p[i];
|
||||
if (c === ".") {
|
||||
i++;
|
||||
if (i >= p.length || p[i] === "." || p[i] === "[") return null;
|
||||
} else if (c === "[") {
|
||||
const close = p.indexOf("]", i);
|
||||
if (close < 0) return null;
|
||||
const inner = p.slice(i + 1, close).trim();
|
||||
i = close + 1;
|
||||
if (/^\d+$/.test(inner)) steps.push(Number(inner));
|
||||
else if (/^(".*"|'.*')$/.test(inner) && inner.length >= 2) steps.push(inner.slice(1, -1));
|
||||
else return null;
|
||||
} else {
|
||||
const m = /^[^.[\]]+/.exec(p.slice(i));
|
||||
if (!m) return null;
|
||||
steps.push(m[0]);
|
||||
i += m[0].length;
|
||||
}
|
||||
}
|
||||
return steps.length > 0 ? steps : null;
|
||||
}
|
||||
|
||||
export const isValuePath = (path: string) => parsePath(path) !== null;
|
||||
|
||||
export type Read = { found: true; value: unknown } | { found: false; at: string };
|
||||
|
||||
/** What is at a path in a parsed answer. `at` says how far it got when it isn't there. */
|
||||
export function readPath(json: unknown, path: string): Read {
|
||||
const steps = parsePath(path);
|
||||
if (!steps) return { found: false, at: path };
|
||||
let here: unknown = json;
|
||||
const walked: string[] = [];
|
||||
for (const step of steps) {
|
||||
walked.push(typeof step === "number" ? `[${step}]` : walked.length === 0 ? step : `.${step}`);
|
||||
if (typeof step === "number") {
|
||||
if (!Array.isArray(here) || step >= here.length) return { found: false, at: walked.join("") };
|
||||
here = here[step];
|
||||
} else {
|
||||
if (here === null || typeof here !== "object" || Array.isArray(here) || !Object.prototype.hasOwnProperty.call(here, step)) return { found: false, at: walked.join("") };
|
||||
here = (here as Record<string, unknown>)[step];
|
||||
}
|
||||
}
|
||||
return { found: true, value: here };
|
||||
}
|
||||
|
||||
/** A value as a number when it is one, or text that is one ("12.5": Prometheus gives its numbers as text); otherwise null. */
|
||||
export function asNumber(value: unknown): number | null {
|
||||
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
||||
if (typeof value === "string" && value.trim() !== "" && Number.isFinite(Number(value))) return Number(value);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** A value as text, short enough to keep and show. */
|
||||
export function showValue(value: unknown): string {
|
||||
const text = typeof value === "string" ? value : typeof value === "number" || typeof value === "boolean" || value === null ? String(value) : JSON.stringify(value);
|
||||
return text.length > 120 ? `${text.slice(0, 117)}…` : text;
|
||||
}
|
||||
|
||||
export interface Verdict {
|
||||
ok: boolean;
|
||||
/** What was read, as text, to show and to remember. */
|
||||
shown: string;
|
||||
/** The number, when it is one. */
|
||||
number: number | null;
|
||||
message: string;
|
||||
}
|
||||
|
||||
const WORDS: Record<ValueOp, string> = { "<": "below", "<=": "at most", ">": "above", ">=": "at least", "=": "equal to", "!=": "anything but", contains: "containing", exists: "there" };
|
||||
|
||||
/** How a limit reads: "below 5000 W". */
|
||||
export function describeLimit(op: ValueOp, expected: string | null, unit: string | null): string {
|
||||
if (op === "exists") return "there";
|
||||
const n = asNumber(expected);
|
||||
const quoted = NUMERIC_OPS.includes(op) || n !== null ? String(expected) : `“${expected}”`;
|
||||
return `${WORDS[op]} ${quoted}${unit && (NUMERIC_OPS.includes(op) || n !== null) ? ` ${unit}` : ""}`;
|
||||
}
|
||||
|
||||
/** Whether what was read meets the limit, and what to say about it. `read` is what readPath found. */
|
||||
export function judgeValue(read: Read, path: string, op: ValueOp, expected: string | null, unit: string | null): Verdict {
|
||||
if (!read.found) {
|
||||
// "exists" is satisfied by the value being there, so its absence is the thing to say; the others cannot be judged without it.
|
||||
return { ok: false, shown: "", number: null, message: `there is no “${path}” in the answer (it stops at ${read.at})` };
|
||||
}
|
||||
const number = asNumber(read.value);
|
||||
const shown = showValue(read.value);
|
||||
const withUnit = number !== null && unit ? `${shown} ${unit}` : shown;
|
||||
const limit = describeLimit(op, expected, unit);
|
||||
if (op === "exists") return { ok: true, shown, number, message: `${path} is ${withUnit}` };
|
||||
let ok: boolean;
|
||||
if (NUMERIC_OPS.includes(op)) {
|
||||
const want = asNumber(expected);
|
||||
if (number === null) return { ok: false, shown, number, message: `${path} is “${shown}”, which isn't a number, so it can't be compared with ${expected}` };
|
||||
if (want === null) return { ok: false, shown, number, message: `the limit “${expected}” isn't a number` };
|
||||
ok = op === "<" ? number < want : op === "<=" ? number <= want : op === ">" ? number > want : number >= want;
|
||||
} else if (op === "contains") {
|
||||
ok = shown.includes(expected ?? "");
|
||||
} else {
|
||||
// Numbers are equal as numbers ("5" and 5.0), anything else as the same text.
|
||||
const want = asNumber(expected);
|
||||
const same = number !== null && want !== null ? number === want : shown === (expected ?? "");
|
||||
ok = op === "=" ? same : !same;
|
||||
}
|
||||
return { ok, shown, number, message: ok ? `${path} is ${withUnit} (should be ${limit})` : `${path} is ${withUnit}, but should be ${limit}` };
|
||||
}
|
||||
|
||||
/** Whether a saved check's value settings make sense together: the reason why not, or null. */
|
||||
export function valueProblem(path: string | null, op: string | null, expected: string | null): string | null {
|
||||
if (!path && !op && !expected) return null;
|
||||
if (!path) return "Say where in the answer the value is, e.g. data.result[0].value[1].";
|
||||
if (!isValuePath(path)) return "That isn't a path I can read. Use names separated by dots, and [0] for a place in a list, like data.result[0].value[1].";
|
||||
if (!op || !isValueOp(op)) return "Say how the value should compare.";
|
||||
if (op !== "exists" && (expected === null || expected.trim() === "")) return "Say what to compare it with.";
|
||||
if (NUMERIC_OPS.includes(op) && asNumber(expected) === null) return "A limit to be above or below has to be a number.";
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
/**
|
||||
* Checking a value in an answer: reading a path out of JSON, judging the value against a limit, asking a local server with and without a header, what is
|
||||
* remembered and charted, and the routes that keep a header's value to themselves.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type IncomingMessage, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { networkInterfaces, tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-values-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, endpointChecks, endpointResults } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const value = await import("../src/services/valueCheck.js");
|
||||
const { probe, runCheck, bucketValues, getEndpointHistory, endpointConditions } = await import("../src/services/endpointChecks.js");
|
||||
const { endpointsRouter } = await import("../src/routes/endpoints.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { endpointsV1 } = await import("../src/services/publicApi.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
describe("where a value is", () => {
|
||||
test("dots, brackets, quoted names and a leading $", () => {
|
||||
assert.deepEqual(value.parsePath("power.now"), ["power", "now"]);
|
||||
assert.deepEqual(value.parsePath("data.result[0].value[1]"), ["data", "result", 0, "value", 1]);
|
||||
assert.deepEqual(value.parsePath("$.items[2].name"), ["items", 2, "name"]);
|
||||
assert.deepEqual(value.parsePath('["a.b"].c'), ["a.b", "c"]);
|
||||
assert.deepEqual(value.parsePath("[0].x"), [0, "x"]);
|
||||
for (const bad of ["", "a..b", "a.", "a[", "a[x]", "a[]", ".", "$"]) assert.equal(value.parsePath(bad), null, `"${bad}"`);
|
||||
});
|
||||
|
||||
test("it reads what is there, and says how far it got when it isn't", () => {
|
||||
const json = { power: { now: 120.5, unit: "W" }, data: { result: [{ value: [1700, "42.5"] }] }, "a.b": { c: true }, nothing: null, list: [] };
|
||||
assert.deepEqual(value.readPath(json, "power.now"), { found: true, value: 120.5 });
|
||||
assert.deepEqual(value.readPath(json, "data.result[0].value[1]"), { found: true, value: "42.5" });
|
||||
assert.deepEqual(value.readPath(json, '["a.b"].c'), { found: true, value: true });
|
||||
assert.deepEqual(value.readPath(json, "nothing"), { found: true, value: null }, "null is a value that is there");
|
||||
assert.deepEqual(value.readPath(json, "power.later"), { found: false, at: "power.later" });
|
||||
assert.deepEqual(value.readPath(json, "data.result[3]"), { found: false, at: "data.result[3]" });
|
||||
assert.deepEqual(value.readPath(json, "list[0]"), { found: false, at: "list[0]" });
|
||||
assert.deepEqual(value.readPath(json, "power.now.deeper"), { found: false, at: "power.now.deeper" }, "a number has nothing inside it");
|
||||
assert.equal(value.readPath(json, "toString").found, false, "nothing is inherited from the language");
|
||||
assert.equal(value.readPath([1, 2], "[1]").found, true);
|
||||
});
|
||||
|
||||
test("numbers are numbers, and so is text that is one", () => {
|
||||
assert.deepEqual([12, "12.5", " 7 ", "", "abc", null, true, Infinity].map(value.asNumber), [12, 12.5, 7, null, null, null, null, null]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("whether a value is fine", () => {
|
||||
const read = (v: unknown) => ({ found: true as const, value: v });
|
||||
const j = (v: unknown, op: (typeof value.VALUE_OPS)[number], expected: string | null, unit: string | null = null) => value.judgeValue(read(v), "p", op, expected, unit);
|
||||
|
||||
test("below, above and the edges", () => {
|
||||
assert.equal(j(4999, "<", "5000").ok, true);
|
||||
assert.equal(j(5000, "<", "5000").ok, false);
|
||||
assert.equal(j(5000, "<=", "5000").ok, true);
|
||||
assert.equal(j(5000, ">", "5000").ok, false);
|
||||
assert.equal(j(5000, ">=", "5000").ok, true);
|
||||
assert.equal(j("42.5", "<", "50").ok, true, "a number given as text, as Prometheus does");
|
||||
});
|
||||
|
||||
test("equal and not equal compare numbers as numbers and anything else as the same text", () => {
|
||||
assert.equal(j(5, "=", "5.0").ok, true);
|
||||
assert.equal(j("5", "=", "5").ok, true);
|
||||
assert.equal(j("playing", "=", "playing").ok, true);
|
||||
assert.equal(j("Playing", "=", "playing").ok, false, "case matters");
|
||||
assert.equal(j("idle", "!=", "error").ok, true);
|
||||
assert.equal(j("error", "!=", "error").ok, false);
|
||||
assert.equal(j(true, "=", "true").ok, true);
|
||||
});
|
||||
|
||||
test("contains, and exists (being there is enough, even a null)", () => {
|
||||
assert.equal(j("all systems go", "contains", "systems").ok, true);
|
||||
assert.equal(j("all systems go", "contains", "fault").ok, false);
|
||||
assert.equal(j(null, "exists", null).ok, true);
|
||||
assert.equal(value.judgeValue({ found: false, at: "x.y" }, "x.y.z", "exists", null, null).ok, false);
|
||||
});
|
||||
|
||||
test("it says what it found and what it wanted, with the unit when the value is a number", () => {
|
||||
assert.equal(j(5200, "<", "5000", "W").message, "p is 5200 W, but should be below 5000 W");
|
||||
assert.equal(j(120, "<", "5000", "W").message, "p is 120 W (should be below 5000 W)");
|
||||
assert.equal(j("error", "=", "playing").message, "p is error, but should be equal to “playing”");
|
||||
assert.equal(j("x", "=", "playing", "W").message, "p is x, but should be equal to “playing”", "no unit on text");
|
||||
assert.match(j("hot", "<", "50").message, /“hot”, which isn't a number/);
|
||||
assert.match(value.judgeValue({ found: false, at: "data" }, "data.x", "<", "5", null).message, /no “data.x” in the answer \(it stops at data\)/);
|
||||
const long = j("x".repeat(500), "contains", "y");
|
||||
assert.ok(long.shown.length <= 120 && long.shown.endsWith("…"), "what is kept is short");
|
||||
assert.equal(j({ a: 1 }, "contains", "a").shown, '{"a":1}', "an object is shown as JSON");
|
||||
});
|
||||
|
||||
test("a saved setting is refused when it doesn't make sense, with the reason", () => {
|
||||
assert.equal(value.valueProblem(null, null, null), null, "no value check at all is fine");
|
||||
assert.match(value.valueProblem(null, ">", "5")!, /where in the answer/);
|
||||
assert.match(value.valueProblem("a..b", ">", "5")!, /path I can read/);
|
||||
assert.match(value.valueProblem("a", null, "5")!, /how the value should compare/);
|
||||
assert.match(value.valueProblem("a", ">", null)!, /compare it with/);
|
||||
assert.match(value.valueProblem("a", ">", "lots")!, /has to be a number/);
|
||||
assert.equal(value.valueProblem("a", "=", "playing"), null);
|
||||
assert.equal(value.valueProblem("a", "exists", null), null, "exists needs nothing to compare with");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the values over time", () => {
|
||||
test("readings fall into slices as their average with the lowest and highest, and nothing outside the stretch", () => {
|
||||
const H = 3_600_000;
|
||||
const pts = bucketValues([{ t: 0, v: 10 }, { t: H, v: 20 }, { t: 2 * H, v: 30 }, { t: 3 * H, v: 50 }, { t: 99 * H, v: 999 }], 0, 4 * H, 2);
|
||||
assert.deepEqual(pts.map((p) => [p.v, p.lo, p.hi]), [[15, 10, 20], [40, 30, 50]]);
|
||||
assert.deepEqual(bucketValues([], 0, 10, 5), []);
|
||||
});
|
||||
});
|
||||
|
||||
/** An address of this machine that isn't loopback (the checks refuse that), if it has one. */
|
||||
function lanAddress(): string | null {
|
||||
for (const list of Object.values(networkInterfaces())) {
|
||||
for (const i of list ?? []) if (i.family === "IPv4" && !i.internal && !i.address.startsWith("169.254.")) return i.address;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const LAN = lanAddress();
|
||||
|
||||
describe("asking for a value", { skip: LAN === null && "this machine has no address other than loopback" }, () => {
|
||||
let web: Server;
|
||||
let other: Server;
|
||||
let port = 0;
|
||||
let otherPort = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let who = "admin";
|
||||
const seen: { url: string; auth: string | undefined }[] = [];
|
||||
let power = 120;
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([
|
||||
{ oidcSub: "admin", name: "Ada", role: "admin" },
|
||||
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
|
||||
]);
|
||||
web = createHttpServer((req: IncomingMessage, res) => {
|
||||
seen.push({ url: req.url ?? "", auth: req.headers.authorization });
|
||||
const json = (o: unknown) => {
|
||||
res.setHeader("content-type", "application/json");
|
||||
res.end(JSON.stringify(o));
|
||||
};
|
||||
if (req.url === "/power") return json({ power: { now: power }, state: "playing", data: { result: [{ value: [1700000000, "42.5"] }] } });
|
||||
if (req.url === "/secure") {
|
||||
if (req.headers.authorization !== "Bearer s3cret") {
|
||||
res.statusCode = 401;
|
||||
return void res.end("no");
|
||||
}
|
||||
return json({ ok: 1 });
|
||||
}
|
||||
if (req.url === "/html") return void res.end("<h1>hello</h1>");
|
||||
if (req.url === "/big") return void res.end(`{"x":"${"y".repeat(100_000)}"}`);
|
||||
if (req.url === "/hop") {
|
||||
res.statusCode = 302;
|
||||
res.setHeader("location", `http://${LAN}:${otherPort}/power`);
|
||||
return void res.end();
|
||||
}
|
||||
res.statusCode = 404;
|
||||
res.end("no");
|
||||
});
|
||||
await new Promise<void>((r) => web.listen(0, "0.0.0.0", r));
|
||||
port = (web.address() as AddressInfo).port;
|
||||
other = createHttpServer((req, res) => {
|
||||
seen.push({ url: `other${req.url}`, auth: req.headers.authorization });
|
||||
res.setHeader("content-type", "application/json");
|
||||
res.end(JSON.stringify({ power: { now: 1 } }));
|
||||
});
|
||||
await new Promise<void>((r) => other.listen(0, "0.0.0.0", r));
|
||||
otherPort = (other.address() as AddressInfo).port;
|
||||
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: who } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/endpoints", endpointsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/endpoints`;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
web.close();
|
||||
other.close();
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
type Row = typeof endpointChecks.$inferSelect;
|
||||
const check = (path: string, extra: Partial<Row> = {}): Row =>
|
||||
({
|
||||
id: 1, name: "Meter", kind: "http", target: `http://${LAN}:${port}${path}`, port: null, expectStatus: null, keyword: null, valuePath: null, valueOp: null, valueExpected: null, valueUnit: null, headerName: null, headerValue: null, lastValue: null,
|
||||
insecure: false, intervalMinutes: 5, timeoutSeconds: 3, failuresBeforeAlert: 2, warnDays: 14, serverId: null, enabled: true, lastCheckedAt: null, lastOk: null, lastMessage: null, lastResponseMs: null, certExpiresAt: null, consecutiveFailures: 0, lastChangeAt: null,
|
||||
createdAt: "2026-10-01 00:00:00", ...extra,
|
||||
}) as Row;
|
||||
|
||||
test("a value under its limit is fine, over it is not, and either way what was read comes back", async () => {
|
||||
const fine = await probe(check("/power", { valuePath: "power.now", valueOp: "<", valueExpected: "5000", valueUnit: "W" }));
|
||||
assert.deepEqual([fine.ok, fine.value, fine.valueNumber], [true, "120", 120]);
|
||||
assert.equal(fine.message, "power.now is 120 W (should be below 5000 W)");
|
||||
const over = await probe(check("/power", { valuePath: "power.now", valueOp: "<", valueExpected: "100", valueUnit: "W" }));
|
||||
assert.deepEqual([over.ok, over.value], [false, "120"]);
|
||||
assert.equal(over.message, "power.now is 120 W, but should be below 100 W");
|
||||
assert.ok(over.ms !== null, "it did answer");
|
||||
});
|
||||
|
||||
test("text, and a number a query gives as text", async () => {
|
||||
assert.equal((await probe(check("/power", { valuePath: "state", valueOp: "=", valueExpected: "playing" }))).ok, true);
|
||||
const bad = await probe(check("/power", { valuePath: "state", valueOp: "=", valueExpected: "paused" }));
|
||||
assert.deepEqual([bad.ok, bad.valueNumber], [false, null]);
|
||||
const prom = await probe(check("/power", { valuePath: "data.result[0].value[1]", valueOp: ">", valueExpected: "40" }));
|
||||
assert.deepEqual([prom.ok, prom.valueNumber], [true, 42.5]);
|
||||
});
|
||||
|
||||
test("an answer that isn't JSON, a value that isn't there and a path that goes nowhere each say so", async () => {
|
||||
assert.match((await probe(check("/html", { valuePath: "a", valueOp: "exists" }))).message, /isn't JSON/);
|
||||
assert.match((await probe(check("/power", { valuePath: "power.gone", valueOp: "exists" }))).message, /no “power.gone” in the answer \(it stops at power.gone\)/);
|
||||
assert.match((await probe(check("/big", { valuePath: "x", valueOp: "exists" }))).message, /more than 64 KB/);
|
||||
const wrongStatus = await probe(check("/nope", { valuePath: "a", valueOp: "exists" }));
|
||||
assert.match(wrongStatus.message, /answered 404/, "the status is judged before the value");
|
||||
});
|
||||
|
||||
test("a header goes with the request, and its value is read back from the sealed form", async () => {
|
||||
const { sealSecret } = await import("../src/services/sealed.js");
|
||||
seen.length = 0;
|
||||
const ok = await probe(check("/secure", { headerName: "Authorization", headerValue: sealSecret("Bearer s3cret") }));
|
||||
assert.equal(ok.ok, true);
|
||||
assert.equal(seen[0].auth, "Bearer s3cret");
|
||||
assert.equal((await probe(check("/secure"))).ok, false, "without it the service says no");
|
||||
const lost = await probe(check("/secure", { headerName: "Authorization", headerValue: "enc:v1:not-decryptable" }));
|
||||
assert.match(lost.message, /can't be read — enter it again/);
|
||||
});
|
||||
|
||||
test("the header is not sent on to another host a redirect leads to", async () => {
|
||||
const { sealSecret } = await import("../src/services/sealed.js");
|
||||
seen.length = 0;
|
||||
const r = await probe(check("/hop", { valuePath: "power.now", valueOp: "<", valueExpected: "5", headerName: "Authorization", headerValue: sealSecret("Bearer s3cret") }));
|
||||
assert.equal(r.ok, true, "the redirect is followed to read the value");
|
||||
assert.deepEqual(seen.map((s) => [s.url, s.auth]), [["/hop", "Bearer s3cret"], ["other/power", undefined]]);
|
||||
});
|
||||
|
||||
test("a run remembers the value on the check and with the result; one that can't reach it keeps the last it read", async () => {
|
||||
const [row] = await db.insert(endpointChecks).values({ name: "Meter", kind: "http", target: `http://${LAN}:${port}/power`, valuePath: "power.now", valueOp: "<", valueExpected: "5000", valueUnit: "W" }).returning();
|
||||
await runCheck(row, new Date("2026-10-18T10:00:00Z"));
|
||||
let [after1] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, row.id));
|
||||
assert.deepEqual([after1.lastOk, after1.lastValue], [true, "120"]);
|
||||
power = 160;
|
||||
await runCheck(after1, new Date("2026-10-18T10:05:00Z"));
|
||||
const results = await db.select().from(endpointResults).where(eq(endpointResults.checkId, row.id)).orderBy(endpointResults.at);
|
||||
assert.deepEqual(results.map((r) => r.value), [120, 160]);
|
||||
[after1] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, row.id));
|
||||
await db.update(endpointChecks).set({ target: `http://${LAN}:1/never` }).where(eq(endpointChecks.id, row.id));
|
||||
const [gone] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, row.id));
|
||||
await runCheck(gone, new Date("2026-10-18T10:10:00Z"));
|
||||
const [kept] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, row.id));
|
||||
assert.deepEqual([kept.lastOk, kept.lastValue], [false, "160"]);
|
||||
power = 120;
|
||||
});
|
||||
|
||||
test("a value that is off is an alert that says what is wrong with it, not that it isn't answering", async () => {
|
||||
const list = endpointConditions([check("/power", { valuePath: "power.now", valueOp: "<", valueExpected: "100", lastOk: false, consecutiveFailures: 2, lastMessage: "power.now is 120, but should be below 100" }), check("/power", { id: 2, lastOk: false, consecutiveFailures: 2, lastMessage: "answered 500" })], new Date());
|
||||
assert.equal(list[0].message, "Meter — power.now is 120, but should be below 100");
|
||||
assert.equal(list[1].message, "Meter isn't answering — answered 500");
|
||||
});
|
||||
|
||||
test("the history carries the values, charted as far back as the results are kept", async () => {
|
||||
const [row] = await db.select().from(endpointChecks).where(eq(endpointChecks.valuePath, "power.now"));
|
||||
const h = await getEndpointHistory(row.id, 7, new Date("2026-10-18T12:00:00Z"));
|
||||
assert.ok(h.values);
|
||||
assert.deepEqual([h.values.unit, h.values.latest, h.values.lowest, h.values.highest], ["W", 160, 120, 160]);
|
||||
assert.equal(h.values.days, 7);
|
||||
assert.ok(h.values.points.length >= 1);
|
||||
const plain = await db.insert(endpointChecks).values({ name: "Plain", kind: "http", target: "https://x/" }).returning();
|
||||
assert.equal((await getEndpointHistory(plain[0].id, 7)).values, null, "a check that reads no value has none");
|
||||
});
|
||||
|
||||
test("/metrics and the read API carry it", async () => {
|
||||
const text = await buildMetrics();
|
||||
assert.match(text, /homelab_endpoint_value\{[^}]*name="Meter"[^}]*\} 160/);
|
||||
const [api] = (await endpointsV1()).filter((e) => e.name === "Meter");
|
||||
assert.deepEqual(api.value, { path: "power.now", operator: "<", expected: "5000", unit: "W", last: "160" });
|
||||
});
|
||||
|
||||
const send = (method: string, path: string, body?: unknown) => fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close" }, body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
|
||||
test("a value check is made over the API; a wrong one is refused with the reason; the header's value never comes back", async () => {
|
||||
const good = { name: "Solar", kind: "http", target: "https://inverter.lan/status", valuePath: "pv.power", valueOp: ">", valueExpected: "10", valueUnit: "W", headerName: "Authorization", headerValue: "Bearer topsecret" };
|
||||
const res = await send("POST", "/", good);
|
||||
assert.equal(res.status, 201);
|
||||
const created = (await res.json()).check;
|
||||
assert.deepEqual([created.valuePath, created.valueOp, created.valueExpected, created.headerName, created.hasHeader], ["pv.power", ">", "10", "Authorization", true]);
|
||||
assert.equal(JSON.stringify(created).includes("topsecret"), false);
|
||||
assert.equal(JSON.stringify(created).includes("headerValue"), false);
|
||||
const [stored] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, created.id));
|
||||
assert.ok(stored.headerValue && !stored.headerValue.includes("topsecret"), "stored sealed");
|
||||
|
||||
const list = (await (await send("GET", "/")).json()) as { checks: Record<string, unknown>[] };
|
||||
assert.equal(JSON.stringify(list).includes("topsecret"), false);
|
||||
assert.ok(list.checks.every((c) => !("headerValue" in c)));
|
||||
|
||||
for (const bad of [
|
||||
{ ...good, valuePath: "a..b" },
|
||||
{ ...good, valuePath: null },
|
||||
{ ...good, valueOp: "~" },
|
||||
{ ...good, valueExpected: "lots" },
|
||||
{ ...good, valueExpected: null },
|
||||
{ ...good, headerName: "Bad Name" },
|
||||
{ ...good, headerName: "Host" },
|
||||
{ name: "x", kind: "http", target: "https://a/", headerName: "Authorization" },
|
||||
]) assert.equal((await send("POST", "/", bad)).status, 400, JSON.stringify(bad));
|
||||
who = "viewer";
|
||||
assert.equal((await send("POST", "/", good)).status, 403);
|
||||
who = "admin";
|
||||
});
|
||||
|
||||
test("editing keeps the header's value unless a new one is given, and clearing the name removes it; changing what is asked forgets what was read", async () => {
|
||||
const [c] = await db.select().from(endpointChecks).where(eq(endpointChecks.name, "Solar"));
|
||||
await db.update(endpointChecks).set({ lastValue: "55", lastOk: true }).where(eq(endpointChecks.id, c.id));
|
||||
const same = { name: "Solar", kind: "http", target: "https://inverter.lan/status", valuePath: "pv.power", valueOp: ">", valueExpected: "10", valueUnit: "W", headerName: "Authorization" };
|
||||
assert.equal((await send("PUT", `/${c.id}`, same)).status, 200);
|
||||
let [now] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, c.id));
|
||||
assert.equal(now.headerValue, c.headerValue, "kept");
|
||||
assert.equal(now.lastValue, "55", "nothing that matters changed, so what was read stands");
|
||||
await send("PUT", `/${c.id}`, { ...same, valueExpected: "20" });
|
||||
[now] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, c.id));
|
||||
assert.deepEqual([now.lastValue, now.lastOk], [null, null]);
|
||||
await send("PUT", `/${c.id}`, { ...same, headerValue: "Bearer another" });
|
||||
const [replaced] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, c.id));
|
||||
assert.notEqual(replaced.headerValue, c.headerValue);
|
||||
await send("PUT", `/${c.id}`, { ...same, headerName: null, valuePath: null, valueOp: null, valueExpected: null });
|
||||
const [cleared] = await db.select().from(endpointChecks).where(eq(endpointChecks.id, c.id));
|
||||
assert.deepEqual([cleared.headerName, cleared.headerValue, cleared.valuePath, cleared.valueOp], [null, null, null, null]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user