Endpoint checks, restore tests, and a change log

- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags
- Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page
- Change log: one filterable timeline made from the audit log and the incident record

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 23:03:56 +02:00
1 parent c3bc6bbf72
commit 92726568b7
37 files changed
+6918 -21

No files matched your search

+35 -1
View File
@@ -11,7 +11,7 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
@@ -34,6 +34,8 @@ import { getFailingSources } from "./integrationHealthMonitor.js";
import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import { endpointConditions } from "./endpointChecks.js";
import { latestTests, restoreAlerts } from "./restoreTests.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
export interface AlertsReport {
@@ -62,6 +64,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
osticket: { label: "osTicket", link: "/osticket" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
};
const SOURCE_TIMEOUT_MS = 20_000;
@@ -268,6 +271,37 @@ export async function collectAlerts(): Promise<AlertsReport> {
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
});
}
// A restore that failed, or hasn't been tried in too long (the limit is in Settings; a failure is always raised).
const { healthChecks } = await getSettings();
for (const r of restoreAlerts(report, await latestTests(), healthChecks.restoreTestDays, now)) {
const g = r.guest;
push({
category: "backup",
severity: r.severity,
type: g ? "proxmox" : "homelab",
message: r.message,
key: r.key,
link: "/backups",
silenced: g ? subjects.has(`integration:${g.integrationId}`) || (g.server !== null && subjects.has(`server:${g.server.id}`)) : false,
target: g ? (g.server ? targetOf("server", g.server.id) : integrationTarget(g.integrationId)) : null,
});
}
}),
// What the endpoint checks have found: an address, port or certificate that has stopped being fine.
check("Endpoint checks", async () => {
for (const c of endpointConditions(await db.select().from(endpointChecks), new Date())) {
push({
category: c.category,
severity: c.severity,
type: "endpoint",
message: c.message,
key: c.key,
link: "/endpoints",
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
});
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
+307
View File
@@ -0,0 +1,307 @@
/**
* The change log: one timeline of what has changed in the homelab and in this app — agents updated, settings changed, maintenance started and
* ended, dependencies edited, hardware added, endpoints set up, alerts raised and cleared — each in a sentence, newest first, and filterable
* by what kind of thing it was, by server, by time and by a word. It is made from the audit log and the incident record, so it needs nothing
* of its own; the Audit Log page remains the raw table behind it.
*/
import { and, desc, gte, or, lte } from "drizzle-orm";
import { db } from "../db/client.js";
import { alertIncidents, auditLog, endpointChecks, integrations, servers } from "../db/schema.js";
import { auditTimeToIso, describeAudit, type ActivityTone } from "./serverActivity.js";
export const CHANGE_KINDS = ["servers", "maintenance", "alerts", "settings", "integrations", "dependencies", "inventory", "backups", "endpoints", "other"] as const;
export type ChangeKind = (typeof CHANGE_KINDS)[number];
export const KIND_LABELS: Record<ChangeKind, string> = {
servers: "Servers and agents",
maintenance: "Maintenance",
alerts: "Alerts and incidents",
settings: "Settings and access",
integrations: "Integrations, DNS and secrets",
dependencies: "Dependencies",
inventory: "Inventory",
backups: "Backups and restore tests",
endpoints: "Endpoint checks",
other: "Other",
};
const KIND_OF_CATEGORY: Record<string, ChangeKind> = {
server: "servers",
task: "servers",
update_run: "servers",
tag: "servers",
maintenance: "maintenance",
alert: "alerts",
incident: "alerts",
alert_ticket: "alerts",
settings: "settings",
user: "settings",
push: "settings",
share_link: "settings",
metrics_token: "settings",
webhook: "settings",
integration: "integrations",
dns: "integrations",
secret: "integrations",
domain: "integrations",
ipam: "integrations",
network: "integrations",
consistency: "integrations",
dependency: "dependencies",
inventory: "inventory",
runbook: "other",
db_backup: "backups",
restore_test: "backups",
endpoint: "endpoints",
};
/** Sign-ins and a person's own data export are not changes to the homelab; the diagnostic log has its own page. */
const LEFT_OUT = new Set(["session", "privacy", "diag_log"]);
export interface ChangeEntry {
/** Stable, so a row can be keyed on it. */
id: string;
/** ISO. */
at: string;
kind: ChangeKind;
tone: ActivityTone;
title: string;
detail: string | null;
/** A person's name, or "system" for the app's own checks; null when nobody did it. */
actor: string | null;
/** What it is about, and where to read more. */
target: { type: string; id: string | null; name: string | null } | null;
link: string | null;
/** The server it concerns, when it concerns one (so the log can be narrowed to a server). */
serverId: number | null;
}
const humanize = (s: string) => {
const t = s.replace(/[_-]+/g, " ");
return t.charAt(0).toUpperCase() + t.slice(1);
};
function parseDetail(detail: string | null): Record<string, unknown> {
if (!detail) return {};
try {
const v = JSON.parse(detail);
return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
} catch {
return {};
}
}
const str = (v: unknown) => (typeof v === "string" && v ? v : null);
const MAX_SUMMARY = 220;
/** "notifications: healthAlerts true → false" for a settings update; nothing for what is only a name. */
function settingsSummary(detail: Record<string, unknown>): string | null {
const changes = detail.changes;
if (!changes || typeof changes !== "object") return null;
const parts: string[] = [];
for (const [section, fields] of Object.entries(changes as Record<string, Record<string, unknown>>)) {
for (const [field, change] of Object.entries(fields ?? {})) {
if (change && typeof change === "object" && "to" in (change as object)) {
const c = change as { from: unknown; to: unknown };
parts.push(`${section}.${field}: ${JSON.stringify(c.from)} → ${JSON.stringify(c.to)}`);
} else parts.push(`${section}.${field} changed`);
}
}
const text = parts.join(" · ");
return text.length > MAX_SUMMARY ? `${text.slice(0, MAX_SUMMARY)}…` : text || null;
}
/** What is left of the detail once the name and what is already in the sentence are taken out. */
function genericSummary(detail: Record<string, unknown>, skip: string[] = []): string | null {
const parts = Object.entries(detail)
.filter(([k, v]) => !["name", "message", ...skip].includes(k) && v !== null && v !== undefined && v !== "" && !(Array.isArray(v) && v.length === 0))
.map(([k, v]) => `${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`);
const text = parts.join(" · ");
return text.length > MAX_SUMMARY ? `${text.slice(0, MAX_SUMMARY)}…` : text || null;
}
/** One sentence for an audit entry. Servers, alerts and update runs have their own wording (the same as a server's Activity card); the rest is made here. */
export function describeChange(category: string, action: string, detail: Record<string, unknown>): { title: string; tone: ActivityTone; detail: string | null } {
const name = str(detail.name);
const minutes = typeof detail.minutes === "number" ? detail.minutes : null;
const forHow = minutes !== null ? (minutes >= 120 ? `${Math.round(minutes / 60)} h` : `${minutes} min`) : null;
switch (`${category}/${action}`) {
case "maintenance/start":
return { title: `Maintenance started${name ? ` on ${name}` : ""}`, tone: "info", detail: [forHow && `for ${forHow}`, str(detail.reason)].filter(Boolean).join(" · ") || null };
case "maintenance/extend":
return { title: `Maintenance extended${name ? ` on ${name}` : ""}`, tone: "info", detail: [forHow && `for ${forHow} from now`, str(detail.reason)].filter(Boolean).join(" · ") || null };
case "maintenance/end":
return { title: `Maintenance ended${name ? ` on ${name}` : ""}`, tone: "info", detail: null };
case "maintenance/start_bulk":
return { title: "Maintenance started on many at once", tone: "info", detail: genericSummary(detail) };
case "dependency/create":
return { title: `${str(detail.from) ?? "Something"} now needs ${str(detail.to) ?? "something"}`, tone: "neutral", detail: null };
case "dependency/delete":
return { title: `${str(detail.from) ?? "Something"} no longer needs ${str(detail.to) ?? "something"}`, tone: "neutral", detail: null };
case "inventory/create":
return { title: `Hardware added${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail, ["kind"]) };
case "inventory/update":
return { title: `Hardware changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
case "inventory/delete":
return { title: `Hardware removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "inventory/refresh_from_agent":
return { title: `Hardware details taken from the agent${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "inventory/update_config":
return { title: "Price of power changed", tone: "neutral", detail: null };
case "endpoint/create":
return { title: `Endpoint check added${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/update":
return { title: `Endpoint check changed${name ? `: ${name}` : ""}`, tone: "neutral", detail: [str(detail.kind), str(detail.target)].filter(Boolean).join(" · ") || null };
case "endpoint/delete":
return { title: `Endpoint check removed${name ? `: ${name}` : ""}`, tone: "neutral", detail: null };
case "restore_test/record": {
const result = str(detail.result);
const tone: ActivityTone = result === "failed" ? "bad" : result === "not_needed" ? "neutral" : "good";
const said = result === "ok" ? "passed" : result === "partial" ? "partly worked" : result === "failed" ? "failed" : "marked as not needed";
return { title: `Restore test of ${name ?? "something"} ${said}`, tone, detail: [str(detail.testedAt), str(detail.note)].filter(Boolean).join(" · ") || null };
}
case "restore_test/delete":
return { title: `A restore test of ${name ?? "something"} was removed`, tone: "neutral", detail: str(detail.testedAt) };
case "settings/update": {
const sections = Array.isArray(detail.sections) ? (detail.sections as string[]).join(", ") : null;
return { title: `Settings changed${sections ? `: ${sections}` : ""}`, tone: "neutral", detail: settingsSummary(detail) };
}
case "push/subscribe":
return { title: `A browser was added for push notifications${str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: "neutral", detail: null };
case "push/unsubscribe":
return { title: `A browser was removed from push notifications${str(detail.label) ? `: ${str(detail.label)}` : ""}`, tone: "neutral", detail: null };
case "alert/escalate":
return { title: "An alert nobody had acknowledged was escalated", tone: "bad", detail: [str(detail.message), typeof detail.afterMinutes === "number" ? `step after ${detail.afterMinutes} min` : null].filter(Boolean).join(" · ") || null };
}
if (category === "server" || category === "alert" || category === "update_run") {
const d = describeAudit(category, action, detail);
return { title: d.title, tone: d.tone, detail: d.detail };
}
return { title: `${humanize(category)}: ${action.replace(/_/g, " ")}${name ? ` — ${name}` : ""}`, tone: "neutral", detail: genericSummary(detail) };
}
export interface ChangeQuery {
/** How far back, in days (default 7). */
days?: number;
kind?: ChangeKind;
serverId?: number;
/** A word or two that must all appear somewhere in the entry. */
q?: string;
/** Only entries older than this (ISO), for the next page. */
before?: string;
limit?: number;
now?: Date;
}
const sqlTime = (d: Date) => d.toISOString().slice(0, 19).replace("T", " ");
function serverIdOf(targetType: string | null, targetId: string | null, detail: Record<string, unknown>): number | null {
if (targetType === "server" && targetId && /^\d+$/.test(targetId)) return Number(targetId);
// Some entries about something else name the servers they touch by key ("server:3").
for (const k of ["fromKey", "toKey", "key"]) {
const m = /^server:(\d+)$/.exec(typeof detail[k] === "string" ? (detail[k] as string) : "");
if (m) return Number(m[1]);
}
return null;
}
const touchesServer = (e: ChangeEntry, id: number, detail: Record<string, unknown>) =>
e.serverId === id || ["fromKey", "toKey"].some((k) => detail[k] === `server:${id}`);
/** The log, newest first, narrowed as asked. `more` says whether older entries are left beyond the ones returned. */
export async function getChangeLog(query: ChangeQuery = {}): Promise<{ entries: ChangeEntry[]; more: boolean }> {
const now = query.now ?? new Date();
const days = Math.min(Math.max(query.days ?? 7, 1), 400);
const limit = Math.min(Math.max(query.limit ?? 200, 1), 500);
const since = new Date(now.getTime() - days * 86_400_000);
const until = query.before ? new Date(query.before) : null;
const [serverRows, integrationRows, endpointRows] = await Promise.all([
db.select({ id: servers.id, name: servers.name }).from(servers),
db.select({ id: integrations.id, name: integrations.name }).from(integrations),
db.select({ id: endpointChecks.id, name: endpointChecks.name }).from(endpointChecks),
]);
const serverName = new Map(serverRows.map((s) => [s.id, s.name]));
const integrationName = new Map(integrationRows.map((i) => [i.id, i.name]));
const endpointName = new Map(endpointRows.map((e) => [e.id, e.name]));
const entries: { entry: ChangeEntry; detail: Record<string, unknown> }[] = [];
const audits = await db
.select()
.from(auditLog)
.where(until ? and(gte(auditLog.createdAt, sqlTime(since)), lte(auditLog.createdAt, sqlTime(until))) : gte(auditLog.createdAt, sqlTime(since)))
.orderBy(desc(auditLog.createdAt), desc(auditLog.id))
.limit(5000);
for (const a of audits) {
if (LEFT_OUT.has(a.category)) continue;
const detail = parseDetail(a.detail);
const d = describeChange(a.category, a.action, detail);
const serverId = serverIdOf(a.targetType, a.targetId, detail);
const idNum = a.targetId && /^\d+$/.test(a.targetId) ? Number(a.targetId) : null;
let targetName = str(detail.name);
let link: string | null = null;
if (a.targetType === "server" && idNum !== null) {
targetName = serverName.get(idNum) ?? targetName;
link = serverName.has(idNum) ? `/servers/${idNum}` : null;
} else if (a.targetType === "integration" && idNum !== null) {
targetName = integrationName.get(idNum) ?? targetName;
link = "/integrations";
} else if (a.targetType === "endpoint" && idNum !== null) {
targetName = endpointName.get(idNum) ?? targetName;
link = "/endpoints";
} else if (a.targetType === "asset" && idNum !== null) link = `/inventory?asset=${idNum}`;
else if (a.targetType === "restore_test") link = "/backups";
else if (a.targetType === "dependency") link = "/dependencies";
else if (a.targetType === "settings") link = "/settings/notifications";
entries.push({
detail,
entry: {
id: `audit-${a.id}`,
at: auditTimeToIso(a.createdAt),
kind: KIND_OF_CATEGORY[a.category] ?? "other",
tone: d.tone,
title: d.title,
detail: d.detail,
actor: a.actorLabel,
target: a.targetType ? { type: a.targetType, id: a.targetId, name: targetName } : null,
link,
serverId,
},
});
}
// Alerts raised and cleared, from the incident record.
const incidents = await db
.select()
.from(alertIncidents)
.where(or(gte(alertIncidents.openedAt, since.toISOString()), gte(alertIncidents.resolvedAt, since.toISOString())))
.orderBy(desc(alertIncidents.openedAt))
.limit(1000);
for (const i of incidents) {
const serverId = i.targetType === "server" ? i.targetId : null;
const base = { kind: "alerts" as const, actor: null, target: i.targetType ? { type: i.targetType, id: i.targetId === null ? null : String(i.targetId), name: i.targetName } : null, link: `/incidents/${i.id}`, serverId };
if (i.openedAt >= since.toISOString() && (!until || i.openedAt <= until.toISOString())) {
entries.push({ detail: {}, entry: { ...base, id: `incident-open-${i.id}`, at: i.openedAt, tone: i.peakSeverity === "critical" ? "bad" : "info", title: `Alert raised: ${i.message}`, detail: i.causedBy ? `explained by ${i.causedBy}` : null } });
}
if (i.resolvedAt && i.resolvedAt >= since.toISOString() && (!until || i.resolvedAt <= until.toISOString())) {
entries.push({ detail: {}, entry: { ...base, id: `incident-close-${i.id}`, at: i.resolvedAt, tone: "good", title: `Alert cleared: ${i.message}`, detail: null } });
}
}
const terms = (query.q ?? "").toLowerCase().split(/\s+/).filter(Boolean);
const kept = entries
.filter(({ entry, detail }) => {
if (query.kind && entry.kind !== query.kind) return false;
if (query.serverId !== undefined && !touchesServer(entry, query.serverId, detail)) return false;
if (until && entry.at >= until.toISOString()) return false; // strictly older than the cursor
if (terms.length > 0) {
const hay = `${entry.title} ${entry.detail ?? ""} ${entry.actor ?? ""} ${entry.target?.name ?? ""} ${KIND_LABELS[entry.kind]}`.toLowerCase();
if (!terms.every((t) => hay.includes(t))) return false;
}
return true;
})
.map((e) => e.entry)
.sort((a, b) => b.at.localeCompare(a.at) || b.id.localeCompare(a.id));
return { entries: kept.slice(0, limit), more: kept.length > limit };
}
+350
View File
@@ -0,0 +1,350 @@
/**
* Endpoint checks: the app asks a web address, a TCP port or a TLS certificate whether it is fine, every few minutes, and remembers how it
* went — answer time, uptime, when the certificate runs out. A check that keeps failing becomes an alert (and a notification), and so does a
* certificate that is about to expire. This is for the simple "is it up?" that doesn't need a monitoring server of its own.
*
* What is asked is typed in by an administrator, so it can't be just anything: loopback, link-local (cloud metadata) and similar addresses
* are refused, and the connection goes to the one address that passed the check rather than to the name again (utils/networkGuard.ts).
* LAN and public hosts are fine; that is what these are for.
*/
import http from "node:http";
import https from "node:https";
import net from "node:net";
import tls from "node:tls";
import type { TLSSocket } from "node:tls";
import { and, asc, eq, gte, inArray, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, endpointResults } from "../db/schema.js";
import { resolveTarget } from "../utils/networkGuard.js";
export const CHECK_KINDS = ["http", "tcp", "tls"] as const;
export type CheckKind = (typeof CHECK_KINDS)[number];
export type CheckRow = typeof endpointChecks.$inferSelect;
export interface ProbeResult {
ok: boolean;
/** How long it took to answer, when it did. */
ms: number | null;
message: string;
/** ISO — when the certificate it showed runs out (https and tls checks). */
certExpiresAt?: string | null;
}
export const KEEP_RESULT_DAYS = 14;
const MAX_BODY_BYTES = 64 * 1024;
const MAX_REDIRECTS = 5;
/** "200-299, 301" → a test for a status code. Anything unreadable falls back to the usual 200–399. */
export function statusAccepter(spec: string | null | undefined): (code: number) => boolean {
const ranges: [number, number][] = [];
for (const part of (spec ?? "").split(",")) {
const m = /^\s*(\d{3})(?:\s*-\s*(\d{3}))?\s*$/.exec(part);
if (m) ranges.push([Number(m[1]), Number(m[2] ?? m[1])]);
}
if (ranges.length === 0) ranges.push([200, 399]);
return (code) => ranges.some(([lo, hi]) => code >= lo && code <= hi);
}
/** Whether a status spec is one statusAccepter would read (for refusing a typo when it is saved). */
export const isStatusSpec = (spec: string) => spec.split(",").every((p) => /^\s*\d{3}(\s*-\s*\d{3})?\s*$/.test(p));
const certExpiry = (socket: unknown): string | null => {
try {
const cert = (socket as TLSSocket).getPeerCertificate?.();
if (!cert || !cert.valid_to) return null;
const t = new Date(cert.valid_to);
return Number.isNaN(t.getTime()) ? null : t.toISOString();
} catch {
return null;
}
};
/** What to tell a person when the certificate can't be trusted: the usual reasons, said plainly. */
function trustProblem(code: string | undefined, text: string): string {
const known: Record<string, string> = {
DEPTH_ZERO_SELF_SIGNED_CERT: "it is self-signed",
SELF_SIGNED_CERT_IN_CHAIN: "it is signed by a certificate authority this server doesn't know",
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "it is signed by a certificate authority this server doesn't know",
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "it is signed by a certificate authority this server doesn't know",
CERT_HAS_EXPIRED: "it has expired",
ERR_TLS_CERT_ALTNAME_INVALID: "it isn't for this name",
};
return `The certificate can't be trusted: ${(code && known[code]) || text}. If that is expected for this service, tick "Accept a certificate that isn't trusted".`;
}
interface OnceResult {
status: number;
location: string | null;
body: string;
cert: string | null;
}
function requestOnce(url: URL, address: string, insecure: boolean, timeoutMs: number, wantBody: boolean): Promise<OnceResult> {
const family = net.isIP(address) === 6 ? 6 : 4;
return new Promise((resolve, reject) => {
const isHttps = url.protocol === "https:";
const bare = url.hostname.replace(/^\[|\]$/g, "");
const req = (isHttps ? https : http).request(
{
host: bare,
port: url.port ? Number(url.port) : isHttps ? 443 : 80,
method: "GET",
path: `${url.pathname}${url.search}`,
headers: { "user-agent": "HomelabManager-endpoint-check", accept: "*/*", host: url.host },
// Connect to the address that passed the check, not to the name again.
lookup: ((_host: string, options: { all?: boolean }, cb: (...args: unknown[]) => void) => (options?.all ? cb(null, [{ address, family }]) : cb(null, address, family))) as never,
timeout: timeoutMs,
...(isHttps ? { rejectUnauthorized: !insecure, servername: net.isIP(bare) === 0 ? bare : undefined } : {}),
},
(res) => {
const cert = isHttps ? certExpiry(res.socket) : null;
const chunks: Buffer[] = [];
let size = 0;
const finish = () => resolve({ status: res.statusCode ?? 0, location: typeof res.headers.location === "string" ? res.headers.location : null, body: Buffer.concat(chunks).toString("utf8"), cert });
if (!wantBody) {
res.resume();
res.on("end", finish);
res.on("close", finish);
return;
}
res.on("data", (c: Buffer) => {
size += c.length;
chunks.push(c);
if (size >= MAX_BODY_BYTES) {
res.destroy();
finish();
}
});
res.on("end", finish);
res.on("close", finish);
},
);
req.on("timeout", () => req.destroy(new Error(`no answer within ${Math.round(timeoutMs / 1000)} s`)));
req.on("error", reject);
req.end();
});
}
async function probeHttp(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
const timeoutMs = c.timeoutSeconds * 1000;
const accepts = statusAccepter(c.expectStatus);
let url: URL;
try {
url = new URL(c.target);
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("not http");
} catch {
return { ok: false, ms: null, message: "The address isn't a web address (it should start with http:// or https://)." };
}
let cert: string | null = null;
for (let hops = 0; ; hops++) {
const left = timeoutMs - (performance.now() - started);
if (left <= 0) return { ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` };
let once: OnceResult;
try {
const address = await resolveTarget(url.hostname.replace(/^\[|\]$/g, ""), { allowPrivate: true });
once = await requestOnce(url, address, c.insecure, left, !!c.keyword);
} catch (err) {
const e = err as NodeJS.ErrnoException;
const text = err instanceof Error ? err.message : String(err);
const trust = url.protocol === "https:" && /^(DEPTH_ZERO|SELF_SIGNED|UNABLE_TO|CERT_|ERR_TLS_CERT)/.test(e.code ?? "");
return { ok: false, ms: null, message: trust ? trustProblem(e.code, text) : text, certExpiresAt: cert };
}
cert = once.cert ?? cert;
// Redirects are only followed when there is a keyword to look for in the page they lead to; otherwise a redirect is an answer.
if (c.keyword && [301, 302, 303, 307, 308].includes(once.status) && once.location && hops < MAX_REDIRECTS) {
url = new URL(once.location, url);
continue;
}
const ms = Math.round(performance.now() - started);
if (!accepts(once.status)) return { ok: false, ms, message: `answered ${once.status}`, certExpiresAt: cert };
if (c.keyword && !once.body.includes(c.keyword)) return { ok: false, ms, message: `answered ${once.status}, but the page doesn't contain “${c.keyword}”`, certExpiresAt: cert };
return { ok: true, ms, message: `answered ${once.status} in ${ms} ms`, certExpiresAt: cert };
}
}
async function probeTcp(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
let address: string;
try {
address = await resolveTarget(c.target, { allowPrivate: true });
} catch (err) {
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
}
return new Promise((resolve) => {
const socket = net.connect({ host: address, port: c.port ?? 0 });
const done = (r: ProbeResult) => {
socket.destroy();
resolve(r);
};
socket.setTimeout(c.timeoutSeconds * 1000, () => done({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` }));
socket.on("connect", () => {
const ms = Math.round(performance.now() - started);
done({ ok: true, ms, message: `port ${c.port} answered in ${ms} ms` });
});
socket.on("error", (err: NodeJS.ErrnoException) => done({ ok: false, ms: null, message: err.code === "ECONNREFUSED" ? `port ${c.port} refused the connection` : err.message }));
});
}
async function probeTls(c: CheckRow): Promise<ProbeResult> {
const started = performance.now();
let address: string;
try {
address = await resolveTarget(c.target, { allowPrivate: true });
} catch (err) {
return { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
}
return new Promise((resolve) => {
const socket = tls.connect({ host: address, port: c.port ?? 443, servername: net.isIP(c.target) === 0 ? c.target : undefined, rejectUnauthorized: false }, () => {
const ms = Math.round(performance.now() - started);
const expires = certExpiry(socket);
const authorized = socket.authorized;
const reason = socket.authorizationError ? String(socket.authorizationError) : "";
socket.end();
if (!expires) return resolve({ ok: false, ms, message: "The server showed no certificate." });
const days = Math.floor((Date.parse(expires) - Date.now()) / 86_400_000);
if (days < 0) return resolve({ ok: false, ms, message: `The certificate expired on ${expires.slice(0, 10)}.`, certExpiresAt: expires });
if (!authorized && !c.insecure) return resolve({ ok: false, ms, message: trustProblem(reason, reason || "it didn't verify"), certExpiresAt: expires });
resolve({ ok: true, ms, message: `The certificate is good until ${expires.slice(0, 10)} (${days} days).`, certExpiresAt: expires });
});
socket.setTimeout(c.timeoutSeconds * 1000, () => {
socket.destroy();
resolve({ ok: false, ms: null, message: `no answer within ${c.timeoutSeconds} s` });
});
socket.on("error", (err) => {
resolve({ ok: false, ms: null, message: /wrong version number|packet length too long/i.test(err.message) ? `${c.target}:${c.port ?? 443} isn't speaking TLS — is that the right port?` : err.message });
});
});
}
export function probe(c: CheckRow): Promise<ProbeResult> {
return c.kind === "tcp" ? probeTcp(c) : c.kind === "tls" ? probeTls(c) : probeHttp(c);
}
/** Asks once, and writes down how it went: the check's current state, and one more point in its history. */
export async function runCheck(c: CheckRow, now: Date = new Date(), probeFn: (c: CheckRow) => Promise<ProbeResult> = probe): Promise<ProbeResult> {
let result: ProbeResult;
try {
result = await probeFn(c);
} catch (err) {
result = { ok: false, ms: null, message: err instanceof Error ? err.message : String(err) };
}
const iso = now.toISOString();
const flipped = c.lastOk === null || c.lastOk !== result.ok;
await db
.update(endpointChecks)
.set({
lastCheckedAt: iso,
lastOk: result.ok,
lastMessage: result.message.slice(0, 300),
lastResponseMs: result.ms,
certExpiresAt: result.certExpiresAt === undefined ? c.certExpiresAt : result.certExpiresAt,
consecutiveFailures: result.ok ? 0 : c.consecutiveFailures + 1,
lastChangeAt: flipped ? iso : c.lastChangeAt,
})
.where(eq(endpointChecks.id, c.id));
await db.insert(endpointResults).values({ checkId: c.id, at: iso, ok: result.ok, ms: result.ms, message: result.message.slice(0, 300) });
return result;
}
/** Runs every enabled check that is due, a few at a time. Returns how many it asked. */
export async function runDueChecks(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<number> {
const rows = await db.select().from(endpointChecks).where(eq(endpointChecks.enabled, true)).orderBy(asc(endpointChecks.id));
const due = rows.filter((c) => !c.lastCheckedAt || now.getTime() - Date.parse(c.lastCheckedAt) >= c.intervalMinutes * 60_000 - 5_000);
let next = 0;
const worker = async () => {
while (next < due.length) {
const c = due[next++];
await runCheck(c, now, probeFn);
}
};
await Promise.all(Array.from({ length: Math.min(5, due.length) }, worker));
return due.length;
}
export async function pruneResults(now: Date = new Date()): Promise<void> {
await db.delete(endpointResults).where(lt(endpointResults.at, new Date(now.getTime() - KEEP_RESULT_DAYS * 86_400_000).toISOString()));
}
// ─── What the history says ─────────────────────────────────────────────────
export interface CheckStats {
uptime24h: number | null;
uptime7d: number | null;
avgMs24h: number | null;
/** The last results, oldest first: how the recent checks went. */
recent: { at: string; ok: boolean; ms: number | null }[];
}
/** Pure: uptime, average answer time and the recent run of results, from a check's stored results. */
export function summariseResults(results: { at: string; ok: boolean; ms: number | null }[], now: Date): CheckStats {
const sorted = [...results].sort((a, b) => a.at.localeCompare(b.at));
const since = (ms: number) => sorted.filter((r) => now.getTime() - Date.parse(r.at) <= ms);
const pct = (list: typeof sorted) => (list.length === 0 ? null : Math.round((list.filter((r) => r.ok).length / list.length) * 1000) / 10);
const day = since(86_400_000);
const timed = day.filter((r) => r.ok && r.ms !== null);
return {
uptime24h: pct(day),
uptime7d: pct(since(7 * 86_400_000)),
avgMs24h: timed.length === 0 ? null : Math.round(timed.reduce((n, r) => n + (r.ms ?? 0), 0) / timed.length),
recent: sorted.slice(-30),
};
}
export async function loadStats(ids: number[], now: Date = new Date()): Promise<Map<number, CheckStats>> {
const out = new Map<number, CheckStats>();
if (ids.length === 0) return out;
const rows = await db
.select({ checkId: endpointResults.checkId, at: endpointResults.at, ok: endpointResults.ok, ms: endpointResults.ms })
.from(endpointResults)
.where(and(inArray(endpointResults.checkId, ids), gte(endpointResults.at, new Date(now.getTime() - 7 * 86_400_000).toISOString())));
const by = new Map<number, { at: string; ok: boolean; ms: number | null }[]>();
for (const r of rows) by.set(r.checkId, [...(by.get(r.checkId) ?? []), r]);
for (const id of ids) out.set(id, summariseResults(by.get(id) ?? [], now));
return out;
}
// ─── What is wrong ─────────────────────────────────────────────────────────
export interface EndpointCondition {
key: string;
source: "endpoint";
message: string;
severity: "critical" | "warning";
/** "monitoring" for a check that fails, "expiry" for a certificate running out. */
category: "monitoring" | "expiry";
checkId: number;
serverId: number | null;
}
const daysLeft = (iso: string, now: Date) => Math.floor((Date.parse(iso) - now.getTime()) / 86_400_000);
/**
* The problems the checks stand for. A check must have failed several times in a row before it is one (so a blip isn't), and a certificate is
* one from its warning time on — critical in the last three days. The key carries the server a check is linked to, so maintenance on it silences them.
*/
export function endpointConditions(rows: CheckRow[], now: Date): EndpointCondition[] {
const out: EndpointCondition[] = [];
for (const c of rows) {
if (!c.enabled) continue;
const suffix = c.serverId !== null ? `:server:${c.serverId}` : "";
const left = c.certExpiresAt ? daysLeft(c.certExpiresAt, now) : null;
const certGone = left !== null && left < 0;
if (c.lastOk === false && c.consecutiveFailures >= c.failuresBeforeAlert && !(c.kind === "tls" && certGone)) {
out.push({ key: `endpoint:${c.id}${suffix}`, source: "endpoint", category: "monitoring", severity: "critical", checkId: c.id, serverId: c.serverId, message: `${c.name} isn't answering — ${c.lastMessage ?? "no answer"}` });
}
if (c.certExpiresAt && left !== null && left <= c.warnDays) {
out.push({
key: `endpoint-cert:${c.id}${suffix}`,
source: "endpoint",
category: "expiry",
severity: left <= 3 ? "critical" : "warning",
checkId: c.id,
serverId: c.serverId,
message: `The certificate for ${c.name} ${left < 0 ? `expired ${-left} day${-left === 1 ? "" : "s"} ago` : left === 0 ? "expires today" : `expires in ${left} day${left === 1 ? "" : "s"}`} (${c.certExpiresAt.slice(0, 10)})`,
});
}
}
return out;
}
+59
View File
@@ -0,0 +1,59 @@
/**
* The checks' own rhythm: every minute, ask the ones that are due, then see what has started or stopped being a problem and say so — a
* notification when a check has failed enough times in a row, another when it works again, the same for a certificate running out. It
* uses the same bookkeeping as the server and automation checks (each problem announced once, remembered across a restart), so
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
*/
import { db } from "../db/client.js";
import { endpointChecks } from "../db/schema.js";
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
import { activeSubjects } from "./maintenance.js";
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
const STATE_FLAG = "endpointActiveConditions";
async function loadState(): Promise<ActiveState> {
const raw = await getInternalFlag(STATE_FLAG);
if (!raw) return {};
try {
return JSON.parse(raw);
} catch {
return {};
}
}
/** One pass: ask what is due, then announce what changed. Returns what it did. */
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
const asked = await runDueChecks(now, probeFn);
const rows = await db.select().from(endpointChecks);
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
if (added.length > 0) await notifyHealthIssues(added);
if (resolved.length > 0) await notifyHealthRecovered(resolved);
return { asked, announced: added.length, recovered: resolved.length };
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Looks every minute (each check has its own interval, so most passes ask nobody). Once a day it clears out old results. */
export function initEndpointMonitor(): void {
if (timer) return;
let lastPrune = 0;
const tick = async () => {
try {
await runEndpointPass();
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneResults();
}
} catch (err) {
console.error("[endpoints] the pass failed:", err);
}
};
setTimeout(() => void tick(), 20_000).unref();
timer = setInterval(() => void tick(), 60_000);
timer.unref();
}
+2
View File
@@ -25,6 +25,8 @@ export async function listUpcomingWindows(now: Date = new Date()): Promise<Activ
export function subjectOfConditionKey(key: string): string | null {
let m = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)(?::|$)/.exec(key);
if (m) return `server:${m[1]}`;
m = /^endpoint(?:-cert)?:\d+:server:(\d+)$/.exec(key);
if (m) return `server:${m[1]}`;
m = /^disk:proxmox:(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
+19 -1
View File
@@ -4,7 +4,8 @@
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { db } from "../db/client.js";
import { assets, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { latestTests } from "./restoreTests.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
@@ -53,6 +54,22 @@ class Exposition {
}
}
/** Endpoint checks and restore tests, for /metrics. */
async function addEndpointMetrics(out: Exposition, now: Date) {
for (const c of await db.select().from(endpointChecks)) {
if (!c.enabled || c.lastOk === null) continue;
const labels = { name: c.name, kind: c.kind };
out.add("homelab_endpoint_up", "gauge", "1 if the endpoint check's last answer was fine, else 0.", c.lastOk ? 1 : 0, labels);
if (c.lastResponseMs !== null) out.add("homelab_endpoint_response_milliseconds", "gauge", "How long the endpoint took to answer the last time.", c.lastResponseMs, labels);
if (c.certExpiresAt) out.add("homelab_endpoint_certificate_days_left", "gauge", "Days until the certificate the endpoint shows expires (negative once it has).", Math.floor((Date.parse(c.certExpiresAt) - now.getTime()) / 86_400_000), labels);
}
for (const r of (await latestTests()).values()) {
if (r.result === "not_needed") continue;
out.add("homelab_restore_test_age_days", "gauge", "Days since a restore of this was last tested.", Math.max(0, Math.floor((now.getTime() - Date.parse(`${r.testedAt}T12:00:00Z`)) / 86_400_000)), { subject: r.label });
out.add("homelab_restore_test_passed", "gauge", "1 if the last restore test passed (fully or partly), 0 if it failed.", r.result === "failed" ? 0 : 1, { subject: r.label });
}
}
export async function buildMetrics(now: Date = new Date()): Promise<string> {
const started = Date.now();
const out = new Exposition();
@@ -177,5 +194,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
await addEndpointMetrics(out, now);
return out.toString();
}
+3 -1
View File
@@ -109,6 +109,8 @@ export function metaOfConditionKey(key: string, severity?: string): NotifyMeta {
else if (key.startsWith("failed-units:")) category = "services";
else if (/^(disk|smart|zfs|synology-volume|synology-disk):/.test(key)) category = "disk";
else if (key.startsWith("automation:")) category = "automation";
const server = /^(?:offline|disk|failed-units|smart|zfs):server:(\d+)/.exec(key);
else if (key.startsWith("endpoint-cert:")) category = "expiry";
else if (key.startsWith("endpoint:")) category = "monitoring";
const server = /^(?:(?:offline|disk|failed-units|smart|zfs):server:|endpoint(?:-cert)?:\d+:server:)(\d+)/.exec(key);
return { category, severity: severity === "critical" ? "critical" : "warning", serverId: server ? Number(server[1]) : null };
}
+94
View File
@@ -0,0 +1,94 @@
/**
* Restore tests: a backup nobody has ever restored is a hope, not a backup. Somebody records when they tried to restore a guest (or anything else
* — a share, a database) and how it went, and the app keeps the history, shows each guest's newest test beside its backups, and speaks up
* when a test failed (always) or when none has been done in too long (if a time limit is set under Settings → Notifications).
*/
import { desc } from "drizzle-orm";
import { db } from "../db/client.js";
import { restoreTests } from "../db/schema.js";
import type { CoverageGuest, CoverageReport } from "./backupCoverage.js";
export const RESULTS = ["ok", "partial", "failed", "not_needed"] as const;
export type RestoreResult = (typeof RESULTS)[number];
export type TestRow = typeof restoreTests.$inferSelect;
/** What a subject's newest test amounts to: how it went, or that none was done (never) or that it is too old (stale). */
export type RestoreStatus = RestoreResult | "never" | "stale";
export interface RestoreInfo {
status: RestoreStatus;
/** YYYY-MM-DD of the newest test; null when there is none. */
testedAt: string | null;
ageDays: number | null;
note: string | null;
testedBy: string | null;
}
export const guestKey = (integrationId: number, vmid: number) => `guest:${integrationId}:${vmid}`;
export const KEY_PATTERN = /^(guest:\d+:\d+|custom:[a-z0-9][a-z0-9-]{0,60})$/;
/** "Photos share" → "custom:photos-share". */
export function customKey(label: string): string | null {
const slug = label
.toLowerCase()
.normalize("NFKD")
.replace(/[̀-ͯ]/g, "")
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 60);
return slug ? `custom:${slug}` : null;
}
const ageInDays = (testedAt: string, now: Date) => Math.floor((now.getTime() - Date.parse(`${testedAt}T12:00:00Z`)) / 86_400_000);
/** Pure: what the newest test (if any) says, against the time limit (0 = no limit). */
export function restoreInfo(latest: Pick<TestRow, "result" | "testedAt" | "note" | "testedBy"> | undefined, now: Date, limitDays: number): RestoreInfo {
if (!latest) return { status: "never", testedAt: null, ageDays: null, note: null, testedBy: null };
const ageDays = ageInDays(latest.testedAt, now);
let status: RestoreStatus = latest.result as RestoreResult;
if ((status === "ok" || status === "partial") && limitDays > 0 && ageDays > limitDays) status = "stale";
return { status, testedAt: latest.testedAt, ageDays, note: latest.note, testedBy: latest.testedBy };
}
/** The newest test of each subject (the one with the latest date, and the later one if two share a date). */
export async function latestTests(): Promise<Map<string, TestRow>> {
const rows = await db.select().from(restoreTests).orderBy(desc(restoreTests.testedAt), desc(restoreTests.id));
const out = new Map<string, TestRow>();
for (const r of rows) if (!out.has(r.subjectKey)) out.set(r.subjectKey, r);
return out;
}
export interface RestoreAlert {
key: string;
severity: "critical" | "warning";
message: string;
guest: CoverageGuest | null;
}
const daysWords = (d: number) => `${d} day${d === 1 ? "" : "s"}`;
/**
* What to speak up about. A test that failed is always a problem (the backups may not be usable) until a newer one passes. With a limit set,
* a guest that has backups but whose newest passing test is older than the limit — or that has never had one — is too, and so is anything
* else somebody chose to track. A guest marked "no need to test" never is.
*/
export function restoreAlerts(report: CoverageReport, latest: Map<string, TestRow>, limitDays: number, now: Date): RestoreAlert[] {
const out: RestoreAlert[] = [];
const seen = new Set<string>();
for (const g of report.guests) {
const key = guestKey(g.integrationId, g.vmid);
seen.add(key);
const info = restoreInfo(latest.get(key), now, limitDays);
const who = `${g.name} (#${g.vmid}) [${g.integrationName}]`;
if (info.status === "failed") out.push({ key: `restore-test:${key}`, severity: "critical", guest: g, message: `${who}: the last restore test (${info.testedAt}) failed${info.note ? ` — ${info.note}` : ""}` });
else if (limitDays > 0 && g.lastBackupAt !== null && info.status === "never") out.push({ key: `restore-test:${key}`, severity: "warning", guest: g, message: `${who}: a restore has never been tested` });
else if (limitDays > 0 && info.status === "stale") out.push({ key: `restore-test:${key}`, severity: "warning", guest: g, message: `${who}: its last restore test was ${daysWords(info.ageDays ?? 0)} ago (the limit is ${daysWords(limitDays)})` });
}
for (const [key, row] of latest) {
if (seen.has(key) || key.startsWith("guest:")) continue;
const info = restoreInfo(row, now, limitDays);
if (info.status === "failed") out.push({ key: `restore-test:${key}`, severity: "critical", guest: null, message: `${row.label}: the last restore test (${row.testedAt}) failed${row.note ? ` — ${row.note}` : ""}` });
else if (info.status === "stale") out.push({ key: `restore-test:${key}`, severity: "warning", guest: null, message: `${row.label}: its last restore test was ${daysWords(info.ageDays ?? 0)} ago (the limit is ${daysWords(limitDays)})` });
}
return out;
}
+3 -1
View File
@@ -111,6 +111,8 @@ export interface HealthCheckSettings {
diskForecastDays: number;
/** A Proxmox guest whose newest backup is older than this many hours is raised as stale. 0 turns it off. */
backupStaleHours: number;
/** A restore test older than this many days (or none at all) is a problem for a guest that has backups; 0 = never (a failed test always is). */
restoreTestDays: number;
}
/** The weekly digest: one message a week through the notification channels. Off until an admin switches it on under Settings → Notifications. */
@@ -257,7 +259,7 @@ const DEFAULTS: AppSettings = {
agents: { autoUpdate: false },
selfBackup: { enabled: true, time: "03:30", keep: 14 },
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48 },
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0 },
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
consistency: { excludedRanges: ["172.16.0.0/12"] },