Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* Suggested endpoint checks: things the app already knows enough about to offer a check for, so setting up monitoring is a click instead of typing.
|
||||
*
|
||||
* - A web port a server's agent sees open (80, 443, 8080, 8443), with a name that points at it in DNS if there is one — "git.example.lan answers on
|
||||
* 443 on nas" — as a web check, linked to that server so its maintenance silences the alerts.
|
||||
* - A secret that already has a host to read the certificate from, as a certificate check: the secret only warns about the expiry, this also hears
|
||||
* when the service stops answering.
|
||||
*
|
||||
* It only reads what is stored (no network), leaves out what a check already covers, and remembers what somebody said no to. Uptime Kuma's monitors are
|
||||
* not offered: they are already being watched, and the adapter only keeps a host and port for them, not the address.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsRecordsCache, endpointChecks, secrets, servers } from "../db/schema.js";
|
||||
import { isLoopback } from "./agentPorts.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export interface Suggestion {
|
||||
/** Stable identity, so a dismissed suggestion stays dismissed and "add" can name it. */
|
||||
key: string;
|
||||
kind: "http" | "tls";
|
||||
name: string;
|
||||
target: string;
|
||||
port: number | null;
|
||||
serverId: number | null;
|
||||
serverName: string | null;
|
||||
/** Why it is offered, in a sentence. */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface SuggestionInput {
|
||||
servers: { id: number; name: string; hostname: string | null; ips: string[]; ports: { protocol?: string; port: number; address?: string }[] }[];
|
||||
dns: { name: string; type: string; content: string }[];
|
||||
secrets: { name: string; checkHost: string | null; checkPort: number | null }[];
|
||||
existing: { kind: string; target: string; port: number | null }[];
|
||||
dismissed: Set<string>;
|
||||
}
|
||||
|
||||
/** The web ports worth offering, and what they speak. */
|
||||
const WEB_PORTS: Record<number, "http" | "https"> = { 80: "http", 443: "https", 8080: "http", 8443: "https" };
|
||||
/** The https twin of a plain port, so a server answering on both gets the one that reads the certificate. */
|
||||
const SECURE_TWIN: Record<number, number> = { 80: 443, 8080: 8443 };
|
||||
const MAX_NAMES_PER_PORT = 3;
|
||||
const MAX_SUGGESTIONS = 200;
|
||||
|
||||
const cleanHost = (h: string) => h.trim().toLowerCase().replace(/\.$/, "");
|
||||
const isIPv4 = (s: string) => /^\d{1,3}(\.\d{1,3}){3}$/.test(s);
|
||||
const usableIPv4 = (s: string) => isIPv4(s) && !s.startsWith("127.") && !s.startsWith("169.254.");
|
||||
|
||||
/** Where a check's address is, in the same words for an existing check and a suggestion: "web:host:port" and "tls:host:port" (a web check on https reads the certificate too). */
|
||||
function coverage(existing: SuggestionInput["existing"]): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const c of existing) {
|
||||
if (c.kind === "http") {
|
||||
try {
|
||||
const u = new URL(c.target);
|
||||
const port = u.port ? Number(u.port) : u.protocol === "https:" ? 443 : 80;
|
||||
const host = cleanHost(u.hostname);
|
||||
out.add(`web:${host}:${port}`);
|
||||
if (u.protocol === "https:") out.add(`tls:${host}:${port}`);
|
||||
} catch {
|
||||
// an address that isn't one covers nothing
|
||||
}
|
||||
} else if (c.kind === "tls" && c.port !== null) out.add(`tls:${cleanHost(c.target)}:${c.port}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the checks worth offering, from what is known. */
|
||||
export function suggestChecks(input: SuggestionInput): { suggestions: Suggestion[]; dismissed: Suggestion[] } {
|
||||
const covered = coverage(input.existing);
|
||||
const all = new Map<string, Suggestion>();
|
||||
|
||||
for (const s of input.servers) {
|
||||
const ips = new Set(s.ips.map((i) => i.trim().toLowerCase()));
|
||||
const hostname = s.hostname ? cleanHost(s.hostname) : null;
|
||||
// Names that point at this server in DNS: an address record with one of its addresses, and an alias to one of those names or to its hostname.
|
||||
const direct = new Set<string>();
|
||||
for (const r of input.dns) {
|
||||
if ((r.type === "A" || r.type === "AAAA") && ips.has(r.content.trim().toLowerCase())) direct.add(cleanHost(r.name));
|
||||
}
|
||||
const names = new Set(direct);
|
||||
for (const r of input.dns) {
|
||||
if (r.type !== "CNAME") continue;
|
||||
const to = cleanHost(r.content);
|
||||
if (direct.has(to) || (hostname !== null && to === hostname)) names.add(cleanHost(r.name));
|
||||
}
|
||||
const dnsNames = [...names].filter((n) => n.includes(".") && !n.startsWith("*") && !n.endsWith(".arpa") && !/\s/.test(n)).sort((a, b) => a.length - b.length || a.localeCompare(b));
|
||||
|
||||
const listening = new Set(s.ports.filter((p) => (p.protocol ?? "tcp") === "tcp" && !(p.address && isLoopback(p.address))).map((p) => p.port));
|
||||
for (const port of Object.keys(WEB_PORTS).map(Number).filter((p) => listening.has(p))) {
|
||||
// Plain and secure on the same server: the secure one is the real service (the plain one is most likely its redirect), and reads the certificate.
|
||||
if (SECURE_TWIN[port] !== undefined && listening.has(SECURE_TWIN[port])) continue;
|
||||
const scheme = WEB_PORTS[port];
|
||||
const fallback = hostname && hostname.includes(".") ? [hostname] : [s.ips.map((i) => i.trim()).find(usableIPv4)].filter((x): x is string => !!x);
|
||||
const hosts = dnsNames.length > 0 ? dnsNames.slice(0, MAX_NAMES_PER_PORT) : fallback;
|
||||
for (const host of hosts) {
|
||||
const defaultPort = scheme === "https" ? 443 : 80;
|
||||
const target = `${scheme}://${host}${port === defaultPort ? "" : `:${port}`}/`;
|
||||
if (covered.has(`web:${host}:${port}`)) continue;
|
||||
const viaDns = dnsNames.includes(host);
|
||||
all.set(`http|${target}`, {
|
||||
key: `http|${target}`,
|
||||
kind: "http",
|
||||
name: `${host}${port === defaultPort ? "" : `:${port}`}`,
|
||||
target,
|
||||
port: null,
|
||||
serverId: s.id,
|
||||
serverName: s.name,
|
||||
reason: `${s.name} is listening on port ${port}${viaDns ? `, and ${host} points at it in DNS` : ""}.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const sec of input.secrets) {
|
||||
if (!sec.checkHost || sec.checkPort === null) continue;
|
||||
const host = cleanHost(sec.checkHost);
|
||||
if (covered.has(`tls:${host}:${sec.checkPort}`)) continue;
|
||||
all.set(`tls|${host}|${sec.checkPort}`, {
|
||||
key: `tls|${host}|${sec.checkPort}`,
|
||||
kind: "tls",
|
||||
name: `Certificate: ${sec.name}`,
|
||||
target: host,
|
||||
port: sec.checkPort,
|
||||
serverId: null,
|
||||
serverName: null,
|
||||
reason: `The secret “${sec.name}” already reads this certificate; a check also tells you when ${host} stops answering.`,
|
||||
});
|
||||
}
|
||||
|
||||
// By server, those about no server last, then by address.
|
||||
const sorted = [...all.values()].sort((a, b) => Number(a.serverName === null) - Number(b.serverName === null) || (a.serverName ?? "").localeCompare(b.serverName ?? "") || a.target.localeCompare(b.target));
|
||||
return {
|
||||
suggestions: sorted.filter((x) => !input.dismissed.has(x.key)).slice(0, MAX_SUGGESTIONS),
|
||||
dismissed: sorted.filter((x) => input.dismissed.has(x.key)),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── What was said no to ───────────────────────────────────────────────────
|
||||
|
||||
const DISMISSED_FLAG = "endpointSuggestionsDismissed";
|
||||
const MAX_DISMISSED = 500;
|
||||
|
||||
export async function loadDismissed(): Promise<Set<string>> {
|
||||
const raw = await getInternalFlag(DISMISSED_FLAG);
|
||||
if (!raw) return new Set();
|
||||
try {
|
||||
const v = JSON.parse(raw);
|
||||
return new Set(Array.isArray(v) ? v.filter((k): k is string => typeof k === "string") : []);
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveDismissed(keys: Set<string>): Promise<void> {
|
||||
await setInternalFlag(DISMISSED_FLAG, JSON.stringify([...keys].slice(-MAX_DISMISSED)));
|
||||
}
|
||||
|
||||
// ─── Reading what is stored ────────────────────────────────────────────────
|
||||
|
||||
function jsonArray<T>(stored: string | null): T[] {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? (v as T[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadSuggestions(): Promise<{ suggestions: Suggestion[]; dismissed: Suggestion[] }> {
|
||||
const [serverRows, dnsRows, secretRows, existing, dismissed] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name, hostname: servers.hostname, ipAddresses: servers.ipAddresses, listeningPorts: servers.listeningPorts }).from(servers),
|
||||
db.select({ name: dnsRecordsCache.name, type: dnsRecordsCache.type, content: dnsRecordsCache.content }).from(dnsRecordsCache),
|
||||
db.select({ name: secrets.name, checkHost: secrets.checkHost, checkPort: secrets.checkPort }).from(secrets),
|
||||
db.select({ kind: endpointChecks.kind, target: endpointChecks.target, port: endpointChecks.port }).from(endpointChecks),
|
||||
loadDismissed(),
|
||||
]);
|
||||
return suggestChecks({
|
||||
servers: serverRows.map((s) => ({ id: s.id, name: s.name, hostname: s.hostname, ips: jsonArray<string>(s.ipAddresses).filter((i) => typeof i === "string"), ports: jsonArray<{ protocol?: string; port: number; address?: string }>(s.listeningPorts).filter((p) => p && Number.isInteger(p.port)) })),
|
||||
dns: dnsRows,
|
||||
secrets: secretRows,
|
||||
existing,
|
||||
dismissed,
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user