Files
Homelab-manager/server/src/services/endpointMonitor.ts
T
bobbanandClaude Sonnet 5.5 4e0c6927b2 Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-19 21:42:34 +02:00

64 lines
3.1 KiB
TypeScript

/**
* The checks' own rhythm: every minute, ask the ones that are due, then see what has started or stopped being a problem and say so — a
* notification when a check has failed enough times in a row, another when it works again, the same for a certificate running out. It
* uses the same bookkeeping as the server and automation checks (each problem announced once, remembered across a restart), so
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
*/
import { db } from "../db/client.js";
import { endpointChecks, heartbeats } from "../db/schema.js";
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
import { activeSubjects } from "./maintenance.js";
import { heartbeatConditions, prunePings } from "./heartbeats.js";
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
const STATE_FLAG = "endpointActiveConditions";
async function loadState(): Promise<ActiveState> {
const raw = await getInternalFlag(STATE_FLAG);
if (!raw) return {};
try {
return JSON.parse(raw);
} catch {
return {};
}
}
/** One pass: ask what is due, then announce what changed. Returns what it did. */
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
const asked = await runDueChecks(now, probeFn);
const rows = await db.select().from(endpointChecks);
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
const beats = await db.select().from(heartbeats);
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
if (added.length > 0) await notifyHealthIssues(added);
if (resolved.length > 0) await notifyHealthRecovered(resolved);
return { asked, announced: added.length, recovered: resolved.length };
}
let timer: ReturnType<typeof setInterval> | null = null;
/** Looks every minute (each check has its own interval, so most passes ask nobody). Once a day it clears out old results. */
export function initEndpointMonitor(): void {
if (timer) return;
let lastPrune = 0;
const tick = async () => {
try {
await runEndpointPass();
if (Date.now() - lastPrune > 24 * 3600_000) {
lastPrune = Date.now();
await pruneResults();
await prunePings();
}
} catch (err) {
console.error("[endpoints] the pass failed:", err);
}
};
setTimeout(() => void tick(), 20_000).unref();
timer = setInterval(() => void tick(), 60_000);
timer.unref();
}