Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
64 lines
3.1 KiB
TypeScript
64 lines
3.1 KiB
TypeScript
/**
|
|
* The checks' own rhythm: every minute, ask the ones that are due, then see what has started or stopped being a problem and say so — a
|
|
* notification when a check has failed enough times in a row, another when it works again, the same for a certificate running out. It
|
|
* uses the same bookkeeping as the server and automation checks (each problem announced once, remembered across a restart), so
|
|
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
|
|
*/
|
|
import { db } from "../db/client.js";
|
|
import { endpointChecks, heartbeats } from "../db/schema.js";
|
|
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
|
|
import { activeSubjects } from "./maintenance.js";
|
|
import { heartbeatConditions, prunePings } from "./heartbeats.js";
|
|
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
|
|
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
|
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
|
|
|
const STATE_FLAG = "endpointActiveConditions";
|
|
|
|
async function loadState(): Promise<ActiveState> {
|
|
const raw = await getInternalFlag(STATE_FLAG);
|
|
if (!raw) return {};
|
|
try {
|
|
return JSON.parse(raw);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
/** One pass: ask what is due, then announce what changed. Returns what it did. */
|
|
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
|
|
const asked = await runDueChecks(now, probeFn);
|
|
const rows = await db.select().from(endpointChecks);
|
|
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
|
|
const beats = await db.select().from(heartbeats);
|
|
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
|
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
|
|
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
|
if (added.length > 0) await notifyHealthIssues(added);
|
|
if (resolved.length > 0) await notifyHealthRecovered(resolved);
|
|
return { asked, announced: added.length, recovered: resolved.length };
|
|
}
|
|
|
|
let timer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
/** Looks every minute (each check has its own interval, so most passes ask nobody). Once a day it clears out old results. */
|
|
export function initEndpointMonitor(): void {
|
|
if (timer) return;
|
|
let lastPrune = 0;
|
|
const tick = async () => {
|
|
try {
|
|
await runEndpointPass();
|
|
if (Date.now() - lastPrune > 24 * 3600_000) {
|
|
lastPrune = Date.now();
|
|
await pruneResults();
|
|
await prunePings();
|
|
}
|
|
} catch (err) {
|
|
console.error("[endpoints] the pass failed:", err);
|
|
}
|
|
};
|
|
setTimeout(() => void tick(), 20_000).unref();
|
|
timer = setInterval(() => void tick(), 60_000);
|
|
timer.unref();
|
|
}
|