Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -5,9 +5,10 @@
|
||||
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats } from "../db/schema.js";
|
||||
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { heartbeatConditions, prunePings } from "./heartbeats.js";
|
||||
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
|
||||
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
@@ -28,7 +29,9 @@ async function loadState(): Promise<ActiveState> {
|
||||
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
|
||||
const asked = await runDueChecks(now, probeFn);
|
||||
const rows = await db.select().from(endpointChecks);
|
||||
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
// The jobs that report in are judged in the same pass, by the same bookkeeping: a call that is overdue is announced once, and again as cleared when it comes.
|
||||
const beats = await db.select().from(heartbeats);
|
||||
const current: HealthCondition[] = [...endpointConditions(rows, now), ...heartbeatConditions(beats, now)].map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
||||
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
if (added.length > 0) await notifyHealthIssues(added);
|
||||
@@ -48,6 +51,7 @@ export function initEndpointMonitor(): void {
|
||||
if (Date.now() - lastPrune > 24 * 3600_000) {
|
||||
lastPrune = Date.now();
|
||||
await pruneResults();
|
||||
await prunePings();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[endpoints] the pass failed:", err);
|
||||
|
||||
Reference in new issue
Block a user