- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
60 lines
2.8 KiB
TypeScript
60 lines
2.8 KiB
TypeScript
/**
|
|
* The checks' own rhythm: every minute, ask the ones that are due, then see what has started or stopped being a problem and say so — a
|
|
* notification when a check has failed enough times in a row, another when it works again, the same for a certificate running out. It
|
|
* uses the same bookkeeping as the server and automation checks (each problem announced once, remembered across a restart), so
|
|
* a maintenance window on the server a check is linked to holds its problems back, and they alert after it if they are still there.
|
|
*/
|
|
import { db } from "../db/client.js";
|
|
import { endpointChecks } from "../db/schema.js";
|
|
import { diffConditions, type ActiveState, type HealthCondition } from "./healthMonitor.js";
|
|
import { activeSubjects } from "./maintenance.js";
|
|
import { endpointConditions, pruneResults, runDueChecks, type CheckRow, type ProbeResult } from "./endpointChecks.js";
|
|
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
|
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
|
|
|
const STATE_FLAG = "endpointActiveConditions";
|
|
|
|
async function loadState(): Promise<ActiveState> {
|
|
const raw = await getInternalFlag(STATE_FLAG);
|
|
if (!raw) return {};
|
|
try {
|
|
return JSON.parse(raw);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
/** One pass: ask what is due, then announce what changed. Returns what it did. */
|
|
export async function runEndpointPass(now: Date = new Date(), probeFn?: (c: CheckRow) => Promise<ProbeResult>): Promise<{ asked: number; announced: number; recovered: number }> {
|
|
const asked = await runDueChecks(now, probeFn);
|
|
const rows = await db.select().from(endpointChecks);
|
|
const current: HealthCondition[] = endpointConditions(rows, now).map((c) => ({ key: c.key, source: c.source, message: c.message, severity: c.severity }));
|
|
const { added, resolved, next } = diffConditions(await loadState(), current, new Set(), await activeSubjects(now));
|
|
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
|
if (added.length > 0) await notifyHealthIssues(added);
|
|
if (resolved.length > 0) await notifyHealthRecovered(resolved);
|
|
return { asked, announced: added.length, recovered: resolved.length };
|
|
}
|
|
|
|
let timer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
/** Looks every minute (each check has its own interval, so most passes ask nobody). Once a day it clears out old results. */
|
|
export function initEndpointMonitor(): void {
|
|
if (timer) return;
|
|
let lastPrune = 0;
|
|
const tick = async () => {
|
|
try {
|
|
await runEndpointPass();
|
|
if (Date.now() - lastPrune > 24 * 3600_000) {
|
|
lastPrune = Date.now();
|
|
await pruneResults();
|
|
}
|
|
} catch (err) {
|
|
console.error("[endpoints] the pass failed:", err);
|
|
}
|
|
};
|
|
setTimeout(() => void tick(), 20_000).unref();
|
|
timer = setInterval(() => void tick(), 60_000);
|
|
timer.unref();
|
|
}
|