Heartbeats, Coming up and suggested checks; Operations menu tidied

Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 21:42:34 +02:00
1 parent c05c847d17
commit 4e0c6927b2
44 files changed
+7092 -75

No files matched your search

+15 -1
View File
@@ -4,8 +4,9 @@
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
*/
import { db } from "../db/client.js";
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
import { env } from "../env.js";
import { getAlerts } from "./alerts.js";
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
@@ -70,6 +71,18 @@ async function addEndpointMetrics(out: Exposition, now: Date) {
}
}
/** Jobs that report in, for /metrics. */
async function addHeartbeatMetrics(out: Exposition, now: Date) {
for (const h of await db.select().from(heartbeats)) {
if (!h.enabled) continue;
const labels = { name: h.name };
const state = heartbeatState(h, now);
out.add("homelab_heartbeat_up", "gauge", "1 if the job has reported in time and not said it failed, else 0.", state === "late" || state === "failed" ? 0 : 1, labels);
if (h.lastPingAt) out.add("homelab_heartbeat_last_ping_timestamp_seconds", "gauge", "When the job last reported that it finished (Unix time).", Math.floor(parseStamp(h.lastPingAt) / 1000), labels);
if (h.lastDurationMs !== null) out.add("homelab_heartbeat_last_duration_seconds", "gauge", "How long the job said its last run took.", Math.round(h.lastDurationMs / 100) / 10, labels);
}
}
export async function buildMetrics(now: Date = new Date()): Promise<string> {
const started = Date.now();
const out = new Exposition();
@@ -195,5 +208,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
await addEndpointMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}