Heartbeats, Coming up and suggested checks; Operations menu tidied

Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app
alerts when the call is late or the job says it failed. Coming up: one agenda of everything
dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests,
waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested
checks: web ports and certificates the app already knows about, added with a click.

Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group
(Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under
Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow
the real clock.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-19 21:42:34 +02:00
1 parent c05c847d17
commit 4e0c6927b2
44 files changed
+7092 -75

No files matched your search

+19 -1
View File
@@ -11,7 +11,7 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
@@ -35,6 +35,7 @@ import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import { endpointConditions } from "./endpointChecks.js";
import { heartbeatConditions } from "./heartbeats.js";
import { latestTests, restoreAlerts } from "./restoreTests.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
@@ -65,6 +66,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
};
const SOURCE_TIMEOUT_MS = 20_000;
@@ -304,6 +306,22 @@ export async function collectAlerts(): Promise<AlertsReport> {
}
}),
// Jobs that report in: one that has not done so in time, or said it failed.
check("Heartbeats", async () => {
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
push({
category: c.category,
severity: c.severity,
type: "heartbeat",
message: c.message,
key: c.key,
link: "/heartbeats",
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
});
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
check("Homelab Manager's own backups", async () => {