Heartbeats, Coming up and suggested checks; Operations menu tidied
Heartbeats: a cron job or script calls /api/heartbeat/<token> (and /start, /fail); the app alerts when the call is late or the job says it failed. Coming up: one agenda of everything dated (expiries, warranties, end of support, planned and recurring maintenance, restore tests, waiting updates), with a dashboard card and an iCalendar feed behind a share link. Suggested checks: web ports and certificates the app already knows about, added with a click. Menu: Incidents become Alerts -> History (old /incidents links redirect); new Monitoring group (Endpoints, Heartbeats, Uptime Kuma); osTicket under Automation, Admin Links under Infrastructure, Generator under Administration. Also makes the restore-test metrics test follow the real clock. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c05c847d17
commit
4e0c6927b2
44 files changed
+7092
-75
No files matched your search
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -35,6 +35,7 @@ import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { heartbeatConditions } from "./heartbeats.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
@@ -65,6 +66,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
heartbeat: { label: "Heartbeat", link: "/heartbeats" },
|
||||
};
|
||||
|
||||
const SOURCE_TIMEOUT_MS = 20_000;
|
||||
@@ -304,6 +306,22 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
}
|
||||
}),
|
||||
|
||||
// Jobs that report in: one that has not done so in time, or said it failed.
|
||||
check("Heartbeats", async () => {
|
||||
for (const c of heartbeatConditions(await db.select().from(heartbeats), new Date())) {
|
||||
push({
|
||||
category: c.category,
|
||||
severity: c.severity,
|
||||
type: "heartbeat",
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: "/heartbeats",
|
||||
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
||||
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
// stopped working — a full disk, a folder that went read-only — into something somebody hears about before they need it.
|
||||
check("Homelab Manager's own backups", async () => {
|
||||
|
||||
Reference in new issue
Block a user