Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3bc6bbf72
commit
92726568b7
37 files changed
+6918
-21
No files matched your search
@@ -4,7 +4,8 @@
|
||||
* integration on its own, so a scrape every fifteen seconds costs a few database reads (the alerts are the app's own minute-old result).
|
||||
*/
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { assets, endpointChecks, integrations, maintenanceWindows, secrets, servers } from "../db/schema.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { env } from "../env.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import { isAgentOutdated, latestAgentVersions } from "./agentVersions.js";
|
||||
@@ -53,6 +54,22 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Endpoint checks and restore tests, for /metrics. */
|
||||
async function addEndpointMetrics(out: Exposition, now: Date) {
|
||||
for (const c of await db.select().from(endpointChecks)) {
|
||||
if (!c.enabled || c.lastOk === null) continue;
|
||||
const labels = { name: c.name, kind: c.kind };
|
||||
out.add("homelab_endpoint_up", "gauge", "1 if the endpoint check's last answer was fine, else 0.", c.lastOk ? 1 : 0, labels);
|
||||
if (c.lastResponseMs !== null) out.add("homelab_endpoint_response_milliseconds", "gauge", "How long the endpoint took to answer the last time.", c.lastResponseMs, labels);
|
||||
if (c.certExpiresAt) out.add("homelab_endpoint_certificate_days_left", "gauge", "Days until the certificate the endpoint shows expires (negative once it has).", Math.floor((Date.parse(c.certExpiresAt) - now.getTime()) / 86_400_000), labels);
|
||||
}
|
||||
for (const r of (await latestTests()).values()) {
|
||||
if (r.result === "not_needed") continue;
|
||||
out.add("homelab_restore_test_age_days", "gauge", "Days since a restore of this was last tested.", Math.max(0, Math.floor((now.getTime() - Date.parse(`${r.testedAt}T12:00:00Z`)) / 86_400_000)), { subject: r.label });
|
||||
out.add("homelab_restore_test_passed", "gauge", "1 if the last restore test passed (fully or partly), 0 if it failed.", r.result === "failed" ? 0 : 1, { subject: r.label });
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
const started = Date.now();
|
||||
const out = new Exposition();
|
||||
@@ -177,5 +194,6 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
|
||||
out.add("homelab_scrape_duration_seconds", "gauge", "How long building this answer took.", (Date.now() - started) / 1000);
|
||||
out.add("homelab_scrape_timestamp_seconds", "gauge", "When this answer was built, as a Unix time.", nowSec);
|
||||
await addEndpointMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user