Endpoint checks, restore tests, and a change log

- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags
- Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page
- Change log: one filterable timeline made from the audit log and the incident record

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-18 23:03:56 +02:00
1 parent c3bc6bbf72
commit 92726568b7
37 files changed
+6918 -21

No files matched your search

+35 -1
View File
@@ -11,7 +11,7 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, secrets, servers } from "../db/schema.js";
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
@@ -34,6 +34,8 @@ import { getFailingSources } from "./integrationHealthMonitor.js";
import { getSettings } from "./settingsStore.js";
import { sourceLabel } from "./notify.js";
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
import { endpointConditions } from "./endpointChecks.js";
import { latestTests, restoreAlerts } from "./restoreTests.js";
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
export interface AlertsReport {
@@ -62,6 +64,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
osticket: { label: "osTicket", link: "/osticket" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
};
const SOURCE_TIMEOUT_MS = 20_000;
@@ -268,6 +271,37 @@ export async function collectAlerts(): Promise<AlertsReport> {
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
});
}
// A restore that failed, or hasn't been tried in too long (the limit is in Settings; a failure is always raised).
const { healthChecks } = await getSettings();
for (const r of restoreAlerts(report, await latestTests(), healthChecks.restoreTestDays, now)) {
const g = r.guest;
push({
category: "backup",
severity: r.severity,
type: g ? "proxmox" : "homelab",
message: r.message,
key: r.key,
link: "/backups",
silenced: g ? subjects.has(`integration:${g.integrationId}`) || (g.server !== null && subjects.has(`server:${g.server.id}`)) : false,
target: g ? (g.server ? targetOf("server", g.server.id) : integrationTarget(g.integrationId)) : null,
});
}
}),
// What the endpoint checks have found: an address, port or certificate that has stopped being fine.
check("Endpoint checks", async () => {
for (const c of endpointConditions(await db.select().from(endpointChecks), new Date())) {
push({
category: c.category,
severity: c.severity,
type: "endpoint",
message: c.message,
key: c.key,
link: "/endpoints",
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
});
}
}),
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly