Endpoint checks, restore tests, and a change log
- Endpoints: HTTP, TCP and TLS checks with answer time, uptime and certificate expiry; failures and expiring certificates become alerts and notifications; linked to a server for maintenance and tags - Restore tests: record when a backup was restored and how it went; failed tests always alert, overdue ones with a limit set; shown on the Backup coverage page - Change log: one filterable timeline made from the audit log and the incident record Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3bc6bbf72
commit
92726568b7
37 files changed
+6918
-21
No files matched your search
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -34,6 +34,8 @@ import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { sourceLabel } from "./notify.js";
|
||||
import { alertIdOfNode, dependencyCover, integrationTypeCause, takenByType } from "./dependencies.js";
|
||||
import { endpointConditions } from "./endpointChecks.js";
|
||||
import { latestTests, restoreAlerts } from "./restoreTests.js";
|
||||
import type { Alert, AlertCategory, AlertSeverity, SourceFailure } from "./alertTypes.js";
|
||||
|
||||
export interface AlertsReport {
|
||||
@@ -62,6 +64,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
};
|
||||
|
||||
const SOURCE_TIMEOUT_MS = 20_000;
|
||||
@@ -268,6 +271,37 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
target: s.guest.server ? targetOf("server", s.guest.server.id) : integrationTarget(s.guest.integrationId),
|
||||
});
|
||||
}
|
||||
// A restore that failed, or hasn't been tried in too long (the limit is in Settings; a failure is always raised).
|
||||
const { healthChecks } = await getSettings();
|
||||
for (const r of restoreAlerts(report, await latestTests(), healthChecks.restoreTestDays, now)) {
|
||||
const g = r.guest;
|
||||
push({
|
||||
category: "backup",
|
||||
severity: r.severity,
|
||||
type: g ? "proxmox" : "homelab",
|
||||
message: r.message,
|
||||
key: r.key,
|
||||
link: "/backups",
|
||||
silenced: g ? subjects.has(`integration:${g.integrationId}`) || (g.server !== null && subjects.has(`server:${g.server.id}`)) : false,
|
||||
target: g ? (g.server ? targetOf("server", g.server.id) : integrationTarget(g.integrationId)) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// What the endpoint checks have found: an address, port or certificate that has stopped being fine.
|
||||
check("Endpoint checks", async () => {
|
||||
for (const c of endpointConditions(await db.select().from(endpointChecks), new Date())) {
|
||||
push({
|
||||
category: c.category,
|
||||
severity: c.severity,
|
||||
type: "endpoint",
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: "/endpoints",
|
||||
silenced: c.serverId !== null && subjects.has(`server:${c.serverId}`),
|
||||
target: c.serverId !== null ? targetOf("server", c.serverId) : null,
|
||||
});
|
||||
}
|
||||
}),
|
||||
|
||||
// Homelab Manager's own backups: if they are switched on, there should be a recent one. This is what turns a backup that has quietly
|
||||
|
||||
Reference in new issue
Block a user