UniFi integration: network devices, Wi-Fi and networks, as part of the dependency map
A UniFi controller (API key on Network 9+, or a read-only local login on any version) is read every five minutes. Its switches, access points and gateways are kept with state, IP, uptime, firmware and uplink; its networks and Wi-Fi names beside them (passphrases are never read). Nothing is written to the controller. Network > Network devices lists them. Devices are a new node type in the dependency map: uplinks become dependencies by themselves, others can be written by hand. A device offline past a limit is an offline alert, folds what depends on it, can be silenced with maintenance and have runbooks. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
73a06ebe63
commit
396e074108
48 files changed
+6893
-102
No files matched your search
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -64,6 +64,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
pbs: { label: "Proxmox Backup", link: "/pbs" },
|
||||
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
unifi: { label: "UniFi", link: "/network/devices" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -80,6 +81,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
|
||||
|
||||
/** Which kind of source, and which one, a health/automation condition key belongs to. */
|
||||
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
||||
const device = /^offline:device:(\d+)$/.exec(key);
|
||||
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
@@ -110,16 +113,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
|
||||
const intById = new Map(intRows.map((r) => [r.id, r]));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name]));
|
||||
const deviceNames = new Map((await db.select({ id: networkDevices.id, name: networkDevices.name }).from(networkDevices)).map((r) => [r.id, r.name]));
|
||||
|
||||
/** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */
|
||||
const targetOf = (type: "server" | "integration" | "dns_provider", id: number): Alert["target"] => ({
|
||||
const targetOf = (type: "server" | "integration" | "dns_provider" | "device", id: number): Alert["target"] => ({
|
||||
type,
|
||||
id,
|
||||
name: (type === "server" ? serverNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
|
||||
name: (type === "server" ? serverNames.get(id) : type === "device" ? deviceNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
|
||||
});
|
||||
const targetOfSubject = (subject: string | null): Alert["target"] => {
|
||||
const m = /^(server|integration|dns_provider):(\d+)$/.exec(subject ?? "");
|
||||
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider", Number(m[2])) : null;
|
||||
const m = /^(server|integration|dns_provider|device):(\d+)$/.exec(subject ?? "");
|
||||
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider" | "device", Number(m[2])) : null;
|
||||
};
|
||||
const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id);
|
||||
/** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */
|
||||
@@ -167,6 +171,9 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
|
||||
let offlineKnown!: (ids: Set<number>) => void;
|
||||
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
|
||||
// The same for network devices, which can explain a service that has stopped answering.
|
||||
let devicesKnown!: (ids: Set<number>) => void;
|
||||
const offlineDeviceIds = new Promise<Set<number>>((resolve) => (devicesKnown = resolve));
|
||||
|
||||
await Promise.all([
|
||||
check("Server and storage health", async () => {
|
||||
@@ -182,38 +189,44 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 });
|
||||
// A server that is down because the one it runs on is down is folded under that one.
|
||||
const offlineIds = new Set<number>();
|
||||
const offlineDevices = new Set<number>();
|
||||
for (const c of conditions) {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
if (m) (m[1] === "server" ? offlineIds : offlineDevices).add(Number(m[2]));
|
||||
}
|
||||
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
|
||||
offlineKnown(offlineIds);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
devicesKnown(offlineDevices);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source), offlineDevices);
|
||||
for (const c of conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
|
||||
const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined;
|
||||
const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
|
||||
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : origin.type === "unifi" ? "/network/devices" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}).finally(() => offlineKnown(new Set())),
|
||||
}).finally(() => {
|
||||
offlineKnown(new Set());
|
||||
devicesKnown(new Set());
|
||||
}),
|
||||
|
||||
check("Automation runs", async () => {
|
||||
const { items, held } = await collectAutomation();
|
||||
@@ -506,7 +519,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const { notifications } = await getSettings();
|
||||
const failing = getFailingSources();
|
||||
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source), await offlineDeviceIds).catch(() => null) : null;
|
||||
for (const f of failing) {
|
||||
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
||||
const affects = cover ? takenByType(f.source, cover) : [];
|
||||
|
||||
Reference in new issue
Block a user