UniFi integration: network devices, Wi-Fi and networks, as part of the dependency map
A UniFi controller (API key on Network 9+, or a read-only local login on any version) is read every five minutes. Its switches, access points and gateways are kept with state, IP, uptime, firmware and uplink; its networks and Wi-Fi names beside them (passphrases are never read). Nothing is written to the controller. Network > Network devices lists them. Devices are a new node type in the dependency map: uplinks become dependencies by themselves, others can be written by hand. A device offline past a limit is an offline alert, folds what depends on it, can be silenced with maintenance and have runbooks. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
73a06ebe63
commit
396e074108
48 files changed
+6893
-102
No files matched your search
@@ -26,7 +26,7 @@ export interface Alert {
|
||||
/** Raised and cleared over and over: its raised and cleared messages are being held back (services/flapGuard.ts). */
|
||||
flapping?: boolean;
|
||||
/** What a maintenance window would have to be on to silence it, when there is such a thing — which is what "Silence" acts on. */
|
||||
target: { type: "server" | "integration" | "dns_provider"; id: number; name: string } | null;
|
||||
target: { type: "server" | "integration" | "dns_provider" | "device"; id: number; name: string } | null;
|
||||
}
|
||||
|
||||
/** One integration that couldn't be read while collecting — so the page never mistakes "couldn't check" for "all clear". */
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { endpointChecks, heartbeats, integrations, secrets, servers } from "../db/schema.js";
|
||||
import { endpointChecks, heartbeats, integrations, networkDevices, secrets, servers } from "../db/schema.js";
|
||||
import { disksFillingSoon, getServerMetrics } from "./serverMetrics.js";
|
||||
import { diskLimit, effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getBackupCoverage, staleBackups } from "./backupCoverage.js";
|
||||
@@ -64,6 +64,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
pbs: { label: "Proxmox Backup", link: "/pbs" },
|
||||
uptimekuma: { label: "Uptime Kuma", link: "/uptime-kuma" },
|
||||
osticket: { label: "osTicket", link: "/osticket" },
|
||||
unifi: { label: "UniFi", link: "/network/devices" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -80,6 +81,8 @@ const SEVERITY_ORDER: Record<AlertSeverity, number> = { critical: 0, warning: 1,
|
||||
|
||||
/** Which kind of source, and which one, a health/automation condition key belongs to. */
|
||||
function originOfConditionKey(key: string): { type: string; id: number | null; category: AlertCategory } {
|
||||
const device = /^offline:device:(\d+)$/.exec(key);
|
||||
if (device) return { type: "unifi", id: Number(device[1]), category: "offline" };
|
||||
let m = /^(?:offline|disk|failed-units|smart|zfs|new-port):server:(\d+)/.exec(key);
|
||||
if (m) return { type: "server", id: Number(m[1]), category: key.startsWith("offline") ? "offline" : key.startsWith("failed-units") ? "services" : key.startsWith("new-port") ? "monitoring" : "disk" };
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
@@ -110,16 +113,17 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const intRows = await db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations);
|
||||
const intById = new Map(intRows.map((r) => [r.id, r]));
|
||||
const serverNames = new Map((await db.select({ id: servers.id, name: servers.name }).from(servers)).map((r) => [r.id, r.name]));
|
||||
const deviceNames = new Map((await db.select({ id: networkDevices.id, name: networkDevices.name }).from(networkDevices)).map((r) => [r.id, r.name]));
|
||||
|
||||
/** The thing a maintenance window would go on to silence an alert about `subject` ("server:3", "integration:1"). */
|
||||
const targetOf = (type: "server" | "integration" | "dns_provider", id: number): Alert["target"] => ({
|
||||
const targetOf = (type: "server" | "integration" | "dns_provider" | "device", id: number): Alert["target"] => ({
|
||||
type,
|
||||
id,
|
||||
name: (type === "server" ? serverNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
|
||||
name: (type === "server" ? serverNames.get(id) : type === "device" ? deviceNames.get(id) : type === "integration" ? intById.get(id)?.name : undefined) ?? `${type} #${id}`,
|
||||
});
|
||||
const targetOfSubject = (subject: string | null): Alert["target"] => {
|
||||
const m = /^(server|integration|dns_provider):(\d+)$/.exec(subject ?? "");
|
||||
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider", Number(m[2])) : null;
|
||||
const m = /^(server|integration|dns_provider|device):(\d+)$/.exec(subject ?? "");
|
||||
return m ? targetOf(m[1] as "server" | "integration" | "dns_provider" | "device", Number(m[2])) : null;
|
||||
};
|
||||
const integrationTarget = (id: number): Alert["target"] => targetOf("integration", id);
|
||||
/** Integration-down alerts are per kind of service; a window on any integration of that kind silences them, so offer the first. */
|
||||
@@ -167,6 +171,9 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
// The servers found to be offline, for the integration failures that one may explain: the server check fills it in (and it is empty if that check fails).
|
||||
let offlineKnown!: (ids: Set<number>) => void;
|
||||
const offlineServers = new Promise<Set<number>>((resolve) => (offlineKnown = resolve));
|
||||
// The same for network devices, which can explain a service that has stopped answering.
|
||||
let devicesKnown!: (ids: Set<number>) => void;
|
||||
const offlineDeviceIds = new Promise<Set<number>>((resolve) => (devicesKnown = resolve));
|
||||
|
||||
await Promise.all([
|
||||
check("Server and storage health", async () => {
|
||||
@@ -182,38 +189,44 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const conditions = evaluateHealth(snapshot, healthChecks, now, { skipServers: grace > 0 });
|
||||
// A server that is down because the one it runs on is down is folded under that one.
|
||||
const offlineIds = new Set<number>();
|
||||
const offlineDevices = new Set<number>();
|
||||
for (const c of conditions) {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
if (m) (m[1] === "server" ? offlineIds : offlineDevices).add(Number(m[2]));
|
||||
}
|
||||
// (Or because of a service it needs that is failing, or a server it needs that needs one.)
|
||||
offlineKnown(offlineIds);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
devicesKnown(offlineDevices);
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source), offlineDevices);
|
||||
for (const c of conditions) {
|
||||
const origin = originOfConditionKey(c.key);
|
||||
const subject = subjectOfConditionKey(c.key);
|
||||
const offlineId = /^offline:server:(\d+)$/.exec(c.key)?.[1];
|
||||
const parentOf = offlineId ? cover.cause.get(`server:${offlineId}`) : undefined;
|
||||
const offlineMatch = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const offlineNode = offlineMatch ? `${offlineMatch[1]}:${offlineMatch[2]}` : undefined;
|
||||
const parentOf = offlineNode ? cover.cause.get(offlineNode) : undefined;
|
||||
push({
|
||||
causedBy: parentOf !== undefined ? { name: cover.graph.names.get(parentOf) ?? parentOf, alertId: alertIdOfNode(parentOf, cover.graph) ?? undefined } : undefined,
|
||||
affects: offlineId ? cover.taken.get(`server:${offlineId}`)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
affects: offlineNode ? cover.taken.get(offlineNode)?.sort((a, b) => a.localeCompare(b)) : undefined,
|
||||
category: origin.category,
|
||||
severity: c.severity ?? "warning",
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}` : origin.type === "unifi" ? "/network/devices" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
// Integrations the check couldn't read this time. (A whole-integration entry covers its nodes, so skip those.)
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
}
|
||||
}).finally(() => offlineKnown(new Set())),
|
||||
}).finally(() => {
|
||||
offlineKnown(new Set());
|
||||
devicesKnown(new Set());
|
||||
}),
|
||||
|
||||
check("Automation runs", async () => {
|
||||
const { items, held } = await collectAutomation();
|
||||
@@ -506,7 +519,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
const { notifications } = await getSettings();
|
||||
const failing = getFailingSources();
|
||||
// A failure explained by something it needs that is down is folded under that, like a guest under its host.
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source)).catch(() => null) : null;
|
||||
const cover = failing.length > 0 ? await dependencyCover(await offlineServers, failing.map((f) => f.source), await offlineDeviceIds).catch(() => null) : null;
|
||||
for (const f of failing) {
|
||||
const cause = cover ? integrationTypeCause(f.source, cover) : null;
|
||||
const affects = cover ? takenByType(f.source, cover) : [];
|
||||
|
||||
@@ -41,7 +41,7 @@ export interface SilenceUndo {
|
||||
restored: { id: number; endsAt: string; reason: string | null }[];
|
||||
}
|
||||
|
||||
type Target = { type: "server" | "integration" | "dns_provider"; id: number };
|
||||
type Target = { type: "server" | "integration" | "dns_provider" | "device"; id: number };
|
||||
|
||||
/** Puts each target under maintenance (and, if asked, what depends on a server). Targets that are already under it have their window extended. */
|
||||
export async function silenceTargets(targets: Target[], minutes: number, reason: string | null, createdBy: string, withDependents: boolean, now: Date = new Date()): Promise<{ silenced: { type: string; id: number; name: string }[]; skipped: Skipped[]; undo: SilenceUndo | null }> {
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
* server that needs a service (a guest that only exists while Proxmox answers). When the thing depended on is down, the things that need it
|
||||
* are down too — and only the one that matters is worth an alert, so the rest are folded under it: shown, not announced.
|
||||
*
|
||||
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts) plus the dependencies written
|
||||
* here. Servers and integrations are the things that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* The graph is the "runs on" links of servers (set by hand or worked out from Proxmox, see serverParents.ts), the uplinks of network devices (which
|
||||
* device each is connected through, as a UniFi controller says), plus the dependencies written here. Servers, network devices and integrations are the things
|
||||
* that can be down and be needed. (DNS providers raise no alerts of their own, so there is
|
||||
* nothing to fold.) An integration is judged per kind of service, as its health is: it is folded only if every enabled instance of that
|
||||
* kind is explained.
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies, integrations, servers } from "../db/schema.js";
|
||||
import { dependencies, integrations, networkDevices, servers } from "../db/schema.js";
|
||||
import { getInternalFlag } from "./settingsStore.js";
|
||||
import { listActiveWindows } from "./maintenance.js";
|
||||
import { loadParentRows, resolveParents, type ParentLink } from "./serverParents.js";
|
||||
@@ -18,7 +19,7 @@ import { loadParentRows, resolveParents, type ParentLink } from "./serverParents
|
||||
/** Where the health monitor keeps the problems it is announcing; read here so a service's alert can tell a server it needs is down. */
|
||||
export const HEALTH_STATE_FLAG = "healthActiveConditions";
|
||||
|
||||
export const nodeTypes = ["server", "integration"] as const;
|
||||
export const nodeTypes = ["server", "integration", "device"] as const;
|
||||
export type NodeType = (typeof nodeTypes)[number];
|
||||
|
||||
export const nodeKey = (type: NodeType, id: number) => `${type}:${id}`;
|
||||
@@ -128,26 +129,43 @@ export interface Graph {
|
||||
}
|
||||
|
||||
export async function loadGraph(): Promise<Graph> {
|
||||
const [serverRows, intRows, depRows, parentRows] = await Promise.all([
|
||||
const [serverRows, intRows, depRows, parentRows, deviceRows] = await Promise.all([
|
||||
db.select({ id: servers.id, name: servers.name }).from(servers),
|
||||
db.select({ id: integrations.id, name: integrations.name, type: integrations.type, enabled: integrations.enabled }).from(integrations),
|
||||
db.select().from(dependencies),
|
||||
loadParentRows(),
|
||||
db.select({ id: networkDevices.id, name: networkDevices.name, integrationId: networkDevices.integrationId, externalId: networkDevices.externalId, uplinkExternalId: networkDevices.uplinkExternalId }).from(networkDevices),
|
||||
]);
|
||||
const names = new Map<string, string>();
|
||||
for (const s of serverRows) names.set(nodeKey("server", s.id), s.name);
|
||||
for (const i of intRows) names.set(nodeKey("integration", i.id), i.name);
|
||||
for (const d of deviceRows) names.set(nodeKey("device", d.id), d.name);
|
||||
const rows = depRows.map((r) => ({ id: r.id, fromType: r.fromType as NodeType, fromId: r.fromId, toType: r.toType as NodeType, toId: r.toId, createdAt: r.createdAt }));
|
||||
const instances = new Map<string, number[]>();
|
||||
for (const i of intRows) if (i.enabled) instances.set(i.type, [...(instances.get(i.type) ?? []), i.id]);
|
||||
return { edges: buildEdges(resolveParents(parentRows), rows, (k) => names.has(k)), names, rows, instances, intRows };
|
||||
// A device needs the one it is connected through: the controller says so, and nobody has to write it.
|
||||
const uplinks = uplinkEdges(deviceRows);
|
||||
return { edges: buildEdges(resolveParents(parentRows), [...rows, ...uplinks], (k) => names.has(k)), names, rows, instances, intRows };
|
||||
}
|
||||
|
||||
/** The edges from each device to the device it is connected through, from what the controller reports; a device whose uplink is not (or no longer) known has none. */
|
||||
export function uplinkEdges(devices: { id: number; integrationId: number; externalId: string; uplinkExternalId: string | null }[]): DependencyEdge[] {
|
||||
const idOf = new Map(devices.map((d) => [`${d.integrationId}:${d.externalId}`, d.id]));
|
||||
const out: DependencyEdge[] = [];
|
||||
for (const d of devices) {
|
||||
if (!d.uplinkExternalId) continue;
|
||||
const to = idOf.get(`${d.integrationId}:${d.uplinkExternalId}`);
|
||||
if (to !== undefined && to !== d.id) out.push({ fromType: "device", fromId: d.id, toType: "device", toId: to });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The id of the alert about a node being down, when it has one (a server's "offline", a kind of service's "failing"). */
|
||||
export function alertIdOfNode(key: string, graph: Graph): string | null {
|
||||
const m = /^(server|integration):(\d+)$/.exec(key);
|
||||
const m = /^(server|integration|device):(\d+)$/.exec(key);
|
||||
if (!m) return null;
|
||||
if (m[1] === "server") return `offline:offline:server:${m[2]}`;
|
||||
if (m[1] === "device") return `offline:offline:device:${m[2]}`;
|
||||
const row = graph.intRows.find((r) => r.id === Number(m[2]));
|
||||
return row ? `integration:failing:${row.type}` : null;
|
||||
}
|
||||
@@ -190,11 +208,12 @@ export interface Cover {
|
||||
graph: Graph;
|
||||
}
|
||||
|
||||
/** Works out what is down because of what, given the servers that are offline and the kinds of integration that are failing. */
|
||||
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>): Promise<Cover> {
|
||||
/** Works out what is down because of what, given the servers and network devices that are offline and the kinds of integration that are failing. */
|
||||
export async function dependencyCover(offlineServerIds: Iterable<number>, failingTypes: Iterable<string>, offlineDeviceIds: Iterable<number> = []): Promise<Cover> {
|
||||
const graph = await loadGraph();
|
||||
const down = new Set<string>();
|
||||
for (const id of offlineServerIds) down.add(nodeKey("server", id));
|
||||
for (const id of offlineDeviceIds) down.add(nodeKey("device", id));
|
||||
const failing = new Set(failingTypes);
|
||||
for (const [type, ids] of graph.instances) if (failing.has(type)) for (const id of ids) down.add(nodeKey("integration", id));
|
||||
const cause = coveredNodes(down, graph.edges);
|
||||
@@ -223,15 +242,15 @@ export function takenByType(type: string, cover: Cover): string[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The servers the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
|
||||
export async function offlineServerIdsFromState(): Promise<number[]> {
|
||||
/** The servers or network devices the health monitor is announcing as offline (or holding back as explained), from what it last stored. */
|
||||
async function offlineIdsFromState(kind: "server" | "device"): Promise<number[]> {
|
||||
try {
|
||||
const raw = await getInternalFlag(HEALTH_STATE_FLAG);
|
||||
if (!raw) return [];
|
||||
const ids: number[] = [];
|
||||
for (const key of Object.keys(JSON.parse(raw) as Record<string, unknown>)) {
|
||||
const m = /^offline:server:(\d+)$/.exec(key);
|
||||
if (m) ids.push(Number(m[1]));
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(key);
|
||||
if (m && m[1] === kind) ids.push(Number(m[2]));
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
@@ -239,13 +258,16 @@ export async function offlineServerIdsFromState(): Promise<number[]> {
|
||||
}
|
||||
}
|
||||
|
||||
export const offlineServerIdsFromState = () => offlineIdsFromState("server");
|
||||
export const offlineDeviceIdsFromState = () => offlineIdsFromState("device");
|
||||
|
||||
/**
|
||||
* Whether a kind of integration that has started failing is failing because of something it needs that is down — a server the health
|
||||
* monitor says is offline, or another service that is failing. Never throws: a problem here must not swallow a real outage.
|
||||
*/
|
||||
export async function integrationCausedBy(type: string, failingTypes: string[]): Promise<{ name: string } | null> {
|
||||
try {
|
||||
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes);
|
||||
const cover = await dependencyCover(await offlineServerIdsFromState(), failingTypes, await offlineDeviceIdsFromState());
|
||||
return integrationTypeCause(type, cover);
|
||||
} catch (err) {
|
||||
console.error("[dependencies] couldn't work out whether a failure is explained:", err);
|
||||
@@ -254,6 +276,7 @@ export async function integrationCausedBy(type: string, failingTypes: string[]):
|
||||
}
|
||||
|
||||
export async function nodeExists(type: NodeType, id: number): Promise<boolean> {
|
||||
if (type === "device") return (await db.select({ id: networkDevices.id }).from(networkDevices).where(eq(networkDevices.id, id)).limit(1)).length > 0;
|
||||
if (type === "server") return (await db.select({ id: servers.id }).from(servers).where(eq(servers.id, id)).limit(1)).length > 0;
|
||||
return (await db.select({ id: integrations.id }).from(integrations).where(eq(integrations.id, id)).limit(1)).length > 0;
|
||||
}
|
||||
@@ -1,12 +1,13 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, serverPorts, servers } from "../db/schema.js";
|
||||
import { integrations, networkDevices, serverPorts, servers } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
import { notifyHealthIssues, notifyHealthRecovered } from "./notify.js";
|
||||
import { recordAudit } from "./audit.js";
|
||||
import { getInternalFlag, getSettings, setInternalFlag } from "./settingsStore.js";
|
||||
import { isFresh, syncStatuses } from "./unifiSync.js";
|
||||
import { activeSubjects, subjectOfConditionKey } from "./maintenance.js";
|
||||
import { dependencyCover, HEALTH_STATE_FLAG } from "./dependencies.js";
|
||||
import { getFailingSources } from "./integrationHealthMonitor.js";
|
||||
@@ -54,14 +55,23 @@ export interface SynologySnapshot {
|
||||
integrationName: string;
|
||||
storage: SynologyStorageInfo;
|
||||
}
|
||||
/** The network equipment of one UniFi controller, as last read; only offered when that reading is recent enough to be believed. */
|
||||
export interface UnifiSnapshot {
|
||||
integrationId: number;
|
||||
integrationName: string;
|
||||
devices: { id: number; name: string; kind: string; model: string | null; state: string; offlineSince: string | null }[];
|
||||
}
|
||||
export interface HealthSnapshot {
|
||||
servers: ServerSnapshot[];
|
||||
proxmox: ProxmoxSnapshot[];
|
||||
synology: SynologySnapshot[];
|
||||
unifi?: UnifiSnapshot[];
|
||||
}
|
||||
export interface Thresholds {
|
||||
serverOfflineMinutes: number;
|
||||
diskUsagePercent: number;
|
||||
/** Minutes a network device may be offline before it is raised (10 if left out). */
|
||||
deviceOfflineMinutes?: number;
|
||||
}
|
||||
|
||||
// ─── Formatting helpers ─────────────────────────────────────────────────────
|
||||
@@ -145,6 +155,21 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
for (const u of snapshot.unifi ?? []) {
|
||||
const limitMs = (thresholds.deviceOfflineMinutes ?? 10) * 60_000;
|
||||
for (const d of u.devices) {
|
||||
if (d.state !== "offline" || !d.offlineSince) continue;
|
||||
const age = now - parseTimestamp(d.offlineSince);
|
||||
if (!Number.isFinite(age) || age <= limitMs) continue;
|
||||
out.push({
|
||||
key: `offline:device:${d.id}`,
|
||||
source: `unifi:${u.integrationId}`,
|
||||
message: `${d.name} (${d.kind.replace("_", " ")}${d.model ? `, ${d.model}` : ""}) has been offline for ${formatDuration(age)}`,
|
||||
severity: "critical",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const px of snapshot.proxmox) {
|
||||
const seenShared = new Set<string>();
|
||||
for (const node of px.nodes) {
|
||||
@@ -331,6 +356,24 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
.from(integrations)
|
||||
.where(eq(integrations.enabled, true));
|
||||
|
||||
// UniFi controllers are read by their own pass every few minutes; what is judged here is what that pass stored, and only while it is recent.
|
||||
const unifiRows = rows.filter((r) => r.type === "unifi");
|
||||
if (unifiRows.length > 0) {
|
||||
const statuses = await syncStatuses();
|
||||
const nowMs = Date.now();
|
||||
for (const row of unifiRows) {
|
||||
if (!isFresh(statuses[row.id], nowMs)) {
|
||||
held.add(`unifi:${row.id}`);
|
||||
continue;
|
||||
}
|
||||
const devices = await db
|
||||
.select({ id: networkDevices.id, name: networkDevices.name, kind: networkDevices.kind, model: networkDevices.model, state: networkDevices.state, offlineSince: networkDevices.offlineSince })
|
||||
.from(networkDevices)
|
||||
.where(eq(networkDevices.integrationId, row.id));
|
||||
(snapshot.unifi ??= []).push({ integrationId: row.id, integrationName: row.name, devices });
|
||||
}
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
if (row.type !== "proxmox" && row.type !== "synology") continue;
|
||||
try {
|
||||
@@ -410,24 +453,25 @@ export async function runHealthCheck(now: number = Date.now()): Promise<{ added:
|
||||
const current = evaluateHealth(snapshot, healthChecks, now, { skipServers: inGrace });
|
||||
// A server that is down because the one it runs on is down is folded under that one: recorded, but not announced on its own.
|
||||
const offlineIds = new Set<number>();
|
||||
const offlineDevices = new Set<number>();
|
||||
for (const c of current) {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
if (m) offlineIds.add(Number(m[1]));
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
if (m) (m[1] === "server" ? offlineIds : offlineDevices).add(Number(m[2]));
|
||||
}
|
||||
// (Also when it needs a service that is failing, or runs on a server that needs one.)
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source));
|
||||
// (Also when it needs a service that is failing, or runs on a server that needs one, or is connected through a device that is down.)
|
||||
const cover = await dependencyCover(offlineIds, getFailingSources().map((f) => f.source), offlineDevices);
|
||||
const coveredKeys = new Set<string>();
|
||||
for (const child of cover.cause.keys()) {
|
||||
const m = /^server:(\d+)$/.exec(child);
|
||||
if (m) coveredKeys.add(`offline:server:${m[1]}`);
|
||||
const m = /^(server|device):(\d+)$/.exec(child);
|
||||
if (m) coveredKeys.add(`offline:${m[1]}:${m[2]}`);
|
||||
}
|
||||
const { added: announced, resolved, next, newlyDown, ended } = diffConditions(await loadState(), current, held, await activeSubjects(new Date(now)), coveredKeys);
|
||||
await setInternalFlag(STATE_FLAG, JSON.stringify(next));
|
||||
|
||||
// A host's message says what else it takes with it.
|
||||
const added = announced.map((c) => {
|
||||
const m = /^offline:server:(\d+)$/.exec(c.key);
|
||||
const children = m ? cover.taken.get(`server:${m[1]}`) : undefined;
|
||||
const m = /^offline:(server|device):(\d+)$/.exec(c.key);
|
||||
const children = m ? cover.taken.get(`${m[1]}:${m[2]}`) : undefined;
|
||||
return children?.length ? { ...c, message: `${c.message} — also takes down ${alsoList(children)}` } : c;
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq, gt, lt, lte } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dnsProviders, integrations, maintenanceWindows, servers } from "../db/schema.js";
|
||||
import { dnsProviders, integrations, maintenanceWindows, networkDevices, servers } from "../db/schema.js";
|
||||
|
||||
export type ActiveWindow = typeof maintenanceWindows.$inferSelect;
|
||||
|
||||
@@ -27,6 +27,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^(?:endpoint(?:-cert)?|heartbeat(?:-fail)?):\d+:server:(\d+)$/.exec(key);
|
||||
if (m) return `server:${m[1]}`;
|
||||
m = /^offline:device:(\d+)$/.exec(key);
|
||||
if (m) return `device:${m[1]}`;
|
||||
m = /^disk:proxmox:(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^(?:synology-volume|synology-disk|disk:synology):(\d+):/.exec(key);
|
||||
@@ -110,6 +112,10 @@ export async function describeTarget(targetType: ActiveWindow["targetType"], tar
|
||||
const [r] = await db.select({ name: servers.name }).from(servers).where(eq(servers.id, targetId)).limit(1);
|
||||
return r ? { name: r.name, kind: "Server" } : null;
|
||||
}
|
||||
if (targetType === "device") {
|
||||
const [r] = await db.select({ name: networkDevices.name, kind: networkDevices.kind }).from(networkDevices).where(eq(networkDevices.id, targetId)).limit(1);
|
||||
return r ? { name: r.name, kind: `Network device (${r.kind.replace("_", " ")})` } : null;
|
||||
}
|
||||
if (targetType === "integration") {
|
||||
const [r] = await db.select({ name: integrations.name, type: integrations.type }).from(integrations).where(eq(integrations.id, targetId)).limit(1);
|
||||
return r ? { name: r.name, kind: `Integration (${r.type})` } : null;
|
||||
|
||||
@@ -606,6 +606,7 @@ const SOURCE_LABELS: Record<string, string> = {
|
||||
phpipam: "phpIPAM",
|
||||
pbs: "Proxmox Backup Server",
|
||||
osticket: "osTicket",
|
||||
unifi: "UniFi",
|
||||
};
|
||||
|
||||
export function sourceLabel(source: string): string {
|
||||
|
||||
@@ -117,6 +117,8 @@ export interface HealthCheckSettings {
|
||||
newPortAlerts: boolean;
|
||||
/** An alert raised this many times within an hour is "flapping": one message says so and its further raised/cleared messages are held until it has been quiet for an hour. 0 turns it off. */
|
||||
flapRaises: number;
|
||||
/** A network device (switch, access point, gateway) that a UniFi controller has not seen online for this many minutes is raised as offline. */
|
||||
deviceOfflineMinutes: number;
|
||||
}
|
||||
|
||||
/** The monthly report: last month's availability, incidents, updates and renewals, in one message on a day of the month. Off until switched on. */
|
||||
@@ -287,7 +289,7 @@ const DEFAULTS: AppSettings = {
|
||||
selfBackup: { enabled: true, time: "03:30", keep: 14 },
|
||||
profilePictures: { gravatar: false, identityProvider: false },
|
||||
signIn: { roleFromGroups: false, groupsClaim: "groups", adminGroups: [], operatorGroups: [], otherwise: "viewer" },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3 },
|
||||
healthChecks: { serverOfflineMinutes: 60, diskUsagePercent: 90, domainWarnDays: 30, diskForecastDays: 14, backupStaleHours: 48, restoreTestDays: 0, newPortAlerts: true, flapRaises: 3, deviceOfflineMinutes: 10 },
|
||||
// Docker's default bridge (172.17.0.0/16) and the pool it hands custom networks from (172.18–31) live here, and every
|
||||
// Docker host repeats them. Shown on the Consistency page, where it can be removed if 172.16/12 is used as a real LAN.
|
||||
consistency: { excludedRanges: ["172.16.0.0/12"] },
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Keeps the network equipment of every UniFi integration in the database: the devices (state, IP, uptime, firmware, uplink), and the networks and Wi-Fi names beside
|
||||
* them. A pass runs every few minutes and on request. The rows are what the Network devices page shows, what the dependency map and the offline alerts are
|
||||
* built from — so none of those talk to the controller themselves, and a controller that is down costs only the freshness of what they show.
|
||||
*
|
||||
* What is kept is small and exact: a device that has gone offline carries `offlineSince` until it is back, which is what an "offline" alert counts from.
|
||||
*/
|
||||
import { and, eq, inArray, or } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { dependencies, integrations, networkDevices, networkSegments } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createUnifiAdapter, type UnifiDevice, type UnifiInventory } from "../integrations/unifi/adapter.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** A device's state is only as good as the last pass: after this long without a good one, offline alerts are held rather than judged. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "unifiSync";
|
||||
|
||||
export interface SyncStatus {
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
sites: string[];
|
||||
mode: "api_key" | "login" | null;
|
||||
devices: number;
|
||||
networks: number;
|
||||
ssids: number;
|
||||
}
|
||||
|
||||
type DeviceRow = typeof networkDevices.$inferSelect;
|
||||
|
||||
/** What a device's offline bookkeeping becomes now. Pure. `offlineSince` starts when it is first seen not online and stays until it is online again. */
|
||||
export function trackOffline(previous: { offlineSince: string | null; lastOnlineAt: string | null } | undefined, state: UnifiDevice["state"], nowIso: string): { offlineSince: string | null; lastOnlineAt: string | null } {
|
||||
const lastOnlineAt = previous?.lastOnlineAt ?? null;
|
||||
if (state === "online") return { offlineSince: null, lastOnlineAt: nowIso };
|
||||
if (state === "offline") return { offlineSince: previous?.offlineSince ?? nowIso, lastOnlineAt };
|
||||
// Updating, being adopted or unknown: neither back nor newly gone, so what was known stays.
|
||||
return { offlineSince: previous?.offlineSince ?? null, lastOnlineAt };
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, SyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, SyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncStatuses(): Promise<Record<number, SyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
async function writeStatus(integrationId: number, status: SyncStatus): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
|
||||
/** Writes one controller's inventory: updates what is there, adds what is new, removes what the controller no longer has. */
|
||||
export async function storeInventory(integrationId: number, inventory: UnifiInventory, now: Date = new Date()): Promise<void> {
|
||||
const nowIso = now.toISOString();
|
||||
const existing = await db.select().from(networkDevices).where(eq(networkDevices.integrationId, integrationId));
|
||||
const byExternal = new Map(existing.map((r) => [r.externalId, r]));
|
||||
const seen = new Set<string>();
|
||||
for (const d of inventory.devices) {
|
||||
if (seen.has(d.externalId)) continue;
|
||||
seen.add(d.externalId);
|
||||
const prev = byExternal.get(d.externalId);
|
||||
const track = trackOffline(prev, d.state, nowIso);
|
||||
const values = {
|
||||
siteName: d.siteName,
|
||||
name: d.name,
|
||||
model: d.model,
|
||||
kind: d.kind,
|
||||
ip: d.ip,
|
||||
mac: d.mac,
|
||||
state: d.state,
|
||||
uptimeSeconds: d.uptimeSeconds,
|
||||
firmware: d.firmware,
|
||||
firmwareUpdatable: d.firmwareUpdatable,
|
||||
uplinkExternalId: d.uplinkExternalId,
|
||||
offlineSince: track.offlineSince,
|
||||
lastOnlineAt: track.lastOnlineAt,
|
||||
updatedAt: nowIso,
|
||||
};
|
||||
if (prev) await db.update(networkDevices).set(values).where(eq(networkDevices.id, prev.id));
|
||||
else await db.insert(networkDevices).values({ integrationId, externalId: d.externalId, firstSeenAt: nowIso, ...values });
|
||||
}
|
||||
const gone = existing.filter((r) => !seen.has(r.externalId)).map((r) => r.id);
|
||||
if (gone.length > 0) await db.delete(networkDevices).where(inArray(networkDevices.id, gone));
|
||||
|
||||
const segments: (typeof networkSegments.$inferInsert)[] = [
|
||||
...inventory.networks.map((n) => ({ integrationId, kind: "network" as const, externalId: n.externalId, siteName: n.siteName, name: n.name, enabled: n.enabled, vlanId: n.vlanId, subnet: n.subnet, purpose: n.purpose, security: null, networkName: null, updatedAt: nowIso })),
|
||||
...inventory.ssids.map((s) => ({ integrationId, kind: "ssid" as const, externalId: s.externalId, siteName: s.siteName, name: s.name, enabled: s.enabled, vlanId: s.vlanId, subnet: null, purpose: null, security: s.security, networkName: s.networkName, updatedAt: nowIso })),
|
||||
];
|
||||
await db.delete(networkSegments).where(eq(networkSegments.integrationId, integrationId));
|
||||
if (segments.length > 0) await db.insert(networkSegments).values(segments);
|
||||
}
|
||||
|
||||
/** Reads one UniFi integration and stores what it says. Never throws: how it went is the returned status, and what was stored before is left alone when it failed. */
|
||||
export async function syncUnifiIntegration(integrationId: number, now: Date = new Date()): Promise<SyncStatus> {
|
||||
let status: SyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "unifi") throw new Error("not a UniFi integration");
|
||||
const inventory = await createUnifiAdapter(loaded.config as never).fetchInventory();
|
||||
await storeInventory(integrationId, inventory, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null, sites: inventory.sites, mode: inventory.mode, devices: inventory.devices.length, networks: inventory.networks.length, ssids: inventory.ssids.length };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
// The time of the last good pass is kept: it is what says how stale the stored states are.
|
||||
status = { at: previous?.ok ? previous.at : (previous?.at ?? now.toISOString()), ok: false, error: err instanceof Error ? err.message : String(err), sites: previous?.sites ?? [], mode: previous?.mode ?? null, devices: previous?.devices ?? 0, networks: previous?.networks ?? 0, ssids: previous?.ssids ?? 0 };
|
||||
console.error(`[unifi] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
await writeStatus(integrationId, status);
|
||||
} catch (err) {
|
||||
console.error("[unifi] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllUnifi(now: Date = new Date()): Promise<Record<number, SyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "unifi"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, SyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncUnifiIntegration(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Forgets everything read from a controller — when its integration is removed. Dependencies written by hand that pointed at one of its devices go too:
|
||||
* they would never be shown again, and a device that comes back under a new integration is a new device.
|
||||
*/
|
||||
export async function removeIntegrationData(integrationId: number): Promise<void> {
|
||||
const ids = (await db.select({ id: networkDevices.id }).from(networkDevices).where(eq(networkDevices.integrationId, integrationId))).map((r) => r.id);
|
||||
if (ids.length > 0) {
|
||||
await db.delete(dependencies).where(or(and(eq(dependencies.fromType, "device"), inArray(dependencies.fromId, ids)), and(eq(dependencies.toType, "device"), inArray(dependencies.toId, ids))));
|
||||
await db.delete(networkDevices).where(inArray(networkDevices.id, ids));
|
||||
}
|
||||
await db.delete(networkSegments).where(eq(networkSegments.integrationId, integrationId));
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the stored states of an integration can be trusted: it has been read successfully within the last while. */
|
||||
export function isFresh(status: SyncStatus | undefined, nowMs: number): boolean {
|
||||
return !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
}
|
||||
|
||||
export type { DeviceRow };
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initUnifiSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllUnifi().catch((err) => console.error("[unifi] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
// Not awaited, and not at once: the controller is not needed to start, and the first pass should not compete with the migrations and the first health pass.
|
||||
setTimeout(pass, 5_000);
|
||||
console.log(`[unifi] reading network equipment from UniFi controllers every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
Reference in new issue
Block a user