Nextcloud, NetBox and n8n integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Nextcloud: database needing upgrade (critical), maintenance mode, and the data disk running low; users, files and app updates with an admin app password. NetBox: no background worker running, counts, and a list of the servers this app manages that have no address in NetBox (a list, not an alert). n8n: an active workflow whose last run failed, clearing when it next succeeds. Disk messages for Immich, Paperless and Nextcloud now use the same units as the pages that show them. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a16996d975
commit
4ff0b44943
46 files changed
+10137
-10
No files matched your search
@@ -64,6 +64,9 @@ import { authentikRouter } from "./routes/authentik.js";
|
||||
import { prometheusRouter } from "./routes/prometheus.js";
|
||||
import { immichRouter } from "./routes/immich.js";
|
||||
import { paperlessRouter } from "./routes/paperless.js";
|
||||
import { n8nRouter } from "./routes/n8n.js";
|
||||
import { netboxRouter } from "./routes/netbox.js";
|
||||
import { nextcloudRouter } from "./routes/nextcloud.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
@@ -204,6 +207,9 @@ export function createApp() {
|
||||
app.use("/api/prometheus", prometheusRouter);
|
||||
app.use("/api/immich", immichRouter);
|
||||
app.use("/api/paperless", paperlessRouter);
|
||||
app.use("/api/n8n", n8nRouter);
|
||||
app.use("/api/netbox", netboxRouter);
|
||||
app.use("/api/nextcloud", nextcloudRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
|
||||
@@ -558,6 +558,9 @@ export const integrationTypes = [
|
||||
"prometheus",
|
||||
"immich",
|
||||
"paperless",
|
||||
"nextcloud",
|
||||
"netbox",
|
||||
"n8n",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1164,6 +1167,58 @@ export const haReadings = sqliteTable(
|
||||
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
|
||||
);
|
||||
|
||||
// ─── Nextcloud ──────────────────────────────────────────────────────────────
|
||||
|
||||
// How a Nextcloud (an integration of type "nextcloud") is right now, one row each, replaced at every read: its version, whether it is in maintenance mode or needs its database
|
||||
// upgrading, and — with an administrator's app password — the room left for the data, the users and files, and the app updates waiting. A column that is null could not be read.
|
||||
// Read by services/nextcloudSync.ts.
|
||||
export const nextcloudState = sqliteTable("nextcloud_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
maintenance: integer("maintenance", { mode: "boolean" }).notNull().default(false),
|
||||
needsUpgrade: integer("needs_upgrade", { mode: "boolean" }).notNull().default(false),
|
||||
detail: integer("detail", { mode: "boolean" }).notNull().default(false), // whether the monitoring app answered
|
||||
freeBytes: integer("free_bytes"),
|
||||
users: integer("users"),
|
||||
files: integer("files"),
|
||||
active24h: integer("active_24h"),
|
||||
appUpdates: integer("app_updates"), // how many apps have an update waiting; null: not asked
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── NetBox ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// How a NetBox (an integration of type "netbox") is right now, one row each, replaced at every read: its version, how many background workers are running, the counts and the IP
|
||||
// addresses it holds (kept so the servers this app manages that are not in it can be listed). A column that is null could not be read. Read by services/netboxSync.ts.
|
||||
export const netboxState = sqliteTable("netbox_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
workersRunning: integer("workers_running"), // null: /status/ did not say
|
||||
devices: integer("devices"),
|
||||
virtualMachines: integer("virtual_machines"),
|
||||
prefixes: integer("prefixes"),
|
||||
ipAddresses: integer("ip_addresses"),
|
||||
ips: text("ips"), // JSON string[] of the addresses (without the prefix length), at most 5000; null: could not be listed
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── n8n ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// How an n8n (an integration of type "n8n") is right now, one row each, replaced at every read: its workflows, each with its last outcome and the failures of the last day (worked out
|
||||
// from the latest 250 runs). Read by services/n8nSync.ts.
|
||||
export const n8nState = sqliteTable("n8n_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
workflows: text("workflows").notNull().default("[]"), // JSON: id, name, active, lastStatus, lastAt, failed24h
|
||||
executionsKnown: integer("executions_known", { mode: "boolean" }).notNull().default(false), // false: the runs could not be asked, so no last outcome is known
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Paperless-ngx ──────────────────────────────────────────────────────────
|
||||
|
||||
// How a Paperless-ngx (an integration of type "paperless") is right now, one row each, replaced at every read: its version, the disk, how each component of its system status is, the
|
||||
|
||||
@@ -34,6 +34,9 @@ import { initAuthentikSync } from "./services/authentikSync.js";
|
||||
import { initPrometheusSync } from "./services/prometheusSync.js";
|
||||
import { initImmichSync } from "./services/immichSync.js";
|
||||
import { initPaperlessSync } from "./services/paperlessSync.js";
|
||||
import { initN8nSync } from "./services/n8nSync.js";
|
||||
import { initNetBoxSync } from "./services/netboxSync.js";
|
||||
import { initNextcloudSync } from "./services/nextcloudSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
@@ -89,6 +92,9 @@ initAuthentikSync();
|
||||
initPrometheusSync();
|
||||
initImmichSync();
|
||||
initPaperlessSync();
|
||||
initN8nSync();
|
||||
initNetBoxSync();
|
||||
initNextcloudSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
|
||||
@@ -99,6 +99,22 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
nextcloud: [
|
||||
{ key: "url", label: "Nextcloud URL", secret: false, placeholder: "https://cloud.example.lan" },
|
||||
{ key: "username", label: "Administrator's name (optional)", secret: false, optional: true, placeholder: "for the disk, the users and the app updates" },
|
||||
{ key: "appPassword", label: "App password (optional)", secret: true, optional: true, type: "password", placeholder: "Personal settings → Security → Devices & sessions" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
netbox: [
|
||||
{ key: "url", label: "NetBox URL", secret: false, placeholder: "https://netbox.example.lan" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "your profile → API Tokens; read-only is enough" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
n8n: [
|
||||
{ key: "url", label: "n8n URL", secret: false, placeholder: "http://n8n.example.lan:5678" },
|
||||
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Settings → n8n API" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
paperless: [
|
||||
{ key: "url", label: "Paperless-ngx URL", secret: false, placeholder: "http://paperless.example.lan:8000" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "your profile → API Auth Token; a superuser's, for the system status" },
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* n8n adapter. Config: url, apiKey (made in n8n under Settings → n8n API); optional: insecure.
|
||||
*
|
||||
* Reading only: the workflows (and which are switched on), and the most recent runs, from which each workflow's **last run** and how many runs failed in the last day are worked out —
|
||||
* an automation that has been failing since Tuesday is what is worth hearing about. Nothing is sent to n8n but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from n8n's published public API (`/api/v1/workflows`, `/api/v1/executions`) and have not been checked against a live server here; every field is
|
||||
* read defensively. Only the newest 250 runs are looked at, so a workflow that runs more rarely than that many other runs may show no run.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface N8nConfig {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface N8nWorkflow {
|
||||
id: string;
|
||||
name: string;
|
||||
active: boolean;
|
||||
/** error, crashed, success, canceled, waiting, running, …; null when no run of it is among those looked at. */
|
||||
lastStatus: string | null;
|
||||
lastAt: string | null;
|
||||
/** Failed (error or crashed) runs in the last 24 hours. */
|
||||
failed24h: number;
|
||||
}
|
||||
|
||||
export interface N8nRead {
|
||||
workflows: N8nWorkflow[];
|
||||
/** Whether the runs could be asked; false means the workflows' last runs are unknown, not fine. */
|
||||
executionsKnown: boolean;
|
||||
}
|
||||
|
||||
export interface N8nAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(now?: Date): Promise<N8nRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : v !== null && typeof v === "number" ? String(v) : null);
|
||||
const iso = (v: unknown): string | null => {
|
||||
const s = typeof v === "string" ? v : null;
|
||||
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
|
||||
};
|
||||
|
||||
export const FAILED_STATUSES = new Set(["error", "crashed"]);
|
||||
const PAGES = 4;
|
||||
|
||||
/** Pure: the workflows of one page. */
|
||||
export function parseWorkflows(raw: unknown): { id: string; name: string; active: boolean }[] {
|
||||
const out: { id: string; name: string; active: boolean }[] = [];
|
||||
for (const item of Array.isArray(obj(raw).data) ? (obj(raw).data as unknown[]) : []) {
|
||||
const w = obj(item);
|
||||
const id = str(w.id);
|
||||
if (!id) continue;
|
||||
out.push({ id, name: str(w.name) ?? id, active: w.active === true });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: each workflow's last run and its failures in the last day, from a list of runs. Runs come newest first, but nothing here depends on it. */
|
||||
export function summariseRuns(workflows: { id: string; name: string; active: boolean }[], raw: unknown, now: Date): N8nWorkflow[] {
|
||||
const since = now.getTime() - 86_400_000;
|
||||
const last = new Map<string, { status: string; t: number; at: string }>();
|
||||
const failed = new Map<string, number>();
|
||||
for (const item of Array.isArray(obj(raw).data) ? (obj(raw).data as unknown[]) : []) {
|
||||
const e = obj(item);
|
||||
const wf = str(e.workflowId);
|
||||
const status = str(e.status)?.toLowerCase() ?? (e.finished === true ? "success" : null);
|
||||
const at = iso(e.stoppedAt) ?? iso(e.startedAt);
|
||||
if (!wf || !status || !at) continue;
|
||||
const t = Date.parse(at);
|
||||
// A run that is still going is not an outcome: the one before it is.
|
||||
if (status !== "running" && status !== "waiting" && status !== "new") {
|
||||
const seen = last.get(wf);
|
||||
if (!seen || t > seen.t) last.set(wf, { status, t, at });
|
||||
if (FAILED_STATUSES.has(status) && t >= since) failed.set(wf, (failed.get(wf) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
return workflows.map((w) => ({ ...w, lastStatus: last.get(w.id)?.status ?? null, lastAt: last.get(w.id)?.at ?? null, failed24h: failed.get(w.id) ?? 0 }));
|
||||
}
|
||||
|
||||
export function createN8nAdapter(config: N8nConfig): N8nAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers = { "X-N8N-API-KEY": config.apiKey, Accept: "application/json" };
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}/api/v1${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`n8n did not accept the API key (HTTP ${res.status}) — make one under Settings → n8n API`);
|
||||
if (res.status === 404) throw new Error("n8n has no public API at this address — check the address, and that the API is switched on");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`n8n error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("n8n answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(now: Date = new Date()): Promise<N8nRead> {
|
||||
// The workflows are the hard requirement: they prove the key is accepted.
|
||||
const workflows: { id: string; name: string; active: boolean }[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < PAGES; page++) {
|
||||
const raw = await get(`/workflows?limit=250${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`);
|
||||
workflows.push(...parseWorkflows(raw));
|
||||
cursor = str(obj(raw).nextCursor);
|
||||
if (!cursor) break;
|
||||
}
|
||||
try {
|
||||
return { workflows: summariseRuns(workflows, await get("/executions?limit=250"), now), executionsKnown: true };
|
||||
} catch {
|
||||
return { workflows: workflows.map((w) => ({ ...w, lastStatus: null, lastAt: null, failed24h: 0 })), executionsKnown: false };
|
||||
}
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/workflows?limit=1");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("n8n", { ping, read });
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* NetBox adapter. Config: url, token (made in NetBox under your profile → API Tokens; read-only is enough — "Write enabled" off); optional: insecure.
|
||||
*
|
||||
* Reading only: NetBox's version, **how many background workers are running** (they run its webhooks, scripts, jobs and reports — and in the Docker image they are a separate
|
||||
* container that can be down while the page still loads), how many devices, virtual machines, prefixes and IP addresses it holds, and the IP addresses themselves — so the app can say
|
||||
* which of the servers it manages have no address in NetBox. Nothing is sent to NetBox but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from NetBox's published REST API (`/api/status/`, `/api/dcim/devices/`, `/api/virtualization/virtual-machines/`, `/api/ipam/prefixes/`,
|
||||
* `/api/ipam/ip-addresses/`) and have not been checked against a live server here; every field is read defensively.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface NetBoxConfig {
|
||||
url: string;
|
||||
token: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface NetBoxRead {
|
||||
version: string | null;
|
||||
/** Null when /status/ did not say, which is not the same as none. */
|
||||
workersRunning: number | null;
|
||||
devices: number | null;
|
||||
virtualMachines: number | null;
|
||||
prefixes: number | null;
|
||||
ipAddresses: number | null;
|
||||
/** The addresses themselves, without their prefix length, up to MAX_IPS; null when they could not be asked. */
|
||||
ips: string[] | null;
|
||||
}
|
||||
|
||||
export interface NetBoxAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<NetBoxRead>;
|
||||
}
|
||||
|
||||
export const MAX_IPS = 5000;
|
||||
const PAGE = 1000;
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
|
||||
/** Pure: what /api/status/ says. */
|
||||
export function parseStatus(raw: unknown): { version: string | null; workersRunning: number | null } {
|
||||
const o = obj(raw);
|
||||
return { version: str(o["netbox-version"]), workersRunning: num(o["rq-workers-running"]) };
|
||||
}
|
||||
|
||||
/** Pure: how many things a list says there are (`count`); null when the answer has none. */
|
||||
export function parseCount(raw: unknown): number | null {
|
||||
return num(obj(raw).count);
|
||||
}
|
||||
|
||||
/** Pure: the addresses of one page of /ipam/ip-addresses/, without the "/24". */
|
||||
export function parseAddresses(raw: unknown): string[] {
|
||||
const results = obj(raw).results;
|
||||
const out: string[] = [];
|
||||
for (const item of Array.isArray(results) ? results : []) {
|
||||
const a = str(obj(item).address);
|
||||
if (a) out.push(a.split("/")[0].toLowerCase());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** NetBox 4.5's v2 tokens (nbt_…) go as a Bearer token; the older ones as "Token". */
|
||||
export const authorization = (token: string) => (token.startsWith("nbt_") ? `Bearer ${token}` : `Token ${token}`);
|
||||
|
||||
export function createNetBoxAdapter(config: NetBoxConfig): NetBoxAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers = { Authorization: authorization(config.token), Accept: "application/json" };
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}/api${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`NetBox did not accept the token (HTTP ${res.status}) — check it, and that it has not expired`);
|
||||
if (res.status === 404) throw new Error("NetBox has no API at this address — check the address (and that it is NetBox's own, not a proxy's)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`NetBox error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("NetBox answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(): Promise<NetBoxRead> {
|
||||
const status = parseStatus(await get("/status/"));
|
||||
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
|
||||
try {
|
||||
return parse(await get(path));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const [devices, virtualMachines, prefixes, ipAddresses] = await Promise.all([
|
||||
soft("/dcim/devices/?limit=1", parseCount),
|
||||
soft("/virtualization/virtual-machines/?limit=1", parseCount),
|
||||
soft("/ipam/prefixes/?limit=1", parseCount),
|
||||
soft("/ipam/ip-addresses/?limit=1", parseCount),
|
||||
]);
|
||||
let ips: string[] | null = null;
|
||||
if (ipAddresses !== null) {
|
||||
const all: string[] = [];
|
||||
try {
|
||||
for (let offset = 0; offset < Math.min(ipAddresses, MAX_IPS); offset += PAGE) {
|
||||
all.push(...parseAddresses(await get(`/ipam/ip-addresses/?brief=1&limit=${PAGE}&offset=${offset}`)));
|
||||
}
|
||||
ips = [...new Set(all)].slice(0, MAX_IPS);
|
||||
} catch {
|
||||
ips = null;
|
||||
}
|
||||
}
|
||||
return { version: status.version, workersRunning: status.workersRunning, devices, virtualMachines, prefixes, ipAddresses, ips };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/status/");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("netbox", { ping, read });
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Nextcloud adapter. Config: url; optional: username and appPassword (an administrator's, made under Personal settings → Security → Devices & sessions, so the server's own
|
||||
* password is never needed), insecure.
|
||||
*
|
||||
* Reading only. Without a login: /status.php, which Nextcloud answers to anyone — its version, whether it is in **maintenance mode** and whether it **needs its database
|
||||
* upgrading** (either way nobody can sign in). With an administrator's app password the *Monitoring* ("serverinfo") app is also asked: how much room is left for the data, how many
|
||||
* users and files, who was active, and how many app updates wait. Nothing is sent to Nextcloud but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Nextcloud's published API (`/status.php`, `/ocs/v2.php/apps/serverinfo/api/v1/info`) and have not been checked against a live server here;
|
||||
* every field is read defensively. Whether background jobs (cron) are running is *not* read: the API this app uses does not say.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface NextcloudConfig {
|
||||
url: string;
|
||||
username?: string;
|
||||
appPassword?: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface NextcloudRead {
|
||||
version: string | null;
|
||||
maintenance: boolean;
|
||||
needsUpgrade: boolean;
|
||||
/** Whether the monitoring app answered; false without a login, or when it refused or is not installed. */
|
||||
detail: boolean;
|
||||
/** Bytes free where the data is kept. */
|
||||
freeBytes: number | null;
|
||||
users: number | null;
|
||||
files: number | null;
|
||||
/** Users active in the last 24 hours. */
|
||||
active24h: number | null;
|
||||
/** How many apps have an update waiting; null when not asked. */
|
||||
appUpdates: number | null;
|
||||
}
|
||||
|
||||
export interface NextcloudAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<NextcloudRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
/** Nextcloud's own JSON gives booleans as booleans, but some answers say 1 or "1". */
|
||||
const flag = (v: unknown): boolean => v === true || v === 1 || v === "1" || v === "true";
|
||||
|
||||
/** Pure: what /status.php says. */
|
||||
export function parseStatus(raw: unknown): Pick<NextcloudRead, "version" | "maintenance" | "needsUpgrade"> & { installed: boolean } {
|
||||
const o = obj(raw);
|
||||
return { version: str(o.versionstring) ?? str(o.version), maintenance: flag(o.maintenance), needsUpgrade: flag(o.needsDbUpgrade), installed: o.installed === undefined ? true : flag(o.installed) };
|
||||
}
|
||||
|
||||
/** Pure: what the monitoring app says (`ocs.data`). */
|
||||
export function parseServerInfo(raw: unknown): Pick<NextcloudRead, "freeBytes" | "users" | "files" | "active24h" | "appUpdates"> | null {
|
||||
const data = obj(obj(raw).ocs).data;
|
||||
if (data === undefined || data === null || typeof data !== "object") return null;
|
||||
const nc = obj(obj(data).nextcloud);
|
||||
const system = obj(nc.system);
|
||||
const apps = obj(system.apps);
|
||||
const listed = Object.keys(obj(apps.app_updates)).length;
|
||||
return {
|
||||
freeBytes: num(system.freespace),
|
||||
users: num(obj(nc.storage).num_users),
|
||||
files: num(obj(nc.storage).num_files),
|
||||
active24h: num(obj(obj(data).activeUsers).last24hours),
|
||||
appUpdates: num(apps.num_updates_available) ?? (listed > 0 ? listed : null),
|
||||
};
|
||||
}
|
||||
|
||||
export function createNextcloudAdapter(config: NextcloudConfig): NextcloudAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const login = config.username && config.appPassword ? { Authorization: `Basic ${Buffer.from(`${config.username}:${config.appPassword}`).toString("base64")}`, "OCS-APIRequest": "true" } : null;
|
||||
|
||||
async function get(path: string, headers: Record<string, string> = {}): Promise<unknown> {
|
||||
const res = await requestJson(`${base}${path}`, { headers: { Accept: "application/json", ...headers }, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Nextcloud did not accept the login (HTTP ${res.status}) — use an administrator's name and an app password`);
|
||||
if (res.status === 404) throw new Error("Nextcloud has no such page at this address — check the address (and that it is Nextcloud's own, not a proxy's)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Nextcloud error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Nextcloud answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(): Promise<NextcloudRead> {
|
||||
const status = parseStatus(await get("/status.php"));
|
||||
if (!status.installed) throw new Error("That Nextcloud says it is not installed yet");
|
||||
let detail: ReturnType<typeof parseServerInfo> = null;
|
||||
if (login) {
|
||||
try {
|
||||
detail = parseServerInfo(await get("/ocs/v2.php/apps/serverinfo/api/v1/info?format=json", login));
|
||||
} catch {
|
||||
detail = null; // not an administrator, or the Monitoring app is off: the rest is still worth having
|
||||
}
|
||||
}
|
||||
return { version: status.version, maintenance: status.maintenance, needsUpgrade: status.needsUpgrade, detail: detail !== null, freeBytes: detail?.freeBytes ?? null, users: detail?.users ?? null, files: detail?.files ?? null, active24h: detail?.active24h ?? null, appUpdates: detail?.appUpdates ?? null };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/status.php");
|
||||
if (login) await get("/ocs/v2.php/apps/serverinfo/api/v1/info?format=json&skipApps=true&skipUpdate=true", login);
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("nextcloud", { ping, read });
|
||||
}
|
||||
@@ -19,6 +19,9 @@ import { createAuthentikAdapter } from "./authentik/adapter.js";
|
||||
import { createPrometheusAdapter } from "./prometheus/adapter.js";
|
||||
import { createImmichAdapter } from "./immich/adapter.js";
|
||||
import { createPaperlessAdapter } from "./paperless/adapter.js";
|
||||
import { createN8nAdapter } from "./n8n/adapter.js";
|
||||
import { createNetBoxAdapter } from "./netbox/adapter.js";
|
||||
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
|
||||
|
||||
export interface PingableAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
@@ -71,6 +74,12 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
return createImmichAdapter(config as any);
|
||||
case "paperless":
|
||||
return createPaperlessAdapter(config as any);
|
||||
case "n8n":
|
||||
return createN8nAdapter(config as any);
|
||||
case "netbox":
|
||||
return createNetBoxAdapter(config as any);
|
||||
case "nextcloud":
|
||||
return createNextcloudAdapter(config as any);
|
||||
default:
|
||||
throw new Error(`Integration type "${type}" is not implemented yet`);
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@ import { forgetAuthentik, syncAuthentik } from "../services/authentikSync.js";
|
||||
import { forgetPrometheus, syncPrometheus } from "../services/prometheusSync.js";
|
||||
import { forgetImmich, syncImmich } from "../services/immichSync.js";
|
||||
import { forgetPaperless, syncPaperless } from "../services/paperlessSync.js";
|
||||
import { forgetN8n, syncN8n } from "../services/n8nSync.js";
|
||||
import { forgetNetBox, syncNetBox } from "../services/netboxSync.js";
|
||||
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
@@ -141,6 +144,9 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
if (type === "prometheus") void syncPrometheus(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "immich") void syncImmich(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "paperless") void syncPaperless(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "n8n") void syncN8n(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "netbox") void syncNetBox(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.status(201).json({
|
||||
integration: {
|
||||
@@ -249,6 +255,9 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (updated.type === "prometheus" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPrometheus(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "immich" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncImmich(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "paperless" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPaperless(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "n8n" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncN8n(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "netbox" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNetBox(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.json({
|
||||
integration: {
|
||||
@@ -333,6 +342,9 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (existing.type === "prometheus") await forgetPrometheus(id);
|
||||
if (existing.type === "immich") await forgetImmich(id);
|
||||
if (existing.type === "paperless") await forgetPaperless(id);
|
||||
if (existing.type === "n8n") await forgetN8n(id);
|
||||
if (existing.type === "netbox") await forgetNetBox(id);
|
||||
if (existing.type === "nextcloud") await forgetNextcloud(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
if (existing.credentialId) {
|
||||
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, n8nState } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { isN8nFresh, n8nProblems, n8nSyncStatuses, workflowsOf } from "../services/n8nSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const n8nRouter = Router();
|
||||
n8nRouter.use(requireAuth);
|
||||
|
||||
/** Every n8n, how it is, from what the last read stored: its workflows with their last runs, and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
n8nRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "n8n")).orderBy(integrations.name),
|
||||
db.select().from(n8nState),
|
||||
n8nSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = new Date();
|
||||
res.json({
|
||||
n8ns: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isN8nFresh(status ?? undefined, now.getTime());
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state ? { workflows: workflowsOf(state), executionsKnown: state.executionsKnown, updatedAt: state.updatedAt } : null,
|
||||
problems: state && fresh ? n8nProblems(r.id, r.name, state, now) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, netboxState, servers } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { makeExclusion } from "../services/consistency.js";
|
||||
import { ipsOf, isNetBoxFresh, netboxProblems, netboxSyncStatuses, serversMissingFromNetBox } from "../services/netboxSync.js";
|
||||
import { getSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const netboxRouter = Router();
|
||||
netboxRouter.use(requireAuth);
|
||||
|
||||
const parseIps = (stored: string | null): string[] => {
|
||||
if (!stored) return [];
|
||||
try {
|
||||
const v = JSON.parse(stored);
|
||||
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
/** Every NetBox, how it is, from what the last read stored: its workers, the counts, and which of the servers this app manages have no address in it. Everyone signed in can see it. */
|
||||
netboxRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses, serverRows, settings] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "netbox")).orderBy(integrations.name),
|
||||
db.select().from(netboxState),
|
||||
netboxSyncStatuses(),
|
||||
db.select({ id: servers.id, name: servers.name, ips: servers.ipAddresses }).from(servers),
|
||||
getSettings(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const serverList = serverRows.map((s) => ({ id: s.id, name: s.name, ips: parseIps(s.ips) }));
|
||||
const excluded = makeExclusion(settings.consistency.excludedRanges);
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
netboxes: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isNetBoxFresh(status ?? undefined, now);
|
||||
const ips = state ? ipsOf(state) : null;
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state
|
||||
? {
|
||||
version: state.version,
|
||||
workersRunning: state.workersRunning,
|
||||
devices: state.devices,
|
||||
virtualMachines: state.virtualMachines,
|
||||
prefixes: state.prefixes,
|
||||
ipAddresses: state.ipAddresses,
|
||||
/** Null when NetBox's addresses could not be listed, so nothing can be said about what is missing from it. */
|
||||
missing: ips ? serversMissingFromNetBox(serverList, ips, excluded) : null,
|
||||
/** More addresses than were kept: what is missing may be in the part that was not read. */
|
||||
truncated: ips !== null && state.ipAddresses !== null && state.ipAddresses > ips.length,
|
||||
updatedAt: state.updatedAt,
|
||||
}
|
||||
: null,
|
||||
problems: state && fresh ? netboxProblems(r.id, r.name, state) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, nextcloudState } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses } from "../services/nextcloudSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const nextcloudRouter = Router();
|
||||
nextcloudRouter.use(requireAuth);
|
||||
|
||||
/** Every Nextcloud, how it is, from what the last read stored: maintenance mode, the data disk, the users, the app updates and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
nextcloudRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "nextcloud")).orderBy(integrations.name),
|
||||
db.select().from(nextcloudState),
|
||||
nextcloudSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
nextclouds: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isNextcloudFresh(status ?? undefined, now);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state
|
||||
? {
|
||||
version: state.version,
|
||||
maintenance: state.maintenance,
|
||||
needsUpgrade: state.needsUpgrade,
|
||||
detail: state.detail,
|
||||
freeBytes: state.freeBytes,
|
||||
users: state.users,
|
||||
files: state.files,
|
||||
active24h: state.active24h,
|
||||
appUpdates: state.appUpdates,
|
||||
updatedAt: state.updatedAt,
|
||||
}
|
||||
: null,
|
||||
problems: state && fresh ? nextcloudProblems(r.id, r.name, state) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -80,6 +80,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
prometheus: { label: "Prometheus", link: "/prometheus" },
|
||||
immich: { label: "Immich", link: "/immich" },
|
||||
paperless: { label: "Paperless-ngx", link: "/paperless" },
|
||||
n8n: { label: "n8n", link: "/n8n" },
|
||||
netbox: { label: "NetBox", link: "/netbox" },
|
||||
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -118,6 +121,12 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return { type: "netbox", id: Number(m[1]), category: "services" };
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
if (m) return { type: "n8n", id: Number(m[1]), category: "automation" };
|
||||
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" };
|
||||
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
||||
@@ -250,7 +259,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -260,7 +269,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -25,6 +25,9 @@ import { authentikProblems, authentikSyncStatuses, isAuthentikFresh, type Authen
|
||||
import { isPrometheusFresh, prometheusProblems, prometheusSyncStatuses, type PrometheusProblem } from "./prometheusSync.js";
|
||||
import { immichProblems, immichSyncStatuses, isImmichFresh, type ImmichProblem } from "./immichSync.js";
|
||||
import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type PaperlessProblem } from "./paperlessSync.js";
|
||||
import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8nSync.js";
|
||||
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -92,6 +95,12 @@ export interface HealthSnapshot {
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
/** What is wrong with each Nextcloud (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/nextcloudSync.ts). */
|
||||
nextcloud?: { integrationId: number; problems: NextcloudProblem[] }[];
|
||||
/** What is wrong with each NetBox (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/netboxSync.ts). */
|
||||
netbox?: { integrationId: number; problems: NetBoxProblem[] }[];
|
||||
/** What is wrong with each n8n (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/n8nSync.ts). */
|
||||
n8n?: { integrationId: number; problems: N8nProblem[] }[];
|
||||
/** What is wrong with each Paperless-ngx (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/paperlessSync.ts). */
|
||||
paperless?: { integrationId: number; problems: PaperlessProblem[] }[];
|
||||
/** What is wrong with each Immich (a disk filling, background jobs failing), only while that reading is recent (services/immichSync.ts). */
|
||||
@@ -213,6 +222,15 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A Nextcloud whose worker has stopped, whose imports fail, or whose disk is filling.
|
||||
for (const p of snapshot.nextcloud ?? []) for (const c of p.problems) out.push({ key: c.key, source: `nextcloud:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// A NetBox whose worker has stopped, whose imports fail, or whose disk is filling.
|
||||
for (const p of snapshot.netbox ?? []) for (const c of p.problems) out.push({ key: c.key, source: `netbox:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// A N8n whose worker has stopped, whose imports fail, or whose disk is filling.
|
||||
for (const p of snapshot.n8n ?? []) for (const c of p.problems) out.push({ key: c.key, source: `n8n:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// A Paperless whose worker has stopped, whose imports fail, or whose disk is filling.
|
||||
for (const p of snapshot.paperless ?? []) for (const c of p.problems) out.push({ key: c.key, source: `paperless:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
@@ -458,6 +476,60 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Nextcloud: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "nextcloud"), eq(integrations.enabled, true)));
|
||||
if (rowsL.length > 0) {
|
||||
const statuses = await nextcloudSyncStatuses();
|
||||
for (const r of rowsL) {
|
||||
const [state] = await db.select().from(nextcloudState).where(eq(nextcloudState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isNextcloudFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`nextcloud:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.nextcloud ??= []).push({ integrationId: r.id, problems: nextcloudProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Nextcloud:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// NetBox: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "netbox"), eq(integrations.enabled, true)));
|
||||
if (rowsL.length > 0) {
|
||||
const statuses = await netboxSyncStatuses();
|
||||
for (const r of rowsL) {
|
||||
const [state] = await db.select().from(netboxState).where(eq(netboxState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isNetBoxFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`netbox:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.netbox ??= []).push({ integrationId: r.id, problems: netboxProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge NetBox:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// N8n: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "n8n"), eq(integrations.enabled, true)));
|
||||
if (rowsL.length > 0) {
|
||||
const statuses = await n8nSyncStatuses();
|
||||
for (const r of rowsL) {
|
||||
const [state] = await db.select().from(n8nState).where(eq(n8nState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isN8nFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`n8n:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.n8n ??= []).push({ integrationId: r.id, problems: n8nProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge N8n:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Paperless: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -121,7 +121,8 @@ export interface ImmichProblem {
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
/** In the same units the pages show (binary, written GB), so an alert and the tile beside it agree. */
|
||||
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Immich, from what is stored for it. */
|
||||
export function immichProblems(id: number, name: string, s: Pick<StateRow, "diskSize" | "diskUsed" | "diskPercent" | "queues">): ImmichProblem[] {
|
||||
|
||||
@@ -33,6 +33,12 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,11 +5,14 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
||||
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
||||
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
|
||||
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
|
||||
@@ -64,6 +67,50 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Nextcloud, for /metrics: whether it is in maintenance mode, the room left for the data, who was active and the app updates waiting. Only while the reading is recent. */
|
||||
async function addNextcloudMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await nextcloudSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "nextcloud"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(nextcloudState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isNextcloudFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
out.add("homelab_nextcloud_maintenance_mode", "gauge", "1 if Nextcloud is in maintenance mode, else 0.", s.maintenance ? 1 : 0, labels);
|
||||
if (s.freeBytes !== null) out.add("homelab_nextcloud_free_bytes", "gauge", "Bytes free where Nextcloud keeps its data.", s.freeBytes, labels);
|
||||
if (s.active24h !== null) out.add("homelab_nextcloud_active_users_24h", "gauge", "Users active in the last 24 hours.", s.active24h, labels);
|
||||
if (s.appUpdates !== null) out.add("homelab_nextcloud_app_updates", "gauge", "How many apps have an update waiting.", s.appUpdates, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each NetBox, for /metrics: the workers running and what it holds. Only while the reading is recent. */
|
||||
async function addNetBoxMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await netboxSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "netbox"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(netboxState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isNetBoxFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
if (s.workersRunning !== null) out.add("homelab_netbox_workers", "gauge", "How many background workers NetBox has running.", s.workersRunning, labels);
|
||||
if (s.devices !== null) out.add("homelab_netbox_devices", "gauge", "How many devices NetBox holds.", s.devices, labels);
|
||||
if (s.ipAddresses !== null) out.add("homelab_netbox_ip_addresses", "gauge", "How many IP addresses NetBox holds.", s.ipAddresses, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each n8n, for /metrics: how many workflows there are, are switched on and have failed on their last run. Only while the reading is recent. */
|
||||
async function addN8nMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await n8nSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "n8n"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(n8nState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isN8nFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
const workflows = n8nWorkflowsOf(s);
|
||||
out.add("homelab_n8n_workflows", "gauge", "How many workflows n8n has.", workflows.length, labels);
|
||||
out.add("homelab_n8n_workflows_active", "gauge", "How many of them are switched on.", workflows.filter((w) => w.active).length, labels);
|
||||
if (s.executionsKnown) out.add("homelab_n8n_workflows_failing", "gauge", "Active workflows whose last run failed.", workflows.filter((w) => w.active && (w.lastStatus === "error" || w.lastStatus === "crashed")).length, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
|
||||
async function addPaperlessMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await paperlessSyncStatuses();
|
||||
@@ -337,6 +384,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addAuthentikMetrics(out, now);
|
||||
await addImmichMetrics(out, now);
|
||||
await addPaperlessMetrics(out, now);
|
||||
await addN8nMetrics(out, now);
|
||||
await addNetBoxMetrics(out, now);
|
||||
await addNextcloudMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Keeps what n8n says up to date: every five minutes each n8n integration is read (its workflows, and from the latest runs each one's last outcome and failures in the last day)
|
||||
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to n8n.
|
||||
*
|
||||
* What is raised is an **active workflow whose last run failed** — an automation that has quietly stopped doing its job — and it clears by itself when the workflow next succeeds.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, n8nState } from "../db/schema.js";
|
||||
import { createN8nAdapter, FAILED_STATUSES, type N8nRead, type N8nWorkflow } from "../integrations/n8n/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
/** A failure older than this is no longer news (a workflow that ran once, failed, and is never run again). */
|
||||
export const FAILED_WITHIN_MS = 7 * 86_400_000;
|
||||
const STATUS_FLAG = "n8nSync";
|
||||
|
||||
export interface N8nSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, N8nSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, N8nSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function n8nSyncStatuses(): Promise<Record<number, N8nSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isN8nFresh = (status: N8nSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof n8nState.$inferSelect;
|
||||
|
||||
export const workflowsOf = (row: Pick<StateRow, "workflows">): N8nWorkflow[] => {
|
||||
try {
|
||||
const v = JSON.parse(row.workflows);
|
||||
return Array.isArray(v) ? (v as N8nWorkflow[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: N8nRead, now: Date = new Date()): Promise<void> {
|
||||
const values = { workflows: JSON.stringify(r.workflows), executionsKnown: r.executionsKnown, updatedAt: now.toISOString() };
|
||||
await db.insert(n8nState).values({ integrationId, ...values }).onConflictDoUpdate({ target: n8nState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one n8n and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncN8n(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<N8nRead>): Promise<N8nSyncStatus> {
|
||||
let status: N8nSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "n8n") throw new Error("not an n8n integration");
|
||||
const result = await (read ? read(loaded.config) : createN8nAdapter(loaded.config as never).read(now));
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[n8n] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[n8n] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllN8n(now: Date = new Date()): Promise<Record<number, N8nSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "n8n"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, N8nSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncN8n(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetN8n(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface N8nProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const ago = (iso: string, now: Date) => {
|
||||
const min = Math.max(0, Math.round((now.getTime() - Date.parse(iso)) / 60_000));
|
||||
return min < 90 ? `${min} min ago` : min < 2 * 1440 ? `${Math.round(min / 60)} h ago` : `${Math.round(min / 1440)} days ago`;
|
||||
};
|
||||
|
||||
/** Pure: the active workflows whose last run failed, within the last week. A workflow that is switched off is a person's choice and says nothing. */
|
||||
export function n8nProblems(id: number, name: string, s: Pick<StateRow, "workflows" | "executionsKnown">, now: Date = new Date()): N8nProblem[] {
|
||||
if (!s.executionsKnown) return [];
|
||||
const out: N8nProblem[] = [];
|
||||
for (const w of workflowsOf(s)) {
|
||||
if (!w.active || !w.lastStatus || !w.lastAt || !FAILED_STATUSES.has(w.lastStatus)) continue;
|
||||
if (now.getTime() - Date.parse(w.lastAt) > FAILED_WITHIN_MS) continue;
|
||||
const many = w.failed24h > 1 ? `, and ${w.failed24h} runs failed in the last day` : "";
|
||||
out.push({ key: `n8n-failed:${id}:${w.id}`, severity: "warning", message: `${name}: the workflow “${w.name}” failed on its last run (${ago(w.lastAt, now)}${w.lastStatus === "crashed" ? ", crashed" : ""})${many}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initN8nSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllN8n().catch((err) => console.error("[n8n] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 25_000);
|
||||
console.log(`[n8n] reading n8n every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* Keeps what NetBox says up to date: every five minutes each NetBox integration is read (its version, how many background workers are running, the counts, and the IP addresses it
|
||||
* holds) and the state is stored. Pages and alerts are built from what is stored, so none of them talks to NetBox.
|
||||
*
|
||||
* Two uses. **Is it working**: the workers (a separate container in the Docker image) run NetBox's webhooks, scripts and jobs, and can be down while the pages load. **Is it complete**:
|
||||
* the servers this app manages whose addresses are nowhere in NetBox, which is a list on the page and not an alert — documentation that is behind is not an outage.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, netboxState } from "../db/schema.js";
|
||||
import { createNetBoxAdapter, type NetBoxRead } from "../integrations/netbox/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "netboxSync";
|
||||
|
||||
export interface NetBoxSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, NetBoxSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, NetBoxSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function netboxSyncStatuses(): Promise<Record<number, NetBoxSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isNetBoxFresh = (status: NetBoxSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof netboxState.$inferSelect;
|
||||
|
||||
/** The addresses NetBox holds; null when they could not be asked. */
|
||||
export const ipsOf = (row: Pick<StateRow, "ips">): string[] | null => {
|
||||
if (row.ips === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.ips);
|
||||
return Array.isArray(v) ? (v as string[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: NetBoxRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
workersRunning: r.workersRunning,
|
||||
devices: r.devices,
|
||||
virtualMachines: r.virtualMachines,
|
||||
prefixes: r.prefixes,
|
||||
ipAddresses: r.ipAddresses,
|
||||
ips: r.ips && JSON.stringify(r.ips),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(netboxState).values({ integrationId, ...values }).onConflictDoUpdate({ target: netboxState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one NetBox and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncNetBox(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<NetBoxRead>): Promise<NetBoxSyncStatus> {
|
||||
let status: NetBoxSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "netbox") throw new Error("not a NetBox integration");
|
||||
const result = await (read ? read(loaded.config) : createNetBoxAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[netbox] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[netbox] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllNetBox(now: Date = new Date()): Promise<Record<number, NetBoxSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "netbox"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, NetBoxSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncNetBox(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetNetBox(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface NetBoxProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** Pure: what is wrong with one NetBox, from what is stored for it. */
|
||||
export function netboxProblems(id: number, name: string, s: Pick<StateRow, "workersRunning">): NetBoxProblem[] {
|
||||
if (s.workersRunning === 0) return [{ key: `netbox-workers:${id}`, severity: "warning", message: `${name}: no background worker is running — its webhooks, scripts, jobs and reports are not being run` }];
|
||||
return [];
|
||||
}
|
||||
|
||||
// ─── Is it complete ─────────────────────────────────────────────────────────
|
||||
|
||||
/** An address that says nothing about whether a machine is documented: loopback, link-local, and the like. */
|
||||
const skip = (ip: string) => ip.startsWith("127.") || ip === "::1" || ip.startsWith("169.254.") || ip.toLowerCase().startsWith("fe80:");
|
||||
|
||||
/**
|
||||
* Pure: the servers none of whose addresses are in NetBox. A server with no usable address is not judged (nothing is known to look for); one is "missing" only when it has
|
||||
* addresses and not one of them is in NetBox's list. `excluded` takes out addresses that are the same on every host (Docker's bridge) before they are weighed.
|
||||
*/
|
||||
export function serversMissingFromNetBox(servers: { id: number; name: string; ips: string[] }[], netboxIps: string[], excluded: (ip: string) => boolean = () => false): { id: number; name: string; ips: string[] }[] {
|
||||
const known = new Set(netboxIps.map((i) => i.trim().toLowerCase()));
|
||||
const out: { id: number; name: string; ips: string[] }[] = [];
|
||||
for (const s of servers) {
|
||||
const own = [...new Set(s.ips.map((i) => i.trim().toLowerCase()).filter((i) => i !== "" && !skip(i) && !excluded(i)))];
|
||||
if (own.length === 0) continue;
|
||||
if (!own.some((i) => known.has(i))) out.push({ id: s.id, name: s.name, ips: own });
|
||||
}
|
||||
return out.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initNetBoxSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllNetBox().catch((err) => console.error("[netbox] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 23_000);
|
||||
console.log(`[netbox] reading NetBox every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Keeps what Nextcloud says up to date: every five minutes each Nextcloud integration is read (whether it is in maintenance mode or needs its database upgrading, and — with an
|
||||
* administrator's app password — how much room is left, the users and files, and the app updates waiting) and the state is stored. Pages and alerts are built from what is stored, so
|
||||
* none of them talks to Nextcloud.
|
||||
*
|
||||
* What is raised is what stops anyone signing in (maintenance mode left on, a database that wants upgrading) and the data disk running out.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, nextcloudState } from "../db/schema.js";
|
||||
import { createNextcloudAdapter, type NextcloudRead } from "../integrations/nextcloud/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
/** Room left for the data: a warning below this, critical below the other. */
|
||||
export const FREE_WARN_BYTES = 20e9;
|
||||
export const FREE_CRITICAL_BYTES = 5e9;
|
||||
const STATUS_FLAG = "nextcloudSync";
|
||||
|
||||
export interface NextcloudSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, NextcloudSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, NextcloudSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function nextcloudSyncStatuses(): Promise<Record<number, NextcloudSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isNextcloudFresh = (status: NextcloudSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof nextcloudState.$inferSelect;
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: NextcloudRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
maintenance: r.maintenance,
|
||||
needsUpgrade: r.needsUpgrade,
|
||||
detail: r.detail,
|
||||
freeBytes: r.freeBytes,
|
||||
users: r.users,
|
||||
files: r.files,
|
||||
active24h: r.active24h,
|
||||
appUpdates: r.appUpdates,
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(nextcloudState).values({ integrationId, ...values }).onConflictDoUpdate({ target: nextcloudState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Nextcloud and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncNextcloud(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<NextcloudRead>): Promise<NextcloudSyncStatus> {
|
||||
let status: NextcloudSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "nextcloud") throw new Error("not a Nextcloud integration");
|
||||
const result = await (read ? read(loaded.config) : createNextcloudAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[nextcloud] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[nextcloud] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllNextcloud(now: Date = new Date()): Promise<Record<number, NextcloudSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "nextcloud"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, NextcloudSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncNextcloud(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetNextcloud(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface NextcloudProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** In the same units the pages show (binary, written GB), so an alert and the tile beside it agree. */
|
||||
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Nextcloud, from what is stored for it. */
|
||||
export function nextcloudProblems(id: number, name: string, s: Pick<StateRow, "maintenance" | "needsUpgrade" | "freeBytes">): NextcloudProblem[] {
|
||||
const out: NextcloudProblem[] = [];
|
||||
if (s.needsUpgrade) out.push({ key: `nextcloud-upgrade:${id}`, severity: "critical", message: `${name}: its database needs upgrading — nobody can use it until "occ upgrade" has been run` });
|
||||
if (s.maintenance) out.push({ key: `nextcloud-maintenance:${id}`, severity: "warning", message: `${name} is in maintenance mode — nobody can sign in ("occ maintenance:mode --off" ends it)` });
|
||||
if (s.freeBytes !== null && s.freeBytes < FREE_WARN_BYTES) {
|
||||
out.push({ key: `nextcloud-disk:${id}`, severity: s.freeBytes < FREE_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: only ${gb(s.freeBytes)} is left where the data is kept` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initNextcloudSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllNextcloud().catch((err) => console.error("[nextcloud] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 21_000);
|
||||
console.log(`[nextcloud] reading Nextcloud every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -122,7 +122,8 @@ export interface PaperlessProblem {
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
/** In the same units the pages show (binary, written GB), so an alert and the tile beside it agree. */
|
||||
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
|
||||
const WORDS: Record<string, string> = { database: "the database", redis: "Redis", celery: "the worker that consumes documents", index: "the search index", classifier: "the classifier", sanity_check: "the sanity check" };
|
||||
|
||||
/** Pure: what is wrong with one Paperless, from what is stored for it. */
|
||||
|
||||
Reference in new issue
Block a user