Nextcloud, NetBox and n8n integrations

Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Nextcloud: database needing upgrade (critical), maintenance mode, and the data
disk running low; users, files and app updates with an admin app password.
NetBox: no background worker running, counts, and a list of the servers this
app manages that have no address in NetBox (a list, not an alert).
n8n: an active workflow whose last run failed, clearing when it next succeeds.

Disk messages for Immich, Paperless and Nextcloud now use the same units as
the pages that show them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 23:13:13 +02:00
1 parent a16996d975
commit 4ff0b44943
46 files changed
+10137 -10

No files matched your search

+11 -2
View File
@@ -80,6 +80,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
prometheus: { label: "Prometheus", link: "/prometheus" },
immich: { label: "Immich", link: "/immich" },
paperless: { label: "Paperless-ngx", link: "/paperless" },
n8n: { label: "n8n", link: "/n8n" },
netbox: { label: "NetBox", link: "/netbox" },
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -118,6 +121,12 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
if (m) return { type: "nextcloud", id: Number(m[1]), category: key.startsWith("nextcloud-disk") ? "disk" : "services" };
m = /^netbox-(?:workers):(\d+)/.exec(key);
if (m) return { type: "netbox", id: Number(m[1]), category: "services" };
m = /^n8n-(?:failed):(\d+)/.exec(key);
if (m) return { type: "n8n", id: Number(m[1]), category: "automation" };
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" };
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
@@ -250,7 +259,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -260,7 +269,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+73 -1
View File
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { authentikState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -25,6 +25,9 @@ import { authentikProblems, authentikSyncStatuses, isAuthentikFresh, type Authen
import { isPrometheusFresh, prometheusProblems, prometheusSyncStatuses, type PrometheusProblem } from "./prometheusSync.js";
import { immichProblems, immichSyncStatuses, isImmichFresh, type ImmichProblem } from "./immichSync.js";
import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type PaperlessProblem } from "./paperlessSync.js";
import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8nSync.js";
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -92,6 +95,12 @@ export interface HealthSnapshot {
baselines?: BaselineFinding[];
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
levels?: LevelFinding[];
/** What is wrong with each Nextcloud (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/nextcloudSync.ts). */
nextcloud?: { integrationId: number; problems: NextcloudProblem[] }[];
/** What is wrong with each NetBox (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/netboxSync.ts). */
netbox?: { integrationId: number; problems: NetBoxProblem[] }[];
/** What is wrong with each n8n (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/n8nSync.ts). */
n8n?: { integrationId: number; problems: N8nProblem[] }[];
/** What is wrong with each Paperless-ngx (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/paperlessSync.ts). */
paperless?: { integrationId: number; problems: PaperlessProblem[] }[];
/** What is wrong with each Immich (a disk filling, background jobs failing), only while that reading is recent (services/immichSync.ts). */
@@ -213,6 +222,15 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// A Nextcloud whose worker has stopped, whose imports fail, or whose disk is filling.
for (const p of snapshot.nextcloud ?? []) for (const c of p.problems) out.push({ key: c.key, source: `nextcloud:${p.integrationId}`, message: c.message, severity: c.severity });
// A NetBox whose worker has stopped, whose imports fail, or whose disk is filling.
for (const p of snapshot.netbox ?? []) for (const c of p.problems) out.push({ key: c.key, source: `netbox:${p.integrationId}`, message: c.message, severity: c.severity });
// A N8n whose worker has stopped, whose imports fail, or whose disk is filling.
for (const p of snapshot.n8n ?? []) for (const c of p.problems) out.push({ key: c.key, source: `n8n:${p.integrationId}`, message: c.message, severity: c.severity });
// A Paperless whose worker has stopped, whose imports fail, or whose disk is filling.
for (const p of snapshot.paperless ?? []) for (const c of p.problems) out.push({ key: c.key, source: `paperless:${p.integrationId}`, message: c.message, severity: c.severity });
@@ -458,6 +476,60 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// Nextcloud: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "nextcloud"), eq(integrations.enabled, true)));
if (rowsL.length > 0) {
const statuses = await nextcloudSyncStatuses();
for (const r of rowsL) {
const [state] = await db.select().from(nextcloudState).where(eq(nextcloudState.integrationId, r.id)).limit(1);
if (!state || !isNextcloudFresh(statuses[r.id], Date.now())) {
held.add(`nextcloud:${r.id}`);
continue;
}
(snapshot.nextcloud ??= []).push({ integrationId: r.id, problems: nextcloudProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge Nextcloud:", err instanceof Error ? err.message : err);
}
// NetBox: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "netbox"), eq(integrations.enabled, true)));
if (rowsL.length > 0) {
const statuses = await netboxSyncStatuses();
for (const r of rowsL) {
const [state] = await db.select().from(netboxState).where(eq(netboxState.integrationId, r.id)).limit(1);
if (!state || !isNetBoxFresh(statuses[r.id], Date.now())) {
held.add(`netbox:${r.id}`);
continue;
}
(snapshot.netbox ??= []).push({ integrationId: r.id, problems: netboxProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge NetBox:", err instanceof Error ? err.message : err);
}
// N8n: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "n8n"), eq(integrations.enabled, true)));
if (rowsL.length > 0) {
const statuses = await n8nSyncStatuses();
for (const r of rowsL) {
const [state] = await db.select().from(n8nState).where(eq(n8nState.integrationId, r.id)).limit(1);
if (!state || !isN8nFresh(statuses[r.id], Date.now())) {
held.add(`n8n:${r.id}`);
continue;
}
(snapshot.n8n ??= []).push({ integrationId: r.id, problems: n8nProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge N8n:", err instanceof Error ? err.message : err);
}
// Paperless: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
+2 -1
View File
@@ -121,7 +121,8 @@ export interface ImmichProblem {
severity: "warning" | "critical";
}
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
/** In the same units the pages show (binary, written GB), so an alert and the tile beside it agree. */
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
/** Pure: what is wrong with one Immich, from what is stored for it. */
export function immichProblems(id: number, name: string, s: Pick<StateRow, "diskSize" | "diskUsed" | "diskPercent" | "queues">): ImmichProblem[] {
+6
View File
@@ -33,6 +33,12 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^nextcloud-(?:maintenance|upgrade|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^netbox-(?:workers):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^n8n-(?:failed):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
+51 -1
View File
@@ -5,11 +5,14 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
@@ -64,6 +67,50 @@ class Exposition {
}
}
/** Each Nextcloud, for /metrics: whether it is in maintenance mode, the room left for the data, who was active and the app updates waiting. Only while the reading is recent. */
async function addNextcloudMetrics(out: Exposition, now: Date) {
const statuses = await nextcloudSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "nextcloud"))).map((r) => [r.id, r]));
for (const s of await db.select().from(nextcloudState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isNextcloudFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
out.add("homelab_nextcloud_maintenance_mode", "gauge", "1 if Nextcloud is in maintenance mode, else 0.", s.maintenance ? 1 : 0, labels);
if (s.freeBytes !== null) out.add("homelab_nextcloud_free_bytes", "gauge", "Bytes free where Nextcloud keeps its data.", s.freeBytes, labels);
if (s.active24h !== null) out.add("homelab_nextcloud_active_users_24h", "gauge", "Users active in the last 24 hours.", s.active24h, labels);
if (s.appUpdates !== null) out.add("homelab_nextcloud_app_updates", "gauge", "How many apps have an update waiting.", s.appUpdates, labels);
}
}
/** Each NetBox, for /metrics: the workers running and what it holds. Only while the reading is recent. */
async function addNetBoxMetrics(out: Exposition, now: Date) {
const statuses = await netboxSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "netbox"))).map((r) => [r.id, r]));
for (const s of await db.select().from(netboxState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isNetBoxFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
if (s.workersRunning !== null) out.add("homelab_netbox_workers", "gauge", "How many background workers NetBox has running.", s.workersRunning, labels);
if (s.devices !== null) out.add("homelab_netbox_devices", "gauge", "How many devices NetBox holds.", s.devices, labels);
if (s.ipAddresses !== null) out.add("homelab_netbox_ip_addresses", "gauge", "How many IP addresses NetBox holds.", s.ipAddresses, labels);
}
}
/** Each n8n, for /metrics: how many workflows there are, are switched on and have failed on their last run. Only while the reading is recent. */
async function addN8nMetrics(out: Exposition, now: Date) {
const statuses = await n8nSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "n8n"))).map((r) => [r.id, r]));
for (const s of await db.select().from(n8nState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isN8nFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
const workflows = n8nWorkflowsOf(s);
out.add("homelab_n8n_workflows", "gauge", "How many workflows n8n has.", workflows.length, labels);
out.add("homelab_n8n_workflows_active", "gauge", "How many of them are switched on.", workflows.filter((w) => w.active).length, labels);
if (s.executionsKnown) out.add("homelab_n8n_workflows_failing", "gauge", "Active workflows whose last run failed.", workflows.filter((w) => w.active && (w.lastStatus === "error" || w.lastStatus === "crashed")).length, labels);
}
}
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
async function addPaperlessMetrics(out: Exposition, now: Date) {
const statuses = await paperlessSyncStatuses();
@@ -337,6 +384,9 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addAuthentikMetrics(out, now);
await addImmichMetrics(out, now);
await addPaperlessMetrics(out, now);
await addN8nMetrics(out, now);
await addNetBoxMetrics(out, now);
await addNextcloudMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+133
View File
@@ -0,0 +1,133 @@
/**
* Keeps what n8n says up to date: every five minutes each n8n integration is read (its workflows, and from the latest runs each one's last outcome and failures in the last day)
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to n8n.
*
* What is raised is an **active workflow whose last run failed** — an automation that has quietly stopped doing its job — and it clears by itself when the workflow next succeeds.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, n8nState } from "../db/schema.js";
import { createN8nAdapter, FAILED_STATUSES, type N8nRead, type N8nWorkflow } from "../integrations/n8n/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
/** A failure older than this is no longer news (a workflow that ran once, failed, and is never run again). */
export const FAILED_WITHIN_MS = 7 * 86_400_000;
const STATUS_FLAG = "n8nSync";
export interface N8nSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, N8nSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, N8nSyncStatus>) : {};
} catch {
return {};
}
}
export async function n8nSyncStatuses(): Promise<Record<number, N8nSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isN8nFresh = (status: N8nSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof n8nState.$inferSelect;
export const workflowsOf = (row: Pick<StateRow, "workflows">): N8nWorkflow[] => {
try {
const v = JSON.parse(row.workflows);
return Array.isArray(v) ? (v as N8nWorkflow[]) : [];
} catch {
return [];
}
};
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: N8nRead, now: Date = new Date()): Promise<void> {
const values = { workflows: JSON.stringify(r.workflows), executionsKnown: r.executionsKnown, updatedAt: now.toISOString() };
await db.insert(n8nState).values({ integrationId, ...values }).onConflictDoUpdate({ target: n8nState.integrationId, set: values });
}
/** Reads one n8n and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncN8n(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<N8nRead>): Promise<N8nSyncStatus> {
let status: N8nSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "n8n") throw new Error("not an n8n integration");
const result = await (read ? read(loaded.config) : createN8nAdapter(loaded.config as never).read(now));
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[n8n] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[n8n] couldn't record the result:", err);
}
return status;
}
export async function syncAllN8n(now: Date = new Date()): Promise<Record<number, N8nSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "n8n"), eq(integrations.enabled, true)));
const out: Record<number, N8nSyncStatus> = {};
for (const r of rows) out[r.id] = await syncN8n(r.id, now);
return out;
}
export async function forgetN8n(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface N8nProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const ago = (iso: string, now: Date) => {
const min = Math.max(0, Math.round((now.getTime() - Date.parse(iso)) / 60_000));
return min < 90 ? `${min} min ago` : min < 2 * 1440 ? `${Math.round(min / 60)} h ago` : `${Math.round(min / 1440)} days ago`;
};
/** Pure: the active workflows whose last run failed, within the last week. A workflow that is switched off is a person's choice and says nothing. */
export function n8nProblems(id: number, name: string, s: Pick<StateRow, "workflows" | "executionsKnown">, now: Date = new Date()): N8nProblem[] {
if (!s.executionsKnown) return [];
const out: N8nProblem[] = [];
for (const w of workflowsOf(s)) {
if (!w.active || !w.lastStatus || !w.lastAt || !FAILED_STATUSES.has(w.lastStatus)) continue;
if (now.getTime() - Date.parse(w.lastAt) > FAILED_WITHIN_MS) continue;
const many = w.failed24h > 1 ? `, and ${w.failed24h} runs failed in the last day` : "";
out.push({ key: `n8n-failed:${id}:${w.id}`, severity: "warning", message: `${name}: the workflow “${w.name}” failed on its last run (${ago(w.lastAt, now)}${w.lastStatus === "crashed" ? ", crashed" : ""})${many}` });
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initN8nSync(): void {
if (timer) return;
const pass = () => void syncAllN8n().catch((err) => console.error("[n8n] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 25_000);
console.log(`[n8n] reading n8n every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+151
View File
@@ -0,0 +1,151 @@
/**
* Keeps what NetBox says up to date: every five minutes each NetBox integration is read (its version, how many background workers are running, the counts, and the IP addresses it
* holds) and the state is stored. Pages and alerts are built from what is stored, so none of them talks to NetBox.
*
* Two uses. **Is it working**: the workers (a separate container in the Docker image) run NetBox's webhooks, scripts and jobs, and can be down while the pages load. **Is it complete**:
* the servers this app manages whose addresses are nowhere in NetBox, which is a list on the page and not an alert — documentation that is behind is not an outage.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, netboxState } from "../db/schema.js";
import { createNetBoxAdapter, type NetBoxRead } from "../integrations/netbox/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
const STATUS_FLAG = "netboxSync";
export interface NetBoxSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, NetBoxSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, NetBoxSyncStatus>) : {};
} catch {
return {};
}
}
export async function netboxSyncStatuses(): Promise<Record<number, NetBoxSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isNetBoxFresh = (status: NetBoxSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof netboxState.$inferSelect;
/** The addresses NetBox holds; null when they could not be asked. */
export const ipsOf = (row: Pick<StateRow, "ips">): string[] | null => {
if (row.ips === null) return null;
try {
const v = JSON.parse(row.ips);
return Array.isArray(v) ? (v as string[]) : null;
} catch {
return null;
}
};
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: NetBoxRead, now: Date = new Date()): Promise<void> {
const values = {
version: r.version,
workersRunning: r.workersRunning,
devices: r.devices,
virtualMachines: r.virtualMachines,
prefixes: r.prefixes,
ipAddresses: r.ipAddresses,
ips: r.ips && JSON.stringify(r.ips),
updatedAt: now.toISOString(),
};
await db.insert(netboxState).values({ integrationId, ...values }).onConflictDoUpdate({ target: netboxState.integrationId, set: values });
}
/** Reads one NetBox and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncNetBox(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<NetBoxRead>): Promise<NetBoxSyncStatus> {
let status: NetBoxSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "netbox") throw new Error("not a NetBox integration");
const result = await (read ? read(loaded.config) : createNetBoxAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[netbox] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[netbox] couldn't record the result:", err);
}
return status;
}
export async function syncAllNetBox(now: Date = new Date()): Promise<Record<number, NetBoxSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "netbox"), eq(integrations.enabled, true)));
const out: Record<number, NetBoxSyncStatus> = {};
for (const r of rows) out[r.id] = await syncNetBox(r.id, now);
return out;
}
export async function forgetNetBox(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface NetBoxProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
/** Pure: what is wrong with one NetBox, from what is stored for it. */
export function netboxProblems(id: number, name: string, s: Pick<StateRow, "workersRunning">): NetBoxProblem[] {
if (s.workersRunning === 0) return [{ key: `netbox-workers:${id}`, severity: "warning", message: `${name}: no background worker is running — its webhooks, scripts, jobs and reports are not being run` }];
return [];
}
// ─── Is it complete ─────────────────────────────────────────────────────────
/** An address that says nothing about whether a machine is documented: loopback, link-local, and the like. */
const skip = (ip: string) => ip.startsWith("127.") || ip === "::1" || ip.startsWith("169.254.") || ip.toLowerCase().startsWith("fe80:");
/**
* Pure: the servers none of whose addresses are in NetBox. A server with no usable address is not judged (nothing is known to look for); one is "missing" only when it has
* addresses and not one of them is in NetBox's list. `excluded` takes out addresses that are the same on every host (Docker's bridge) before they are weighed.
*/
export function serversMissingFromNetBox(servers: { id: number; name: string; ips: string[] }[], netboxIps: string[], excluded: (ip: string) => boolean = () => false): { id: number; name: string; ips: string[] }[] {
const known = new Set(netboxIps.map((i) => i.trim().toLowerCase()));
const out: { id: number; name: string; ips: string[] }[] = [];
for (const s of servers) {
const own = [...new Set(s.ips.map((i) => i.trim().toLowerCase()).filter((i) => i !== "" && !skip(i) && !excluded(i)))];
if (own.length === 0) continue;
if (!own.some((i) => known.has(i))) out.push({ id: s.id, name: s.name, ips: own });
}
return out.sort((a, b) => a.name.localeCompare(b.name));
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initNetBoxSync(): void {
if (timer) return;
const pass = () => void syncAllNetBox().catch((err) => console.error("[netbox] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 23_000);
console.log(`[netbox] reading NetBox every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+133
View File
@@ -0,0 +1,133 @@
/**
* Keeps what Nextcloud says up to date: every five minutes each Nextcloud integration is read (whether it is in maintenance mode or needs its database upgrading, and — with an
* administrator's app password — how much room is left, the users and files, and the app updates waiting) and the state is stored. Pages and alerts are built from what is stored, so
* none of them talks to Nextcloud.
*
* What is raised is what stops anyone signing in (maintenance mode left on, a database that wants upgrading) and the data disk running out.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, nextcloudState } from "../db/schema.js";
import { createNextcloudAdapter, type NextcloudRead } from "../integrations/nextcloud/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
/** Room left for the data: a warning below this, critical below the other. */
export const FREE_WARN_BYTES = 20e9;
export const FREE_CRITICAL_BYTES = 5e9;
const STATUS_FLAG = "nextcloudSync";
export interface NextcloudSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, NextcloudSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, NextcloudSyncStatus>) : {};
} catch {
return {};
}
}
export async function nextcloudSyncStatuses(): Promise<Record<number, NextcloudSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isNextcloudFresh = (status: NextcloudSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof nextcloudState.$inferSelect;
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: NextcloudRead, now: Date = new Date()): Promise<void> {
const values = {
version: r.version,
maintenance: r.maintenance,
needsUpgrade: r.needsUpgrade,
detail: r.detail,
freeBytes: r.freeBytes,
users: r.users,
files: r.files,
active24h: r.active24h,
appUpdates: r.appUpdates,
updatedAt: now.toISOString(),
};
await db.insert(nextcloudState).values({ integrationId, ...values }).onConflictDoUpdate({ target: nextcloudState.integrationId, set: values });
}
/** Reads one Nextcloud and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncNextcloud(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<NextcloudRead>): Promise<NextcloudSyncStatus> {
let status: NextcloudSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "nextcloud") throw new Error("not a Nextcloud integration");
const result = await (read ? read(loaded.config) : createNextcloudAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[nextcloud] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[nextcloud] couldn't record the result:", err);
}
return status;
}
export async function syncAllNextcloud(now: Date = new Date()): Promise<Record<number, NextcloudSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "nextcloud"), eq(integrations.enabled, true)));
const out: Record<number, NextcloudSyncStatus> = {};
for (const r of rows) out[r.id] = await syncNextcloud(r.id, now);
return out;
}
export async function forgetNextcloud(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface NextcloudProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
/** In the same units the pages show (binary, written GB), so an alert and the tile beside it agree. */
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
/** Pure: what is wrong with one Nextcloud, from what is stored for it. */
export function nextcloudProblems(id: number, name: string, s: Pick<StateRow, "maintenance" | "needsUpgrade" | "freeBytes">): NextcloudProblem[] {
const out: NextcloudProblem[] = [];
if (s.needsUpgrade) out.push({ key: `nextcloud-upgrade:${id}`, severity: "critical", message: `${name}: its database needs upgrading — nobody can use it until "occ upgrade" has been run` });
if (s.maintenance) out.push({ key: `nextcloud-maintenance:${id}`, severity: "warning", message: `${name} is in maintenance mode — nobody can sign in ("occ maintenance:mode --off" ends it)` });
if (s.freeBytes !== null && s.freeBytes < FREE_WARN_BYTES) {
out.push({ key: `nextcloud-disk:${id}`, severity: s.freeBytes < FREE_CRITICAL_BYTES ? "critical" : "warning", message: `${name}: only ${gb(s.freeBytes)} is left where the data is kept` });
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initNextcloudSync(): void {
if (timer) return;
const pass = () => void syncAllNextcloud().catch((err) => console.error("[nextcloud] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 21_000);
console.log(`[nextcloud] reading Nextcloud every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+2 -1
View File
@@ -122,7 +122,8 @@ export interface PaperlessProblem {
severity: "warning" | "critical";
}
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
/** In the same units the pages show (binary, written GB), so an alert and the tile beside it agree. */
const gb = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(bytes >= 100 * 1024 ** 3 ? 0 : 1)} GB`;
const WORDS: Record<string, string> = { database: "the database", redis: "Redis", celery: "the worker that consumes documents", index: "the search index", classifier: "the classifier", sanity_check: "the sanity check" };
/** Pure: what is wrong with one Paperless, from what is stored for it. */