Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebe98bf15
commit
251a4a9d73
47 files changed
+24148
-7
No files matched your search
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE `prometheus_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`version` text,
|
||||
`jobs` text,
|
||||
`down` text,
|
||||
`alerts` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE `immich_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`version` text,
|
||||
`latest_version` text,
|
||||
`update_available` integer DEFAULT false NOT NULL,
|
||||
`disk_size` integer,
|
||||
`disk_used` integer,
|
||||
`disk_percent` real,
|
||||
`photos` integer,
|
||||
`videos` integer,
|
||||
`usage_bytes` integer,
|
||||
`queues` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE `paperless_state` (
|
||||
`integration_id` integer PRIMARY KEY NOT NULL,
|
||||
`version` text,
|
||||
`latest_version` text,
|
||||
`update_available` integer DEFAULT false NOT NULL,
|
||||
`storage_total` integer,
|
||||
`storage_available` integer,
|
||||
`components` text,
|
||||
`documents` integer,
|
||||
`inbox` integer,
|
||||
`failed` text,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -414,6 +414,27 @@
|
||||
"when": 1792872166034,
|
||||
"tag": "0058_flippant_sasquatch",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 59,
|
||||
"version": "6",
|
||||
"when": 1792872843134,
|
||||
"tag": "0059_secret_bullseye",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 60,
|
||||
"version": "6",
|
||||
"when": 1792873156389,
|
||||
"tag": "0060_big_kylun",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 61,
|
||||
"version": "6",
|
||||
"when": 1792873347697,
|
||||
"tag": "0061_reflective_daimon_hellstrom",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -60,6 +60,9 @@ import { awayRouter } from "./routes/away.js";
|
||||
import { duplicatiReportRouter, duplicatiRouter } from "./routes/duplicati.js";
|
||||
import { jellyfinRouter } from "./routes/jellyfin.js";
|
||||
import { authentikRouter } from "./routes/authentik.js";
|
||||
import { prometheusRouter } from "./routes/prometheus.js";
|
||||
import { immichRouter } from "./routes/immich.js";
|
||||
import { paperlessRouter } from "./routes/paperless.js";
|
||||
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
|
||||
import { publicApiRouter } from "./routes/publicApi.js";
|
||||
import { generatorRouter } from "./routes/generator.js";
|
||||
@@ -197,6 +200,9 @@ export function createApp() {
|
||||
app.use("/api/duplicati", duplicatiRouter);
|
||||
app.use("/api/jellyfin", jellyfinRouter);
|
||||
app.use("/api/authentik", authentikRouter);
|
||||
app.use("/api/prometheus", prometheusRouter);
|
||||
app.use("/api/immich", immichRouter);
|
||||
app.use("/api/paperless", paperlessRouter);
|
||||
app.use("/api/service-levels", serviceLevelsRouter);
|
||||
app.use("/api/todos", todosRouter);
|
||||
app.use("/api/notes", notesRouter);
|
||||
|
||||
@@ -555,6 +555,9 @@ export const integrationTypes = [
|
||||
"ups",
|
||||
"jellyfin",
|
||||
"authentik",
|
||||
"prometheus",
|
||||
"immich",
|
||||
"paperless",
|
||||
] as const;
|
||||
export type IntegrationType = (typeof integrationTypes)[number];
|
||||
|
||||
@@ -1161,6 +1164,62 @@ export const haReadings = sqliteTable(
|
||||
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
|
||||
);
|
||||
|
||||
// ─── Paperless-ngx ──────────────────────────────────────────────────────────
|
||||
|
||||
// How a Paperless-ngx (an integration of type "paperless") is right now, one row each, replaced at every read: its version, the disk, how each component of its system status is, the
|
||||
// document counts and the imports that failed. A column that is null could not be read. Read by services/paperlessSync.ts.
|
||||
export const paperlessState = sqliteTable("paperless_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
latestVersion: text("latest_version"),
|
||||
updateAvailable: integer("update_available", { mode: "boolean" }).notNull().default(false),
|
||||
storageTotal: integer("storage_total"), // bytes
|
||||
storageAvailable: integer("storage_available"), // bytes
|
||||
components: text("components"), // JSON: name, status, error — null: the status could not be asked (it needs a superuser)
|
||||
documents: integer("documents"),
|
||||
inbox: integer("inbox"),
|
||||
failed: text("failed"), // JSON: file, error, at — imports that failed in the last week and were not dismissed; null: could not be asked
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Immich ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// How an Immich (an integration of type "immich") is right now, one row each, replaced at every read: its version, the disk under the library, the counts and the background job queues.
|
||||
// A column that is null could not be read. Read by services/immichSync.ts.
|
||||
export const immichState = sqliteTable("immich_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
latestVersion: text("latest_version"),
|
||||
updateAvailable: integer("update_available", { mode: "boolean" }).notNull().default(false),
|
||||
diskSize: integer("disk_size"), // bytes
|
||||
diskUsed: integer("disk_used"), // bytes
|
||||
diskPercent: real("disk_percent"),
|
||||
photos: integer("photos"),
|
||||
videos: integer("videos"),
|
||||
usageBytes: integer("usage_bytes"),
|
||||
queues: text("queues"), // JSON: name, active, waiting, failed, paused — null: could not be asked
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Prometheus ─────────────────────────────────────────────────────────────
|
||||
|
||||
// How a Prometheus (an integration of type "prometheus") is right now, one row each, replaced at every read: its version, how many targets each job has up and down, which are down
|
||||
// (and since when), and the alerts its rules have firing. A column that is null could not be read. Read by services/prometheusSync.ts.
|
||||
export const prometheusState = sqliteTable("prometheus_state", {
|
||||
integrationId: integer("integration_id")
|
||||
.primaryKey()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
version: text("version"),
|
||||
jobs: text("jobs"), // JSON: job, up, down — null: the targets could not be asked
|
||||
down: text("down"), // JSON: job, instance, error, since
|
||||
alerts: text("alerts"), // JSON: name, severity, state, summary, instance, activeAt — null: could not be asked
|
||||
updatedAt: text("updated_at").notNull(), // ISO
|
||||
});
|
||||
|
||||
// ─── Authentik ──────────────────────────────────────────────────────────────
|
||||
|
||||
// How an Authentik (an integration of type "authentik") is right now, one row each, replaced at every read: its version, whether a worker is connected, how each outpost is, the
|
||||
|
||||
@@ -31,6 +31,9 @@ import { initUpsSync } from "./services/upsSync.js";
|
||||
import { initAwayMonitor } from "./services/awayMonitor.js";
|
||||
import { initJellyfinSync } from "./services/jellyfinSync.js";
|
||||
import { initAuthentikSync } from "./services/authentikSync.js";
|
||||
import { initPrometheusSync } from "./services/prometheusSync.js";
|
||||
import { initImmichSync } from "./services/immichSync.js";
|
||||
import { initPaperlessSync } from "./services/paperlessSync.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
import { backfillEndpointHourly } from "./services/endpointChecks.js";
|
||||
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
|
||||
@@ -83,6 +86,9 @@ initUpsSync();
|
||||
initAwayMonitor();
|
||||
initJellyfinSync();
|
||||
initAuthentikSync();
|
||||
initPrometheusSync();
|
||||
initImmichSync();
|
||||
initPaperlessSync();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
|
||||
|
||||
|
||||
@@ -99,6 +99,21 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
|
||||
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
paperless: [
|
||||
{ key: "url", label: "Paperless-ngx URL", secret: false, placeholder: "http://paperless.example.lan:8000" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "your profile → API Auth Token; a superuser's, for the system status" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
immich: [
|
||||
{ key: "url", label: "Immich URL", secret: false, placeholder: "http://immich.example.lan:2283" },
|
||||
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Account Settings → API Keys; read permissions are enough" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
prometheus: [
|
||||
{ key: "url", label: "Prometheus URL", secret: false, placeholder: "http://prometheus.example.lan:9090" },
|
||||
{ key: "token", label: "Bearer token (optional)", secret: true, optional: true, type: "password", placeholder: "only if something in front of Prometheus asks for one" },
|
||||
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
|
||||
],
|
||||
authentik: [
|
||||
{ key: "url", label: "Authentik URL", secret: false, placeholder: "https://auth.example.lan" },
|
||||
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Directory → Tokens and App passwords, intent API Token" },
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Immich adapter. Config: url, apiKey (made in Immich under Account Settings → API Keys; read permissions are enough: server.about, server.storage, server.statistics and
|
||||
* job.read — or queue.read on newer versions); optional: insecure.
|
||||
*
|
||||
* Reading only: the version (and whether a newer one is out), how full the disk under the library is, how many photos and videos there are and how much space they take, and the
|
||||
* background job queues — above all how many jobs have **failed** (thumbnails or metadata that were never made). Nothing is sent to Immich but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Immich's published API (`/api/server/about`, `/server/storage`, `/server/statistics`, `/server/version-check`, `/jobs` — `/queues` on newer
|
||||
* versions) and have not been checked against a live server here; every field is read defensively, and a part that cannot be read (statistics need an administrator's key) is left out.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface ImmichConfig {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface ImmichQueue {
|
||||
name: string;
|
||||
active: number;
|
||||
waiting: number;
|
||||
failed: number;
|
||||
paused: boolean;
|
||||
}
|
||||
|
||||
export interface ImmichRead {
|
||||
version: string | null;
|
||||
latestVersion: string | null;
|
||||
updateAvailable: boolean;
|
||||
/** The disk under the library; null when it could not be asked. */
|
||||
disk: { size: number; used: number; percent: number } | null;
|
||||
photos: number | null;
|
||||
videos: number | null;
|
||||
/** What the library takes, in bytes. */
|
||||
usageBytes: number | null;
|
||||
/** Null when the queues could not be asked, which is not the same as having none. */
|
||||
queues: ImmichQueue[] | null;
|
||||
}
|
||||
|
||||
export interface ImmichAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<ImmichRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
const count = (v: unknown): number => num(v) ?? 0;
|
||||
|
||||
/** Pure: "v1.120.2" and "1.120.2" are the same version, and 1.120.10 is later than 1.120.9. Null when either is not a version. */
|
||||
export function newerVersion(latest: string | null, current: string | null): boolean {
|
||||
const parts = (s: string | null) => (s ? /^v?(\d+)\.(\d+)\.(\d+)/.exec(s.trim())?.slice(1).map(Number) : undefined);
|
||||
const a = parts(latest);
|
||||
const b = parts(current);
|
||||
if (!a || !b) return false;
|
||||
for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] > b[i];
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Pure: the version from /server/about (a string like "v1.120.2"), or from /server/version ({ major, minor, patch }). */
|
||||
export function parseVersion(raw: unknown): string | null {
|
||||
const o = obj(raw);
|
||||
const v = str(o.version);
|
||||
if (v) return v;
|
||||
const [major, minor, patch] = [num(o.major), num(o.minor), num(o.patch)];
|
||||
return major !== null && minor !== null && patch !== null ? `v${major}.${minor}.${patch}` : null;
|
||||
}
|
||||
|
||||
/** Pure: the disk the library is on. */
|
||||
export function parseStorage(raw: unknown): ImmichRead["disk"] {
|
||||
const o = obj(raw);
|
||||
const size = num(o.diskSizeRaw);
|
||||
const used = num(o.diskUseRaw);
|
||||
if (size === null || used === null || size <= 0) return null;
|
||||
return { size, used, percent: num(o.diskUsagePercentage) ?? Math.round((used / size) * 1000) / 10 };
|
||||
}
|
||||
|
||||
/** Pure: the queues, from /jobs (an object keyed by queue name) or /queues (a list); each with its counts. Null when the answer is neither. */
|
||||
export function parseQueues(raw: unknown): ImmichQueue[] | null {
|
||||
const read = (name: string, q: Record<string, unknown>): ImmichQueue => {
|
||||
const counts = obj(q.jobCounts ?? q.statistics);
|
||||
const status = obj(q.queueStatus);
|
||||
return { name, active: count(counts.active), waiting: count(counts.waiting), failed: count(counts.failed), paused: q.isPaused === true || status.isPaused === true };
|
||||
};
|
||||
if (Array.isArray(raw)) return raw.map((q) => read(str(obj(q).name) ?? "queue", obj(q)));
|
||||
const o = obj(raw);
|
||||
const entries = Object.entries(o).filter(([, v]) => v !== null && typeof v === "object");
|
||||
return entries.length > 0 ? entries.map(([name, q]) => read(name, obj(q))) : null;
|
||||
}
|
||||
|
||||
export function createImmichAdapter(config: ImmichConfig): ImmichAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers = { "x-api-key": config.apiKey, Accept: "application/json" };
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}/api${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Immich did not accept the API key (HTTP ${res.status}) — check it, and that it may read the server, its storage and its jobs`);
|
||||
if (res.status === 404) throw new Error("Immich has no API at this address — check the address and port (2283)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Immich error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Immich answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(): Promise<ImmichRead> {
|
||||
// The version first and as a hard requirement: it also proves the key is accepted.
|
||||
let version: string | null;
|
||||
try {
|
||||
version = parseVersion(await get("/server/about"));
|
||||
} catch (err) {
|
||||
// Older versions have no /about; their /version is open, so a refused key is only found out by the next call.
|
||||
if (err instanceof Error && /did not accept/.test(err.message)) throw err;
|
||||
version = parseVersion(await get("/server/version"));
|
||||
}
|
||||
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
|
||||
try {
|
||||
return parse(await get(path));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const [disk, stats, check, jobs] = await Promise.all([
|
||||
soft("/server/storage", parseStorage),
|
||||
soft("/server/statistics", obj),
|
||||
soft("/server/version-check", obj),
|
||||
soft("/jobs", parseQueues),
|
||||
]);
|
||||
const queues = jobs ?? (await soft("/queues", parseQueues));
|
||||
const latest = check ? str(check.releaseVersion) : null;
|
||||
return {
|
||||
version,
|
||||
latestVersion: latest,
|
||||
updateAvailable: newerVersion(latest, version),
|
||||
disk,
|
||||
photos: stats ? num(stats.photos) : null,
|
||||
videos: stats ? num(stats.videos) : null,
|
||||
usageBytes: stats ? num(stats.usage) : null,
|
||||
queues,
|
||||
};
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/server/about").catch((err) => {
|
||||
if (err instanceof Error && /did not accept/.test(err.message)) throw err;
|
||||
return get("/server/version");
|
||||
});
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("immich", { ping, read });
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Paperless-ngx adapter. Config: url, token (made in Paperless under your profile → API Auth Token, or in the admin pages under Auth Tokens; a superuser's token is needed for the
|
||||
* system status); optional: insecure.
|
||||
*
|
||||
* Reading only: the version (and whether a newer one is out), the system status Paperless itself keeps — the database, Redis, the Celery worker that consumes documents, the search
|
||||
* index, the classifier and the sanity check — the disk, how many documents there are and how many wait in the inbox, and the **documents that failed to import**. Nothing is sent
|
||||
* to Paperless but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Paperless-ngx's published API (`/api/status/`, `/api/statistics/`, `/api/tasks/`, `/api/remote_version/`) and have not been checked against a
|
||||
* live server here; every field is read defensively, and a part that cannot be read is left out.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface PaperlessConfig {
|
||||
url: string;
|
||||
token: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface PaperlessComponent {
|
||||
/** database, redis, celery, index, classifier, sanity_check. */
|
||||
name: string;
|
||||
/** OK, WARNING or ERROR, as Paperless says. */
|
||||
status: string;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface PaperlessFailure {
|
||||
file: string;
|
||||
error: string | null;
|
||||
at: string | null;
|
||||
}
|
||||
|
||||
export interface PaperlessRead {
|
||||
version: string | null;
|
||||
latestVersion: string | null;
|
||||
updateAvailable: boolean;
|
||||
/** The disk the documents are on; null when it could not be asked. */
|
||||
storage: { total: number; available: number } | null;
|
||||
/** Null when the status could not be asked (it needs a superuser), which is not the same as everything being well. */
|
||||
components: PaperlessComponent[] | null;
|
||||
documents: number | null;
|
||||
inbox: number | null;
|
||||
/** Documents whose import failed in the last week and that nobody has dismissed; null when the tasks could not be asked. */
|
||||
failed: PaperlessFailure[] | null;
|
||||
}
|
||||
|
||||
export interface PaperlessAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(now?: Date): Promise<PaperlessRead>;
|
||||
}
|
||||
|
||||
/** A failed import older than this is no longer news. */
|
||||
export const FAILED_WITHIN_MS = 7 * 86_400_000;
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
||||
const iso = (v: unknown): string | null => {
|
||||
const s = str(v);
|
||||
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
|
||||
};
|
||||
|
||||
/** The components /status/ reports on, with the key their status and error fields start with. */
|
||||
const COMPONENTS: { name: string; where: "database" | "tasks"; key: string }[] = [
|
||||
{ name: "database", where: "database", key: "" },
|
||||
{ name: "redis", where: "tasks", key: "redis" },
|
||||
{ name: "celery", where: "tasks", key: "celery" },
|
||||
{ name: "index", where: "tasks", key: "index" },
|
||||
{ name: "classifier", where: "tasks", key: "classifier" },
|
||||
{ name: "sanity_check", where: "tasks", key: "sanity_check" },
|
||||
];
|
||||
|
||||
/** Pure: the system status — version, disk and each component with how it is. Components is null when the answer holds none of them. */
|
||||
export function parseStatus(raw: unknown): { version: string | null; storage: PaperlessRead["storage"]; components: PaperlessComponent[] | null } {
|
||||
const o = obj(raw);
|
||||
const storage = obj(o.storage);
|
||||
const total = num(storage.total);
|
||||
const available = num(storage.available);
|
||||
const components: PaperlessComponent[] = [];
|
||||
for (const c of COMPONENTS) {
|
||||
const section = obj(o[c.where]);
|
||||
const status = str(c.where === "database" ? section.status : section[`${c.key}_status`]);
|
||||
if (!status) continue;
|
||||
components.push({ name: c.name, status: status.toUpperCase(), error: str(c.where === "database" ? section.error : section[`${c.key}_error`]) });
|
||||
}
|
||||
return {
|
||||
version: str(o.pngx_version),
|
||||
storage: total !== null && available !== null && total > 0 ? { total, available } : null,
|
||||
components: components.length > 0 ? components : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Pure: the documents and how many are in the inbox. */
|
||||
export function parseStatistics(raw: unknown): { documents: number | null; inbox: number | null } {
|
||||
const o = obj(raw);
|
||||
return { documents: num(o.documents_total), inbox: num(o.documents_inbox) };
|
||||
}
|
||||
|
||||
/** Pure: the imports that failed in the last week and have not been dismissed, newest first. Null when the answer holds no list of tasks. */
|
||||
export function parseFailures(raw: unknown, now: Date): PaperlessFailure[] | null {
|
||||
const list = Array.isArray(raw) ? raw : Array.isArray(obj(raw).results) ? (obj(raw).results as unknown[]) : null;
|
||||
if (!list) return null;
|
||||
const since = now.getTime() - FAILED_WITHIN_MS;
|
||||
const out: (PaperlessFailure & { t: number })[] = [];
|
||||
for (const item of list) {
|
||||
const t = obj(item);
|
||||
if (str(t.status)?.toUpperCase() !== "FAILURE" || t.acknowledged === true) continue;
|
||||
const at = iso(t.date_done) ?? iso(t.date_created);
|
||||
const time = at ? Date.parse(at) : now.getTime();
|
||||
if (time < since) continue;
|
||||
out.push({ file: str(t.task_file_name) ?? str(t.task_name) ?? "a document", error: str(t.result), at, t: time });
|
||||
}
|
||||
return out.sort((a, b) => b.t - a.t).map(({ t: _t, ...f }) => f);
|
||||
}
|
||||
|
||||
/** Pure: whether Paperless says a newer version exists. */
|
||||
export function parseRemoteVersion(raw: unknown): { latestVersion: string | null; updateAvailable: boolean } {
|
||||
const o = obj(raw);
|
||||
return { latestVersion: str(o.version), updateAvailable: o.update_available === true };
|
||||
}
|
||||
|
||||
export function createPaperlessAdapter(config: PaperlessConfig): PaperlessAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers = { Authorization: `Token ${config.token}`, Accept: "application/json" };
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}/api${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Paperless did not accept the token (HTTP ${res.status}) — check it, and that it belongs to a superuser if the system status is wanted`);
|
||||
if (res.status === 404) throw new Error("Paperless has no API at this address — check the address and port (8000)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Paperless error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Paperless answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(now: Date = new Date()): Promise<PaperlessRead> {
|
||||
// The statistics are open to any user, so they are the hard requirement that proves the token works; the status needs a superuser and is optional.
|
||||
const stats = parseStatistics(await get("/statistics/"));
|
||||
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
|
||||
try {
|
||||
return parse(await get(path));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const [status, failed, remote] = await Promise.all([soft("/status/", parseStatus), soft("/tasks/", (r) => parseFailures(r, now)), soft("/remote_version/", parseRemoteVersion)]);
|
||||
return {
|
||||
version: status?.version ?? null,
|
||||
latestVersion: remote?.latestVersion ?? null,
|
||||
updateAvailable: remote?.updateAvailable ?? false,
|
||||
storage: status?.storage ?? null,
|
||||
components: status?.components ?? null,
|
||||
documents: stats.documents,
|
||||
inbox: stats.inbox,
|
||||
failed,
|
||||
};
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/statistics/");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("paperless", { ping, read });
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Prometheus adapter. Config: url; optional: token (a bearer token, when something in front of Prometheus asks for one), insecure.
|
||||
*
|
||||
* Reading only: Prometheus's version, every scrape target and whether its last scrape worked (a node exporter that stopped answering), and the alerts that Prometheus's own rules have
|
||||
* pending or firing. That makes Prometheus a source: whatever you already measure there is seen here too. Nothing is sent to Prometheus but GET requests for these.
|
||||
*
|
||||
* Endpoints and field names are from Prometheus's published HTTP API (`/api/v1/status/buildinfo`, `/api/v1/targets`, `/api/v1/alerts`) and have not been checked against a live server
|
||||
* here; every field is read defensively. Alerts that Grafana manages itself are not in Prometheus's list and are not seen.
|
||||
*/
|
||||
import { requestJson } from "../httpJson.js";
|
||||
import { withDiagLogging } from "../../services/diagLog.js";
|
||||
|
||||
export interface PrometheusConfig {
|
||||
url: string;
|
||||
token?: string;
|
||||
insecure?: boolean;
|
||||
}
|
||||
|
||||
export interface PrometheusTarget {
|
||||
job: string;
|
||||
instance: string;
|
||||
health: "up" | "down" | "unknown";
|
||||
/** Why the last scrape failed. */
|
||||
error: string | null;
|
||||
lastScrape: string | null;
|
||||
}
|
||||
|
||||
export interface PrometheusAlert {
|
||||
name: string;
|
||||
/** From the alert's own `severity` label, lower-case; null when it has none. */
|
||||
severity: string | null;
|
||||
state: "firing" | "pending";
|
||||
summary: string | null;
|
||||
instance: string | null;
|
||||
activeAt: string | null;
|
||||
}
|
||||
|
||||
export interface PrometheusRead {
|
||||
version: string | null;
|
||||
/** Null when the targets could not be asked, which is not the same as there being none. */
|
||||
targets: PrometheusTarget[] | null;
|
||||
alerts: PrometheusAlert[] | null;
|
||||
}
|
||||
|
||||
export interface PrometheusAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
read(): Promise<PrometheusRead>;
|
||||
}
|
||||
|
||||
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
|
||||
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
|
||||
const iso = (v: unknown): string | null => {
|
||||
const s = str(v);
|
||||
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
|
||||
};
|
||||
|
||||
/** Pure: the version from /status/buildinfo. */
|
||||
export function parseBuildInfo(raw: unknown): string | null {
|
||||
return str(obj(obj(raw).data).version);
|
||||
}
|
||||
|
||||
/** Pure: the active targets from /targets; null when the answer has no list in it. */
|
||||
export function parseTargets(raw: unknown): PrometheusTarget[] | null {
|
||||
const list = obj(obj(raw).data).activeTargets;
|
||||
if (!Array.isArray(list)) return null;
|
||||
const out: PrometheusTarget[] = [];
|
||||
for (const item of list) {
|
||||
const t = obj(item);
|
||||
const labels = obj(t.labels);
|
||||
const job = str(labels.job) ?? str(t.scrapePool);
|
||||
if (!job) continue;
|
||||
const health = str(t.health)?.toLowerCase();
|
||||
out.push({
|
||||
job,
|
||||
instance: str(labels.instance) ?? str(t.scrapeUrl) ?? "?",
|
||||
health: health === "up" || health === "down" ? health : "unknown",
|
||||
error: str(t.lastError),
|
||||
lastScrape: iso(t.lastScrape),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Pure: the alerts from /alerts; null when the answer has no list in it. */
|
||||
export function parseAlerts(raw: unknown): PrometheusAlert[] | null {
|
||||
const list = obj(obj(raw).data).alerts;
|
||||
if (!Array.isArray(list)) return null;
|
||||
const out: PrometheusAlert[] = [];
|
||||
for (const item of list) {
|
||||
const a = obj(item);
|
||||
const labels = obj(a.labels);
|
||||
const annotations = obj(a.annotations);
|
||||
const name = str(labels.alertname);
|
||||
const state = str(a.state)?.toLowerCase();
|
||||
if (!name || (state !== "firing" && state !== "pending")) continue;
|
||||
out.push({ name, severity: str(labels.severity)?.toLowerCase() ?? null, state, summary: str(annotations.summary) ?? str(annotations.description) ?? str(annotations.message), instance: str(labels.instance), activeAt: iso(a.activeAt) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function createPrometheusAdapter(config: PrometheusConfig): PrometheusAdapter {
|
||||
const base = String(config.url ?? "").replace(/\/+$/, "");
|
||||
const insecure = config.insecure === true;
|
||||
const headers: Record<string, string> = config.token ? { Authorization: `Bearer ${config.token}` } : {};
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const res = await requestJson(`${base}${path}`, { headers, insecure });
|
||||
if (res.status === 401 || res.status === 403) throw new Error(`Prometheus did not accept the request (HTTP ${res.status}) — something in front of it wants a token`);
|
||||
if (res.status === 404) throw new Error("Prometheus has no API at this address — check the address and port (9090)");
|
||||
if (res.status < 200 || res.status >= 300) throw new Error(`Prometheus error: HTTP ${res.status}`);
|
||||
if (res.json === null) throw new Error("Prometheus answered with something that isn't JSON — is the address its own?");
|
||||
return res.json;
|
||||
}
|
||||
|
||||
async function read(): Promise<PrometheusRead> {
|
||||
const version = parseBuildInfo(await get("/api/v1/status/buildinfo"));
|
||||
// The rest is worth having but not worth failing for.
|
||||
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
|
||||
try {
|
||||
return parse(await get(path));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const [targets, alerts] = await Promise.all([soft("/api/v1/targets?state=active", parseTargets), soft("/api/v1/alerts", parseAlerts)]);
|
||||
return { version, targets, alerts };
|
||||
}
|
||||
|
||||
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await get("/api/v1/status/buildinfo");
|
||||
return { ok: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
return withDiagLogging("prometheus", { ping, read });
|
||||
}
|
||||
@@ -16,6 +16,9 @@ import { createHomeAssistantAdapter } from "./homeassistant/adapter.js";
|
||||
import { createUpsAdapter } from "./ups/adapter.js";
|
||||
import { createJellyfinAdapter } from "./jellyfin/adapter.js";
|
||||
import { createAuthentikAdapter } from "./authentik/adapter.js";
|
||||
import { createPrometheusAdapter } from "./prometheus/adapter.js";
|
||||
import { createImmichAdapter } from "./immich/adapter.js";
|
||||
import { createPaperlessAdapter } from "./paperless/adapter.js";
|
||||
|
||||
export interface PingableAdapter {
|
||||
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
|
||||
@@ -62,6 +65,12 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
|
||||
return createJellyfinAdapter(config as any);
|
||||
case "authentik":
|
||||
return createAuthentikAdapter(config as any);
|
||||
case "prometheus":
|
||||
return createPrometheusAdapter(config as any);
|
||||
case "immich":
|
||||
return createImmichAdapter(config as any);
|
||||
case "paperless":
|
||||
return createPaperlessAdapter(config as any);
|
||||
default:
|
||||
throw new Error(`Integration type "${type}" is not implemented yet`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { immichState, integrations } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { immichProblems, immichSyncStatuses, isImmichFresh, queuesOf } from "../services/immichSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const immichRouter = Router();
|
||||
immichRouter.use(requireAuth);
|
||||
|
||||
/** Every Immich, how it is, from what the last read stored: the disk under the library, the counts, the job queues and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
immichRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich")).orderBy(integrations.name),
|
||||
db.select().from(immichState),
|
||||
immichSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
immiches: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isImmichFresh(status ?? undefined, now);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state
|
||||
? {
|
||||
version: state.version,
|
||||
latestVersion: state.latestVersion,
|
||||
updateAvailable: state.updateAvailable,
|
||||
disk: state.diskSize !== null && state.diskUsed !== null ? { size: state.diskSize, used: state.diskUsed, percent: state.diskPercent } : null,
|
||||
photos: state.photos,
|
||||
videos: state.videos,
|
||||
usageBytes: state.usageBytes,
|
||||
queues: queuesOf(state),
|
||||
updatedAt: state.updatedAt,
|
||||
}
|
||||
: null,
|
||||
problems: state && fresh ? immichProblems(r.id, r.name, state) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -31,6 +31,9 @@ import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssi
|
||||
import { forgetUps, syncUps } from "../services/upsSync.js";
|
||||
import { forgetJellyfin, syncJellyfin } from "../services/jellyfinSync.js";
|
||||
import { forgetAuthentik, syncAuthentik } from "../services/authentikSync.js";
|
||||
import { forgetPrometheus, syncPrometheus } from "../services/prometheusSync.js";
|
||||
import { forgetImmich, syncImmich } from "../services/immichSync.js";
|
||||
import { forgetPaperless, syncPaperless } from "../services/paperlessSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
@@ -135,6 +138,9 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
|
||||
if (type === "ups") void syncUps(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "jellyfin") void syncJellyfin(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "authentik") void syncAuthentik(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "prometheus") void syncPrometheus(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "immich") void syncImmich(created.id).then(() => invalidateAlertsCache());
|
||||
if (type === "paperless") void syncPaperless(created.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.status(201).json({
|
||||
integration: {
|
||||
@@ -240,6 +246,9 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (updated.type === "ups" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUps(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "jellyfin" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncJellyfin(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "authentik" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAuthentik(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "prometheus" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPrometheus(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "immich" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncImmich(updated.id).then(() => invalidateAlertsCache());
|
||||
if (updated.type === "paperless" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPaperless(updated.id).then(() => invalidateAlertsCache());
|
||||
|
||||
res.json({
|
||||
integration: {
|
||||
@@ -321,6 +330,9 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
|
||||
if (existing.type === "ups") await forgetUps(id);
|
||||
if (existing.type === "jellyfin") await forgetJellyfin(id);
|
||||
if (existing.type === "authentik") await forgetAuthentik(id);
|
||||
if (existing.type === "prometheus") await forgetPrometheus(id);
|
||||
if (existing.type === "immich") await forgetImmich(id);
|
||||
if (existing.type === "paperless") await forgetPaperless(id);
|
||||
await db.delete(integrations).where(eq(integrations.id, id));
|
||||
if (existing.credentialId) {
|
||||
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, paperlessState } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { componentsOf, failedOf, isPaperlessFresh, paperlessProblems, paperlessSyncStatuses } from "../services/paperlessSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const paperlessRouter = Router();
|
||||
paperlessRouter.use(requireAuth);
|
||||
|
||||
/** Every Paperless-ngx, how it is, from what the last read stored: its status components, the disk, the counts, the imports that failed and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
paperlessRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless")).orderBy(integrations.name),
|
||||
db.select().from(paperlessState),
|
||||
paperlessSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = Date.now();
|
||||
res.json({
|
||||
paperlesses: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isPaperlessFresh(status ?? undefined, now);
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state
|
||||
? {
|
||||
version: state.version,
|
||||
latestVersion: state.latestVersion,
|
||||
updateAvailable: state.updateAvailable,
|
||||
storage: state.storageTotal !== null && state.storageAvailable !== null ? { total: state.storageTotal, available: state.storageAvailable } : null,
|
||||
components: componentsOf(state),
|
||||
documents: state.documents,
|
||||
inbox: state.inbox,
|
||||
failed: failedOf(state),
|
||||
updatedAt: state.updatedAt,
|
||||
}
|
||||
: null,
|
||||
problems: state && fresh ? paperlessProblems(r.id, r.name, state) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Router } from "express";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, prometheusState } from "../db/schema.js";
|
||||
import { requireAuth } from "../auth/middleware.js";
|
||||
import { alertsOf, downOf, isPrometheusFresh, jobsOf, prometheusProblems, prometheusSyncStatuses } from "../services/prometheusSync.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
|
||||
export const prometheusRouter = Router();
|
||||
prometheusRouter.use(requireAuth);
|
||||
|
||||
/** Every Prometheus, how it is, from what the last read stored: targets by job, the ones that are down, the alerts firing, and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
|
||||
prometheusRouter.get("/", asyncHandler(async (_req, res) => {
|
||||
const [rows, states, statuses] = await Promise.all([
|
||||
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "prometheus")).orderBy(integrations.name),
|
||||
db.select().from(prometheusState),
|
||||
prometheusSyncStatuses(),
|
||||
]);
|
||||
const byId = new Map(states.map((s) => [s.integrationId, s]));
|
||||
const now = new Date();
|
||||
res.json({
|
||||
prometheuses: rows.map((r) => {
|
||||
const state = byId.get(r.id) ?? null;
|
||||
const status = statuses[r.id] ?? null;
|
||||
const fresh = isPrometheusFresh(status ?? undefined, now.getTime());
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
enabled: r.enabled,
|
||||
status,
|
||||
fresh,
|
||||
state: state ? { version: state.version, jobs: jobsOf(state), down: downOf(state), alerts: alertsOf(state), updatedAt: state.updatedAt } : null,
|
||||
problems: state && fresh ? prometheusProblems(r.id, r.name, state, now) : [],
|
||||
};
|
||||
}),
|
||||
});
|
||||
}));
|
||||
@@ -77,6 +77,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
duplicati: { label: "Duplicati", link: "/duplicati" },
|
||||
jellyfin: { label: "Jellyfin", link: "/jellyfin" },
|
||||
authentik: { label: "Authentik", link: "/authentik" },
|
||||
prometheus: { label: "Prometheus", link: "/prometheus" },
|
||||
immich: { label: "Immich", link: "/immich" },
|
||||
paperless: { label: "Paperless-ngx", link: "/paperless" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -115,6 +118,12 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" };
|
||||
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
||||
if (m) return { type: "immich", id: Number(m[1]), category: key.startsWith("immich-disk") ? "disk" : "services" };
|
||||
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
|
||||
if (m) return { type: "prometheus", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return { type: "authentik", id: Number(m[1]), category: /^authentik-(?:cert|token):/.test(key) ? "expiry" : /^authentik-(?:outpost|worker)/.test(key) ? "services" : "monitoring" };
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
@@ -241,7 +250,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -251,7 +260,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { authentikState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -22,6 +22,9 @@ import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
|
||||
import { currentProblems as duplicatiProblemsNow, type DuplicatiProblem } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, type JellyfinProblem } from "./jellyfinSync.js";
|
||||
import { authentikProblems, authentikSyncStatuses, isAuthentikFresh, type AuthentikProblem } from "./authentikSync.js";
|
||||
import { isPrometheusFresh, prometheusProblems, prometheusSyncStatuses, type PrometheusProblem } from "./prometheusSync.js";
|
||||
import { immichProblems, immichSyncStatuses, isImmichFresh, type ImmichProblem } from "./immichSync.js";
|
||||
import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type PaperlessProblem } from "./paperlessSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -89,6 +92,12 @@ export interface HealthSnapshot {
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
/** What is wrong with each Paperless-ngx (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/paperlessSync.ts). */
|
||||
paperless?: { integrationId: number; problems: PaperlessProblem[] }[];
|
||||
/** What is wrong with each Immich (a disk filling, background jobs failing), only while that reading is recent (services/immichSync.ts). */
|
||||
immich?: { integrationId: number; problems: ImmichProblem[] }[];
|
||||
/** What is wrong with each Prometheus (a target that stopped answering, an alert its rules fire), only while that reading is recent (services/prometheusSync.ts). */
|
||||
prometheus?: { integrationId: number; problems: PrometheusProblem[] }[];
|
||||
/** What is wrong with each Authentik, from what was stored, only while that reading is recent (services/authentikSync.ts). */
|
||||
authentik?: { integrationId: number; problems: AuthentikProblem[] }[];
|
||||
/** What is wrong with each Jellyfin, from what was stored, only while that reading is recent (services/jellyfinSync.ts). */
|
||||
@@ -204,6 +213,15 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A Paperless whose worker has stopped, whose imports fail, or whose disk is filling.
|
||||
for (const p of snapshot.paperless ?? []) for (const c of p.problems) out.push({ key: c.key, source: `paperless:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// An Immich whose disk is filling or whose background jobs are failing.
|
||||
for (const i of snapshot.immich ?? []) for (const c of i.problems) out.push({ key: c.key, source: `immich:${i.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// A Prometheus target that has stopped answering, an alert that Prometheus's own rules are firing.
|
||||
for (const p of snapshot.prometheus ?? []) for (const c of p.problems) out.push({ key: c.key, source: `prometheus:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// An Authentik outpost that went quiet, a certificate or token about to run out, no worker, failed sign-ins.
|
||||
for (const a of snapshot.authentik ?? []) for (const p of a.problems) out.push({ key: p.key, source: `authentik:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -440,6 +458,60 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Paperless: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
|
||||
if (rowsL.length > 0) {
|
||||
const statuses = await paperlessSyncStatuses();
|
||||
for (const r of rowsL) {
|
||||
const [state] = await db.select().from(paperlessState).where(eq(paperlessState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isPaperlessFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`paperless:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.paperless ??= []).push({ integrationId: r.id, problems: paperlessProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Paperless:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Immich: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsI = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
|
||||
if (rowsI.length > 0) {
|
||||
const statuses = await immichSyncStatuses();
|
||||
for (const r of rowsI) {
|
||||
const [state] = await db.select().from(immichState).where(eq(immichState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isImmichFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`immich:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.immich ??= []).push({ integrationId: r.id, problems: immichProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Immich:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Prometheus: read by its own pass every couple of minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsP = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
|
||||
if (rowsP.length > 0) {
|
||||
const statuses = await prometheusSyncStatuses();
|
||||
for (const r of rowsP) {
|
||||
const [state] = await db.select().from(prometheusState).where(eq(prometheusState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isPrometheusFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`prometheus:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.prometheus ??= []).push({ integrationId: r.id, problems: prometheusProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Prometheus:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Authentik: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsA = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "authentik"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Keeps what Immich says up to date: every five minutes each Immich integration is read (version, the disk under the library, the photo and video counts, the background job
|
||||
* queues) and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Immich.
|
||||
*
|
||||
* Photos are what cannot be made again, and Immich fails quietly: a disk that fills up, or thumbnail and metadata jobs that fail one by one and are never looked at. Those are
|
||||
* what is raised.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { immichState, integrations } from "../db/schema.js";
|
||||
import { createImmichAdapter, type ImmichQueue, type ImmichRead } from "../integrations/immich/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
/** This many failed background jobs, together, is a warning. */
|
||||
export const FAILED_JOBS_WARN = 10;
|
||||
export const DISK_WARN_PERCENT = 90;
|
||||
export const DISK_CRITICAL_PERCENT = 95;
|
||||
const STATUS_FLAG = "immichSync";
|
||||
|
||||
export interface ImmichSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, ImmichSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, ImmichSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function immichSyncStatuses(): Promise<Record<number, ImmichSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isImmichFresh = (status: ImmichSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof immichState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
export const queuesOf = (row: Pick<StateRow, "queues">): ImmichQueue[] | null => {
|
||||
if (row.queues === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.queues);
|
||||
return Array.isArray(v) ? (v as ImmichQueue[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: ImmichRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
latestVersion: r.latestVersion,
|
||||
updateAvailable: r.updateAvailable,
|
||||
diskSize: r.disk?.size ?? null,
|
||||
diskUsed: r.disk?.used ?? null,
|
||||
diskPercent: r.disk?.percent ?? null,
|
||||
photos: r.photos,
|
||||
videos: r.videos,
|
||||
usageBytes: r.usageBytes,
|
||||
queues: r.queues && JSON.stringify(r.queues),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(immichState).values({ integrationId, ...values }).onConflictDoUpdate({ target: immichState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Immich and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncImmich(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<ImmichRead>): Promise<ImmichSyncStatus> {
|
||||
let status: ImmichSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "immich") throw new Error("not an Immich integration");
|
||||
const result = await (read ? read(loaded.config) : createImmichAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[immich] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[immich] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllImmich(now: Date = new Date()): Promise<Record<number, ImmichSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, ImmichSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncImmich(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetImmich(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImmichProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Immich, from what is stored for it. */
|
||||
export function immichProblems(id: number, name: string, s: Pick<StateRow, "diskSize" | "diskUsed" | "diskPercent" | "queues">): ImmichProblem[] {
|
||||
const out: ImmichProblem[] = [];
|
||||
if (s.diskPercent !== null && s.diskPercent >= DISK_WARN_PERCENT) {
|
||||
out.push({
|
||||
key: `immich-disk:${id}`,
|
||||
severity: s.diskPercent >= DISK_CRITICAL_PERCENT ? "critical" : "warning",
|
||||
message: `${name}: the disk under the library is ${Math.round(s.diskPercent)} % full${s.diskSize !== null && s.diskUsed !== null ? ` (${gb(s.diskUsed)} of ${gb(s.diskSize)})` : ""}`,
|
||||
});
|
||||
}
|
||||
const queues = queuesOf(s) ?? [];
|
||||
const failed = queues.filter((q) => q.failed > 0);
|
||||
const total = failed.reduce((n, q) => n + q.failed, 0);
|
||||
if (total >= FAILED_JOBS_WARN) {
|
||||
const worst = [...failed].sort((a, b) => b.failed - a.failed).slice(0, 3).map((q) => `${q.name} ${q.failed}`).join(", ");
|
||||
out.push({ key: `immich-jobs:${id}`, severity: "warning", message: `${name}: ${total} background jobs have failed (${worst}) — thumbnails or metadata that were never made; look at Administration → Jobs and run the failed ones again` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initImmichSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllImmich().catch((err) => console.error("[immich] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 17_000);
|
||||
console.log(`[immich] reading Immich every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -33,6 +33,12 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
|
||||
|
||||
@@ -5,11 +5,13 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
||||
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
|
||||
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
@@ -62,6 +64,39 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
|
||||
async function addPaperlessMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await paperlessSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(paperlessState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isPaperlessFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
if (s.documents !== null) out.add("homelab_paperless_documents", "gauge", "How many documents Paperless holds.", s.documents, labels);
|
||||
if (s.inbox !== null) out.add("homelab_paperless_inbox", "gauge", "How many documents are waiting in Paperless's inbox.", s.inbox, labels);
|
||||
const failed = paperlessFailedOf(s);
|
||||
if (failed) out.add("homelab_paperless_failed_imports", "gauge", "Imports that failed in the last week and were not dismissed.", failed.length, labels);
|
||||
for (const c of paperlessComponentsOf(s) ?? []) out.add("homelab_paperless_component_ok", "gauge", "1 if the part of Paperless named is in order, else 0.", c.status === "OK" ? 1 : 0, { name: row.name, component: c.name });
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Immich, for /metrics: the photos and videos, what they take, how full the disk is and the jobs that failed. Only while the reading is recent. */
|
||||
async function addImmichMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await immichSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(immichState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isImmichFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
if (s.photos !== null) out.add("homelab_immich_photos", "gauge", "How many photos Immich holds.", s.photos, labels);
|
||||
if (s.videos !== null) out.add("homelab_immich_videos", "gauge", "How many videos Immich holds.", s.videos, labels);
|
||||
if (s.usageBytes !== null) out.add("homelab_immich_library_bytes", "gauge", "What Immich's library takes, in bytes.", s.usageBytes, labels);
|
||||
if (s.diskPercent !== null) out.add("homelab_immich_disk_used_percent", "gauge", "How full the disk under Immich's library is.", s.diskPercent, labels);
|
||||
const queues = immichQueuesOf(s);
|
||||
if (queues) out.add("homelab_immich_failed_jobs", "gauge", "Background jobs that have failed, all queues together.", queues.reduce((n, q) => n + q.failed, 0), labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Authentik, for /metrics: whether each outpost that serves something is reporting, how many workers there are, the failed sign-ins and the days left of each certificate. Only while the reading is recent. */
|
||||
async function addAuthentikMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await authentikSyncStatuses();
|
||||
@@ -300,6 +335,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addDuplicatiMetrics(out, now);
|
||||
await addJellyfinMetrics(out, now);
|
||||
await addAuthentikMetrics(out, now);
|
||||
await addImmichMetrics(out, now);
|
||||
await addPaperlessMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* Keeps what Paperless-ngx says up to date: every five minutes each Paperless integration is read (its system status, the disk, the document counts and the imports that failed)
|
||||
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Paperless.
|
||||
*
|
||||
* Paperless fails quietly in two ways: the worker that consumes documents stops (a scan dropped in the folder just sits there), or one import fails and nobody looks. Both are raised,
|
||||
* and so is anything Paperless's own status check says is in error.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, paperlessState } from "../db/schema.js";
|
||||
import { createPaperlessAdapter, type PaperlessComponent, type PaperlessFailure, type PaperlessRead } from "../integrations/paperless/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
export const DISK_WARN_PERCENT = 90;
|
||||
export const DISK_CRITICAL_PERCENT = 95;
|
||||
/** If one of these is in error, nothing can be consumed or found: critical. The others are a warning. */
|
||||
const ESSENTIAL = new Set(["database", "redis", "celery"]);
|
||||
const STATUS_FLAG = "paperlessSync";
|
||||
|
||||
export interface PaperlessSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, PaperlessSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, PaperlessSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function paperlessSyncStatuses(): Promise<Record<number, PaperlessSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isPaperlessFresh = (status: PaperlessSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof paperlessState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
const parse = <T>(text: string | null): T[] | null => {
|
||||
if (text === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(text);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
export const componentsOf = (row: Pick<StateRow, "components">) => parse<PaperlessComponent>(row.components);
|
||||
export const failedOf = (row: Pick<StateRow, "failed">) => parse<PaperlessFailure>(row.failed);
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: PaperlessRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
latestVersion: r.latestVersion,
|
||||
updateAvailable: r.updateAvailable,
|
||||
storageTotal: r.storage?.total ?? null,
|
||||
storageAvailable: r.storage?.available ?? null,
|
||||
components: r.components && JSON.stringify(r.components),
|
||||
documents: r.documents,
|
||||
inbox: r.inbox,
|
||||
failed: r.failed && JSON.stringify(r.failed),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(paperlessState).values({ integrationId, ...values }).onConflictDoUpdate({ target: paperlessState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Paperless and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncPaperless(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PaperlessRead>): Promise<PaperlessSyncStatus> {
|
||||
let status: PaperlessSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "paperless") throw new Error("not a Paperless integration");
|
||||
const result = await (read ? read(loaded.config) : createPaperlessAdapter(loaded.config as never).read(now));
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[paperless] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[paperless] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllPaperless(now: Date = new Date()): Promise<Record<number, PaperlessSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, PaperlessSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncPaperless(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetPaperless(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface PaperlessProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
const WORDS: Record<string, string> = { database: "the database", redis: "Redis", celery: "the worker that consumes documents", index: "the search index", classifier: "the classifier", sanity_check: "the sanity check" };
|
||||
|
||||
/** Pure: what is wrong with one Paperless, from what is stored for it. */
|
||||
export function paperlessProblems(id: number, name: string, s: Pick<StateRow, "storageTotal" | "storageAvailable" | "components" | "failed">): PaperlessProblem[] {
|
||||
const out: PaperlessProblem[] = [];
|
||||
for (const c of componentsOf(s) ?? []) {
|
||||
const word = WORDS[c.name] ?? c.name;
|
||||
if (c.status === "ERROR") {
|
||||
out.push({ key: `paperless-component:${id}:${c.name}`, severity: ESSENTIAL.has(c.name) ? "critical" : "warning", message: `${name}: ${word} is in error${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
|
||||
} else if (c.status === "WARNING" && c.name === "sanity_check") {
|
||||
out.push({ key: `paperless-component:${id}:${c.name}`, severity: "warning", message: `${name}: the sanity check found problems with some documents${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
|
||||
}
|
||||
}
|
||||
const failed = failedOf(s) ?? [];
|
||||
if (failed.length > 0) {
|
||||
const files = failed.slice(0, 3).map((f) => f.file).join(", ") + (failed.length > 3 ? ` and ${failed.length - 3} more` : "");
|
||||
const why = failed[0].error ? ` — the latest: ${failed[0].error.length > 140 ? `${failed[0].error.slice(0, 137)}…` : failed[0].error}` : "";
|
||||
out.push({ key: `paperless-failed:${id}`, severity: "warning", message: `${name}: ${failed.length} document${failed.length === 1 ? "" : "s"} failed to import (${files})${why}` });
|
||||
}
|
||||
if (s.storageTotal !== null && s.storageAvailable !== null && s.storageTotal > 0) {
|
||||
const percent = ((s.storageTotal - s.storageAvailable) / s.storageTotal) * 100;
|
||||
if (percent >= DISK_WARN_PERCENT) {
|
||||
out.push({ key: `paperless-disk:${id}`, severity: percent >= DISK_CRITICAL_PERCENT ? "critical" : "warning", message: `${name}: the disk under the documents is ${Math.round(percent)} % full (${gb(s.storageTotal - s.storageAvailable)} of ${gb(s.storageTotal)})` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initPaperlessSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllPaperless().catch((err) => console.error("[paperless] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 19_000);
|
||||
console.log(`[paperless] reading Paperless every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* Keeps what Prometheus says up to date: every two minutes each Prometheus integration is read (its scrape targets and whether they answer, and the alerts its rules have firing)
|
||||
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Prometheus.
|
||||
*
|
||||
* Prometheus is where the things you already measure live — a node exporter on each host, a service's own metrics — so this makes them part of the one list of what is wrong:
|
||||
* a **target that has stopped answering** (for five minutes, so one failed scrape is not news) and **an alert that Prometheus's rules are firing**.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, prometheusState } from "../db/schema.js";
|
||||
import { createPrometheusAdapter, type PrometheusAlert, type PrometheusRead } from "../integrations/prometheus/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 2 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 10 * 60_000;
|
||||
/** A target must have been down this long before it is a problem; one failed scrape is not. */
|
||||
export const TARGET_DOWN_AFTER_MS = 5 * 60_000;
|
||||
const STATUS_FLAG = "prometheusSync";
|
||||
|
||||
export interface PrometheusSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface JobCount {
|
||||
job: string;
|
||||
up: number;
|
||||
down: number;
|
||||
}
|
||||
export interface DownTarget {
|
||||
job: string;
|
||||
instance: string;
|
||||
error: string | null;
|
||||
/** When it was first seen down, carried over from one read to the next. */
|
||||
since: string;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, PrometheusSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, PrometheusSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function prometheusSyncStatuses(): Promise<Record<number, PrometheusSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isPrometheusFresh = (status: PrometheusSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof prometheusState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
const parse = <T>(text: string | null): T[] | null => {
|
||||
if (text === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(text);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
export const jobsOf = (row: Pick<StateRow, "jobs">) => parse<JobCount>(row.jobs);
|
||||
export const downOf = (row: Pick<StateRow, "down">) => parse<DownTarget>(row.down);
|
||||
export const alertsOf = (row: Pick<StateRow, "alerts">) => parse<PrometheusAlert>(row.alerts);
|
||||
|
||||
/** Writes down one read, replacing the last; a target that was down before keeps the time it was first seen down. */
|
||||
export async function storeRead(integrationId: number, r: PrometheusRead, now: Date = new Date()): Promise<void> {
|
||||
const [before] = await db.select({ down: prometheusState.down }).from(prometheusState).where(eq(prometheusState.integrationId, integrationId)).limit(1);
|
||||
const earlier = new Map((before ? downOf(before) ?? [] : []).map((d) => [`${d.job}\u0000${d.instance}`, d.since]));
|
||||
let jobs: JobCount[] | null = null;
|
||||
let down: DownTarget[] | null = null;
|
||||
if (r.targets) {
|
||||
const counts = new Map<string, JobCount>();
|
||||
down = [];
|
||||
for (const t of r.targets) {
|
||||
const c = counts.get(t.job) ?? { job: t.job, up: 0, down: 0 };
|
||||
if (t.health === "down") {
|
||||
c.down += 1;
|
||||
down.push({ job: t.job, instance: t.instance, error: t.error, since: earlier.get(`${t.job}\u0000${t.instance}`) ?? now.toISOString() });
|
||||
} else if (t.health === "up") c.up += 1;
|
||||
counts.set(t.job, c);
|
||||
}
|
||||
jobs = [...counts.values()].sort((a, b) => a.job.localeCompare(b.job));
|
||||
}
|
||||
const values = { version: r.version, jobs: jobs && JSON.stringify(jobs), down: down && JSON.stringify(down), alerts: r.alerts && JSON.stringify(r.alerts), updatedAt: now.toISOString() };
|
||||
await db.insert(prometheusState).values({ integrationId, ...values }).onConflictDoUpdate({ target: prometheusState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Prometheus and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncPrometheus(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PrometheusRead>): Promise<PrometheusSyncStatus> {
|
||||
let status: PrometheusSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "prometheus") throw new Error("not a Prometheus integration");
|
||||
const result = await (read ? read(loaded.config) : createPrometheusAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[prometheus] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[prometheus] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllPrometheus(now: Date = new Date()): Promise<Record<number, PrometheusSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, PrometheusSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncPrometheus(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetPrometheus(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface PrometheusProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** Alerts that are firing by design (a heartbeat to prove the pipeline works) and so say nothing. */
|
||||
const BY_DESIGN = /^(watchdog|deadmansswitch|deadmanssnitch|infoinhibitor)$/i;
|
||||
|
||||
/** Pure: what is wrong, from what is stored. */
|
||||
export function prometheusProblems(id: number, name: string, s: Pick<StateRow, "jobs" | "down" | "alerts">, now: Date = new Date()): PrometheusProblem[] {
|
||||
const out: PrometheusProblem[] = [];
|
||||
const jobs = new Map((jobsOf(s) ?? []).map((j) => [j.job, j]));
|
||||
const byJob = new Map<string, DownTarget[]>();
|
||||
for (const d of downOf(s) ?? []) {
|
||||
if (now.getTime() - Date.parse(d.since) < TARGET_DOWN_AFTER_MS) continue;
|
||||
byJob.set(d.job, [...(byJob.get(d.job) ?? []), d]);
|
||||
}
|
||||
for (const [job, list] of byJob) {
|
||||
const c = jobs.get(job);
|
||||
const total = c ? c.up + c.down : list.length;
|
||||
const who = list.slice(0, 3).map((d) => d.instance).join(", ") + (list.length > 3 ? ` and ${list.length - 3} more` : "");
|
||||
const why = list[0].error ? ` — ${list[0].error.length > 120 ? `${list[0].error.slice(0, 117)}…` : list[0].error}` : "";
|
||||
out.push({ key: `prometheus-target:${id}:${job}`, severity: "warning", message: `${name}: ${list.length} of ${total} “${job}” target${total === 1 ? "" : "s"} not answering (${who})${why}` });
|
||||
}
|
||||
const firing = new Map<string, PrometheusAlert[]>();
|
||||
for (const a of alertsOf(s) ?? []) {
|
||||
if (a.state !== "firing" || BY_DESIGN.test(a.name) || a.severity === "info" || a.severity === "none") continue;
|
||||
firing.set(a.name, [...(firing.get(a.name) ?? []), a]);
|
||||
}
|
||||
for (const [alertname, list] of firing) {
|
||||
const critical = list.some((a) => a.severity === "critical" || a.severity === "page");
|
||||
const first = list.find((a) => a.summary) ?? list[0];
|
||||
out.push({
|
||||
key: `prometheus-alert:${id}:${alertname}`,
|
||||
severity: critical ? "critical" : "warning",
|
||||
message: `${name}: ${alertname}${list.length > 1 ? ` (${list.length} firing)` : first.instance ? ` on ${first.instance}` : ""}${first.summary ? ` — ${first.summary}` : ""}`,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initPrometheusSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllPrometheus().catch((err) => console.error("[prometheus] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 15_000);
|
||||
console.log(`[prometheus] reading Prometheus every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
/**
|
||||
* Immich: reading what its API says (version, storage, statistics, job queues in either shape), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-immich-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, immichState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/immich/adapter.js");
|
||||
const im = await import("../src/services/immichSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { immichRouter } = await import("../src/routes/immich.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
|
||||
type Read = import("../src/integrations/immich/adapter.js").ImmichRead;
|
||||
const queue = (name: string, failed = 0, extra: Partial<import("../src/integrations/immich/adapter.js").ImmichQueue> = {}) => ({ name, active: 0, waiting: 0, failed, paused: false, ...extra });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "v1.120.2", latestVersion: "v1.120.2", updateAvailable: false, disk: { size: 4e12, used: 1e12, percent: 25 }, photos: 52000, videos: 1800, usageBytes: 7e11, queues: [queue("thumbnailGeneration")], ...extra });
|
||||
|
||||
describe("what Immich's API says", () => {
|
||||
test("a version can be a string or a major, minor and patch; and newer is told by number, not by text", () => {
|
||||
assert.equal(adapter.parseVersion({ version: "v1.120.2" }), "v1.120.2");
|
||||
assert.equal(adapter.parseVersion({ major: 1, minor: 120, patch: 2 }), "v1.120.2");
|
||||
assert.equal(adapter.parseVersion({}), null);
|
||||
assert.equal(adapter.newerVersion("v1.120.10", "v1.120.9"), true, "10 is later than 9");
|
||||
assert.equal(adapter.newerVersion("1.120.2", "v1.120.2"), false);
|
||||
assert.equal(adapter.newerVersion("v2.0.0", "v1.999.9"), true);
|
||||
assert.equal(adapter.newerVersion(null, "v1.0.0"), false);
|
||||
assert.equal(adapter.newerVersion("nightly", "v1.0.0"), false);
|
||||
});
|
||||
|
||||
test("the disk under the library", () => {
|
||||
assert.deepEqual(adapter.parseStorage({ diskSizeRaw: 4e12, diskUseRaw: 1e12, diskUsagePercentage: 25 }), { size: 4e12, used: 1e12, percent: 25 });
|
||||
assert.equal(adapter.parseStorage({ diskSizeRaw: 1000, diskUseRaw: 333 })!.percent, 33.3, "worked out when Immich does not say");
|
||||
assert.equal(adapter.parseStorage({}), null);
|
||||
});
|
||||
|
||||
test("job queues: as an object keyed by name (/jobs) and as a list (/queues)", () => {
|
||||
const asObject = adapter.parseQueues({ thumbnailGeneration: { jobCounts: { active: 1, waiting: 5, failed: 2 }, queueStatus: { isPaused: false } }, library: { jobCounts: { failed: 0 }, queueStatus: { isPaused: true } } })!;
|
||||
assert.deepEqual(asObject.map((q) => [q.name, q.active, q.waiting, q.failed, q.paused]), [["thumbnailGeneration", 1, 5, 2, false], ["library", 0, 0, 0, true]]);
|
||||
const asList = adapter.parseQueues([{ name: "metadataExtraction", isPaused: false, statistics: { failed: 7, waiting: 1 } }])!;
|
||||
assert.deepEqual(asList.map((q) => [q.name, q.failed, q.waiting]), [["metadataExtraction", 7, 1]]);
|
||||
assert.equal(adapter.parseQueues({}), null);
|
||||
assert.equal(adapter.parseQueues("x"), null);
|
||||
});
|
||||
|
||||
describe("the adapter against an Immich that answers", () => {
|
||||
let server: Server;
|
||||
let url = "";
|
||||
let newStyle = false;
|
||||
const seen: string[] = [];
|
||||
before(async () => {
|
||||
server = createHttpServer((req, res) => {
|
||||
const path = (req.url ?? "").split("?")[0];
|
||||
seen.push(`${req.headers["x-api-key"]} ${path}`);
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
if (req.headers["x-api-key"] !== "key") return send(401, {});
|
||||
if (path === "/api/server/about") return send(200, { version: "v1.120.2" });
|
||||
if (path === "/api/server/storage") return send(200, { diskSizeRaw: 4e12, diskUseRaw: 3.8e12, diskUsagePercentage: 95 });
|
||||
if (path === "/api/server/statistics") return send(200, { photos: 100, videos: 5, usage: 123456 });
|
||||
if (path === "/api/server/version-check") return send(200, { releaseVersion: "v1.121.0" });
|
||||
if (path === "/api/jobs") return newStyle ? send(404, {}) : send(200, { thumbnailGeneration: { jobCounts: { failed: 4 } } });
|
||||
if (path === "/api/queues") return newStyle ? send(200, [{ name: "thumbnailGeneration", statistics: { failed: 9 } }]) : send(404, {});
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => server.close());
|
||||
|
||||
test("it reads everything, sends the key, and says a newer version is out", async () => {
|
||||
const r = await adapter.createImmichAdapter({ url: `${url}/`, apiKey: "key" }).read();
|
||||
assert.deepEqual([r.version, r.latestVersion, r.updateAvailable, r.photos, r.videos, r.usageBytes], ["v1.120.2", "v1.121.0", true, 100, 5, 123456]);
|
||||
assert.deepEqual(r.disk, { size: 4e12, used: 3.8e12, percent: 95 });
|
||||
assert.deepEqual(r.queues?.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 4]]);
|
||||
assert.ok(seen.every((s) => s.startsWith("key ")));
|
||||
});
|
||||
|
||||
test("a newer Immich with /queues in place of /jobs is read all the same", async () => {
|
||||
newStyle = true;
|
||||
const r = await adapter.createImmichAdapter({ url, apiKey: "key" }).read();
|
||||
assert.deepEqual(r.queues?.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 9]]);
|
||||
});
|
||||
|
||||
test("a refused key and a dead address are said plainly", async () => {
|
||||
const bad = await adapter.createImmichAdapter({ url, apiKey: "nope" }).ping();
|
||||
assert.equal(bad.ok, false);
|
||||
assert.match(bad.error ?? "", /did not accept the API key/);
|
||||
assert.equal((await adapter.createImmichAdapter({ url, apiKey: "key" }).ping()).ok, true);
|
||||
assert.equal((await adapter.createImmichAdapter({ url: "http://127.0.0.1:1", apiKey: "key" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "immich", name: "Photos", baseUrl: "http://10.9.9.3:2283", config: JSON.stringify({ url: "http://10.9.9.3:2283", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const state = async () => (await db.select().from(immichState).where(eq(immichState.integrationId, id)))[0];
|
||||
|
||||
test("the state is replaced at each read; queues that could not be read stay null, not empty", async () => {
|
||||
await im.storeRead(id, read({ queues: [queue("a", 3)] }), NOW);
|
||||
assert.deepEqual(im.queuesOf(await state())?.map((q) => q.name), ["a"]);
|
||||
await im.storeRead(id, read({ queues: null, disk: null, photos: null }), NOW);
|
||||
const s = await state();
|
||||
assert.deepEqual([im.queuesOf(s), s.diskPercent, s.photos], [null, null, null]);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await im.syncImmich(id, new Date(), async () => read({ version: "v1.121.0" }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await im.syncImmich(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal((await state()).version, "v1.121.0");
|
||||
assert.equal(im.isImmichFresh((await im.immichSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const row = (extra: Partial<Parameters<typeof im.immichProblems>[2]> = {}) => ({ diskSize: 4e12, diskUsed: 1e12, diskPercent: 25 as number | null, queues: JSON.stringify([queue("thumbnailGeneration")]) as string | null, ...extra });
|
||||
const keys = (r: ReturnType<typeof row>) => im.immichProblems(1, "Photos", r).map((p) => `${p.key}=${p.severity}`);
|
||||
|
||||
test("nothing wrong says nothing; what could not be read is not a problem", () => {
|
||||
assert.deepEqual(keys(row()), []);
|
||||
assert.deepEqual(keys(row({ diskPercent: null, queues: null })), []);
|
||||
});
|
||||
|
||||
test("the disk: a warning at 90 %, critical at 95 %", () => {
|
||||
assert.deepEqual(keys(row({ diskPercent: 89.9 })), []);
|
||||
assert.deepEqual(keys(row({ diskPercent: 90 })), ["immich-disk:1=warning"]);
|
||||
assert.deepEqual(keys(row({ diskPercent: 95 })), ["immich-disk:1=critical"]);
|
||||
assert.match(im.immichProblems(1, "Photos", row({ diskPercent: 96, diskUsed: 3.84e12 }))[0].message, /96 % full \(3840 GB of 4000 GB\)/);
|
||||
});
|
||||
|
||||
test("failed jobs are one alert when there are ten or more together, naming the worst queues", () => {
|
||||
const q = (a: number, b: number) => JSON.stringify([queue("thumbnailGeneration", a), queue("metadataExtraction", b), queue("library", 0)]);
|
||||
assert.deepEqual(keys(row({ queues: q(5, 4) })), [], "nine is not yet");
|
||||
const p = im.immichProblems(1, "Photos", row({ queues: q(8, 4) }));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["immich-jobs:1", "warning"]]);
|
||||
assert.match(p[0].message, /12 background jobs have failed \(thumbnailGeneration 8, metadataExtraction 4\)/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let imId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("immichSync", JSON.stringify({ [imId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("immichSync", JSON.stringify({ [imId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
imId = (await db.insert(integrations).values({ type: "immich", name: "Family photos", baseUrl: "http://10.9.9.2:2283", config: JSON.stringify({ url: "http://10.9.9.2:2283", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
|
||||
await im.storeRead(imId, read({ disk: { size: 4e12, used: 3.9e12, percent: 97.5 }, queues: [queue("thumbnailGeneration", 14, { waiting: 3 }), queue("library")] }), new Date());
|
||||
await fresh();
|
||||
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/immich", immichRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/immich`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.immich?.map((i) => i.integrationId), [imId]);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `immich-disk:${imId}`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `immich:${imId}`]);
|
||||
assert.equal(subjectOfConditionKey(c.key), `integration:${imId}`, "maintenance on it silences its alerts");
|
||||
assert.equal(subjectOfConditionKey(`immich-jobs:${imId}`), `integration:${imId}`);
|
||||
assert.ok(list.some((x) => x.key === `immich-jobs:${imId}`));
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.immich ?? [], held.has(`immich:${imId}`)], [[], true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page shows them: the disk under Disk, the jobs under Services, with where to look", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const disk = report.alerts.find((x) => x.id === `disk:immich-disk:${imId}`)!;
|
||||
assert.deepEqual([disk.severity, disk.category, disk.source, disk.link, disk.target?.type, disk.target?.id], ["critical", "disk", "Immich", "/immich", "integration", imId]);
|
||||
assert.equal(report.alerts.find((x) => x.id === `services:immich-jobs:${imId}`)?.category, "services");
|
||||
});
|
||||
|
||||
test("anyone signed in sees each Immich, the counts, the disk and the queues; an old reading says nothing is wrong", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { immiches: { id: number; fresh: boolean; state: { photos: number; disk: { percent: number } | null; queues: { name: string; failed: number }[] } | null; problems: unknown[] }[] };
|
||||
const i = body.immiches.find((x) => x.id === imId)!;
|
||||
assert.equal(i.fresh, true);
|
||||
assert.deepEqual([i.state?.photos, i.state?.disk?.percent], [52000, 97.5]);
|
||||
assert.deepEqual(i.state?.queues.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 14], ["library", 0]]);
|
||||
assert.equal(i.problems.length, 2);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.immiches.find((x) => x.id === imId)!.problems, []);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("/metrics has the counts, what the library takes, the disk and the failed jobs — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_immich_photos\{name="Family photos"\} 52000/);
|
||||
assert.match(text, /homelab_immich_library_bytes\{name="Family photos"\} 7e\+11|homelab_immich_library_bytes\{name="Family photos"\} 700000000000/);
|
||||
assert.match(text, /homelab_immich_disk_used_percent\{name="Family photos"\} 97\.5/);
|
||||
assert.match(text, /homelab_immich_failed_jobs\{name="Family photos"\} 14/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_immich/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* Paperless-ngx: reading what its API says (status components, statistics, failed imports), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-paperless-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, paperlessState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/paperless/adapter.js");
|
||||
const pl = await import("../src/services/paperlessSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { paperlessRouter } = await import("../src/routes/paperless.js");
|
||||
const { buildMetrics } = await import("../src/services/metrics.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const DAY = 86_400_000;
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
|
||||
type Read = import("../src/integrations/paperless/adapter.js").PaperlessRead;
|
||||
const comp = (name: string, status = "OK", error: string | null = null) => ({ name, status, error });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({
|
||||
version: "2.14.7", latestVersion: "2.14.7", updateAvailable: false, storage: { total: 1e12, available: 6e11 },
|
||||
components: [comp("database"), comp("redis"), comp("celery"), comp("index"), comp("classifier"), comp("sanity_check")], documents: 4200, inbox: 3, failed: [], ...extra,
|
||||
});
|
||||
|
||||
const STATUS = {
|
||||
pngx_version: "2.14.7",
|
||||
storage: { total: 1e12, available: 6e11 },
|
||||
database: { type: "sqlite", status: "OK", error: null },
|
||||
tasks: { redis_status: "OK", celery_status: "ERROR", celery_error: "No celery workers responded", index_status: "OK", classifier_status: "WARNING", classifier_error: "Classifier has not been trained yet", sanity_check_status: "OK" },
|
||||
};
|
||||
|
||||
describe("what Paperless's API says", () => {
|
||||
test("the system status: the version, the disk and each component with how it is", () => {
|
||||
const s = adapter.parseStatus(STATUS);
|
||||
assert.equal(s.version, "2.14.7");
|
||||
assert.deepEqual(s.storage, { total: 1e12, available: 6e11 });
|
||||
assert.deepEqual(s.components?.map((c) => [c.name, c.status, c.error]), [["database", "OK", null], ["redis", "OK", null], ["celery", "ERROR", "No celery workers responded"], ["index", "OK", null], ["classifier", "WARNING", "Classifier has not been trained yet"], ["sanity_check", "OK", null]]);
|
||||
assert.deepEqual(adapter.parseStatus({}), { version: null, storage: null, components: null });
|
||||
});
|
||||
|
||||
test("the documents and the inbox", () => {
|
||||
assert.deepEqual(adapter.parseStatistics({ documents_total: 4200, documents_inbox: 3 }), { documents: 4200, inbox: 3 });
|
||||
assert.deepEqual(adapter.parseStatistics(null), { documents: null, inbox: null });
|
||||
});
|
||||
|
||||
test("failed imports: only failures, not dismissed, from the last week, newest first; an answer without a list is not 'none'", () => {
|
||||
const f = adapter.parseFailures(
|
||||
[
|
||||
{ status: "FAILURE", task_file_name: "old.pdf", date_done: new Date(NOW.getTime() - 10 * DAY).toISOString(), result: "x" },
|
||||
{ status: "FAILURE", task_file_name: "a.pdf", date_done: new Date(NOW.getTime() - 2 * DAY).toISOString(), result: "Not a valid PDF" },
|
||||
{ status: "FAILURE", task_file_name: "dismissed.pdf", acknowledged: true, date_done: NOW.toISOString() },
|
||||
{ status: "SUCCESS", task_file_name: "fine.pdf", date_done: NOW.toISOString() },
|
||||
{ status: "FAILURE", task_file_name: "b.pdf", date_done: new Date(NOW.getTime() - 3600_000).toISOString() },
|
||||
],
|
||||
NOW,
|
||||
)!;
|
||||
assert.deepEqual(f.map((x) => [x.file, x.error]), [["b.pdf", null], ["a.pdf", "Not a valid PDF"]]);
|
||||
assert.equal(adapter.parseFailures({ results: [] }, NOW)?.length, 0, "a paginated answer is read too");
|
||||
assert.equal(adapter.parseFailures({ detail: "no" }, NOW), null);
|
||||
});
|
||||
|
||||
test("a newer version", () => {
|
||||
assert.deepEqual(adapter.parseRemoteVersion({ version: "v2.15.0", update_available: true }), { latestVersion: "v2.15.0", updateAvailable: true });
|
||||
assert.deepEqual(adapter.parseRemoteVersion(null), { latestVersion: null, updateAvailable: false });
|
||||
});
|
||||
|
||||
describe("the adapter against a Paperless that answers", () => {
|
||||
let server: Server;
|
||||
let url = "";
|
||||
let superuser = true;
|
||||
const seen: (string | undefined)[] = [];
|
||||
before(async () => {
|
||||
server = createHttpServer((req, res) => {
|
||||
seen.push(req.headers.authorization);
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
if (req.headers.authorization !== "Token tok") return send(401, {});
|
||||
const path = (req.url ?? "").split("?")[0];
|
||||
if (path === "/api/statistics/") return send(200, { documents_total: 100, documents_inbox: 2 });
|
||||
if (path === "/api/status/") return superuser ? send(200, STATUS) : send(403, {});
|
||||
if (path === "/api/tasks/") return send(200, [{ status: "FAILURE", task_file_name: "x.pdf", date_done: new Date().toISOString(), result: "boom" }]);
|
||||
if (path === "/api/remote_version/") return send(200, { version: "2.15.0", update_available: true });
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => server.close());
|
||||
|
||||
test("it reads everything and sends the token", async () => {
|
||||
const r = await adapter.createPaperlessAdapter({ url: `${url}/`, token: "tok" }).read(new Date());
|
||||
assert.deepEqual([r.version, r.documents, r.inbox, r.updateAvailable, r.latestVersion], ["2.14.7", 100, 2, true, "2.15.0"]);
|
||||
assert.deepEqual(r.failed?.map((f) => f.file), ["x.pdf"]);
|
||||
assert.equal(r.components?.find((c) => c.name === "celery")?.status, "ERROR");
|
||||
assert.ok(seen.every((s) => s === "Token tok"));
|
||||
});
|
||||
|
||||
test("a token that is not a superuser's still reads the rest; the status is unknown, not fine", async () => {
|
||||
superuser = false;
|
||||
const r = await adapter.createPaperlessAdapter({ url, token: "tok" }).read(new Date());
|
||||
assert.deepEqual([r.components, r.storage, r.version], [null, null, null]);
|
||||
assert.equal(r.documents, 100);
|
||||
});
|
||||
|
||||
test("a refused token and a dead address are said plainly", async () => {
|
||||
const bad = await adapter.createPaperlessAdapter({ url, token: "nope" }).ping();
|
||||
assert.equal(bad.ok, false);
|
||||
assert.match(bad.error ?? "", /did not accept the token/);
|
||||
assert.equal((await adapter.createPaperlessAdapter({ url, token: "tok" }).ping()).ok, true);
|
||||
assert.equal((await adapter.createPaperlessAdapter({ url: "http://127.0.0.1:1", token: "tok" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "paperless", name: "Documents", baseUrl: "http://10.9.9.1:8000", config: JSON.stringify({ url: "http://10.9.9.1:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const state = async () => (await db.select().from(paperlessState).where(eq(paperlessState.integrationId, id)))[0];
|
||||
|
||||
test("the state is replaced at each read; what could not be read stays null, not empty", async () => {
|
||||
await pl.storeRead(id, read({ failed: [{ file: "a.pdf", error: null, at: null }] }), NOW);
|
||||
assert.deepEqual(pl.failedOf(await state())?.map((f) => f.file), ["a.pdf"]);
|
||||
await pl.storeRead(id, read({ components: null, failed: null, storage: null }), NOW);
|
||||
const s = await state();
|
||||
assert.deepEqual([pl.componentsOf(s), pl.failedOf(s), s.storageTotal], [null, null, null]);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await pl.syncPaperless(id, new Date(), async () => read({ version: "2.15.0" }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await pl.syncPaperless(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal((await state()).version, "2.15.0");
|
||||
assert.equal(pl.isPaperlessFresh((await pl.paperlessSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const row = (extra: { storageTotal?: number | null; storageAvailable?: number | null; components?: unknown; failed?: unknown } = {}) => ({
|
||||
storageTotal: 1e12 as number | null,
|
||||
storageAvailable: 6e11 as number | null,
|
||||
components: JSON.stringify(extra.components ?? [comp("database"), comp("celery")]) as string | null,
|
||||
failed: JSON.stringify(extra.failed ?? []) as string | null,
|
||||
...("storageTotal" in extra ? { storageTotal: extra.storageTotal ?? null } : {}),
|
||||
...("storageAvailable" in extra ? { storageAvailable: extra.storageAvailable ?? null } : {}),
|
||||
});
|
||||
const keys = (r: ReturnType<typeof row>) => pl.paperlessProblems(1, "Docs", r).map((p) => `${p.key}=${p.severity}`);
|
||||
|
||||
test("nothing wrong says nothing; what could not be read is not a problem", () => {
|
||||
assert.deepEqual(keys(row()), []);
|
||||
assert.deepEqual(pl.paperlessProblems(1, "Docs", { storageTotal: null, storageAvailable: null, components: null, failed: null }), []);
|
||||
});
|
||||
|
||||
test("the database, Redis and the worker in error are critical; the index and the classifier a warning; an untrained classifier ('warning') says nothing", () => {
|
||||
assert.deepEqual(keys(row({ components: [comp("celery", "ERROR", "No celery workers responded"), comp("index", "ERROR"), comp("classifier", "WARNING", "not trained")] })), ["paperless-component:1:celery=critical", "paperless-component:1:index=warning"]);
|
||||
assert.match(pl.paperlessProblems(1, "Docs", row({ components: [comp("celery", "ERROR", "No celery workers responded")] }))[0].message, /the worker that consumes documents is in error — No celery workers responded/);
|
||||
assert.deepEqual(keys(row({ components: [comp("database", "ERROR")] })), ["paperless-component:1:database=critical"]);
|
||||
});
|
||||
|
||||
test("a sanity check that found problems is a warning", () => {
|
||||
assert.deepEqual(keys(row({ components: [comp("sanity_check", "WARNING", "2 documents are missing their file")] })), ["paperless-component:1:sanity_check=warning"]);
|
||||
});
|
||||
|
||||
test("imports that failed are one alert, naming the files and the latest error", () => {
|
||||
const failed = ["a.pdf", "b.pdf", "c.pdf", "d.pdf"].map((file, i) => ({ file, error: i === 0 ? "Not a valid PDF" : null, at: null }));
|
||||
const p = pl.paperlessProblems(1, "Docs", row({ failed }));
|
||||
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["paperless-failed:1", "warning"]]);
|
||||
assert.match(p[0].message, /4 documents failed to import \(a\.pdf, b\.pdf, c\.pdf and 1 more\) — the latest: Not a valid PDF/);
|
||||
assert.match(pl.paperlessProblems(1, "Docs", row({ failed: [failed[1]] }))[0].message, /1 document failed to import \(b\.pdf\)$/);
|
||||
});
|
||||
|
||||
test("the disk: a warning at 90 %, critical at 95 %", () => {
|
||||
assert.deepEqual(keys(row({ storageAvailable: 1.1e11 })), []);
|
||||
assert.deepEqual(keys(row({ storageAvailable: 1e11 })), ["paperless-disk:1=warning"]);
|
||||
assert.deepEqual(keys(row({ storageAvailable: 4e10 })), ["paperless-disk:1=critical"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass, the page and /metrics", () => {
|
||||
let plId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
plId = (await db.insert(integrations).values({ type: "paperless", name: "Archive", baseUrl: "http://10.9.9.0:8000", config: JSON.stringify({ url: "http://10.9.9.0:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
|
||||
await pl.storeRead(
|
||||
plId,
|
||||
read({ storage: { total: 1e12, available: 3e10 }, components: [comp("database"), comp("celery", "ERROR", "No celery workers responded")], failed: [{ file: "scan.pdf", error: "Not a valid PDF", at: new Date().toISOString() }] }),
|
||||
new Date(),
|
||||
);
|
||||
await fresh();
|
||||
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/paperless", paperlessRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/paperless`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.paperless?.map((p) => p.integrationId), [plId]);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `paperless-component:${plId}:celery`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `paperless:${plId}`]);
|
||||
assert.equal(subjectOfConditionKey(c.key), `integration:${plId}`, "maintenance on it silences its alerts");
|
||||
for (const k of ["paperless-failed", "paperless-disk"]) assert.equal(subjectOfConditionKey(`${k}:${plId}`), `integration:${plId}`);
|
||||
assert.ok(list.some((x) => x.key === `paperless-failed:${plId}`) && list.some((x) => x.key === `paperless-disk:${plId}`));
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.paperless ?? [], held.has(`paperless:${plId}`)], [[], true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page shows them: the worker and the failed imports under Services, the disk under Disk, with where to look", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const worker = report.alerts.find((x) => x.id === `services:paperless-component:${plId}:celery`)!;
|
||||
assert.deepEqual([worker.severity, worker.category, worker.source, worker.link, worker.target?.type, worker.target?.id], ["critical", "services", "Paperless-ngx", "/paperless", "integration", plId]);
|
||||
assert.equal(report.alerts.find((x) => x.id === `services:paperless-failed:${plId}`)?.category, "services");
|
||||
assert.equal(report.alerts.find((x) => x.id === `disk:paperless-disk:${plId}`)?.severity, "critical");
|
||||
});
|
||||
|
||||
test("anyone signed in sees each Paperless, its components and failed imports; an old reading says nothing is wrong", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { paperlesses: { id: number; fresh: boolean; state: { documents: number; components: { name: string; status: string }[]; failed: { file: string }[] } | null; problems: unknown[] }[] };
|
||||
const p = body.paperlesses.find((x) => x.id === plId)!;
|
||||
assert.equal(p.fresh, true);
|
||||
assert.equal(p.state?.documents, 4200);
|
||||
assert.deepEqual(p.state?.components.map((c) => [c.name, c.status]), [["database", "OK"], ["celery", "ERROR"]]);
|
||||
assert.deepEqual(p.state?.failed.map((f) => f.file), ["scan.pdf"]);
|
||||
assert.equal(p.problems.length, 3);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.paperlesses.find((x) => x.id === plId)!.problems, []);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("/metrics has the documents, the inbox, the failed imports and each component — only while the reading is recent", async () => {
|
||||
let text = await buildMetrics(new Date());
|
||||
assert.match(text, /homelab_paperless_documents\{name="Archive"\} 4200/);
|
||||
assert.match(text, /homelab_paperless_inbox\{name="Archive"\} 3/);
|
||||
assert.match(text, /homelab_paperless_failed_imports\{name="Archive"\} 1/);
|
||||
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="celery"\} 0/);
|
||||
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="database"\} 1/);
|
||||
await stale();
|
||||
text = await buildMetrics(new Date());
|
||||
assert.doesNotMatch(text, /homelab_paperless/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Prometheus: reading what its API says (targets, alerts), what is stored and how long a target has been down, what is wrong, the health pass, the alerts, the page's route.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createHttpServer, type Server } from "node:http";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-prometheus-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, prometheusState } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const adapter = await import("../src/integrations/prometheus/adapter.js");
|
||||
const pm = await import("../src/services/prometheusSync.js");
|
||||
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
|
||||
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
|
||||
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
|
||||
const { prometheusRouter } = await import("../src/routes/prometheus.js");
|
||||
const { setInternalFlag } = await import("../src/services/settingsStore.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const MIN = 60_000;
|
||||
const NOW = new Date("2026-10-20T12:00:00Z");
|
||||
const ago = (min: number, from: Date = NOW) => new Date(from.getTime() - min * MIN);
|
||||
|
||||
type Read = import("../src/integrations/prometheus/adapter.js").PrometheusRead;
|
||||
type Target = import("../src/integrations/prometheus/adapter.js").PrometheusTarget;
|
||||
const target = (job: string, instance: string, health: Target["health"] = "up", error: string | null = null): Target => ({ job, instance, health, error, lastScrape: NOW.toISOString() });
|
||||
const alert = (name: string, extra: Partial<import("../src/integrations/prometheus/adapter.js").PrometheusAlert> = {}) => ({ name, severity: "warning", state: "firing" as const, summary: null, instance: null, activeAt: NOW.toISOString(), ...extra });
|
||||
const read = (extra: Partial<Read> = {}): Read => ({ version: "3.1.0", targets: [], alerts: [], ...extra });
|
||||
|
||||
describe("what Prometheus's API says", () => {
|
||||
test("the version", () => {
|
||||
assert.equal(adapter.parseBuildInfo({ status: "success", data: { version: "3.1.0" } }), "3.1.0");
|
||||
assert.equal(adapter.parseBuildInfo(null), null);
|
||||
});
|
||||
|
||||
test("targets, with their health and why a scrape failed; an answer without a list is not 'none'", () => {
|
||||
const t = adapter.parseTargets({
|
||||
data: {
|
||||
activeTargets: [
|
||||
{ labels: { job: "node", instance: "jackie:9100" }, health: "up", lastError: "", lastScrape: "2026-10-20T11:59:50Z" },
|
||||
{ labels: { job: "node", instance: "nova:9100" }, health: "down", lastError: "connection refused", lastScrape: "2026-10-20T11:59:50Z" },
|
||||
{ labels: { instance: "x" }, scrapePool: "pool", health: "weird" },
|
||||
{ labels: {} },
|
||||
],
|
||||
},
|
||||
})!;
|
||||
assert.deepEqual(t.map((x) => [x.job, x.instance, x.health, x.error]), [["node", "jackie:9100", "up", null], ["node", "nova:9100", "down", "connection refused"], ["pool", "x", "unknown", null]]);
|
||||
assert.equal(adapter.parseTargets({ status: "error" }), null);
|
||||
assert.deepEqual(adapter.parseTargets({ data: { activeTargets: [] } }), []);
|
||||
});
|
||||
|
||||
test("alerts: firing and pending, with severity and the summary", () => {
|
||||
const a = adapter.parseAlerts({
|
||||
data: {
|
||||
alerts: [
|
||||
{ labels: { alertname: "DiskFull", severity: "Critical", instance: "jackie:9100" }, annotations: { summary: "Disk is 95% full" }, state: "firing", activeAt: "2026-10-20T11:00:00Z" },
|
||||
{ labels: { alertname: "Slow" }, annotations: { description: "slow" }, state: "pending" },
|
||||
{ labels: { alertname: "Odd" }, state: "inactive" },
|
||||
{ labels: {}, state: "firing" },
|
||||
],
|
||||
},
|
||||
})!;
|
||||
assert.deepEqual(a.map((x) => [x.name, x.severity, x.state, x.summary, x.instance]), [["DiskFull", "critical", "firing", "Disk is 95% full", "jackie:9100"], ["Slow", null, "pending", "slow", null]]);
|
||||
assert.equal(adapter.parseAlerts({}), null);
|
||||
});
|
||||
|
||||
describe("the adapter against a Prometheus that answers", () => {
|
||||
let server: Server;
|
||||
let url = "";
|
||||
const seen: (string | undefined)[] = [];
|
||||
before(async () => {
|
||||
server = createHttpServer((req, res) => {
|
||||
seen.push(req.headers.authorization);
|
||||
const send = (code: number, body: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(body));
|
||||
};
|
||||
const path = (req.url ?? "").split("?")[0];
|
||||
if (path === "/api/v1/status/buildinfo") return send(200, { data: { version: "3.1.0" } });
|
||||
if (path === "/api/v1/targets") return send(200, { data: { activeTargets: [{ labels: { job: "node", instance: "a" }, health: "down", lastError: "timeout" }] } });
|
||||
if (path === "/api/v1/alerts") return send(500, {}); // the rest is worth having, not worth failing for
|
||||
send(404, {});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
after(() => server.close());
|
||||
|
||||
test("it reads, and an alert list that cannot be read is null rather than empty", async () => {
|
||||
const r = await adapter.createPrometheusAdapter({ url: `${url}/` }).read();
|
||||
assert.equal(r.version, "3.1.0");
|
||||
assert.equal(r.targets?.length, 1);
|
||||
assert.equal(r.alerts, null);
|
||||
assert.ok(seen.every((s) => s === undefined), "no token, none sent");
|
||||
});
|
||||
|
||||
test("a token is sent when there is one; a dead address is said", async () => {
|
||||
await adapter.createPrometheusAdapter({ url, token: "t0k" }).ping();
|
||||
assert.equal(seen.at(-1), "Bearer t0k");
|
||||
assert.equal((await adapter.createPrometheusAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is stored", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
id = (await db.insert(integrations).values({ type: "prometheus", name: "Metrics", baseUrl: "http://10.9.9.5:9090", config: JSON.stringify({ url: "http://10.9.9.5:9090" }) }).returning({ id: integrations.id }))[0].id;
|
||||
});
|
||||
const state = async () => (await db.select().from(prometheusState).where(eq(prometheusState.integrationId, id)))[0];
|
||||
|
||||
test("targets are counted by job, and a target that stays down keeps the time it was first seen down", async () => {
|
||||
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), ago(10));
|
||||
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), NOW);
|
||||
const s = await state();
|
||||
assert.deepEqual(pm.jobsOf(s), [{ job: "cadvisor", up: 1, down: 0 }, { job: "node", up: 1, down: 1 }]);
|
||||
assert.equal(pm.downOf(s)?.[0].since, ago(10).toISOString(), "since the first look, not the last");
|
||||
// It comes back and goes down again: the clock starts over.
|
||||
await pm.storeRead(id, read({ targets: [target("node", "b")] }), NOW);
|
||||
assert.deepEqual(pm.downOf(await state()), []);
|
||||
await pm.storeRead(id, read({ targets: [target("node", "b", "down")] }), new Date(NOW.getTime() + MIN));
|
||||
assert.equal(pm.downOf(await state())?.[0].since, new Date(NOW.getTime() + MIN).toISOString());
|
||||
});
|
||||
|
||||
test("what could not be read stays null, not empty", async () => {
|
||||
await pm.storeRead(id, read({ targets: null, alerts: null }), NOW);
|
||||
const s = await state();
|
||||
assert.deepEqual([pm.jobsOf(s), pm.downOf(s), pm.alertsOf(s)], [null, null, null]);
|
||||
});
|
||||
|
||||
test("a read that failed is said, and what was stored is left alone", async () => {
|
||||
const ok = await pm.syncPrometheus(id, new Date(), async () => read({ version: "3.2.0" }));
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await pm.syncPrometheus(id, new Date(), async () => {
|
||||
throw new Error("connection refused");
|
||||
});
|
||||
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
|
||||
assert.equal(failed.at, ok.at);
|
||||
assert.equal((await state()).version, "3.2.0");
|
||||
assert.equal(pm.isPrometheusFresh((await pm.prometheusSyncStatuses())[id], Date.now()), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("what is wrong", () => {
|
||||
const row = (r: { jobs?: unknown; down?: unknown; alerts?: unknown }) => ({ jobs: JSON.stringify(r.jobs ?? []), down: JSON.stringify(r.down ?? []), alerts: JSON.stringify(r.alerts ?? []) });
|
||||
const keys = (r: ReturnType<typeof row>) => pm.prometheusProblems(1, "Metrics", r, NOW).map((p) => `${p.key}=${p.severity}`);
|
||||
|
||||
test("a target is a problem only once it has been down five minutes; the job's targets are one alert", () => {
|
||||
const jobs = [{ job: "node", up: 1, down: 3 }];
|
||||
const d = (inst: string, mins: number) => ({ job: "node", instance: inst, error: "connection refused", since: ago(mins).toISOString() });
|
||||
assert.deepEqual(keys(row({ jobs, down: [d("a", 4)] })), [], "one failed scrape is not news");
|
||||
assert.deepEqual(keys(row({ jobs, down: [d("a", 6)] })), ["prometheus-target:1:node=warning"]);
|
||||
const p = pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9), d("e", 1)] }), NOW);
|
||||
assert.equal(p.length, 1);
|
||||
assert.match(p[0].message, /3 of 4 “node” targets not answering \(a, b, c\) — connection refused/);
|
||||
assert.equal(pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9)] }), NOW)[0].message.includes("and "), false);
|
||||
});
|
||||
|
||||
test("firing alerts, one for each name; critical when any is; heartbeat alerts, pending and info ones are left out", () => {
|
||||
const alerts = [
|
||||
alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" }),
|
||||
alert("DiskFull", { severity: "warning", instance: "nova:9100" }),
|
||||
alert("HighLoad", { instance: "nova:9100" }),
|
||||
alert("Watchdog", { severity: "none" }),
|
||||
alert("Pending", { state: "pending" }),
|
||||
alert("FYI", { severity: "info" }),
|
||||
];
|
||||
assert.deepEqual(keys(row({ alerts })), ["prometheus-alert:1:DiskFull=critical", "prometheus-alert:1:HighLoad=warning"]);
|
||||
const p = pm.prometheusProblems(1, "Metrics", row({ alerts }), NOW);
|
||||
assert.equal(p[0].message, "Metrics: DiskFull (2 firing) — Disk is 95% full");
|
||||
assert.equal(p[1].message, "Metrics: HighLoad on nova:9100");
|
||||
});
|
||||
|
||||
test("nothing read says nothing is wrong", () => {
|
||||
assert.deepEqual(pm.prometheusProblems(1, "Metrics", { jobs: null, down: null, alerts: null }, NOW), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the health pass and the page", () => {
|
||||
let pmId = 0;
|
||||
let app: Server;
|
||||
let base = "";
|
||||
const fresh = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date().toISOString(), ok: true, error: null } }));
|
||||
const stale = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
|
||||
|
||||
before(async () => {
|
||||
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
|
||||
pmId = (await db.insert(integrations).values({ type: "prometheus", name: "Prom", baseUrl: "http://10.9.9.4:9090", config: JSON.stringify({ url: "http://10.9.9.4:9090" }) }).returning({ id: integrations.id }))[0].id;
|
||||
const t0 = new Date(Date.now() - 12 * MIN);
|
||||
const r = read({ targets: [target("node", "jackie:9100", "down", "refused"), target("node", "nova:9100")], alerts: [alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" })] });
|
||||
await pm.storeRead(pmId, r, t0);
|
||||
await pm.storeRead(pmId, r, new Date());
|
||||
await fresh();
|
||||
|
||||
const a = express();
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/prometheus", prometheusRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const s = a.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/prometheus`;
|
||||
});
|
||||
after(() => {
|
||||
app.close();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
|
||||
let { snapshot, held } = await collectSnapshot();
|
||||
assert.deepEqual(snapshot.prometheus?.map((p) => p.integrationId), [pmId]);
|
||||
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
|
||||
const c = list.find((x) => x.key === `prometheus-alert:${pmId}:DiskFull`)!;
|
||||
assert.deepEqual([c.severity, c.source], ["critical", `prometheus:${pmId}`]);
|
||||
assert.equal(subjectOfConditionKey(c.key), `integration:${pmId}`, "maintenance on it silences its alerts");
|
||||
assert.equal(subjectOfConditionKey(`prometheus-target:${pmId}:node`), `integration:${pmId}`);
|
||||
assert.ok(list.some((x) => x.key === `prometheus-target:${pmId}:node`));
|
||||
await stale();
|
||||
({ snapshot, held } = await collectSnapshot());
|
||||
assert.deepEqual([snapshot.prometheus ?? [], held.has(`prometheus:${pmId}`)], [[], true]);
|
||||
await fresh();
|
||||
});
|
||||
|
||||
test("the Alerts page shows them in the Monitoring kind, with where to look", async () => {
|
||||
invalidateAlertsCache();
|
||||
const report = await getAlerts(true);
|
||||
const a = report.alerts.find((x) => x.id === `monitoring:prometheus-alert:${pmId}:DiskFull`)!;
|
||||
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "Prometheus", "/prometheus", "integration", pmId]);
|
||||
assert.ok(report.alerts.some((x) => x.id === `monitoring:prometheus-target:${pmId}:node`));
|
||||
});
|
||||
|
||||
test("anyone signed in sees each Prometheus, its jobs, what is down and what fires; an old reading says nothing is wrong", async () => {
|
||||
const body = (await (await fetch(base)).json()) as { prometheuses: { id: number; fresh: boolean; state: { jobs: { job: string; up: number; down: number }[]; down: unknown[]; alerts: unknown[] } | null; problems: unknown[] }[] };
|
||||
const p = body.prometheuses.find((x) => x.id === pmId)!;
|
||||
assert.equal(p.fresh, true);
|
||||
assert.deepEqual(p.state?.jobs, [{ job: "node", up: 1, down: 1 }]);
|
||||
assert.equal(p.state?.down.length, 1);
|
||||
assert.equal(p.problems.length, 2);
|
||||
await stale();
|
||||
const old = (await (await fetch(base)).json()) as typeof body;
|
||||
assert.deepEqual(old.prometheuses.find((x) => x.id === pmId)!.problems, []);
|
||||
await fresh();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user