Prometheus, Immich and Paperless-ngx integrations

Three read-only integrations, each with a monitoring page, alerts, maintenance
silencing and held-when-unreadable handling.

Prometheus: scrape targets that stop answering (5 minutes, one alert per job)
and alerts that Prometheus's own rules fire (critical when labelled so).
Immich: disk under the library filling, failed background jobs, counts.
Paperless-ngx: worker/database/Redis in error, documents that failed to
import, index and sanity check, disk. Immich and Paperless also in /metrics.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-24 22:33:17 +02:00
1 parent aebe98bf15
commit 251a4a9d73
47 files changed
+24148 -7

No files matched your search

+9
View File
@@ -0,0 +1,9 @@
CREATE TABLE `prometheus_state` (
`integration_id` integer PRIMARY KEY NOT NULL,
`version` text,
`jobs` text,
`down` text,
`alerts` text,
`updated_at` text NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
+15
View File
@@ -0,0 +1,15 @@
CREATE TABLE `immich_state` (
`integration_id` integer PRIMARY KEY NOT NULL,
`version` text,
`latest_version` text,
`update_available` integer DEFAULT false NOT NULL,
`disk_size` integer,
`disk_used` integer,
`disk_percent` real,
`photos` integer,
`videos` integer,
`usage_bytes` integer,
`queues` text,
`updated_at` text NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
@@ -0,0 +1,14 @@
CREATE TABLE `paperless_state` (
`integration_id` integer PRIMARY KEY NOT NULL,
`version` text,
`latest_version` text,
`update_available` integer DEFAULT false NOT NULL,
`storage_total` integer,
`storage_available` integer,
`components` text,
`documents` integer,
`inbox` integer,
`failed` text,
`updated_at` text NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+21
View File
@@ -414,6 +414,27 @@
"when": 1792872166034,
"tag": "0058_flippant_sasquatch",
"breakpoints": true
},
{
"idx": 59,
"version": "6",
"when": 1792872843134,
"tag": "0059_secret_bullseye",
"breakpoints": true
},
{
"idx": 60,
"version": "6",
"when": 1792873156389,
"tag": "0060_big_kylun",
"breakpoints": true
},
{
"idx": 61,
"version": "6",
"when": 1792873347697,
"tag": "0061_reflective_daimon_hellstrom",
"breakpoints": true
}
]
}
+6
View File
@@ -60,6 +60,9 @@ import { awayRouter } from "./routes/away.js";
import { duplicatiReportRouter, duplicatiRouter } from "./routes/duplicati.js";
import { jellyfinRouter } from "./routes/jellyfin.js";
import { authentikRouter } from "./routes/authentik.js";
import { prometheusRouter } from "./routes/prometheus.js";
import { immichRouter } from "./routes/immich.js";
import { paperlessRouter } from "./routes/paperless.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
import { publicApiRouter } from "./routes/publicApi.js";
import { generatorRouter } from "./routes/generator.js";
@@ -197,6 +200,9 @@ export function createApp() {
app.use("/api/duplicati", duplicatiRouter);
app.use("/api/jellyfin", jellyfinRouter);
app.use("/api/authentik", authentikRouter);
app.use("/api/prometheus", prometheusRouter);
app.use("/api/immich", immichRouter);
app.use("/api/paperless", paperlessRouter);
app.use("/api/service-levels", serviceLevelsRouter);
app.use("/api/todos", todosRouter);
app.use("/api/notes", notesRouter);
+59
View File
@@ -555,6 +555,9 @@ export const integrationTypes = [
"ups",
"jellyfin",
"authentik",
"prometheus",
"immich",
"paperless",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1161,6 +1164,62 @@ export const haReadings = sqliteTable(
(t) => [index("ha_readings_lookup_idx").on(t.integrationId, t.entityId, t.at)],
);
// ─── Paperless-ngx ──────────────────────────────────────────────────────────
// How a Paperless-ngx (an integration of type "paperless") is right now, one row each, replaced at every read: its version, the disk, how each component of its system status is, the
// document counts and the imports that failed. A column that is null could not be read. Read by services/paperlessSync.ts.
export const paperlessState = sqliteTable("paperless_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
latestVersion: text("latest_version"),
updateAvailable: integer("update_available", { mode: "boolean" }).notNull().default(false),
storageTotal: integer("storage_total"), // bytes
storageAvailable: integer("storage_available"), // bytes
components: text("components"), // JSON: name, status, error — null: the status could not be asked (it needs a superuser)
documents: integer("documents"),
inbox: integer("inbox"),
failed: text("failed"), // JSON: file, error, at — imports that failed in the last week and were not dismissed; null: could not be asked
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Immich ─────────────────────────────────────────────────────────────────
// How an Immich (an integration of type "immich") is right now, one row each, replaced at every read: its version, the disk under the library, the counts and the background job queues.
// A column that is null could not be read. Read by services/immichSync.ts.
export const immichState = sqliteTable("immich_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
latestVersion: text("latest_version"),
updateAvailable: integer("update_available", { mode: "boolean" }).notNull().default(false),
diskSize: integer("disk_size"), // bytes
diskUsed: integer("disk_used"), // bytes
diskPercent: real("disk_percent"),
photos: integer("photos"),
videos: integer("videos"),
usageBytes: integer("usage_bytes"),
queues: text("queues"), // JSON: name, active, waiting, failed, paused — null: could not be asked
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Prometheus ─────────────────────────────────────────────────────────────
// How a Prometheus (an integration of type "prometheus") is right now, one row each, replaced at every read: its version, how many targets each job has up and down, which are down
// (and since when), and the alerts its rules have firing. A column that is null could not be read. Read by services/prometheusSync.ts.
export const prometheusState = sqliteTable("prometheus_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
version: text("version"),
jobs: text("jobs"), // JSON: job, up, down — null: the targets could not be asked
down: text("down"), // JSON: job, instance, error, since
alerts: text("alerts"), // JSON: name, severity, state, summary, instance, activeAt — null: could not be asked
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Authentik ──────────────────────────────────────────────────────────────
// How an Authentik (an integration of type "authentik") is right now, one row each, replaced at every read: its version, whether a worker is connected, how each outpost is, the
+6
View File
@@ -31,6 +31,9 @@ import { initUpsSync } from "./services/upsSync.js";
import { initAwayMonitor } from "./services/awayMonitor.js";
import { initJellyfinSync } from "./services/jellyfinSync.js";
import { initAuthentikSync } from "./services/authentikSync.js";
import { initPrometheusSync } from "./services/prometheusSync.js";
import { initImmichSync } from "./services/immichSync.js";
import { initPaperlessSync } from "./services/paperlessSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -83,6 +86,9 @@ initUpsSync();
initAwayMonitor();
initJellyfinSync();
initAuthentikSync();
initPrometheusSync();
initImmichSync();
initPaperlessSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+15
View File
@@ -99,6 +99,21 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "token", label: "Long-lived access token", secret: true, type: "password", placeholder: "made on your profile page in Home Assistant" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
paperless: [
{ key: "url", label: "Paperless-ngx URL", secret: false, placeholder: "http://paperless.example.lan:8000" },
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "your profile → API Auth Token; a superuser's, for the system status" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
immich: [
{ key: "url", label: "Immich URL", secret: false, placeholder: "http://immich.example.lan:2283" },
{ key: "apiKey", label: "API key", secret: true, type: "password", placeholder: "Account Settings → API Keys; read permissions are enough" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
prometheus: [
{ key: "url", label: "Prometheus URL", secret: false, placeholder: "http://prometheus.example.lan:9090" },
{ key: "token", label: "Bearer token (optional)", secret: true, optional: true, type: "password", placeholder: "only if something in front of Prometheus asks for one" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
authentik: [
{ key: "url", label: "Authentik URL", secret: false, placeholder: "https://auth.example.lan" },
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Directory → Tokens and App passwords, intent API Token" },
+158
View File
@@ -0,0 +1,158 @@
/**
* Immich adapter. Config: url, apiKey (made in Immich under Account Settings → API Keys; read permissions are enough: server.about, server.storage, server.statistics and
* job.read — or queue.read on newer versions); optional: insecure.
*
* Reading only: the version (and whether a newer one is out), how full the disk under the library is, how many photos and videos there are and how much space they take, and the
* background job queues — above all how many jobs have **failed** (thumbnails or metadata that were never made). Nothing is sent to Immich but GET requests for these.
*
* Endpoints and field names are from Immich's published API (`/api/server/about`, `/server/storage`, `/server/statistics`, `/server/version-check`, `/jobs` — `/queues` on newer
* versions) and have not been checked against a live server here; every field is read defensively, and a part that cannot be read (statistics need an administrator's key) is left out.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface ImmichConfig {
url: string;
apiKey: string;
insecure?: boolean;
}
export interface ImmichQueue {
name: string;
active: number;
waiting: number;
failed: number;
paused: boolean;
}
export interface ImmichRead {
version: string | null;
latestVersion: string | null;
updateAvailable: boolean;
/** The disk under the library; null when it could not be asked. */
disk: { size: number; used: number; percent: number } | null;
photos: number | null;
videos: number | null;
/** What the library takes, in bytes. */
usageBytes: number | null;
/** Null when the queues could not be asked, which is not the same as having none. */
queues: ImmichQueue[] | null;
}
export interface ImmichAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<ImmichRead>;
}
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
const count = (v: unknown): number => num(v) ?? 0;
/** Pure: "v1.120.2" and "1.120.2" are the same version, and 1.120.10 is later than 1.120.9. Null when either is not a version. */
export function newerVersion(latest: string | null, current: string | null): boolean {
const parts = (s: string | null) => (s ? /^v?(\d+)\.(\d+)\.(\d+)/.exec(s.trim())?.slice(1).map(Number) : undefined);
const a = parts(latest);
const b = parts(current);
if (!a || !b) return false;
for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] > b[i];
return false;
}
/** Pure: the version from /server/about (a string like "v1.120.2"), or from /server/version ({ major, minor, patch }). */
export function parseVersion(raw: unknown): string | null {
const o = obj(raw);
const v = str(o.version);
if (v) return v;
const [major, minor, patch] = [num(o.major), num(o.minor), num(o.patch)];
return major !== null && minor !== null && patch !== null ? `v${major}.${minor}.${patch}` : null;
}
/** Pure: the disk the library is on. */
export function parseStorage(raw: unknown): ImmichRead["disk"] {
const o = obj(raw);
const size = num(o.diskSizeRaw);
const used = num(o.diskUseRaw);
if (size === null || used === null || size <= 0) return null;
return { size, used, percent: num(o.diskUsagePercentage) ?? Math.round((used / size) * 1000) / 10 };
}
/** Pure: the queues, from /jobs (an object keyed by queue name) or /queues (a list); each with its counts. Null when the answer is neither. */
export function parseQueues(raw: unknown): ImmichQueue[] | null {
const read = (name: string, q: Record<string, unknown>): ImmichQueue => {
const counts = obj(q.jobCounts ?? q.statistics);
const status = obj(q.queueStatus);
return { name, active: count(counts.active), waiting: count(counts.waiting), failed: count(counts.failed), paused: q.isPaused === true || status.isPaused === true };
};
if (Array.isArray(raw)) return raw.map((q) => read(str(obj(q).name) ?? "queue", obj(q)));
const o = obj(raw);
const entries = Object.entries(o).filter(([, v]) => v !== null && typeof v === "object");
return entries.length > 0 ? entries.map(([name, q]) => read(name, obj(q))) : null;
}
export function createImmichAdapter(config: ImmichConfig): ImmichAdapter {
const base = String(config.url ?? "").replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers = { "x-api-key": config.apiKey, Accept: "application/json" };
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}/api${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`Immich did not accept the API key (HTTP ${res.status}) — check it, and that it may read the server, its storage and its jobs`);
if (res.status === 404) throw new Error("Immich has no API at this address — check the address and port (2283)");
if (res.status < 200 || res.status >= 300) throw new Error(`Immich error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Immich answered with something that isn't JSON — is the address its own?");
return res.json;
}
async function read(): Promise<ImmichRead> {
// The version first and as a hard requirement: it also proves the key is accepted.
let version: string | null;
try {
version = parseVersion(await get("/server/about"));
} catch (err) {
// Older versions have no /about; their /version is open, so a refused key is only found out by the next call.
if (err instanceof Error && /did not accept/.test(err.message)) throw err;
version = parseVersion(await get("/server/version"));
}
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
try {
return parse(await get(path));
} catch {
return null;
}
};
const [disk, stats, check, jobs] = await Promise.all([
soft("/server/storage", parseStorage),
soft("/server/statistics", obj),
soft("/server/version-check", obj),
soft("/jobs", parseQueues),
]);
const queues = jobs ?? (await soft("/queues", parseQueues));
const latest = check ? str(check.releaseVersion) : null;
return {
version,
latestVersion: latest,
updateAvailable: newerVersion(latest, version),
disk,
photos: stats ? num(stats.photos) : null,
videos: stats ? num(stats.videos) : null,
usageBytes: stats ? num(stats.usage) : null,
queues,
};
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/server/about").catch((err) => {
if (err instanceof Error && /did not accept/.test(err.message)) throw err;
return get("/server/version");
});
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("immich", { ping, read });
}
@@ -0,0 +1,172 @@
/**
* Paperless-ngx adapter. Config: url, token (made in Paperless under your profile → API Auth Token, or in the admin pages under Auth Tokens; a superuser's token is needed for the
* system status); optional: insecure.
*
* Reading only: the version (and whether a newer one is out), the system status Paperless itself keeps — the database, Redis, the Celery worker that consumes documents, the search
* index, the classifier and the sanity check — the disk, how many documents there are and how many wait in the inbox, and the **documents that failed to import**. Nothing is sent
* to Paperless but GET requests for these.
*
* Endpoints and field names are from Paperless-ngx's published API (`/api/status/`, `/api/statistics/`, `/api/tasks/`, `/api/remote_version/`) and have not been checked against a
* live server here; every field is read defensively, and a part that cannot be read is left out.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface PaperlessConfig {
url: string;
token: string;
insecure?: boolean;
}
export interface PaperlessComponent {
/** database, redis, celery, index, classifier, sanity_check. */
name: string;
/** OK, WARNING or ERROR, as Paperless says. */
status: string;
error: string | null;
}
export interface PaperlessFailure {
file: string;
error: string | null;
at: string | null;
}
export interface PaperlessRead {
version: string | null;
latestVersion: string | null;
updateAvailable: boolean;
/** The disk the documents are on; null when it could not be asked. */
storage: { total: number; available: number } | null;
/** Null when the status could not be asked (it needs a superuser), which is not the same as everything being well. */
components: PaperlessComponent[] | null;
documents: number | null;
inbox: number | null;
/** Documents whose import failed in the last week and that nobody has dismissed; null when the tasks could not be asked. */
failed: PaperlessFailure[] | null;
}
export interface PaperlessAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(now?: Date): Promise<PaperlessRead>;
}
/** A failed import older than this is no longer news. */
export const FAILED_WITHIN_MS = 7 * 86_400_000;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
const iso = (v: unknown): string | null => {
const s = str(v);
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
};
/** The components /status/ reports on, with the key their status and error fields start with. */
const COMPONENTS: { name: string; where: "database" | "tasks"; key: string }[] = [
{ name: "database", where: "database", key: "" },
{ name: "redis", where: "tasks", key: "redis" },
{ name: "celery", where: "tasks", key: "celery" },
{ name: "index", where: "tasks", key: "index" },
{ name: "classifier", where: "tasks", key: "classifier" },
{ name: "sanity_check", where: "tasks", key: "sanity_check" },
];
/** Pure: the system status — version, disk and each component with how it is. Components is null when the answer holds none of them. */
export function parseStatus(raw: unknown): { version: string | null; storage: PaperlessRead["storage"]; components: PaperlessComponent[] | null } {
const o = obj(raw);
const storage = obj(o.storage);
const total = num(storage.total);
const available = num(storage.available);
const components: PaperlessComponent[] = [];
for (const c of COMPONENTS) {
const section = obj(o[c.where]);
const status = str(c.where === "database" ? section.status : section[`${c.key}_status`]);
if (!status) continue;
components.push({ name: c.name, status: status.toUpperCase(), error: str(c.where === "database" ? section.error : section[`${c.key}_error`]) });
}
return {
version: str(o.pngx_version),
storage: total !== null && available !== null && total > 0 ? { total, available } : null,
components: components.length > 0 ? components : null,
};
}
/** Pure: the documents and how many are in the inbox. */
export function parseStatistics(raw: unknown): { documents: number | null; inbox: number | null } {
const o = obj(raw);
return { documents: num(o.documents_total), inbox: num(o.documents_inbox) };
}
/** Pure: the imports that failed in the last week and have not been dismissed, newest first. Null when the answer holds no list of tasks. */
export function parseFailures(raw: unknown, now: Date): PaperlessFailure[] | null {
const list = Array.isArray(raw) ? raw : Array.isArray(obj(raw).results) ? (obj(raw).results as unknown[]) : null;
if (!list) return null;
const since = now.getTime() - FAILED_WITHIN_MS;
const out: (PaperlessFailure & { t: number })[] = [];
for (const item of list) {
const t = obj(item);
if (str(t.status)?.toUpperCase() !== "FAILURE" || t.acknowledged === true) continue;
const at = iso(t.date_done) ?? iso(t.date_created);
const time = at ? Date.parse(at) : now.getTime();
if (time < since) continue;
out.push({ file: str(t.task_file_name) ?? str(t.task_name) ?? "a document", error: str(t.result), at, t: time });
}
return out.sort((a, b) => b.t - a.t).map(({ t: _t, ...f }) => f);
}
/** Pure: whether Paperless says a newer version exists. */
export function parseRemoteVersion(raw: unknown): { latestVersion: string | null; updateAvailable: boolean } {
const o = obj(raw);
return { latestVersion: str(o.version), updateAvailable: o.update_available === true };
}
export function createPaperlessAdapter(config: PaperlessConfig): PaperlessAdapter {
const base = String(config.url ?? "").replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers = { Authorization: `Token ${config.token}`, Accept: "application/json" };
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}/api${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`Paperless did not accept the token (HTTP ${res.status}) — check it, and that it belongs to a superuser if the system status is wanted`);
if (res.status === 404) throw new Error("Paperless has no API at this address — check the address and port (8000)");
if (res.status < 200 || res.status >= 300) throw new Error(`Paperless error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Paperless answered with something that isn't JSON — is the address its own?");
return res.json;
}
async function read(now: Date = new Date()): Promise<PaperlessRead> {
// The statistics are open to any user, so they are the hard requirement that proves the token works; the status needs a superuser and is optional.
const stats = parseStatistics(await get("/statistics/"));
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
try {
return parse(await get(path));
} catch {
return null;
}
};
const [status, failed, remote] = await Promise.all([soft("/status/", parseStatus), soft("/tasks/", (r) => parseFailures(r, now)), soft("/remote_version/", parseRemoteVersion)]);
return {
version: status?.version ?? null,
latestVersion: remote?.latestVersion ?? null,
updateAvailable: remote?.updateAvailable ?? false,
storage: status?.storage ?? null,
components: status?.components ?? null,
documents: stats.documents,
inbox: stats.inbox,
failed,
};
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/statistics/");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("paperless", { ping, read });
}
@@ -0,0 +1,140 @@
/**
* Prometheus adapter. Config: url; optional: token (a bearer token, when something in front of Prometheus asks for one), insecure.
*
* Reading only: Prometheus's version, every scrape target and whether its last scrape worked (a node exporter that stopped answering), and the alerts that Prometheus's own rules have
* pending or firing. That makes Prometheus a source: whatever you already measure there is seen here too. Nothing is sent to Prometheus but GET requests for these.
*
* Endpoints and field names are from Prometheus's published HTTP API (`/api/v1/status/buildinfo`, `/api/v1/targets`, `/api/v1/alerts`) and have not been checked against a live server
* here; every field is read defensively. Alerts that Grafana manages itself are not in Prometheus's list and are not seen.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface PrometheusConfig {
url: string;
token?: string;
insecure?: boolean;
}
export interface PrometheusTarget {
job: string;
instance: string;
health: "up" | "down" | "unknown";
/** Why the last scrape failed. */
error: string | null;
lastScrape: string | null;
}
export interface PrometheusAlert {
name: string;
/** From the alert's own `severity` label, lower-case; null when it has none. */
severity: string | null;
state: "firing" | "pending";
summary: string | null;
instance: string | null;
activeAt: string | null;
}
export interface PrometheusRead {
version: string | null;
/** Null when the targets could not be asked, which is not the same as there being none. */
targets: PrometheusTarget[] | null;
alerts: PrometheusAlert[] | null;
}
export interface PrometheusAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<PrometheusRead>;
}
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const iso = (v: unknown): string | null => {
const s = str(v);
return s && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
};
/** Pure: the version from /status/buildinfo. */
export function parseBuildInfo(raw: unknown): string | null {
return str(obj(obj(raw).data).version);
}
/** Pure: the active targets from /targets; null when the answer has no list in it. */
export function parseTargets(raw: unknown): PrometheusTarget[] | null {
const list = obj(obj(raw).data).activeTargets;
if (!Array.isArray(list)) return null;
const out: PrometheusTarget[] = [];
for (const item of list) {
const t = obj(item);
const labels = obj(t.labels);
const job = str(labels.job) ?? str(t.scrapePool);
if (!job) continue;
const health = str(t.health)?.toLowerCase();
out.push({
job,
instance: str(labels.instance) ?? str(t.scrapeUrl) ?? "?",
health: health === "up" || health === "down" ? health : "unknown",
error: str(t.lastError),
lastScrape: iso(t.lastScrape),
});
}
return out;
}
/** Pure: the alerts from /alerts; null when the answer has no list in it. */
export function parseAlerts(raw: unknown): PrometheusAlert[] | null {
const list = obj(obj(raw).data).alerts;
if (!Array.isArray(list)) return null;
const out: PrometheusAlert[] = [];
for (const item of list) {
const a = obj(item);
const labels = obj(a.labels);
const annotations = obj(a.annotations);
const name = str(labels.alertname);
const state = str(a.state)?.toLowerCase();
if (!name || (state !== "firing" && state !== "pending")) continue;
out.push({ name, severity: str(labels.severity)?.toLowerCase() ?? null, state, summary: str(annotations.summary) ?? str(annotations.description) ?? str(annotations.message), instance: str(labels.instance), activeAt: iso(a.activeAt) });
}
return out;
}
export function createPrometheusAdapter(config: PrometheusConfig): PrometheusAdapter {
const base = String(config.url ?? "").replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers: Record<string, string> = config.token ? { Authorization: `Bearer ${config.token}` } : {};
async function get(path: string): Promise<unknown> {
const res = await requestJson(`${base}${path}`, { headers, insecure });
if (res.status === 401 || res.status === 403) throw new Error(`Prometheus did not accept the request (HTTP ${res.status}) — something in front of it wants a token`);
if (res.status === 404) throw new Error("Prometheus has no API at this address — check the address and port (9090)");
if (res.status < 200 || res.status >= 300) throw new Error(`Prometheus error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Prometheus answered with something that isn't JSON — is the address its own?");
return res.json;
}
async function read(): Promise<PrometheusRead> {
const version = parseBuildInfo(await get("/api/v1/status/buildinfo"));
// The rest is worth having but not worth failing for.
const soft = async <T>(path: string, parse: (raw: unknown) => T): Promise<T | null> => {
try {
return parse(await get(path));
} catch {
return null;
}
};
const [targets, alerts] = await Promise.all([soft("/api/v1/targets?state=active", parseTargets), soft("/api/v1/alerts", parseAlerts)]);
return { version, targets, alerts };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
await get("/api/v1/status/buildinfo");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("prometheus", { ping, read });
}
+9
View File
@@ -16,6 +16,9 @@ import { createHomeAssistantAdapter } from "./homeassistant/adapter.js";
import { createUpsAdapter } from "./ups/adapter.js";
import { createJellyfinAdapter } from "./jellyfin/adapter.js";
import { createAuthentikAdapter } from "./authentik/adapter.js";
import { createPrometheusAdapter } from "./prometheus/adapter.js";
import { createImmichAdapter } from "./immich/adapter.js";
import { createPaperlessAdapter } from "./paperless/adapter.js";
export interface PingableAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
@@ -62,6 +65,12 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createJellyfinAdapter(config as any);
case "authentik":
return createAuthentikAdapter(config as any);
case "prometheus":
return createPrometheusAdapter(config as any);
case "immich":
return createImmichAdapter(config as any);
case "paperless":
return createPaperlessAdapter(config as any);
default:
throw new Error(`Integration type "${type}" is not implemented yet`);
}
+49
View File
@@ -0,0 +1,49 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { immichState, integrations } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { immichProblems, immichSyncStatuses, isImmichFresh, queuesOf } from "../services/immichSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const immichRouter = Router();
immichRouter.use(requireAuth);
/** Every Immich, how it is, from what the last read stored: the disk under the library, the counts, the job queues and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
immichRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich")).orderBy(integrations.name),
db.select().from(immichState),
immichSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
immiches: rows.map((r) => {
const state = byId.get(r.id) ?? null;
const status = statuses[r.id] ?? null;
const fresh = isImmichFresh(status ?? undefined, now);
return {
id: r.id,
name: r.name,
enabled: r.enabled,
status,
fresh,
state: state
? {
version: state.version,
latestVersion: state.latestVersion,
updateAvailable: state.updateAvailable,
disk: state.diskSize !== null && state.diskUsed !== null ? { size: state.diskSize, used: state.diskUsed, percent: state.diskPercent } : null,
photos: state.photos,
videos: state.videos,
usageBytes: state.usageBytes,
queues: queuesOf(state),
updatedAt: state.updatedAt,
}
: null,
problems: state && fresh ? immichProblems(r.id, r.name, state) : [],
};
}),
});
}));
+12
View File
@@ -31,6 +31,9 @@ import { removeHomeAssistantData, syncHomeAssistant } from "../services/homeAssi
import { forgetUps, syncUps } from "../services/upsSync.js";
import { forgetJellyfin, syncJellyfin } from "../services/jellyfinSync.js";
import { forgetAuthentik, syncAuthentik } from "../services/authentikSync.js";
import { forgetPrometheus, syncPrometheus } from "../services/prometheusSync.js";
import { forgetImmich, syncImmich } from "../services/immichSync.js";
import { forgetPaperless, syncPaperless } from "../services/paperlessSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { getSettings, updateSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
@@ -135,6 +138,9 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "ups") void syncUps(created.id).then(() => invalidateAlertsCache());
if (type === "jellyfin") void syncJellyfin(created.id).then(() => invalidateAlertsCache());
if (type === "authentik") void syncAuthentik(created.id).then(() => invalidateAlertsCache());
if (type === "prometheus") void syncPrometheus(created.id).then(() => invalidateAlertsCache());
if (type === "immich") void syncImmich(created.id).then(() => invalidateAlertsCache());
if (type === "paperless") void syncPaperless(created.id).then(() => invalidateAlertsCache());
res.status(201).json({
integration: {
@@ -240,6 +246,9 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "ups" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncUps(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "jellyfin" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncJellyfin(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "authentik" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncAuthentik(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "prometheus" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPrometheus(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "immich" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncImmich(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "paperless" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncPaperless(updated.id).then(() => invalidateAlertsCache());
res.json({
integration: {
@@ -321,6 +330,9 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "ups") await forgetUps(id);
if (existing.type === "jellyfin") await forgetJellyfin(id);
if (existing.type === "authentik") await forgetAuthentik(id);
if (existing.type === "prometheus") await forgetPrometheus(id);
if (existing.type === "immich") await forgetImmich(id);
if (existing.type === "paperless") await forgetPaperless(id);
await db.delete(integrations).where(eq(integrations.id, id));
if (existing.credentialId) {
await db.delete(integrationCredentials).where(eq(integrationCredentials.id, existing.credentialId));
+49
View File
@@ -0,0 +1,49 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, paperlessState } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { componentsOf, failedOf, isPaperlessFresh, paperlessProblems, paperlessSyncStatuses } from "../services/paperlessSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const paperlessRouter = Router();
paperlessRouter.use(requireAuth);
/** Every Paperless-ngx, how it is, from what the last read stored: its status components, the disk, the counts, the imports that failed and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
paperlessRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless")).orderBy(integrations.name),
db.select().from(paperlessState),
paperlessSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
res.json({
paperlesses: rows.map((r) => {
const state = byId.get(r.id) ?? null;
const status = statuses[r.id] ?? null;
const fresh = isPaperlessFresh(status ?? undefined, now);
return {
id: r.id,
name: r.name,
enabled: r.enabled,
status,
fresh,
state: state
? {
version: state.version,
latestVersion: state.latestVersion,
updateAvailable: state.updateAvailable,
storage: state.storageTotal !== null && state.storageAvailable !== null ? { total: state.storageTotal, available: state.storageAvailable } : null,
components: componentsOf(state),
documents: state.documents,
inbox: state.inbox,
failed: failedOf(state),
updatedAt: state.updatedAt,
}
: null,
problems: state && fresh ? paperlessProblems(r.id, r.name, state) : [],
};
}),
});
}));
+37
View File
@@ -0,0 +1,37 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, prometheusState } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { alertsOf, downOf, isPrometheusFresh, jobsOf, prometheusProblems, prometheusSyncStatuses } from "../services/prometheusSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const prometheusRouter = Router();
prometheusRouter.use(requireAuth);
/** Every Prometheus, how it is, from what the last read stored: targets by job, the ones that are down, the alerts firing, and what is wrong. Everyone signed in can see it, like the other pages made of stored readings. */
prometheusRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "prometheus")).orderBy(integrations.name),
db.select().from(prometheusState),
prometheusSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = new Date();
res.json({
prometheuses: rows.map((r) => {
const state = byId.get(r.id) ?? null;
const status = statuses[r.id] ?? null;
const fresh = isPrometheusFresh(status ?? undefined, now.getTime());
return {
id: r.id,
name: r.name,
enabled: r.enabled,
status,
fresh,
state: state ? { version: state.version, jobs: jobsOf(state), down: downOf(state), alerts: alertsOf(state), updatedAt: state.updatedAt } : null,
problems: state && fresh ? prometheusProblems(r.id, r.name, state, now) : [],
};
}),
});
}));
+11 -2
View File
@@ -77,6 +77,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
duplicati: { label: "Duplicati", link: "/duplicati" },
jellyfin: { label: "Jellyfin", link: "/jellyfin" },
authentik: { label: "Authentik", link: "/authentik" },
prometheus: { label: "Prometheus", link: "/prometheus" },
immich: { label: "Immich", link: "/immich" },
paperless: { label: "Paperless-ngx", link: "/paperless" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -115,6 +118,12 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" };
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
if (m) return { type: "immich", id: Number(m[1]), category: key.startsWith("immich-disk") ? "disk" : "services" };
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
if (m) return { type: "prometheus", id: Number(m[1]), category: "monitoring" };
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
if (m) return { type: "authentik", id: Number(m[1]), category: /^authentik-(?:cert|token):/.test(key) ? "expiry" : /^authentik-(?:outpost|worker)/.test(key) ? "services" : "monitoring" };
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
@@ -241,7 +250,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -251,7 +260,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+73 -1
View File
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { authentikState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { authentikState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -22,6 +22,9 @@ import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
import { currentProblems as duplicatiProblemsNow, type DuplicatiProblem } from "./duplicati.js";
import { isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, type JellyfinProblem } from "./jellyfinSync.js";
import { authentikProblems, authentikSyncStatuses, isAuthentikFresh, type AuthentikProblem } from "./authentikSync.js";
import { isPrometheusFresh, prometheusProblems, prometheusSyncStatuses, type PrometheusProblem } from "./prometheusSync.js";
import { immichProblems, immichSyncStatuses, isImmichFresh, type ImmichProblem } from "./immichSync.js";
import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type PaperlessProblem } from "./paperlessSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -89,6 +92,12 @@ export interface HealthSnapshot {
baselines?: BaselineFinding[];
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
levels?: LevelFinding[];
/** What is wrong with each Paperless-ngx (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/paperlessSync.ts). */
paperless?: { integrationId: number; problems: PaperlessProblem[] }[];
/** What is wrong with each Immich (a disk filling, background jobs failing), only while that reading is recent (services/immichSync.ts). */
immich?: { integrationId: number; problems: ImmichProblem[] }[];
/** What is wrong with each Prometheus (a target that stopped answering, an alert its rules fire), only while that reading is recent (services/prometheusSync.ts). */
prometheus?: { integrationId: number; problems: PrometheusProblem[] }[];
/** What is wrong with each Authentik, from what was stored, only while that reading is recent (services/authentikSync.ts). */
authentik?: { integrationId: number; problems: AuthentikProblem[] }[];
/** What is wrong with each Jellyfin, from what was stored, only while that reading is recent (services/jellyfinSync.ts). */
@@ -204,6 +213,15 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
}
}
// A Paperless whose worker has stopped, whose imports fail, or whose disk is filling.
for (const p of snapshot.paperless ?? []) for (const c of p.problems) out.push({ key: c.key, source: `paperless:${p.integrationId}`, message: c.message, severity: c.severity });
// An Immich whose disk is filling or whose background jobs are failing.
for (const i of snapshot.immich ?? []) for (const c of i.problems) out.push({ key: c.key, source: `immich:${i.integrationId}`, message: c.message, severity: c.severity });
// A Prometheus target that has stopped answering, an alert that Prometheus's own rules are firing.
for (const p of snapshot.prometheus ?? []) for (const c of p.problems) out.push({ key: c.key, source: `prometheus:${p.integrationId}`, message: c.message, severity: c.severity });
// An Authentik outpost that went quiet, a certificate or token about to run out, no worker, failed sign-ins.
for (const a of snapshot.authentik ?? []) for (const p of a.problems) out.push({ key: p.key, source: `authentik:${a.integrationId}`, message: p.message, severity: p.severity });
@@ -440,6 +458,60 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
}
}
// Paperless: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
if (rowsL.length > 0) {
const statuses = await paperlessSyncStatuses();
for (const r of rowsL) {
const [state] = await db.select().from(paperlessState).where(eq(paperlessState.integrationId, r.id)).limit(1);
if (!state || !isPaperlessFresh(statuses[r.id], Date.now())) {
held.add(`paperless:${r.id}`);
continue;
}
(snapshot.paperless ??= []).push({ integrationId: r.id, problems: paperlessProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge Paperless:", err instanceof Error ? err.message : err);
}
// Immich: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsI = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
if (rowsI.length > 0) {
const statuses = await immichSyncStatuses();
for (const r of rowsI) {
const [state] = await db.select().from(immichState).where(eq(immichState.integrationId, r.id)).limit(1);
if (!state || !isImmichFresh(statuses[r.id], Date.now())) {
held.add(`immich:${r.id}`);
continue;
}
(snapshot.immich ??= []).push({ integrationId: r.id, problems: immichProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge Immich:", err instanceof Error ? err.message : err);
}
// Prometheus: read by its own pass every couple of minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsP = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
if (rowsP.length > 0) {
const statuses = await prometheusSyncStatuses();
for (const r of rowsP) {
const [state] = await db.select().from(prometheusState).where(eq(prometheusState.integrationId, r.id)).limit(1);
if (!state || !isPrometheusFresh(statuses[r.id], Date.now())) {
held.add(`prometheus:${r.id}`);
continue;
}
(snapshot.prometheus ??= []).push({ integrationId: r.id, problems: prometheusProblems(r.id, r.name, state) });
}
}
} catch (err) {
console.error("[health] couldn't judge Prometheus:", err instanceof Error ? err.message : err);
}
// Authentik: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsA = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "authentik"), eq(integrations.enabled, true)));
+154
View File
@@ -0,0 +1,154 @@
/**
* Keeps what Immich says up to date: every five minutes each Immich integration is read (version, the disk under the library, the photo and video counts, the background job
* queues) and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Immich.
*
* Photos are what cannot be made again, and Immich fails quietly: a disk that fills up, or thumbnail and metadata jobs that fail one by one and are never looked at. Those are
* what is raised.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { immichState, integrations } from "../db/schema.js";
import { createImmichAdapter, type ImmichQueue, type ImmichRead } from "../integrations/immich/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
/** This many failed background jobs, together, is a warning. */
export const FAILED_JOBS_WARN = 10;
export const DISK_WARN_PERCENT = 90;
export const DISK_CRITICAL_PERCENT = 95;
const STATUS_FLAG = "immichSync";
export interface ImmichSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, ImmichSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, ImmichSyncStatus>) : {};
} catch {
return {};
}
}
export async function immichSyncStatuses(): Promise<Record<number, ImmichSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isImmichFresh = (status: ImmichSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof immichState.$inferSelect;
/** Null stays null: a list that could not be read is not an empty one. */
export const queuesOf = (row: Pick<StateRow, "queues">): ImmichQueue[] | null => {
if (row.queues === null) return null;
try {
const v = JSON.parse(row.queues);
return Array.isArray(v) ? (v as ImmichQueue[]) : null;
} catch {
return null;
}
};
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: ImmichRead, now: Date = new Date()): Promise<void> {
const values = {
version: r.version,
latestVersion: r.latestVersion,
updateAvailable: r.updateAvailable,
diskSize: r.disk?.size ?? null,
diskUsed: r.disk?.used ?? null,
diskPercent: r.disk?.percent ?? null,
photos: r.photos,
videos: r.videos,
usageBytes: r.usageBytes,
queues: r.queues && JSON.stringify(r.queues),
updatedAt: now.toISOString(),
};
await db.insert(immichState).values({ integrationId, ...values }).onConflictDoUpdate({ target: immichState.integrationId, set: values });
}
/** Reads one Immich and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncImmich(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<ImmichRead>): Promise<ImmichSyncStatus> {
let status: ImmichSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "immich") throw new Error("not an Immich integration");
const result = await (read ? read(loaded.config) : createImmichAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[immich] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[immich] couldn't record the result:", err);
}
return status;
}
export async function syncAllImmich(now: Date = new Date()): Promise<Record<number, ImmichSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
const out: Record<number, ImmichSyncStatus> = {};
for (const r of rows) out[r.id] = await syncImmich(r.id, now);
return out;
}
export async function forgetImmich(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface ImmichProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
/** Pure: what is wrong with one Immich, from what is stored for it. */
export function immichProblems(id: number, name: string, s: Pick<StateRow, "diskSize" | "diskUsed" | "diskPercent" | "queues">): ImmichProblem[] {
const out: ImmichProblem[] = [];
if (s.diskPercent !== null && s.diskPercent >= DISK_WARN_PERCENT) {
out.push({
key: `immich-disk:${id}`,
severity: s.diskPercent >= DISK_CRITICAL_PERCENT ? "critical" : "warning",
message: `${name}: the disk under the library is ${Math.round(s.diskPercent)} % full${s.diskSize !== null && s.diskUsed !== null ? ` (${gb(s.diskUsed)} of ${gb(s.diskSize)})` : ""}`,
});
}
const queues = queuesOf(s) ?? [];
const failed = queues.filter((q) => q.failed > 0);
const total = failed.reduce((n, q) => n + q.failed, 0);
if (total >= FAILED_JOBS_WARN) {
const worst = [...failed].sort((a, b) => b.failed - a.failed).slice(0, 3).map((q) => `${q.name} ${q.failed}`).join(", ");
out.push({ key: `immich-jobs:${id}`, severity: "warning", message: `${name}: ${total} background jobs have failed (${worst}) — thumbnails or metadata that were never made; look at Administration → Jobs and run the failed ones again` });
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initImmichSync(): void {
if (timer) return;
const pass = () => void syncAllImmich().catch((err) => console.error("[immich] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 17_000);
console.log(`[immich] reading Immich every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+6
View File
@@ -33,6 +33,12 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
+38 -1
View File
@@ -5,11 +5,13 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
import { latestTests } from "./restoreTests.js";
import { heartbeatState, parseStamp } from "./heartbeats.js";
@@ -62,6 +64,39 @@ class Exposition {
}
}
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
async function addPaperlessMetrics(out: Exposition, now: Date) {
const statuses = await paperlessSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless"))).map((r) => [r.id, r]));
for (const s of await db.select().from(paperlessState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isPaperlessFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
if (s.documents !== null) out.add("homelab_paperless_documents", "gauge", "How many documents Paperless holds.", s.documents, labels);
if (s.inbox !== null) out.add("homelab_paperless_inbox", "gauge", "How many documents are waiting in Paperless's inbox.", s.inbox, labels);
const failed = paperlessFailedOf(s);
if (failed) out.add("homelab_paperless_failed_imports", "gauge", "Imports that failed in the last week and were not dismissed.", failed.length, labels);
for (const c of paperlessComponentsOf(s) ?? []) out.add("homelab_paperless_component_ok", "gauge", "1 if the part of Paperless named is in order, else 0.", c.status === "OK" ? 1 : 0, { name: row.name, component: c.name });
}
}
/** Each Immich, for /metrics: the photos and videos, what they take, how full the disk is and the jobs that failed. Only while the reading is recent. */
async function addImmichMetrics(out: Exposition, now: Date) {
const statuses = await immichSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich"))).map((r) => [r.id, r]));
for (const s of await db.select().from(immichState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isImmichFresh(statuses[s.integrationId], now.getTime())) continue;
const labels = { name: row.name };
if (s.photos !== null) out.add("homelab_immich_photos", "gauge", "How many photos Immich holds.", s.photos, labels);
if (s.videos !== null) out.add("homelab_immich_videos", "gauge", "How many videos Immich holds.", s.videos, labels);
if (s.usageBytes !== null) out.add("homelab_immich_library_bytes", "gauge", "What Immich's library takes, in bytes.", s.usageBytes, labels);
if (s.diskPercent !== null) out.add("homelab_immich_disk_used_percent", "gauge", "How full the disk under Immich's library is.", s.diskPercent, labels);
const queues = immichQueuesOf(s);
if (queues) out.add("homelab_immich_failed_jobs", "gauge", "Background jobs that have failed, all queues together.", queues.reduce((n, q) => n + q.failed, 0), labels);
}
}
/** Each Authentik, for /metrics: whether each outpost that serves something is reporting, how many workers there are, the failed sign-ins and the days left of each certificate. Only while the reading is recent. */
async function addAuthentikMetrics(out: Exposition, now: Date) {
const statuses = await authentikSyncStatuses();
@@ -300,6 +335,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addDuplicatiMetrics(out, now);
await addJellyfinMetrics(out, now);
await addAuthentikMetrics(out, now);
await addImmichMetrics(out, now);
await addPaperlessMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+162
View File
@@ -0,0 +1,162 @@
/**
* Keeps what Paperless-ngx says up to date: every five minutes each Paperless integration is read (its system status, the disk, the document counts and the imports that failed)
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Paperless.
*
* Paperless fails quietly in two ways: the worker that consumes documents stops (a scan dropped in the folder just sits there), or one import fails and nobody looks. Both are raised,
* and so is anything Paperless's own status check says is in error.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, paperlessState } from "../db/schema.js";
import { createPaperlessAdapter, type PaperlessComponent, type PaperlessFailure, type PaperlessRead } from "../integrations/paperless/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
export const DISK_WARN_PERCENT = 90;
export const DISK_CRITICAL_PERCENT = 95;
/** If one of these is in error, nothing can be consumed or found: critical. The others are a warning. */
const ESSENTIAL = new Set(["database", "redis", "celery"]);
const STATUS_FLAG = "paperlessSync";
export interface PaperlessSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, PaperlessSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, PaperlessSyncStatus>) : {};
} catch {
return {};
}
}
export async function paperlessSyncStatuses(): Promise<Record<number, PaperlessSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isPaperlessFresh = (status: PaperlessSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof paperlessState.$inferSelect;
/** Null stays null: a list that could not be read is not an empty one. */
const parse = <T>(text: string | null): T[] | null => {
if (text === null) return null;
try {
const v = JSON.parse(text);
return Array.isArray(v) ? (v as T[]) : null;
} catch {
return null;
}
};
export const componentsOf = (row: Pick<StateRow, "components">) => parse<PaperlessComponent>(row.components);
export const failedOf = (row: Pick<StateRow, "failed">) => parse<PaperlessFailure>(row.failed);
/** Writes down one read, replacing the last. */
export async function storeRead(integrationId: number, r: PaperlessRead, now: Date = new Date()): Promise<void> {
const values = {
version: r.version,
latestVersion: r.latestVersion,
updateAvailable: r.updateAvailable,
storageTotal: r.storage?.total ?? null,
storageAvailable: r.storage?.available ?? null,
components: r.components && JSON.stringify(r.components),
documents: r.documents,
inbox: r.inbox,
failed: r.failed && JSON.stringify(r.failed),
updatedAt: now.toISOString(),
};
await db.insert(paperlessState).values({ integrationId, ...values }).onConflictDoUpdate({ target: paperlessState.integrationId, set: values });
}
/** Reads one Paperless and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncPaperless(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PaperlessRead>): Promise<PaperlessSyncStatus> {
let status: PaperlessSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "paperless") throw new Error("not a Paperless integration");
const result = await (read ? read(loaded.config) : createPaperlessAdapter(loaded.config as never).read(now));
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[paperless] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[paperless] couldn't record the result:", err);
}
return status;
}
export async function syncAllPaperless(now: Date = new Date()): Promise<Record<number, PaperlessSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
const out: Record<number, PaperlessSyncStatus> = {};
for (const r of rows) out[r.id] = await syncPaperless(r.id, now);
return out;
}
export async function forgetPaperless(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface PaperlessProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
const WORDS: Record<string, string> = { database: "the database", redis: "Redis", celery: "the worker that consumes documents", index: "the search index", classifier: "the classifier", sanity_check: "the sanity check" };
/** Pure: what is wrong with one Paperless, from what is stored for it. */
export function paperlessProblems(id: number, name: string, s: Pick<StateRow, "storageTotal" | "storageAvailable" | "components" | "failed">): PaperlessProblem[] {
const out: PaperlessProblem[] = [];
for (const c of componentsOf(s) ?? []) {
const word = WORDS[c.name] ?? c.name;
if (c.status === "ERROR") {
out.push({ key: `paperless-component:${id}:${c.name}`, severity: ESSENTIAL.has(c.name) ? "critical" : "warning", message: `${name}: ${word} is in error${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
} else if (c.status === "WARNING" && c.name === "sanity_check") {
out.push({ key: `paperless-component:${id}:${c.name}`, severity: "warning", message: `${name}: the sanity check found problems with some documents${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
}
}
const failed = failedOf(s) ?? [];
if (failed.length > 0) {
const files = failed.slice(0, 3).map((f) => f.file).join(", ") + (failed.length > 3 ? ` and ${failed.length - 3} more` : "");
const why = failed[0].error ? ` — the latest: ${failed[0].error.length > 140 ? `${failed[0].error.slice(0, 137)}…` : failed[0].error}` : "";
out.push({ key: `paperless-failed:${id}`, severity: "warning", message: `${name}: ${failed.length} document${failed.length === 1 ? "" : "s"} failed to import (${files})${why}` });
}
if (s.storageTotal !== null && s.storageAvailable !== null && s.storageTotal > 0) {
const percent = ((s.storageTotal - s.storageAvailable) / s.storageTotal) * 100;
if (percent >= DISK_WARN_PERCENT) {
out.push({ key: `paperless-disk:${id}`, severity: percent >= DISK_CRITICAL_PERCENT ? "critical" : "warning", message: `${name}: the disk under the documents is ${Math.round(percent)} % full (${gb(s.storageTotal - s.storageAvailable)} of ${gb(s.storageTotal)})` });
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initPaperlessSync(): void {
if (timer) return;
const pass = () => void syncAllPaperless().catch((err) => console.error("[paperless] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 19_000);
console.log(`[paperless] reading Paperless every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+187
View File
@@ -0,0 +1,187 @@
/**
* Keeps what Prometheus says up to date: every two minutes each Prometheus integration is read (its scrape targets and whether they answer, and the alerts its rules have firing)
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Prometheus.
*
* Prometheus is where the things you already measure live — a node exporter on each host, a service's own metrics — so this makes them part of the one list of what is wrong:
* a **target that has stopped answering** (for five minutes, so one failed scrape is not news) and **an alert that Prometheus's rules are firing**.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, prometheusState } from "../db/schema.js";
import { createPrometheusAdapter, type PrometheusAlert, type PrometheusRead } from "../integrations/prometheus/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 2 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 10 * 60_000;
/** A target must have been down this long before it is a problem; one failed scrape is not. */
export const TARGET_DOWN_AFTER_MS = 5 * 60_000;
const STATUS_FLAG = "prometheusSync";
export interface PrometheusSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
export interface JobCount {
job: string;
up: number;
down: number;
}
export interface DownTarget {
job: string;
instance: string;
error: string | null;
/** When it was first seen down, carried over from one read to the next. */
since: string;
}
async function readStatuses(): Promise<Record<string, PrometheusSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, PrometheusSyncStatus>) : {};
} catch {
return {};
}
}
export async function prometheusSyncStatuses(): Promise<Record<number, PrometheusSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isPrometheusFresh = (status: PrometheusSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof prometheusState.$inferSelect;
/** Null stays null: a list that could not be read is not an empty one. */
const parse = <T>(text: string | null): T[] | null => {
if (text === null) return null;
try {
const v = JSON.parse(text);
return Array.isArray(v) ? (v as T[]) : null;
} catch {
return null;
}
};
export const jobsOf = (row: Pick<StateRow, "jobs">) => parse<JobCount>(row.jobs);
export const downOf = (row: Pick<StateRow, "down">) => parse<DownTarget>(row.down);
export const alertsOf = (row: Pick<StateRow, "alerts">) => parse<PrometheusAlert>(row.alerts);
/** Writes down one read, replacing the last; a target that was down before keeps the time it was first seen down. */
export async function storeRead(integrationId: number, r: PrometheusRead, now: Date = new Date()): Promise<void> {
const [before] = await db.select({ down: prometheusState.down }).from(prometheusState).where(eq(prometheusState.integrationId, integrationId)).limit(1);
const earlier = new Map((before ? downOf(before) ?? [] : []).map((d) => [`${d.job}\u0000${d.instance}`, d.since]));
let jobs: JobCount[] | null = null;
let down: DownTarget[] | null = null;
if (r.targets) {
const counts = new Map<string, JobCount>();
down = [];
for (const t of r.targets) {
const c = counts.get(t.job) ?? { job: t.job, up: 0, down: 0 };
if (t.health === "down") {
c.down += 1;
down.push({ job: t.job, instance: t.instance, error: t.error, since: earlier.get(`${t.job}\u0000${t.instance}`) ?? now.toISOString() });
} else if (t.health === "up") c.up += 1;
counts.set(t.job, c);
}
jobs = [...counts.values()].sort((a, b) => a.job.localeCompare(b.job));
}
const values = { version: r.version, jobs: jobs && JSON.stringify(jobs), down: down && JSON.stringify(down), alerts: r.alerts && JSON.stringify(r.alerts), updatedAt: now.toISOString() };
await db.insert(prometheusState).values({ integrationId, ...values }).onConflictDoUpdate({ target: prometheusState.integrationId, set: values });
}
/** Reads one Prometheus and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncPrometheus(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PrometheusRead>): Promise<PrometheusSyncStatus> {
let status: PrometheusSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "prometheus") throw new Error("not a Prometheus integration");
const result = await (read ? read(loaded.config) : createPrometheusAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[prometheus] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[prometheus] couldn't record the result:", err);
}
return status;
}
export async function syncAllPrometheus(now: Date = new Date()): Promise<Record<number, PrometheusSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
const out: Record<number, PrometheusSyncStatus> = {};
for (const r of rows) out[r.id] = await syncPrometheus(r.id, now);
return out;
}
export async function forgetPrometheus(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface PrometheusProblem {
key: string;
message: string;
severity: "warning" | "critical";
}
/** Alerts that are firing by design (a heartbeat to prove the pipeline works) and so say nothing. */
const BY_DESIGN = /^(watchdog|deadmansswitch|deadmanssnitch|infoinhibitor)$/i;
/** Pure: what is wrong, from what is stored. */
export function prometheusProblems(id: number, name: string, s: Pick<StateRow, "jobs" | "down" | "alerts">, now: Date = new Date()): PrometheusProblem[] {
const out: PrometheusProblem[] = [];
const jobs = new Map((jobsOf(s) ?? []).map((j) => [j.job, j]));
const byJob = new Map<string, DownTarget[]>();
for (const d of downOf(s) ?? []) {
if (now.getTime() - Date.parse(d.since) < TARGET_DOWN_AFTER_MS) continue;
byJob.set(d.job, [...(byJob.get(d.job) ?? []), d]);
}
for (const [job, list] of byJob) {
const c = jobs.get(job);
const total = c ? c.up + c.down : list.length;
const who = list.slice(0, 3).map((d) => d.instance).join(", ") + (list.length > 3 ? ` and ${list.length - 3} more` : "");
const why = list[0].error ? ` — ${list[0].error.length > 120 ? `${list[0].error.slice(0, 117)}…` : list[0].error}` : "";
out.push({ key: `prometheus-target:${id}:${job}`, severity: "warning", message: `${name}: ${list.length} of ${total} “${job}” target${total === 1 ? "" : "s"} not answering (${who})${why}` });
}
const firing = new Map<string, PrometheusAlert[]>();
for (const a of alertsOf(s) ?? []) {
if (a.state !== "firing" || BY_DESIGN.test(a.name) || a.severity === "info" || a.severity === "none") continue;
firing.set(a.name, [...(firing.get(a.name) ?? []), a]);
}
for (const [alertname, list] of firing) {
const critical = list.some((a) => a.severity === "critical" || a.severity === "page");
const first = list.find((a) => a.summary) ?? list[0];
out.push({
key: `prometheus-alert:${id}:${alertname}`,
severity: critical ? "critical" : "warning",
message: `${name}: ${alertname}${list.length > 1 ? ` (${list.length} firing)` : first.instance ? ` on ${first.instance}` : ""}${first.summary ? ` — ${first.summary}` : ""}`,
});
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initPrometheusSync(): void {
if (timer) return;
const pass = () => void syncAllPrometheus().catch((err) => console.error("[prometheus] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 15_000);
console.log(`[prometheus] reading Prometheus every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+253
View File
@@ -0,0 +1,253 @@
/**
* Immich: reading what its API says (version, storage, statistics, job queues in either shape), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-immich-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, immichState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/immich/adapter.js");
const im = await import("../src/services/immichSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { immichRouter } = await import("../src/routes/immich.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const NOW = new Date("2026-10-20T12:00:00Z");
type Read = import("../src/integrations/immich/adapter.js").ImmichRead;
const queue = (name: string, failed = 0, extra: Partial<import("../src/integrations/immich/adapter.js").ImmichQueue> = {}) => ({ name, active: 0, waiting: 0, failed, paused: false, ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ version: "v1.120.2", latestVersion: "v1.120.2", updateAvailable: false, disk: { size: 4e12, used: 1e12, percent: 25 }, photos: 52000, videos: 1800, usageBytes: 7e11, queues: [queue("thumbnailGeneration")], ...extra });
describe("what Immich's API says", () => {
test("a version can be a string or a major, minor and patch; and newer is told by number, not by text", () => {
assert.equal(adapter.parseVersion({ version: "v1.120.2" }), "v1.120.2");
assert.equal(adapter.parseVersion({ major: 1, minor: 120, patch: 2 }), "v1.120.2");
assert.equal(adapter.parseVersion({}), null);
assert.equal(adapter.newerVersion("v1.120.10", "v1.120.9"), true, "10 is later than 9");
assert.equal(adapter.newerVersion("1.120.2", "v1.120.2"), false);
assert.equal(adapter.newerVersion("v2.0.0", "v1.999.9"), true);
assert.equal(adapter.newerVersion(null, "v1.0.0"), false);
assert.equal(adapter.newerVersion("nightly", "v1.0.0"), false);
});
test("the disk under the library", () => {
assert.deepEqual(adapter.parseStorage({ diskSizeRaw: 4e12, diskUseRaw: 1e12, diskUsagePercentage: 25 }), { size: 4e12, used: 1e12, percent: 25 });
assert.equal(adapter.parseStorage({ diskSizeRaw: 1000, diskUseRaw: 333 })!.percent, 33.3, "worked out when Immich does not say");
assert.equal(adapter.parseStorage({}), null);
});
test("job queues: as an object keyed by name (/jobs) and as a list (/queues)", () => {
const asObject = adapter.parseQueues({ thumbnailGeneration: { jobCounts: { active: 1, waiting: 5, failed: 2 }, queueStatus: { isPaused: false } }, library: { jobCounts: { failed: 0 }, queueStatus: { isPaused: true } } })!;
assert.deepEqual(asObject.map((q) => [q.name, q.active, q.waiting, q.failed, q.paused]), [["thumbnailGeneration", 1, 5, 2, false], ["library", 0, 0, 0, true]]);
const asList = adapter.parseQueues([{ name: "metadataExtraction", isPaused: false, statistics: { failed: 7, waiting: 1 } }])!;
assert.deepEqual(asList.map((q) => [q.name, q.failed, q.waiting]), [["metadataExtraction", 7, 1]]);
assert.equal(adapter.parseQueues({}), null);
assert.equal(adapter.parseQueues("x"), null);
});
describe("the adapter against an Immich that answers", () => {
let server: Server;
let url = "";
let newStyle = false;
const seen: string[] = [];
before(async () => {
server = createHttpServer((req, res) => {
const path = (req.url ?? "").split("?")[0];
seen.push(`${req.headers["x-api-key"]} ${path}`);
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (req.headers["x-api-key"] !== "key") return send(401, {});
if (path === "/api/server/about") return send(200, { version: "v1.120.2" });
if (path === "/api/server/storage") return send(200, { diskSizeRaw: 4e12, diskUseRaw: 3.8e12, diskUsagePercentage: 95 });
if (path === "/api/server/statistics") return send(200, { photos: 100, videos: 5, usage: 123456 });
if (path === "/api/server/version-check") return send(200, { releaseVersion: "v1.121.0" });
if (path === "/api/jobs") return newStyle ? send(404, {}) : send(200, { thumbnailGeneration: { jobCounts: { failed: 4 } } });
if (path === "/api/queues") return newStyle ? send(200, [{ name: "thumbnailGeneration", statistics: { failed: 9 } }]) : send(404, {});
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads everything, sends the key, and says a newer version is out", async () => {
const r = await adapter.createImmichAdapter({ url: `${url}/`, apiKey: "key" }).read();
assert.deepEqual([r.version, r.latestVersion, r.updateAvailable, r.photos, r.videos, r.usageBytes], ["v1.120.2", "v1.121.0", true, 100, 5, 123456]);
assert.deepEqual(r.disk, { size: 4e12, used: 3.8e12, percent: 95 });
assert.deepEqual(r.queues?.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 4]]);
assert.ok(seen.every((s) => s.startsWith("key ")));
});
test("a newer Immich with /queues in place of /jobs is read all the same", async () => {
newStyle = true;
const r = await adapter.createImmichAdapter({ url, apiKey: "key" }).read();
assert.deepEqual(r.queues?.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 9]]);
});
test("a refused key and a dead address are said plainly", async () => {
const bad = await adapter.createImmichAdapter({ url, apiKey: "nope" }).ping();
assert.equal(bad.ok, false);
assert.match(bad.error ?? "", /did not accept the API key/);
assert.equal((await adapter.createImmichAdapter({ url, apiKey: "key" }).ping()).ok, true);
assert.equal((await adapter.createImmichAdapter({ url: "http://127.0.0.1:1", apiKey: "key" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "immich", name: "Photos", baseUrl: "http://10.9.9.3:2283", config: JSON.stringify({ url: "http://10.9.9.3:2283", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(immichState).where(eq(immichState.integrationId, id)))[0];
test("the state is replaced at each read; queues that could not be read stay null, not empty", async () => {
await im.storeRead(id, read({ queues: [queue("a", 3)] }), NOW);
assert.deepEqual(im.queuesOf(await state())?.map((q) => q.name), ["a"]);
await im.storeRead(id, read({ queues: null, disk: null, photos: null }), NOW);
const s = await state();
assert.deepEqual([im.queuesOf(s), s.diskPercent, s.photos], [null, null, null]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await im.syncImmich(id, new Date(), async () => read({ version: "v1.121.0" }));
assert.equal(ok.ok, true);
const failed = await im.syncImmich(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "v1.121.0");
assert.equal(im.isImmichFresh((await im.immichSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (extra: Partial<Parameters<typeof im.immichProblems>[2]> = {}) => ({ diskSize: 4e12, diskUsed: 1e12, diskPercent: 25 as number | null, queues: JSON.stringify([queue("thumbnailGeneration")]) as string | null, ...extra });
const keys = (r: ReturnType<typeof row>) => im.immichProblems(1, "Photos", r).map((p) => `${p.key}=${p.severity}`);
test("nothing wrong says nothing; what could not be read is not a problem", () => {
assert.deepEqual(keys(row()), []);
assert.deepEqual(keys(row({ diskPercent: null, queues: null })), []);
});
test("the disk: a warning at 90 %, critical at 95 %", () => {
assert.deepEqual(keys(row({ diskPercent: 89.9 })), []);
assert.deepEqual(keys(row({ diskPercent: 90 })), ["immich-disk:1=warning"]);
assert.deepEqual(keys(row({ diskPercent: 95 })), ["immich-disk:1=critical"]);
assert.match(im.immichProblems(1, "Photos", row({ diskPercent: 96, diskUsed: 3.84e12 }))[0].message, /96 % full \(3840 GB of 4000 GB\)/);
});
test("failed jobs are one alert when there are ten or more together, naming the worst queues", () => {
const q = (a: number, b: number) => JSON.stringify([queue("thumbnailGeneration", a), queue("metadataExtraction", b), queue("library", 0)]);
assert.deepEqual(keys(row({ queues: q(5, 4) })), [], "nine is not yet");
const p = im.immichProblems(1, "Photos", row({ queues: q(8, 4) }));
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["immich-jobs:1", "warning"]]);
assert.match(p[0].message, /12 background jobs have failed \(thumbnailGeneration 8, metadataExtraction 4\)/);
});
});
describe("the health pass, the page and /metrics", () => {
let imId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("immichSync", JSON.stringify({ [imId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("immichSync", JSON.stringify({ [imId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
imId = (await db.insert(integrations).values({ type: "immich", name: "Family photos", baseUrl: "http://10.9.9.2:2283", config: JSON.stringify({ url: "http://10.9.9.2:2283", apiKey: "key" }) }).returning({ id: integrations.id }))[0].id;
await im.storeRead(imId, read({ disk: { size: 4e12, used: 3.9e12, percent: 97.5 }, queues: [queue("thumbnailGeneration", 14, { waiting: 3 }), queue("library")] }), new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/immich", immichRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/immich`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.immich?.map((i) => i.integrationId), [imId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `immich-disk:${imId}`)!;
assert.deepEqual([c.severity, c.source], ["critical", `immich:${imId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${imId}`, "maintenance on it silences its alerts");
assert.equal(subjectOfConditionKey(`immich-jobs:${imId}`), `integration:${imId}`);
assert.ok(list.some((x) => x.key === `immich-jobs:${imId}`));
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.immich ?? [], held.has(`immich:${imId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them: the disk under Disk, the jobs under Services, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const disk = report.alerts.find((x) => x.id === `disk:immich-disk:${imId}`)!;
assert.deepEqual([disk.severity, disk.category, disk.source, disk.link, disk.target?.type, disk.target?.id], ["critical", "disk", "Immich", "/immich", "integration", imId]);
assert.equal(report.alerts.find((x) => x.id === `services:immich-jobs:${imId}`)?.category, "services");
});
test("anyone signed in sees each Immich, the counts, the disk and the queues; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { immiches: { id: number; fresh: boolean; state: { photos: number; disk: { percent: number } | null; queues: { name: string; failed: number }[] } | null; problems: unknown[] }[] };
const i = body.immiches.find((x) => x.id === imId)!;
assert.equal(i.fresh, true);
assert.deepEqual([i.state?.photos, i.state?.disk?.percent], [52000, 97.5]);
assert.deepEqual(i.state?.queues.map((q) => [q.name, q.failed]), [["thumbnailGeneration", 14], ["library", 0]]);
assert.equal(i.problems.length, 2);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.immiches.find((x) => x.id === imId)!.problems, []);
await fresh();
});
test("/metrics has the counts, what the library takes, the disk and the failed jobs — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_immich_photos\{name="Family photos"\} 52000/);
assert.match(text, /homelab_immich_library_bytes\{name="Family photos"\} 7e\+11|homelab_immich_library_bytes\{name="Family photos"\} 700000000000/);
assert.match(text, /homelab_immich_disk_used_percent\{name="Family photos"\} 97\.5/);
assert.match(text, /homelab_immich_failed_jobs\{name="Family photos"\} 14/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_immich/);
});
});
+294
View File
@@ -0,0 +1,294 @@
/**
* Paperless-ngx: reading what its API says (status components, statistics, failed imports), what is stored, what is wrong, the health pass, the alerts, the page's route and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-paperless-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, paperlessState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/paperless/adapter.js");
const pl = await import("../src/services/paperlessSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { paperlessRouter } = await import("../src/routes/paperless.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const DAY = 86_400_000;
const NOW = new Date("2026-10-20T12:00:00Z");
type Read = import("../src/integrations/paperless/adapter.js").PaperlessRead;
const comp = (name: string, status = "OK", error: string | null = null) => ({ name, status, error });
const read = (extra: Partial<Read> = {}): Read => ({
version: "2.14.7", latestVersion: "2.14.7", updateAvailable: false, storage: { total: 1e12, available: 6e11 },
components: [comp("database"), comp("redis"), comp("celery"), comp("index"), comp("classifier"), comp("sanity_check")], documents: 4200, inbox: 3, failed: [], ...extra,
});
const STATUS = {
pngx_version: "2.14.7",
storage: { total: 1e12, available: 6e11 },
database: { type: "sqlite", status: "OK", error: null },
tasks: { redis_status: "OK", celery_status: "ERROR", celery_error: "No celery workers responded", index_status: "OK", classifier_status: "WARNING", classifier_error: "Classifier has not been trained yet", sanity_check_status: "OK" },
};
describe("what Paperless's API says", () => {
test("the system status: the version, the disk and each component with how it is", () => {
const s = adapter.parseStatus(STATUS);
assert.equal(s.version, "2.14.7");
assert.deepEqual(s.storage, { total: 1e12, available: 6e11 });
assert.deepEqual(s.components?.map((c) => [c.name, c.status, c.error]), [["database", "OK", null], ["redis", "OK", null], ["celery", "ERROR", "No celery workers responded"], ["index", "OK", null], ["classifier", "WARNING", "Classifier has not been trained yet"], ["sanity_check", "OK", null]]);
assert.deepEqual(adapter.parseStatus({}), { version: null, storage: null, components: null });
});
test("the documents and the inbox", () => {
assert.deepEqual(adapter.parseStatistics({ documents_total: 4200, documents_inbox: 3 }), { documents: 4200, inbox: 3 });
assert.deepEqual(adapter.parseStatistics(null), { documents: null, inbox: null });
});
test("failed imports: only failures, not dismissed, from the last week, newest first; an answer without a list is not 'none'", () => {
const f = adapter.parseFailures(
[
{ status: "FAILURE", task_file_name: "old.pdf", date_done: new Date(NOW.getTime() - 10 * DAY).toISOString(), result: "x" },
{ status: "FAILURE", task_file_name: "a.pdf", date_done: new Date(NOW.getTime() - 2 * DAY).toISOString(), result: "Not a valid PDF" },
{ status: "FAILURE", task_file_name: "dismissed.pdf", acknowledged: true, date_done: NOW.toISOString() },
{ status: "SUCCESS", task_file_name: "fine.pdf", date_done: NOW.toISOString() },
{ status: "FAILURE", task_file_name: "b.pdf", date_done: new Date(NOW.getTime() - 3600_000).toISOString() },
],
NOW,
)!;
assert.deepEqual(f.map((x) => [x.file, x.error]), [["b.pdf", null], ["a.pdf", "Not a valid PDF"]]);
assert.equal(adapter.parseFailures({ results: [] }, NOW)?.length, 0, "a paginated answer is read too");
assert.equal(adapter.parseFailures({ detail: "no" }, NOW), null);
});
test("a newer version", () => {
assert.deepEqual(adapter.parseRemoteVersion({ version: "v2.15.0", update_available: true }), { latestVersion: "v2.15.0", updateAvailable: true });
assert.deepEqual(adapter.parseRemoteVersion(null), { latestVersion: null, updateAvailable: false });
});
describe("the adapter against a Paperless that answers", () => {
let server: Server;
let url = "";
let superuser = true;
const seen: (string | undefined)[] = [];
before(async () => {
server = createHttpServer((req, res) => {
seen.push(req.headers.authorization);
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (req.headers.authorization !== "Token tok") return send(401, {});
const path = (req.url ?? "").split("?")[0];
if (path === "/api/statistics/") return send(200, { documents_total: 100, documents_inbox: 2 });
if (path === "/api/status/") return superuser ? send(200, STATUS) : send(403, {});
if (path === "/api/tasks/") return send(200, [{ status: "FAILURE", task_file_name: "x.pdf", date_done: new Date().toISOString(), result: "boom" }]);
if (path === "/api/remote_version/") return send(200, { version: "2.15.0", update_available: true });
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads everything and sends the token", async () => {
const r = await adapter.createPaperlessAdapter({ url: `${url}/`, token: "tok" }).read(new Date());
assert.deepEqual([r.version, r.documents, r.inbox, r.updateAvailable, r.latestVersion], ["2.14.7", 100, 2, true, "2.15.0"]);
assert.deepEqual(r.failed?.map((f) => f.file), ["x.pdf"]);
assert.equal(r.components?.find((c) => c.name === "celery")?.status, "ERROR");
assert.ok(seen.every((s) => s === "Token tok"));
});
test("a token that is not a superuser's still reads the rest; the status is unknown, not fine", async () => {
superuser = false;
const r = await adapter.createPaperlessAdapter({ url, token: "tok" }).read(new Date());
assert.deepEqual([r.components, r.storage, r.version], [null, null, null]);
assert.equal(r.documents, 100);
});
test("a refused token and a dead address are said plainly", async () => {
const bad = await adapter.createPaperlessAdapter({ url, token: "nope" }).ping();
assert.equal(bad.ok, false);
assert.match(bad.error ?? "", /did not accept the token/);
assert.equal((await adapter.createPaperlessAdapter({ url, token: "tok" }).ping()).ok, true);
assert.equal((await adapter.createPaperlessAdapter({ url: "http://127.0.0.1:1", token: "tok" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "paperless", name: "Documents", baseUrl: "http://10.9.9.1:8000", config: JSON.stringify({ url: "http://10.9.9.1:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(paperlessState).where(eq(paperlessState.integrationId, id)))[0];
test("the state is replaced at each read; what could not be read stays null, not empty", async () => {
await pl.storeRead(id, read({ failed: [{ file: "a.pdf", error: null, at: null }] }), NOW);
assert.deepEqual(pl.failedOf(await state())?.map((f) => f.file), ["a.pdf"]);
await pl.storeRead(id, read({ components: null, failed: null, storage: null }), NOW);
const s = await state();
assert.deepEqual([pl.componentsOf(s), pl.failedOf(s), s.storageTotal], [null, null, null]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await pl.syncPaperless(id, new Date(), async () => read({ version: "2.15.0" }));
assert.equal(ok.ok, true);
const failed = await pl.syncPaperless(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "2.15.0");
assert.equal(pl.isPaperlessFresh((await pl.paperlessSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (extra: { storageTotal?: number | null; storageAvailable?: number | null; components?: unknown; failed?: unknown } = {}) => ({
storageTotal: 1e12 as number | null,
storageAvailable: 6e11 as number | null,
components: JSON.stringify(extra.components ?? [comp("database"), comp("celery")]) as string | null,
failed: JSON.stringify(extra.failed ?? []) as string | null,
...("storageTotal" in extra ? { storageTotal: extra.storageTotal ?? null } : {}),
...("storageAvailable" in extra ? { storageAvailable: extra.storageAvailable ?? null } : {}),
});
const keys = (r: ReturnType<typeof row>) => pl.paperlessProblems(1, "Docs", r).map((p) => `${p.key}=${p.severity}`);
test("nothing wrong says nothing; what could not be read is not a problem", () => {
assert.deepEqual(keys(row()), []);
assert.deepEqual(pl.paperlessProblems(1, "Docs", { storageTotal: null, storageAvailable: null, components: null, failed: null }), []);
});
test("the database, Redis and the worker in error are critical; the index and the classifier a warning; an untrained classifier ('warning') says nothing", () => {
assert.deepEqual(keys(row({ components: [comp("celery", "ERROR", "No celery workers responded"), comp("index", "ERROR"), comp("classifier", "WARNING", "not trained")] })), ["paperless-component:1:celery=critical", "paperless-component:1:index=warning"]);
assert.match(pl.paperlessProblems(1, "Docs", row({ components: [comp("celery", "ERROR", "No celery workers responded")] }))[0].message, /the worker that consumes documents is in error — No celery workers responded/);
assert.deepEqual(keys(row({ components: [comp("database", "ERROR")] })), ["paperless-component:1:database=critical"]);
});
test("a sanity check that found problems is a warning", () => {
assert.deepEqual(keys(row({ components: [comp("sanity_check", "WARNING", "2 documents are missing their file")] })), ["paperless-component:1:sanity_check=warning"]);
});
test("imports that failed are one alert, naming the files and the latest error", () => {
const failed = ["a.pdf", "b.pdf", "c.pdf", "d.pdf"].map((file, i) => ({ file, error: i === 0 ? "Not a valid PDF" : null, at: null }));
const p = pl.paperlessProblems(1, "Docs", row({ failed }));
assert.deepEqual(p.map((x) => [x.key, x.severity]), [["paperless-failed:1", "warning"]]);
assert.match(p[0].message, /4 documents failed to import \(a\.pdf, b\.pdf, c\.pdf and 1 more\) — the latest: Not a valid PDF/);
assert.match(pl.paperlessProblems(1, "Docs", row({ failed: [failed[1]] }))[0].message, /1 document failed to import \(b\.pdf\)$/);
});
test("the disk: a warning at 90 %, critical at 95 %", () => {
assert.deepEqual(keys(row({ storageAvailable: 1.1e11 })), []);
assert.deepEqual(keys(row({ storageAvailable: 1e11 })), ["paperless-disk:1=warning"]);
assert.deepEqual(keys(row({ storageAvailable: 4e10 })), ["paperless-disk:1=critical"]);
});
});
describe("the health pass, the page and /metrics", () => {
let plId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("paperlessSync", JSON.stringify({ [plId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
plId = (await db.insert(integrations).values({ type: "paperless", name: "Archive", baseUrl: "http://10.9.9.0:8000", config: JSON.stringify({ url: "http://10.9.9.0:8000", token: "tok" }) }).returning({ id: integrations.id }))[0].id;
await pl.storeRead(
plId,
read({ storage: { total: 1e12, available: 3e10 }, components: [comp("database"), comp("celery", "ERROR", "No celery workers responded")], failed: [{ file: "scan.pdf", error: "Not a valid PDF", at: new Date().toISOString() }] }),
new Date(),
);
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/paperless", paperlessRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/paperless`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.paperless?.map((p) => p.integrationId), [plId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `paperless-component:${plId}:celery`)!;
assert.deepEqual([c.severity, c.source], ["critical", `paperless:${plId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${plId}`, "maintenance on it silences its alerts");
for (const k of ["paperless-failed", "paperless-disk"]) assert.equal(subjectOfConditionKey(`${k}:${plId}`), `integration:${plId}`);
assert.ok(list.some((x) => x.key === `paperless-failed:${plId}`) && list.some((x) => x.key === `paperless-disk:${plId}`));
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.paperless ?? [], held.has(`paperless:${plId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them: the worker and the failed imports under Services, the disk under Disk, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const worker = report.alerts.find((x) => x.id === `services:paperless-component:${plId}:celery`)!;
assert.deepEqual([worker.severity, worker.category, worker.source, worker.link, worker.target?.type, worker.target?.id], ["critical", "services", "Paperless-ngx", "/paperless", "integration", plId]);
assert.equal(report.alerts.find((x) => x.id === `services:paperless-failed:${plId}`)?.category, "services");
assert.equal(report.alerts.find((x) => x.id === `disk:paperless-disk:${plId}`)?.severity, "critical");
});
test("anyone signed in sees each Paperless, its components and failed imports; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { paperlesses: { id: number; fresh: boolean; state: { documents: number; components: { name: string; status: string }[]; failed: { file: string }[] } | null; problems: unknown[] }[] };
const p = body.paperlesses.find((x) => x.id === plId)!;
assert.equal(p.fresh, true);
assert.equal(p.state?.documents, 4200);
assert.deepEqual(p.state?.components.map((c) => [c.name, c.status]), [["database", "OK"], ["celery", "ERROR"]]);
assert.deepEqual(p.state?.failed.map((f) => f.file), ["scan.pdf"]);
assert.equal(p.problems.length, 3);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.paperlesses.find((x) => x.id === plId)!.problems, []);
await fresh();
});
test("/metrics has the documents, the inbox, the failed imports and each component — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_paperless_documents\{name="Archive"\} 4200/);
assert.match(text, /homelab_paperless_inbox\{name="Archive"\} 3/);
assert.match(text, /homelab_paperless_failed_imports\{name="Archive"\} 1/);
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="celery"\} 0/);
assert.match(text, /homelab_paperless_component_ok\{name="Archive",component="database"\} 1/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_paperless/);
});
});
+265
View File
@@ -0,0 +1,265 @@
/**
* Prometheus: reading what its API says (targets, alerts), what is stored and how long a target has been down, what is wrong, the health pass, the alerts, the page's route.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-prometheus-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, prometheusState } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/prometheus/adapter.js");
const pm = await import("../src/services/prometheusSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { prometheusRouter } = await import("../src/routes/prometheus.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const NOW = new Date("2026-10-20T12:00:00Z");
const ago = (min: number, from: Date = NOW) => new Date(from.getTime() - min * MIN);
type Read = import("../src/integrations/prometheus/adapter.js").PrometheusRead;
type Target = import("../src/integrations/prometheus/adapter.js").PrometheusTarget;
const target = (job: string, instance: string, health: Target["health"] = "up", error: string | null = null): Target => ({ job, instance, health, error, lastScrape: NOW.toISOString() });
const alert = (name: string, extra: Partial<import("../src/integrations/prometheus/adapter.js").PrometheusAlert> = {}) => ({ name, severity: "warning", state: "firing" as const, summary: null, instance: null, activeAt: NOW.toISOString(), ...extra });
const read = (extra: Partial<Read> = {}): Read => ({ version: "3.1.0", targets: [], alerts: [], ...extra });
describe("what Prometheus's API says", () => {
test("the version", () => {
assert.equal(adapter.parseBuildInfo({ status: "success", data: { version: "3.1.0" } }), "3.1.0");
assert.equal(adapter.parseBuildInfo(null), null);
});
test("targets, with their health and why a scrape failed; an answer without a list is not 'none'", () => {
const t = adapter.parseTargets({
data: {
activeTargets: [
{ labels: { job: "node", instance: "jackie:9100" }, health: "up", lastError: "", lastScrape: "2026-10-20T11:59:50Z" },
{ labels: { job: "node", instance: "nova:9100" }, health: "down", lastError: "connection refused", lastScrape: "2026-10-20T11:59:50Z" },
{ labels: { instance: "x" }, scrapePool: "pool", health: "weird" },
{ labels: {} },
],
},
})!;
assert.deepEqual(t.map((x) => [x.job, x.instance, x.health, x.error]), [["node", "jackie:9100", "up", null], ["node", "nova:9100", "down", "connection refused"], ["pool", "x", "unknown", null]]);
assert.equal(adapter.parseTargets({ status: "error" }), null);
assert.deepEqual(adapter.parseTargets({ data: { activeTargets: [] } }), []);
});
test("alerts: firing and pending, with severity and the summary", () => {
const a = adapter.parseAlerts({
data: {
alerts: [
{ labels: { alertname: "DiskFull", severity: "Critical", instance: "jackie:9100" }, annotations: { summary: "Disk is 95% full" }, state: "firing", activeAt: "2026-10-20T11:00:00Z" },
{ labels: { alertname: "Slow" }, annotations: { description: "slow" }, state: "pending" },
{ labels: { alertname: "Odd" }, state: "inactive" },
{ labels: {}, state: "firing" },
],
},
})!;
assert.deepEqual(a.map((x) => [x.name, x.severity, x.state, x.summary, x.instance]), [["DiskFull", "critical", "firing", "Disk is 95% full", "jackie:9100"], ["Slow", null, "pending", "slow", null]]);
assert.equal(adapter.parseAlerts({}), null);
});
describe("the adapter against a Prometheus that answers", () => {
let server: Server;
let url = "";
const seen: (string | undefined)[] = [];
before(async () => {
server = createHttpServer((req, res) => {
seen.push(req.headers.authorization);
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
const path = (req.url ?? "").split("?")[0];
if (path === "/api/v1/status/buildinfo") return send(200, { data: { version: "3.1.0" } });
if (path === "/api/v1/targets") return send(200, { data: { activeTargets: [{ labels: { job: "node", instance: "a" }, health: "down", lastError: "timeout" }] } });
if (path === "/api/v1/alerts") return send(500, {}); // the rest is worth having, not worth failing for
send(404, {});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
});
after(() => server.close());
test("it reads, and an alert list that cannot be read is null rather than empty", async () => {
const r = await adapter.createPrometheusAdapter({ url: `${url}/` }).read();
assert.equal(r.version, "3.1.0");
assert.equal(r.targets?.length, 1);
assert.equal(r.alerts, null);
assert.ok(seen.every((s) => s === undefined), "no token, none sent");
});
test("a token is sent when there is one; a dead address is said", async () => {
await adapter.createPrometheusAdapter({ url, token: "t0k" }).ping();
assert.equal(seen.at(-1), "Bearer t0k");
assert.equal((await adapter.createPrometheusAdapter({ url: "http://127.0.0.1:1" }).ping()).ok, false);
});
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "prometheus", name: "Metrics", baseUrl: "http://10.9.9.5:9090", config: JSON.stringify({ url: "http://10.9.9.5:9090" }) }).returning({ id: integrations.id }))[0].id;
});
const state = async () => (await db.select().from(prometheusState).where(eq(prometheusState.integrationId, id)))[0];
test("targets are counted by job, and a target that stays down keeps the time it was first seen down", async () => {
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), ago(10));
await pm.storeRead(id, read({ targets: [target("node", "a"), target("node", "b", "down", "refused"), target("cadvisor", "c")] }), NOW);
const s = await state();
assert.deepEqual(pm.jobsOf(s), [{ job: "cadvisor", up: 1, down: 0 }, { job: "node", up: 1, down: 1 }]);
assert.equal(pm.downOf(s)?.[0].since, ago(10).toISOString(), "since the first look, not the last");
// It comes back and goes down again: the clock starts over.
await pm.storeRead(id, read({ targets: [target("node", "b")] }), NOW);
assert.deepEqual(pm.downOf(await state()), []);
await pm.storeRead(id, read({ targets: [target("node", "b", "down")] }), new Date(NOW.getTime() + MIN));
assert.equal(pm.downOf(await state())?.[0].since, new Date(NOW.getTime() + MIN).toISOString());
});
test("what could not be read stays null, not empty", async () => {
await pm.storeRead(id, read({ targets: null, alerts: null }), NOW);
const s = await state();
assert.deepEqual([pm.jobsOf(s), pm.downOf(s), pm.alertsOf(s)], [null, null, null]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await pm.syncPrometheus(id, new Date(), async () => read({ version: "3.2.0" }));
assert.equal(ok.ok, true);
const failed = await pm.syncPrometheus(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal((await state()).version, "3.2.0");
assert.equal(pm.isPrometheusFresh((await pm.prometheusSyncStatuses())[id], Date.now()), false);
});
});
describe("what is wrong", () => {
const row = (r: { jobs?: unknown; down?: unknown; alerts?: unknown }) => ({ jobs: JSON.stringify(r.jobs ?? []), down: JSON.stringify(r.down ?? []), alerts: JSON.stringify(r.alerts ?? []) });
const keys = (r: ReturnType<typeof row>) => pm.prometheusProblems(1, "Metrics", r, NOW).map((p) => `${p.key}=${p.severity}`);
test("a target is a problem only once it has been down five minutes; the job's targets are one alert", () => {
const jobs = [{ job: "node", up: 1, down: 3 }];
const d = (inst: string, mins: number) => ({ job: "node", instance: inst, error: "connection refused", since: ago(mins).toISOString() });
assert.deepEqual(keys(row({ jobs, down: [d("a", 4)] })), [], "one failed scrape is not news");
assert.deepEqual(keys(row({ jobs, down: [d("a", 6)] })), ["prometheus-target:1:node=warning"]);
const p = pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9), d("e", 1)] }), NOW);
assert.equal(p.length, 1);
assert.match(p[0].message, /3 of 4 “node” targets not answering \(a, b, c\) — connection refused/);
assert.equal(pm.prometheusProblems(1, "Metrics", row({ jobs, down: [d("a", 9), d("b", 9), d("c", 9)] }), NOW)[0].message.includes("and "), false);
});
test("firing alerts, one for each name; critical when any is; heartbeat alerts, pending and info ones are left out", () => {
const alerts = [
alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" }),
alert("DiskFull", { severity: "warning", instance: "nova:9100" }),
alert("HighLoad", { instance: "nova:9100" }),
alert("Watchdog", { severity: "none" }),
alert("Pending", { state: "pending" }),
alert("FYI", { severity: "info" }),
];
assert.deepEqual(keys(row({ alerts })), ["prometheus-alert:1:DiskFull=critical", "prometheus-alert:1:HighLoad=warning"]);
const p = pm.prometheusProblems(1, "Metrics", row({ alerts }), NOW);
assert.equal(p[0].message, "Metrics: DiskFull (2 firing) — Disk is 95% full");
assert.equal(p[1].message, "Metrics: HighLoad on nova:9100");
});
test("nothing read says nothing is wrong", () => {
assert.deepEqual(pm.prometheusProblems(1, "Metrics", { jobs: null, down: null, alerts: null }, NOW), []);
});
});
describe("the health pass and the page", () => {
let pmId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("prometheusSync", JSON.stringify({ [pmId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
pmId = (await db.insert(integrations).values({ type: "prometheus", name: "Prom", baseUrl: "http://10.9.9.4:9090", config: JSON.stringify({ url: "http://10.9.9.4:9090" }) }).returning({ id: integrations.id }))[0].id;
const t0 = new Date(Date.now() - 12 * MIN);
const r = read({ targets: [target("node", "jackie:9100", "down", "refused"), target("node", "nova:9100")], alerts: [alert("DiskFull", { severity: "critical", instance: "jackie:9100", summary: "Disk is 95% full" })] });
await pm.storeRead(pmId, r, t0);
await pm.storeRead(pmId, r, new Date());
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/prometheus", prometheusRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/prometheus`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.prometheus?.map((p) => p.integrationId), [pmId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `prometheus-alert:${pmId}:DiskFull`)!;
assert.deepEqual([c.severity, c.source], ["critical", `prometheus:${pmId}`]);
assert.equal(subjectOfConditionKey(c.key), `integration:${pmId}`, "maintenance on it silences its alerts");
assert.equal(subjectOfConditionKey(`prometheus-target:${pmId}:node`), `integration:${pmId}`);
assert.ok(list.some((x) => x.key === `prometheus-target:${pmId}:node`));
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.prometheus ?? [], held.has(`prometheus:${pmId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them in the Monitoring kind, with where to look", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const a = report.alerts.find((x) => x.id === `monitoring:prometheus-alert:${pmId}:DiskFull`)!;
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["critical", "monitoring", "Prometheus", "/prometheus", "integration", pmId]);
assert.ok(report.alerts.some((x) => x.id === `monitoring:prometheus-target:${pmId}:node`));
});
test("anyone signed in sees each Prometheus, its jobs, what is down and what fires; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { prometheuses: { id: number; fresh: boolean; state: { jobs: { job: string; up: number; down: number }[]; down: unknown[]; alerts: unknown[] } | null; problems: unknown[] }[] };
const p = body.prometheuses.find((x) => x.id === pmId)!;
assert.equal(p.fresh, true);
assert.deepEqual(p.state?.jobs, [{ job: "node", up: 1, down: 1 }]);
assert.equal(p.state?.down.length, 1);
assert.equal(p.problems.length, 2);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.prometheuses.find((x) => x.id === pmId)!.problems, []);
await fresh();
});
});