Prometheus, Immich and Paperless-ngx integrations
Three read-only integrations, each with a monitoring page, alerts, maintenance silencing and held-when-unreadable handling. Prometheus: scrape targets that stop answering (5 minutes, one alert per job) and alerts that Prometheus's own rules fire (critical when labelled so). Immich: disk under the library filling, failed background jobs, counts. Paperless-ngx: worker/database/Redis in error, documents that failed to import, index and sanity check, disk. Immich and Paperless also in /metrics. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aebe98bf15
commit
251a4a9d73
47 files changed
+24148
-7
No files matched your search
@@ -77,6 +77,9 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
duplicati: { label: "Duplicati", link: "/duplicati" },
|
||||
jellyfin: { label: "Jellyfin", link: "/jellyfin" },
|
||||
authentik: { label: "Authentik", link: "/authentik" },
|
||||
prometheus: { label: "Prometheus", link: "/prometheus" },
|
||||
immich: { label: "Immich", link: "/immich" },
|
||||
paperless: { label: "Paperless-ngx", link: "/paperless" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -115,6 +118,12 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
||||
if (m) return { type: "paperless", id: Number(m[1]), category: key.startsWith("paperless-disk") ? "disk" : "services" };
|
||||
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
||||
if (m) return { type: "immich", id: Number(m[1]), category: key.startsWith("immich-disk") ? "disk" : "services" };
|
||||
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
|
||||
if (m) return { type: "prometheus", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return { type: "authentik", id: Number(m[1]), category: /^authentik-(?:cert|token):/.test(key) ? "expiry" : /^authentik-(?:outpost|worker)/.test(key) ? "services" : "monitoring" };
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
@@ -241,7 +250,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -251,7 +260,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { authentikState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -22,6 +22,9 @@ import { createDockhandAdapter } from "../integrations/dockhand/adapter.js";
|
||||
import { currentProblems as duplicatiProblemsNow, type DuplicatiProblem } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinProblems, jellyfinSyncStatuses, type JellyfinProblem } from "./jellyfinSync.js";
|
||||
import { authentikProblems, authentikSyncStatuses, isAuthentikFresh, type AuthentikProblem } from "./authentikSync.js";
|
||||
import { isPrometheusFresh, prometheusProblems, prometheusSyncStatuses, type PrometheusProblem } from "./prometheusSync.js";
|
||||
import { immichProblems, immichSyncStatuses, isImmichFresh, type ImmichProblem } from "./immichSync.js";
|
||||
import { isPaperlessFresh, paperlessProblems, paperlessSyncStatuses, type PaperlessProblem } from "./paperlessSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -89,6 +92,12 @@ export interface HealthSnapshot {
|
||||
baselines?: BaselineFinding[];
|
||||
/** Service-level targets whose error budget for the month is mostly or wholly used (services/serviceLevels.ts). */
|
||||
levels?: LevelFinding[];
|
||||
/** What is wrong with each Paperless-ngx (a component in error, documents that failed to import, a full disk), only while that reading is recent (services/paperlessSync.ts). */
|
||||
paperless?: { integrationId: number; problems: PaperlessProblem[] }[];
|
||||
/** What is wrong with each Immich (a disk filling, background jobs failing), only while that reading is recent (services/immichSync.ts). */
|
||||
immich?: { integrationId: number; problems: ImmichProblem[] }[];
|
||||
/** What is wrong with each Prometheus (a target that stopped answering, an alert its rules fire), only while that reading is recent (services/prometheusSync.ts). */
|
||||
prometheus?: { integrationId: number; problems: PrometheusProblem[] }[];
|
||||
/** What is wrong with each Authentik, from what was stored, only while that reading is recent (services/authentikSync.ts). */
|
||||
authentik?: { integrationId: number; problems: AuthentikProblem[] }[];
|
||||
/** What is wrong with each Jellyfin, from what was stored, only while that reading is recent (services/jellyfinSync.ts). */
|
||||
@@ -204,6 +213,15 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
}
|
||||
}
|
||||
|
||||
// A Paperless whose worker has stopped, whose imports fail, or whose disk is filling.
|
||||
for (const p of snapshot.paperless ?? []) for (const c of p.problems) out.push({ key: c.key, source: `paperless:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// An Immich whose disk is filling or whose background jobs are failing.
|
||||
for (const i of snapshot.immich ?? []) for (const c of i.problems) out.push({ key: c.key, source: `immich:${i.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// A Prometheus target that has stopped answering, an alert that Prometheus's own rules are firing.
|
||||
for (const p of snapshot.prometheus ?? []) for (const c of p.problems) out.push({ key: c.key, source: `prometheus:${p.integrationId}`, message: c.message, severity: c.severity });
|
||||
|
||||
// An Authentik outpost that went quiet, a certificate or token about to run out, no worker, failed sign-ins.
|
||||
for (const a of snapshot.authentik ?? []) for (const p of a.problems) out.push({ key: p.key, source: `authentik:${a.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -440,6 +458,60 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
}
|
||||
}
|
||||
|
||||
// Paperless: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsL = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
|
||||
if (rowsL.length > 0) {
|
||||
const statuses = await paperlessSyncStatuses();
|
||||
for (const r of rowsL) {
|
||||
const [state] = await db.select().from(paperlessState).where(eq(paperlessState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isPaperlessFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`paperless:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.paperless ??= []).push({ integrationId: r.id, problems: paperlessProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Paperless:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Immich: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsI = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
|
||||
if (rowsI.length > 0) {
|
||||
const statuses = await immichSyncStatuses();
|
||||
for (const r of rowsI) {
|
||||
const [state] = await db.select().from(immichState).where(eq(immichState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isImmichFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`immich:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.immich ??= []).push({ integrationId: r.id, problems: immichProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Immich:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Prometheus: read by its own pass every couple of minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsP = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
|
||||
if (rowsP.length > 0) {
|
||||
const statuses = await prometheusSyncStatuses();
|
||||
for (const r of rowsP) {
|
||||
const [state] = await db.select().from(prometheusState).where(eq(prometheusState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isPrometheusFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`prometheus:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
(snapshot.prometheus ??= []).push({ integrationId: r.id, problems: prometheusProblems(r.id, r.name, state) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Prometheus:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Authentik: read by its own pass every few minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsA = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "authentik"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Keeps what Immich says up to date: every five minutes each Immich integration is read (version, the disk under the library, the photo and video counts, the background job
|
||||
* queues) and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Immich.
|
||||
*
|
||||
* Photos are what cannot be made again, and Immich fails quietly: a disk that fills up, or thumbnail and metadata jobs that fail one by one and are never looked at. Those are
|
||||
* what is raised.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { immichState, integrations } from "../db/schema.js";
|
||||
import { createImmichAdapter, type ImmichQueue, type ImmichRead } from "../integrations/immich/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
/** This many failed background jobs, together, is a warning. */
|
||||
export const FAILED_JOBS_WARN = 10;
|
||||
export const DISK_WARN_PERCENT = 90;
|
||||
export const DISK_CRITICAL_PERCENT = 95;
|
||||
const STATUS_FLAG = "immichSync";
|
||||
|
||||
export interface ImmichSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, ImmichSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, ImmichSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function immichSyncStatuses(): Promise<Record<number, ImmichSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isImmichFresh = (status: ImmichSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof immichState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
export const queuesOf = (row: Pick<StateRow, "queues">): ImmichQueue[] | null => {
|
||||
if (row.queues === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(row.queues);
|
||||
return Array.isArray(v) ? (v as ImmichQueue[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: ImmichRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
latestVersion: r.latestVersion,
|
||||
updateAvailable: r.updateAvailable,
|
||||
diskSize: r.disk?.size ?? null,
|
||||
diskUsed: r.disk?.used ?? null,
|
||||
diskPercent: r.disk?.percent ?? null,
|
||||
photos: r.photos,
|
||||
videos: r.videos,
|
||||
usageBytes: r.usageBytes,
|
||||
queues: r.queues && JSON.stringify(r.queues),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(immichState).values({ integrationId, ...values }).onConflictDoUpdate({ target: immichState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Immich and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncImmich(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<ImmichRead>): Promise<ImmichSyncStatus> {
|
||||
let status: ImmichSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "immich") throw new Error("not an Immich integration");
|
||||
const result = await (read ? read(loaded.config) : createImmichAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[immich] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[immich] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllImmich(now: Date = new Date()): Promise<Record<number, ImmichSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "immich"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, ImmichSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncImmich(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetImmich(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImmichProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
|
||||
/** Pure: what is wrong with one Immich, from what is stored for it. */
|
||||
export function immichProblems(id: number, name: string, s: Pick<StateRow, "diskSize" | "diskUsed" | "diskPercent" | "queues">): ImmichProblem[] {
|
||||
const out: ImmichProblem[] = [];
|
||||
if (s.diskPercent !== null && s.diskPercent >= DISK_WARN_PERCENT) {
|
||||
out.push({
|
||||
key: `immich-disk:${id}`,
|
||||
severity: s.diskPercent >= DISK_CRITICAL_PERCENT ? "critical" : "warning",
|
||||
message: `${name}: the disk under the library is ${Math.round(s.diskPercent)} % full${s.diskSize !== null && s.diskUsed !== null ? ` (${gb(s.diskUsed)} of ${gb(s.diskSize)})` : ""}`,
|
||||
});
|
||||
}
|
||||
const queues = queuesOf(s) ?? [];
|
||||
const failed = queues.filter((q) => q.failed > 0);
|
||||
const total = failed.reduce((n, q) => n + q.failed, 0);
|
||||
if (total >= FAILED_JOBS_WARN) {
|
||||
const worst = [...failed].sort((a, b) => b.failed - a.failed).slice(0, 3).map((q) => `${q.name} ${q.failed}`).join(", ");
|
||||
out.push({ key: `immich-jobs:${id}`, severity: "warning", message: `${name}: ${total} background jobs have failed (${worst}) — thumbnails or metadata that were never made; look at Administration → Jobs and run the failed ones again` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initImmichSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllImmich().catch((err) => console.error("[immich] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 17_000);
|
||||
console.log(`[immich] reading Immich every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -33,6 +33,12 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^jellyfin-(?:restart|task|logins):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^paperless-(?:component|failed|disk):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^immich-(?:disk|jobs):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^prometheus-(?:target|alert):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^authentik-(?:outpost|outpost-version|cert|token|worker|logins|errors):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^duplicati-(?:failed|stale):\d+:server:(\d+)$/.exec(key);
|
||||
|
||||
@@ -5,11 +5,13 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
||||
import { immichSyncStatuses, isImmichFresh, queuesOf as immichQueuesOf } from "./immichSync.js";
|
||||
import { authentikSyncStatuses, certificatesOf, daysUntil, isAuthentikFresh, outpostsOf } from "./authentikSync.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { heartbeatState, parseStamp } from "./heartbeats.js";
|
||||
@@ -62,6 +64,39 @@ class Exposition {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Paperless-ngx, for /metrics: the documents, the inbox, the imports that failed and whether each component of its status is in order. Only while the reading is recent. */
|
||||
async function addPaperlessMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await paperlessSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "paperless"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(paperlessState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isPaperlessFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
if (s.documents !== null) out.add("homelab_paperless_documents", "gauge", "How many documents Paperless holds.", s.documents, labels);
|
||||
if (s.inbox !== null) out.add("homelab_paperless_inbox", "gauge", "How many documents are waiting in Paperless's inbox.", s.inbox, labels);
|
||||
const failed = paperlessFailedOf(s);
|
||||
if (failed) out.add("homelab_paperless_failed_imports", "gauge", "Imports that failed in the last week and were not dismissed.", failed.length, labels);
|
||||
for (const c of paperlessComponentsOf(s) ?? []) out.add("homelab_paperless_component_ok", "gauge", "1 if the part of Paperless named is in order, else 0.", c.status === "OK" ? 1 : 0, { name: row.name, component: c.name });
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Immich, for /metrics: the photos and videos, what they take, how full the disk is and the jobs that failed. Only while the reading is recent. */
|
||||
async function addImmichMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await immichSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "immich"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(immichState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isImmichFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
const labels = { name: row.name };
|
||||
if (s.photos !== null) out.add("homelab_immich_photos", "gauge", "How many photos Immich holds.", s.photos, labels);
|
||||
if (s.videos !== null) out.add("homelab_immich_videos", "gauge", "How many videos Immich holds.", s.videos, labels);
|
||||
if (s.usageBytes !== null) out.add("homelab_immich_library_bytes", "gauge", "What Immich's library takes, in bytes.", s.usageBytes, labels);
|
||||
if (s.diskPercent !== null) out.add("homelab_immich_disk_used_percent", "gauge", "How full the disk under Immich's library is.", s.diskPercent, labels);
|
||||
const queues = immichQueuesOf(s);
|
||||
if (queues) out.add("homelab_immich_failed_jobs", "gauge", "Background jobs that have failed, all queues together.", queues.reduce((n, q) => n + q.failed, 0), labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Authentik, for /metrics: whether each outpost that serves something is reporting, how many workers there are, the failed sign-ins and the days left of each certificate. Only while the reading is recent. */
|
||||
async function addAuthentikMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await authentikSyncStatuses();
|
||||
@@ -300,6 +335,8 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addDuplicatiMetrics(out, now);
|
||||
await addJellyfinMetrics(out, now);
|
||||
await addAuthentikMetrics(out, now);
|
||||
await addImmichMetrics(out, now);
|
||||
await addPaperlessMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* Keeps what Paperless-ngx says up to date: every five minutes each Paperless integration is read (its system status, the disk, the document counts and the imports that failed)
|
||||
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Paperless.
|
||||
*
|
||||
* Paperless fails quietly in two ways: the worker that consumes documents stops (a scan dropped in the folder just sits there), or one import fails and nobody looks. Both are raised,
|
||||
* and so is anything Paperless's own status check says is in error.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, paperlessState } from "../db/schema.js";
|
||||
import { createPaperlessAdapter, type PaperlessComponent, type PaperlessFailure, type PaperlessRead } from "../integrations/paperless/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
export const DISK_WARN_PERCENT = 90;
|
||||
export const DISK_CRITICAL_PERCENT = 95;
|
||||
/** If one of these is in error, nothing can be consumed or found: critical. The others are a warning. */
|
||||
const ESSENTIAL = new Set(["database", "redis", "celery"]);
|
||||
const STATUS_FLAG = "paperlessSync";
|
||||
|
||||
export interface PaperlessSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, PaperlessSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, PaperlessSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function paperlessSyncStatuses(): Promise<Record<number, PaperlessSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isPaperlessFresh = (status: PaperlessSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof paperlessState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
const parse = <T>(text: string | null): T[] | null => {
|
||||
if (text === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(text);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
export const componentsOf = (row: Pick<StateRow, "components">) => parse<PaperlessComponent>(row.components);
|
||||
export const failedOf = (row: Pick<StateRow, "failed">) => parse<PaperlessFailure>(row.failed);
|
||||
|
||||
/** Writes down one read, replacing the last. */
|
||||
export async function storeRead(integrationId: number, r: PaperlessRead, now: Date = new Date()): Promise<void> {
|
||||
const values = {
|
||||
version: r.version,
|
||||
latestVersion: r.latestVersion,
|
||||
updateAvailable: r.updateAvailable,
|
||||
storageTotal: r.storage?.total ?? null,
|
||||
storageAvailable: r.storage?.available ?? null,
|
||||
components: r.components && JSON.stringify(r.components),
|
||||
documents: r.documents,
|
||||
inbox: r.inbox,
|
||||
failed: r.failed && JSON.stringify(r.failed),
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await db.insert(paperlessState).values({ integrationId, ...values }).onConflictDoUpdate({ target: paperlessState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Paperless and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncPaperless(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PaperlessRead>): Promise<PaperlessSyncStatus> {
|
||||
let status: PaperlessSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "paperless") throw new Error("not a Paperless integration");
|
||||
const result = await (read ? read(loaded.config) : createPaperlessAdapter(loaded.config as never).read(now));
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[paperless] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[paperless] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllPaperless(now: Date = new Date()): Promise<Record<number, PaperlessSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "paperless"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, PaperlessSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncPaperless(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetPaperless(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface PaperlessProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
const gb = (bytes: number) => `${(bytes / 1e9).toFixed(bytes >= 1e11 ? 0 : 1)} GB`;
|
||||
const WORDS: Record<string, string> = { database: "the database", redis: "Redis", celery: "the worker that consumes documents", index: "the search index", classifier: "the classifier", sanity_check: "the sanity check" };
|
||||
|
||||
/** Pure: what is wrong with one Paperless, from what is stored for it. */
|
||||
export function paperlessProblems(id: number, name: string, s: Pick<StateRow, "storageTotal" | "storageAvailable" | "components" | "failed">): PaperlessProblem[] {
|
||||
const out: PaperlessProblem[] = [];
|
||||
for (const c of componentsOf(s) ?? []) {
|
||||
const word = WORDS[c.name] ?? c.name;
|
||||
if (c.status === "ERROR") {
|
||||
out.push({ key: `paperless-component:${id}:${c.name}`, severity: ESSENTIAL.has(c.name) ? "critical" : "warning", message: `${name}: ${word} is in error${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
|
||||
} else if (c.status === "WARNING" && c.name === "sanity_check") {
|
||||
out.push({ key: `paperless-component:${id}:${c.name}`, severity: "warning", message: `${name}: the sanity check found problems with some documents${c.error ? ` — ${c.error.length > 140 ? `${c.error.slice(0, 137)}…` : c.error}` : ""}` });
|
||||
}
|
||||
}
|
||||
const failed = failedOf(s) ?? [];
|
||||
if (failed.length > 0) {
|
||||
const files = failed.slice(0, 3).map((f) => f.file).join(", ") + (failed.length > 3 ? ` and ${failed.length - 3} more` : "");
|
||||
const why = failed[0].error ? ` — the latest: ${failed[0].error.length > 140 ? `${failed[0].error.slice(0, 137)}…` : failed[0].error}` : "";
|
||||
out.push({ key: `paperless-failed:${id}`, severity: "warning", message: `${name}: ${failed.length} document${failed.length === 1 ? "" : "s"} failed to import (${files})${why}` });
|
||||
}
|
||||
if (s.storageTotal !== null && s.storageAvailable !== null && s.storageTotal > 0) {
|
||||
const percent = ((s.storageTotal - s.storageAvailable) / s.storageTotal) * 100;
|
||||
if (percent >= DISK_WARN_PERCENT) {
|
||||
out.push({ key: `paperless-disk:${id}`, severity: percent >= DISK_CRITICAL_PERCENT ? "critical" : "warning", message: `${name}: the disk under the documents is ${Math.round(percent)} % full (${gb(s.storageTotal - s.storageAvailable)} of ${gb(s.storageTotal)})` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initPaperlessSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllPaperless().catch((err) => console.error("[paperless] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 19_000);
|
||||
console.log(`[paperless] reading Paperless every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* Keeps what Prometheus says up to date: every two minutes each Prometheus integration is read (its scrape targets and whether they answer, and the alerts its rules have firing)
|
||||
* and the state is stored. Pages and alerts are built from what is stored, so none of them talks to Prometheus.
|
||||
*
|
||||
* Prometheus is where the things you already measure live — a node exporter on each host, a service's own metrics — so this makes them part of the one list of what is wrong:
|
||||
* a **target that has stopped answering** (for five minutes, so one failed scrape is not news) and **an alert that Prometheus's rules are firing**.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, prometheusState } from "../db/schema.js";
|
||||
import { createPrometheusAdapter, type PrometheusAlert, type PrometheusRead } from "../integrations/prometheus/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 2 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 10 * 60_000;
|
||||
/** A target must have been down this long before it is a problem; one failed scrape is not. */
|
||||
export const TARGET_DOWN_AFTER_MS = 5 * 60_000;
|
||||
const STATUS_FLAG = "prometheusSync";
|
||||
|
||||
export interface PrometheusSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface JobCount {
|
||||
job: string;
|
||||
up: number;
|
||||
down: number;
|
||||
}
|
||||
export interface DownTarget {
|
||||
job: string;
|
||||
instance: string;
|
||||
error: string | null;
|
||||
/** When it was first seen down, carried over from one read to the next. */
|
||||
since: string;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, PrometheusSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, PrometheusSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function prometheusSyncStatuses(): Promise<Record<number, PrometheusSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isPrometheusFresh = (status: PrometheusSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof prometheusState.$inferSelect;
|
||||
|
||||
/** Null stays null: a list that could not be read is not an empty one. */
|
||||
const parse = <T>(text: string | null): T[] | null => {
|
||||
if (text === null) return null;
|
||||
try {
|
||||
const v = JSON.parse(text);
|
||||
return Array.isArray(v) ? (v as T[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
export const jobsOf = (row: Pick<StateRow, "jobs">) => parse<JobCount>(row.jobs);
|
||||
export const downOf = (row: Pick<StateRow, "down">) => parse<DownTarget>(row.down);
|
||||
export const alertsOf = (row: Pick<StateRow, "alerts">) => parse<PrometheusAlert>(row.alerts);
|
||||
|
||||
/** Writes down one read, replacing the last; a target that was down before keeps the time it was first seen down. */
|
||||
export async function storeRead(integrationId: number, r: PrometheusRead, now: Date = new Date()): Promise<void> {
|
||||
const [before] = await db.select({ down: prometheusState.down }).from(prometheusState).where(eq(prometheusState.integrationId, integrationId)).limit(1);
|
||||
const earlier = new Map((before ? downOf(before) ?? [] : []).map((d) => [`${d.job}\u0000${d.instance}`, d.since]));
|
||||
let jobs: JobCount[] | null = null;
|
||||
let down: DownTarget[] | null = null;
|
||||
if (r.targets) {
|
||||
const counts = new Map<string, JobCount>();
|
||||
down = [];
|
||||
for (const t of r.targets) {
|
||||
const c = counts.get(t.job) ?? { job: t.job, up: 0, down: 0 };
|
||||
if (t.health === "down") {
|
||||
c.down += 1;
|
||||
down.push({ job: t.job, instance: t.instance, error: t.error, since: earlier.get(`${t.job}\u0000${t.instance}`) ?? now.toISOString() });
|
||||
} else if (t.health === "up") c.up += 1;
|
||||
counts.set(t.job, c);
|
||||
}
|
||||
jobs = [...counts.values()].sort((a, b) => a.job.localeCompare(b.job));
|
||||
}
|
||||
const values = { version: r.version, jobs: jobs && JSON.stringify(jobs), down: down && JSON.stringify(down), alerts: r.alerts && JSON.stringify(r.alerts), updatedAt: now.toISOString() };
|
||||
await db.insert(prometheusState).values({ integrationId, ...values }).onConflictDoUpdate({ target: prometheusState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one Prometheus and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncPrometheus(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<PrometheusRead>): Promise<PrometheusSyncStatus> {
|
||||
let status: PrometheusSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "prometheus") throw new Error("not a Prometheus integration");
|
||||
const result = await (read ? read(loaded.config) : createPrometheusAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[prometheus] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[prometheus] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllPrometheus(now: Date = new Date()): Promise<Record<number, PrometheusSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "prometheus"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, PrometheusSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncPrometheus(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetPrometheus(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface PrometheusProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** Alerts that are firing by design (a heartbeat to prove the pipeline works) and so say nothing. */
|
||||
const BY_DESIGN = /^(watchdog|deadmansswitch|deadmanssnitch|infoinhibitor)$/i;
|
||||
|
||||
/** Pure: what is wrong, from what is stored. */
|
||||
export function prometheusProblems(id: number, name: string, s: Pick<StateRow, "jobs" | "down" | "alerts">, now: Date = new Date()): PrometheusProblem[] {
|
||||
const out: PrometheusProblem[] = [];
|
||||
const jobs = new Map((jobsOf(s) ?? []).map((j) => [j.job, j]));
|
||||
const byJob = new Map<string, DownTarget[]>();
|
||||
for (const d of downOf(s) ?? []) {
|
||||
if (now.getTime() - Date.parse(d.since) < TARGET_DOWN_AFTER_MS) continue;
|
||||
byJob.set(d.job, [...(byJob.get(d.job) ?? []), d]);
|
||||
}
|
||||
for (const [job, list] of byJob) {
|
||||
const c = jobs.get(job);
|
||||
const total = c ? c.up + c.down : list.length;
|
||||
const who = list.slice(0, 3).map((d) => d.instance).join(", ") + (list.length > 3 ? ` and ${list.length - 3} more` : "");
|
||||
const why = list[0].error ? ` — ${list[0].error.length > 120 ? `${list[0].error.slice(0, 117)}…` : list[0].error}` : "";
|
||||
out.push({ key: `prometheus-target:${id}:${job}`, severity: "warning", message: `${name}: ${list.length} of ${total} “${job}” target${total === 1 ? "" : "s"} not answering (${who})${why}` });
|
||||
}
|
||||
const firing = new Map<string, PrometheusAlert[]>();
|
||||
for (const a of alertsOf(s) ?? []) {
|
||||
if (a.state !== "firing" || BY_DESIGN.test(a.name) || a.severity === "info" || a.severity === "none") continue;
|
||||
firing.set(a.name, [...(firing.get(a.name) ?? []), a]);
|
||||
}
|
||||
for (const [alertname, list] of firing) {
|
||||
const critical = list.some((a) => a.severity === "critical" || a.severity === "page");
|
||||
const first = list.find((a) => a.summary) ?? list[0];
|
||||
out.push({
|
||||
key: `prometheus-alert:${id}:${alertname}`,
|
||||
severity: critical ? "critical" : "warning",
|
||||
message: `${name}: ${alertname}${list.length > 1 ? ` (${list.length} firing)` : first.instance ? ` on ${first.instance}` : ""}${first.summary ? ` — ${first.summary}` : ""}`,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initPrometheusSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllPrometheus().catch((err) => console.error("[prometheus] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 15_000);
|
||||
console.log(`[prometheus] reading Prometheus every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
Reference in new issue
Block a user