/** * The history of integration figures: what is taken out of each kind of integration's stored reading, how a reading is kept (and not kept twice, or when stale), * the hours and days beyond five weeks, what is thrown away, and how it is asked for. * * Run with: npm test --workspace server */ import { test, describe, before, after, beforeEach } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; const dir = mkdtempSync(join(tmpdir(), "hm-figures-")); process.env.DATABASE_PATH = join(dir, "test.sqlite"); process.env.SESSION_DIR = join(dir, "sessions"); process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32); process.env.HEALTH_STARTUP_GRACE_MS = "0"; const express = (await import("express")).default; const { db } = await import("../src/db/client.js"); const { users, integrations, adguardState, integrationFigures, integrationFigureSamples, integrationFigureRollup } = await import("../src/db/schema.js"); const { runMigrations } = await import("../src/db/migrate.js"); const F = await import("../src/services/integrationFigures.js"); const { integrationsRouter } = await import("../src/routes/integrations.js"); const { eq } = await import("drizzle-orm"); await runMigrations(); const NOW = new Date("2026-11-10T12:00:00.000Z"); const ago = (minutes: number) => new Date(NOW.getTime() - minutes * 60_000); const byKey = (list: { key: string; value: number }[]) => Object.fromEntries(list.map((f) => [f.key, f.value])); describe("what is taken from each kind of integration", () => { test("AdGuard and Technitium: the day's queries and blocked share, and how fast the test question was answered — only when it was", () => { const stats = JSON.stringify({ queries: 12000, blocked: 3000, blockedPercent: 25, avgMs: 11.5 }); assert.deepEqual(byKey(F.extractAdguard({ stats, probe: JSON.stringify({ ok: true, ms: 8 }) })), { queries: 12000, blocked: 3000, blockedPercent: 25, avgMs: 11.5, probeMs: 8 }); assert.deepEqual(byKey(F.extractAdguard({ stats, probe: JSON.stringify({ ok: false, ms: null }) })), { queries: 12000, blocked: 3000, blockedPercent: 25, avgMs: 11.5 }, "an unanswered question has no time"); assert.deepEqual(F.extractAdguard({ stats: null, probe: null }), [], "a reading that could not be taken is nothing, not zeros"); assert.deepEqual(byKey(F.extractTechnitium({ stats: JSON.stringify({ queries: 5, blocked: 1, blockedPercent: null, clients: 3, cachedEntries: 40 }), probe: null })), { queries: 5, blocked: 1, clients: 3, cachedEntries: 40 }); }); test("Sonarr/Radarr/Prowlarr and the download clients", () => { assert.deepEqual(byKey(F.extractArr({ queue: JSON.stringify({ total: 7, items: [] }), missing: 120 })), { queue: 7, missing: 120 }); assert.deepEqual(byKey(F.extractArr({ queue: null, missing: null })), {}, "Prowlarr has neither"); const counts = JSON.stringify({ downloading: 2, queued: 1, seeding: 10, completed: 5, failed: 1, paused: 0 }); assert.deepEqual(byKey(F.extractDownload({ speedDown: 5_000_000, speedUp: 0, counts, total: 19 })), { speedDown: 5_000_000, speedUp: 0, downloading: 2, pending: 3, failed: 1 }); assert.deepEqual(byKey(F.extractDownload({ speedDown: null, speedUp: null, counts: "not json", total: 0 })), {}, "nothing is made up"); }); test("Paperless, Immich, Nextcloud, NetBox", () => { assert.deepEqual(byKey(F.extractPaperless({ documents: 900, inbox: 4, storageTotal: 1000, storageAvailable: 250 })), { documents: 900, inbox: 4, storagePercent: 75 }); assert.deepEqual(byKey(F.extractPaperless({ documents: 900, inbox: null, storageTotal: null, storageAvailable: null })), { documents: 900 }); const queues = JSON.stringify([{ name: "thumbs", active: 1, waiting: 10, failed: 2, paused: false }, { name: "video", active: 0, waiting: 5, failed: 1, paused: false }]); assert.deepEqual(byKey(F.extractImmich({ photos: 100, videos: 7, usageBytes: 5e9, diskPercent: 41.5, queues })), { photos: 100, videos: 7, usageBytes: 5e9, diskPercent: 41.5, waiting: 15, failedJobs: 3 }); assert.deepEqual(byKey(F.extractNextcloud({ users: 3, files: 5000, active24h: 2, freeBytes: 1e11 })), { users: 3, active24h: 2, files: 5000, freeBytes: 1e11 }); assert.deepEqual(byKey(F.extractNetbox({ devices: 12, virtualMachines: 30, ipAddresses: 200 })), { devices: 12, virtualMachines: 30, ipAddresses: 200 }); }); test("n8n, Grafana, Prometheus, Authentik, Jellyfin", () => { const workflows = JSON.stringify([{ active: true, failed24h: 2 }, { active: true, failed24h: 0 }, { active: false, failed24h: 1 }]); assert.deepEqual(byKey(F.extractN8n({ workflows, executionsKnown: true })), { active: 2, failed24h: 3 }); assert.deepEqual(byKey(F.extractN8n({ workflows, executionsKnown: false })), { active: 2 }, "failed runs are not known when the runs could not be asked"); assert.deepEqual(byKey(F.extractGrafana({ rules: JSON.stringify({ total: 20, items: [{ state: "firing", health: "ok" }, { state: "inactive", health: "error" }, { state: "pending", health: "ok" }] }) })), { rules: 20, firing: 1, broken: 1 }); assert.deepEqual(F.extractGrafana({ rules: null }), []); assert.deepEqual(byKey(F.extractPrometheus({ jobs: JSON.stringify([{ job: "a", up: 5, down: 1 }, { job: "b", up: 3, down: 0 }]), down: "[]", alerts: JSON.stringify([{ state: "firing" }, { state: "pending" }]) })), { targetsUp: 8, targetsDown: 1, firing: 1 }); assert.deepEqual(byKey(F.extractAuthentik({ failedLogins24h: 4, configErrors24h: 0 })), { failedLogins24h: 4, configErrors24h: 0 }); assert.deepEqual(byKey(F.extractJellyfin({ movies: 300, series: 40, episodes: 900, songs: null, failedLogins24h: 1 })), { movies: 300, series: 40, episodes: 900, failedLogins24h: 1 }); }); test("Hetzner: its servers and the monthly estimate in the currency it was worked out in", () => { const list = F.extractHetzner({ servers: JSON.stringify([{}, {}, {}]), cost: JSON.stringify({ currency: "eur", total: 42.5 }) }); assert.deepEqual(byKey(list), { servers: 3, cost: 42.5 }); assert.equal(list.find((f) => f.key === "cost")!.unit, "money:EUR"); assert.deepEqual(byKey(F.extractHetzner({ servers: "[]", cost: null })), { servers: 0 }, "no price list, no cost"); }); test("which kinds have figures", () => { for (const t of ["adguard", "technitium", "sonarr", "radarr", "prowlarr", "qbittorrent", "sabnzbd", "paperless", "immich", "nextcloud", "netbox", "n8n", "grafana", "prometheus", "authentik", "jellyfin", "hetzner"] as const) assert.equal(F.hasFigures(t), true, t); for (const t of ["proxmox", "synology", "tailscale", "ups", "homeassistant", "speedtest"] as const) assert.equal(F.hasFigures(t), false, `${t} has history of its own, or nothing stored`); }); }); describe("keeping the figures", () => { let adguard = 0; before(async () => { [{ id: adguard }] = await db.insert(integrations).values({ type: "adguard", name: "AdGuard", baseUrl: "http://x" }).returning({ id: integrations.id }); }); beforeEach(async () => { await db.delete(integrationFigures); await db.delete(adguardState); await db.update(integrations).set({ enabled: true }).where(eq(integrations.id, adguard)); }); after(() => { try { rmSync(dir, { recursive: true, force: true }); } catch { // Windows keeps the database file locked until the process exits. } }); const figs = (queries: number): import("../src/services/integrationFigures.js").Figure[] => [{ key: "queries", label: "DNS queries", unit: "count", value: queries }]; const samples = async () => db.select().from(integrationFigureSamples).orderBy(integrationFigureSamples.at); const state = (readAt: Date, queries = 100) => db.insert(adguardState).values({ integrationId: adguard, stats: JSON.stringify({ queries, blocked: 10, blockedPercent: 10, avgMs: 5 }), probe: null, updatedAt: readAt.toISOString() }); test("a reading is a sample, an hour and a day; the figure remembers its latest, label and unit", async () => { assert.equal(await F.recordFigures(adguard, figs(100), ago(60)), 1); assert.equal(await F.recordFigures(adguard, [{ key: "queries", label: "Queries a day", unit: "count", value: 150 }], ago(30)), 1); const [fig] = await db.select().from(integrationFigures); assert.deepEqual([fig.label, fig.unit, fig.latest, fig.updatedAt], ["Queries a day", "count", 150, ago(30).toISOString()]); assert.equal((await samples()).length, 2); const rollup = await db.select().from(integrationFigureRollup).orderBy(integrationFigureRollup.grain); assert.deepEqual(rollup.map((r) => r.grain), ["d", "h", "h"].slice(0, rollup.length), "a day, and an hour each (the two readings are in different hours)"); const day = rollup.find((r) => r.grain === "d")!; assert.deepEqual([day.n, day.sum, day.min, day.max], [2, 250, 100, 150]); }); test("two readings close together are one, and the same reading seen twice is not kept twice", async () => { assert.equal(await F.recordFigures(adguard, figs(100), ago(30)), 1); assert.equal(await F.recordFigures(adguard, figs(101), ago(30)), 0, "the same moment"); assert.equal(await F.recordFigures(adguard, figs(102), ago(25)), 0, "five minutes later"); assert.equal(await F.recordFigures(adguard, figs(103), ago(10)), 1, "twenty minutes later"); assert.equal((await samples()).length, 2); assert.equal((await db.select().from(integrationFigures))[0].latest, 103, "the latest is the last reading given"); }); test("a pass keeps the figures of a reading that is recent (one for each, and none for a test question that was not asked)", async () => { await state(ago(3)); assert.equal(await F.recordAllFigures(NOW), 4); assert.equal(await F.recordAllFigures(NOW), 0, "the same reading again is not kept twice"); const keys = (await db.select().from(integrationFigures)).map((f) => f.key).sort(); assert.deepEqual(keys, ["avgMs", "blocked", "blockedPercent", "queries"]); }); test("a stale reading is not recorded — a gap is a gap", async () => { await state(ago(F.STALE_MS / 60_000 + 5)); assert.equal(await F.recordAllFigures(NOW), 0); assert.equal((await db.select().from(integrationFigures)).length, 0); }); test("a switched-off integration and a kind with no figures are left alone", async () => { await state(ago(3)); await db.update(integrations).set({ enabled: false }).where(eq(integrations.id, adguard)); assert.equal(await F.recordAllFigures(NOW), 0); await db.update(integrations).set({ enabled: true }).where(eq(integrations.id, adguard)); const [pve] = await db.insert(integrations).values({ type: "proxmox", name: "pve", baseUrl: "http://p" }).returning({ id: integrations.id }); await F.recordAllFigures(NOW); assert.equal((await db.select().from(integrationFigures).where(eq(integrationFigures.integrationId, pve.id))).length, 0); await db.delete(integrations).where(eq(integrations.id, pve.id)); }); test("an integration that has not been read yet has nothing to keep", async () => { assert.equal(await F.recordAllFigures(NOW), 0); }); test("the figures go when the integration does", async () => { const [other] = await db.insert(integrations).values({ type: "adguard", name: "Other", baseUrl: "http://y" }).returning({ id: integrations.id }); await F.recordFigures(other.id, figs(1), ago(10)); assert.equal((await db.select().from(integrationFigureSamples)).length, 1); await db.delete(integrations).where(eq(integrations.id, other.id)); assert.equal((await db.select().from(integrationFigures).where(eq(integrationFigures.integrationId, other.id))).length, 0); assert.equal((await db.select().from(integrationFigureSamples)).length, 0, "its readings and rollups with it"); assert.equal((await db.select().from(integrationFigureRollup)).length, 0); }); test("what is older than it is kept is thrown away: readings after five weeks, hours after four months, days after about two years", async () => { const day = (n: number) => new Date(NOW.getTime() - n * 86_400_000); for (const n of [1, 30, 40, 100, 150, 500, 900]) await F.recordFigures(adguard, figs(n), day(n)); await F.pruneFigures(NOW); assert.deepEqual((await samples()).map((s) => s.value).sort((a, b) => a - b), [1, 30], "five weeks of readings"); const rollup = await db.select().from(integrationFigureRollup); assert.deepEqual(rollup.filter((r) => r.grain === "h").map((r) => r.sum).sort((a, b) => a - b), [1, 30, 40, 100], "four months of hours"); assert.deepEqual(rollup.filter((r) => r.grain === "d").map((r) => r.sum).sort((a, b) => a - b), [1, 30, 40, 100, 150, 500], "two years and a bit of days"); }); }); describe("asking for a stretch", () => { let id = 0; before(async () => { [{ id }] = await db.insert(integrations).values({ type: "immich", name: "Photos", baseUrl: "http://i" }).returning({ id: integrations.id }); }); beforeEach(async () => { await db.delete(integrationFigures).where(eq(integrationFigures.integrationId, id)); }); const day = (n: number) => new Date(NOW.getTime() - n * 86_400_000); test("up to five weeks it is the readings; the lowest, highest and first are those of the stretch, the latest is the latest of all", async () => { for (let d = 20; d >= 0; d--) await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: 1000 + (20 - d) * 10 }], day(d)); const week = await F.getIntegrationFigures(id, 7, NOW); assert.equal(week.length, 1); const s = week[0]; assert.deepEqual([s.key, s.label, s.unit, s.latest], ["photos", "Photos", "count", 1200]); assert.deepEqual([s.first, s.low, s.high], [1130, 1130, 1200]); assert.ok(s.points.length >= 7 && s.points.length <= 96); assert.ok(s.points.every((p) => Date.parse(p.t) >= day(7).getTime()), "only the last week"); const month = (await F.getIntegrationFigures(id, 30, NOW))[0]; assert.equal(month.first, 1000); }); test("beyond five weeks it is the hours, and beyond four months the days, each point with its lowest and highest", async () => { for (const n of [200, 150, 100, 80, 60]) { await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: n }], new Date(day(n).getTime() - 3600_000)); await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: n + 10 }], day(n)); } const sixty = (await F.getIntegrationFigures(id, 90, NOW))[0]; assert.ok(sixty.points.every((p) => p.lo !== undefined && p.hi !== undefined), "from the hours"); assert.ok(sixty.points.length >= 2, "the 80 and 60 day ones (the 100 day one is outside ninety days)"); const year = (await F.getIntegrationFigures(id, 365, NOW))[0]; assert.ok(year.points.length >= 5, "the days reach back further"); assert.equal(year.low, 60); }); test("a figure with nothing in the stretch is left out; the stretch is limited to a year", async () => { await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: 5 }], day(200)); assert.deepEqual(await F.getIntegrationFigures(id, 7, NOW), []); assert.equal((await F.getIntegrationFigures(id, 5000, NOW)).length, 1, "asked for more than a year it is a year, and the 200 day one is in it"); await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: 6 }], day(400)); assert.equal((await F.getIntegrationFigures(id, 5000, NOW))[0].points.length, 1, "but not the 400 day one"); assert.equal((await F.getIntegrationFigures(id, 0, NOW)).length, 0, "at least a day: the last day has nothing"); }); }); describe("the route", () => { let app: Server; let base = ""; let id = 0; let proxmox = 0; before(async () => { await db.insert(users).values({ oidcSub: "viewer", name: "Vera", role: "viewer" }); [{ id }] = await db.insert(integrations).values({ type: "adguard", name: "AdGuard route", baseUrl: "http://r" }).returning({ id: integrations.id }); [{ id: proxmox }] = await db.insert(integrations).values({ type: "proxmox", name: "pve route", baseUrl: "http://pp" }).returning({ id: integrations.id }); await F.recordFigures(id, [{ key: "queries", label: "DNS queries", unit: "count", value: 321 }], new Date(Date.now() - 3600_000)); const a = express(); a.use(express.json()); a.use((req, _res, next) => { (req as unknown as { session: unknown }).session = { user: { sub: "viewer" } }; next(); }); a.use("/api/integrations", integrationsRouter); app = await new Promise((resolve) => { const x = a.listen(0, "127.0.0.1", () => resolve(x)); }); base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations`; }); after(() => app.close()); test("anyone signed in can ask; the days are limited; a kind with no figures says so; a missing integration is a 404", async () => { const res = await fetch(`${base}/${id}/figures?days=7`); assert.equal(res.status, 200); const body = await res.json(); assert.equal(body.kept, true); assert.equal(body.days, 7); assert.equal(body.figures.length, 1); assert.deepEqual([body.figures[0].key, body.figures[0].latest, body.figures[0].unit], ["queries", 321, "count"]); assert.equal((await (await fetch(`${base}/${id}/figures?days=99999`)).json()).days, 365); assert.equal((await (await fetch(`${base}/${id}/figures?days=abc`)).json()).days, 7); const none = await (await fetch(`${base}/${proxmox}/figures`)).json(); assert.deepEqual([none.kept, none.figures], [false, []]); assert.equal((await fetch(`${base}/9999/figures`)).status, 404); assert.equal((await fetch(`${base}/abc/figures`)).status, 400); }); });