Keep a history of integration figures and draw it on each integration's page
The numbers integrations report (queries and blocked share, queue and missing items, speeds, photos, documents, failed runs, firing rules, targets down, monthly cost and more) are kept about every quarter of an hour for five weeks, then as hours and days for up to two years, taken from each integration's stored reading. Every card on the 14 integration pages gets a Show the history button with a chart per figure, how it changed and its lowest and highest. A stretch when the integration could not be read is a gap. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e4f3a54189
commit
e7f23ba13f
30 files changed
+10532
-1
No files matched your search
@@ -0,0 +1,33 @@
|
||||
CREATE TABLE `integration_figure_rollup` (
|
||||
`figure_id` integer NOT NULL,
|
||||
`grain` text NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`sum` real NOT NULL,
|
||||
`n` integer NOT NULL,
|
||||
`min` real NOT NULL,
|
||||
`max` real NOT NULL,
|
||||
PRIMARY KEY(`figure_id`, `grain`, `at`),
|
||||
FOREIGN KEY (`figure_id`) REFERENCES `integration_figures`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE `integration_figure_samples` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`figure_id` integer NOT NULL,
|
||||
`at` text NOT NULL,
|
||||
`value` real NOT NULL,
|
||||
FOREIGN KEY (`figure_id`) REFERENCES `integration_figures`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `integration_figure_samples_idx` ON `integration_figure_samples` (`figure_id`,`at`);--> statement-breakpoint
|
||||
CREATE TABLE `integration_figures` (
|
||||
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||
`integration_id` integer NOT NULL,
|
||||
`key` text NOT NULL,
|
||||
`label` text NOT NULL,
|
||||
`unit` text NOT NULL,
|
||||
`latest` real NOT NULL,
|
||||
`updated_at` text NOT NULL,
|
||||
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX `integration_figures_key_idx` ON `integration_figures` (`integration_id`,`key`);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -659,6 +659,13 @@
|
||||
"when": 1794349689431,
|
||||
"tag": "0093_reboot_after_updates",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 94,
|
||||
"version": "6",
|
||||
"when": 1794350837756,
|
||||
"tag": "0094_integration_figures",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1991,6 +1991,57 @@ export const storageSamples = sqliteTable(
|
||||
(t) => [index("storage_samples_series_idx").on(t.seriesId, t.at)],
|
||||
);
|
||||
|
||||
// ─── The figures of an integration, over time ──────────────────────────────
|
||||
|
||||
// A number an integration reports (queries a day, items in a queue, photos, documents, a monthly cost), one row for each, so how it has moved can be drawn — see services/integrationFigures.ts.
|
||||
// `unit` says how to write it (count, percent, ms, bytes, bytes_per_s, money:SEK …); `latest` is the last reading.
|
||||
export const integrationFigures = sqliteTable(
|
||||
"integration_figures",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
integrationId: integer("integration_id")
|
||||
.notNull()
|
||||
.references(() => integrations.id, { onDelete: "cascade" }),
|
||||
key: text("key").notNull(), // "queries", "queue", "photos" …: stable within a kind of integration
|
||||
label: text("label").notNull(),
|
||||
unit: text("unit").notNull(),
|
||||
latest: real("latest").notNull(),
|
||||
updatedAt: text("updated_at").notNull(), // ISO; the last reading
|
||||
},
|
||||
(t) => [uniqueIndex("integration_figures_key_idx").on(t.integrationId, t.key)],
|
||||
);
|
||||
|
||||
// The readings themselves, one about every quarter of an hour, kept five weeks.
|
||||
export const integrationFigureSamples = sqliteTable(
|
||||
"integration_figure_samples",
|
||||
{
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
figureId: integer("figure_id")
|
||||
.notNull()
|
||||
.references(() => integrationFigures.id, { onDelete: "cascade" }),
|
||||
at: text("at").notNull(), // ISO
|
||||
value: real("value").notNull(),
|
||||
},
|
||||
(t) => [index("integration_figure_samples_idx").on(t.figureId, t.at)],
|
||||
);
|
||||
|
||||
// The same, coarser and kept longer: one row for each figure and hour (grain "h", four months) or day (grain "d", over two years), like the servers' (server_metrics_rollup).
|
||||
export const integrationFigureRollup = sqliteTable(
|
||||
"integration_figure_rollup",
|
||||
{
|
||||
figureId: integer("figure_id")
|
||||
.notNull()
|
||||
.references(() => integrationFigures.id, { onDelete: "cascade" }),
|
||||
grain: text("grain").$type<"h" | "d">().notNull(),
|
||||
at: text("at").notNull(),
|
||||
sum: real("sum").notNull(),
|
||||
n: integer("n").notNull(),
|
||||
min: real("min").notNull(),
|
||||
max: real("max").notNull(),
|
||||
},
|
||||
(t) => [primaryKey({ columns: [t.figureId, t.grain, t.at] })],
|
||||
);
|
||||
|
||||
// "Remind me tomorrow at 09:00": an alert hidden until a time, and announced again then if it is still there — see services/alertSnoozes.ts. Keyed like an acknowledgement.
|
||||
export const alertSnoozes = sqliteTable(
|
||||
"alert_snoozes",
|
||||
|
||||
@@ -47,6 +47,7 @@ import { initTechnitiumSync } from "./services/technitiumSync.js";
|
||||
import { initDownloadsSync } from "./services/downloadsSync.js";
|
||||
import { initHetznerRobotSync } from "./services/hetznerRobotSync.js";
|
||||
import { initCostHistory } from "./services/costHistory.js";
|
||||
import { initIntegrationFigures } from "./services/integrationFigures.js";
|
||||
import { initGrafanaSync } from "./services/grafanaSync.js";
|
||||
import { initVersionCheck } from "./services/appVersions.js";
|
||||
import { backfillMetricRollups } from "./services/serverMetrics.js";
|
||||
@@ -117,6 +118,7 @@ initTechnitiumSync();
|
||||
initDownloadsSync();
|
||||
initHetznerRobotSync();
|
||||
initCostHistory();
|
||||
initIntegrationFigures();
|
||||
initGrafanaSync();
|
||||
initVersionCheck();
|
||||
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
|
||||
|
||||
@@ -30,6 +30,7 @@ import { getSourceStatus } from "../services/integrationHealthMonitor.js";
|
||||
import { dockerVulnSyncStatuses, findingsOf, imageAgeDays, imagesOf, isDockerVulnFresh } from "../services/dockerVulnSync.js";
|
||||
import { invalidateAlertsCache } from "../services/alerts.js";
|
||||
import { forgetReading, readAfterChange, readIntegrationNow } from "../services/integrationSyncs.js";
|
||||
import { getIntegrationFigures, hasFigures } from "../services/integrationFigures.js";
|
||||
import { getSettings, updateSettings } from "../services/settingsStore.js";
|
||||
import { asyncHandler } from "../utils/asyncHandler.js";
|
||||
import { ID_PATTERNS, isValidId } from "../utils/pathSegment.js";
|
||||
@@ -293,6 +294,17 @@ integrationsRouter.post("/:id/test", requireRole("admin"), asyncHandler(async (r
|
||||
|
||||
// Reads one integration now, instead of waiting for its next turn: a kind with a stored reading is read the way its schedule reads it (and what it says is judged again at once); a kind that is read live
|
||||
// is asked whether it answers. Operators and administrators — it only reads, but some systems limit how often they may be asked, so the same integration is not read again within 20 seconds.
|
||||
// How the figures an integration reports (queries a day, items in a queue, photos, documents …) have moved: a day to a year, drawn from what is kept (services/integrationFigures.ts). Anyone signed in.
|
||||
integrationsRouter.get("/:id/figures", asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
const [row] = await db.select({ type: integrations.type }).from(integrations).where(eq(integrations.id, id)).limit(1);
|
||||
if (!row) return res.status(404).json({ error: "not_found" });
|
||||
const asked = Number(req.query.days);
|
||||
const days = Number.isFinite(asked) ? asked : 7;
|
||||
res.json({ kept: hasFigures(row.type), days: Math.min(365, Math.max(1, Math.round(days))), figures: hasFigures(row.type) ? await getIntegrationFigures(id, days) : [] });
|
||||
}));
|
||||
|
||||
integrationsRouter.post("/:id/read-now", requireRole("operator"), asyncHandler(async (req, res) => {
|
||||
const id = Number(req.params.id);
|
||||
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
/**
|
||||
* A history of the numbers integrations report — queries a day and how many were blocked, items waiting in a download queue, photos, documents, failed runs, the monthly cost at Hetzner — so a page
|
||||
* can show how a figure has moved instead of only what it is now.
|
||||
*
|
||||
* What is kept is what each integration's own reading already stores (its "state" row, read on a schedule by its sync): every five minutes this looks at each enabled integration that has one, takes
|
||||
* its figures out of it (the `extract*` functions below, one for each kind), and keeps one reading of each about every quarter of an hour for five weeks — and, beyond that, its hours (four months) and
|
||||
* days (over two years) as a sum, a count, a lowest and a highest, as the servers' figures are (services/serverMetrics.ts, whose way of drawing them this shares). A reading that is not recent — the
|
||||
* integration could not be read for a while — is not recorded, so a gap is a gap and not a flat line.
|
||||
*
|
||||
* What it does not cover: the integrations that already keep their own history (UPS, Home Assistant, speedtest, who is streaming from Jellyfin, the storage of Proxmox and Synology) and those that
|
||||
* are only read live when a page is opened (Proxmox, Synology, Tailscale, Gitea, Semaphore, Uptime Kuma …) — there is nothing stored to take figures from.
|
||||
*/
|
||||
import { and, eq, gte, lt, sql } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { adguardState, arrState, authentikState, downloadState, grafanaState, hetznerState, immichState, integrationFigureRollup, integrationFigureSamples, integrationFigures, integrations, jellyfinState, n8nState, netboxState, nextcloudState, paperlessState, prometheusState, technitiumState, type IntegrationType } from "../db/schema.js";
|
||||
import { ARR_KINDS } from "../integrations/arr/adapter.js";
|
||||
import { DOWNLOAD_KINDS } from "../integrations/downloads/adapter.js";
|
||||
import { bucketPoints, bucketRollups, DAILY_KEEP_DAYS, dayStart, HOURLY_KEEP_DAYS, hourStart, MAX_RANGE_DAYS, RETENTION_DAYS, sourceFor, type Point, type RollupRow } from "./serverMetrics.js";
|
||||
|
||||
const DAY_MS = 24 * 3600_000;
|
||||
/** Readings closer together than this add no sample (a read twice, or two passes over the same read). */
|
||||
export const MIN_GAP_MS = 14 * 60_000;
|
||||
/** A reading older than this is not recorded: it says nothing about now. */
|
||||
export const STALE_MS = 45 * 60_000;
|
||||
export const PASS_INTERVAL_MS = 5 * 60_000;
|
||||
|
||||
/** How a figure is written: a plain count, a percentage, milliseconds, bytes, bytes a second, or money in a currency ("money:SEK"). */
|
||||
export type FigureUnit = "count" | "percent" | "ms" | "bytes" | "bytes_per_s" | `money:${string}`;
|
||||
|
||||
export interface Figure {
|
||||
key: string;
|
||||
label: string;
|
||||
unit: FigureUnit;
|
||||
value: number;
|
||||
}
|
||||
|
||||
// ─── What each kind of integration has ─────────────────────────────────────
|
||||
|
||||
const parse = <T>(s: string | null | undefined): T | null => {
|
||||
if (!s) return null;
|
||||
try {
|
||||
return JSON.parse(s) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const isNum = (n: unknown): n is number => typeof n === "number" && Number.isFinite(n);
|
||||
const round2 = (n: number) => Math.round(n * 100) / 100;
|
||||
|
||||
/** Keeps the figures that are numbers: a column that is null could not be read, which is not the same as zero. */
|
||||
function figures(list: [key: string, label: string, unit: FigureUnit, value: unknown][]): Figure[] {
|
||||
return list.filter((f): f is [string, string, FigureUnit, number] => isNum(f[3])).map(([key, label, unit, value]) => ({ key, label, unit, value }));
|
||||
}
|
||||
|
||||
type Row<T extends { $inferSelect: unknown }> = T["$inferSelect"];
|
||||
|
||||
export const extractAdguard = (r: Pick<Row<typeof adguardState>, "stats" | "probe">): Figure[] => {
|
||||
const s = parse<{ queries?: number; blocked?: number; blockedPercent?: number | null; avgMs?: number | null }>(r.stats);
|
||||
const p = parse<{ ok?: boolean; ms?: number | null }>(r.probe);
|
||||
return figures([
|
||||
["queries", "DNS queries", "count", s?.queries],
|
||||
["blocked", "Blocked", "count", s?.blocked],
|
||||
["blockedPercent", "Blocked share", "percent", s?.blockedPercent],
|
||||
["avgMs", "Average answer time", "ms", s?.avgMs],
|
||||
["probeMs", "Test question answered in", "ms", p?.ok ? p.ms : null],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractTechnitium = (r: Pick<Row<typeof technitiumState>, "stats" | "probe">): Figure[] => {
|
||||
const s = parse<{ queries?: number; blocked?: number; blockedPercent?: number | null; clients?: number; cachedEntries?: number | null }>(r.stats);
|
||||
const p = parse<{ ok?: boolean; ms?: number | null }>(r.probe);
|
||||
return figures([
|
||||
["queries", "DNS queries", "count", s?.queries],
|
||||
["blocked", "Blocked", "count", s?.blocked],
|
||||
["blockedPercent", "Blocked share", "percent", s?.blockedPercent],
|
||||
["clients", "Clients", "count", s?.clients],
|
||||
["cachedEntries", "Cache entries", "count", s?.cachedEntries],
|
||||
["probeMs", "Test question answered in", "ms", p?.ok ? p.ms : null],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractArr = (r: Pick<Row<typeof arrState>, "queue" | "missing">): Figure[] =>
|
||||
figures([
|
||||
["queue", "In the download queue", "count", parse<{ total?: number }>(r.queue)?.total],
|
||||
["missing", "Wanted, missing", "count", r.missing],
|
||||
]);
|
||||
|
||||
export const extractDownload = (r: Pick<Row<typeof downloadState>, "speedDown" | "speedUp" | "counts" | "total">): Figure[] => {
|
||||
const c = parse<Record<string, number>>(r.counts);
|
||||
const pending = c ? r.total - (c.seeding ?? 0) - (c.completed ?? 0) - (c.failed ?? 0) : null;
|
||||
return figures([
|
||||
["speedDown", "Download speed", "bytes_per_s", r.speedDown],
|
||||
["speedUp", "Upload speed", "bytes_per_s", r.speedUp],
|
||||
["downloading", "Downloading", "count", c?.downloading],
|
||||
["pending", "Not finished", "count", pending],
|
||||
["failed", "Failed", "count", c?.failed],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractPaperless = (r: Pick<Row<typeof paperlessState>, "documents" | "inbox" | "storageTotal" | "storageAvailable">): Figure[] =>
|
||||
figures([
|
||||
["documents", "Documents", "count", r.documents],
|
||||
["inbox", "In the inbox", "count", r.inbox],
|
||||
["storagePercent", "Storage used", "percent", isNum(r.storageTotal) && r.storageTotal > 0 && isNum(r.storageAvailable) ? round2(((r.storageTotal - r.storageAvailable) / r.storageTotal) * 100) : null],
|
||||
]);
|
||||
|
||||
export const extractImmich = (r: Pick<Row<typeof immichState>, "photos" | "videos" | "usageBytes" | "diskPercent" | "queues">): Figure[] => {
|
||||
const q = parse<{ waiting?: number; failed?: number }[]>(r.queues);
|
||||
const sum = (k: "waiting" | "failed") => (Array.isArray(q) ? q.reduce((n, x) => n + (isNum(x?.[k]) ? (x[k] as number) : 0), 0) : null);
|
||||
return figures([
|
||||
["photos", "Photos", "count", r.photos],
|
||||
["videos", "Videos", "count", r.videos],
|
||||
["usageBytes", "Library size", "bytes", r.usageBytes],
|
||||
["diskPercent", "Disk used", "percent", r.diskPercent],
|
||||
["waiting", "Jobs waiting", "count", sum("waiting")],
|
||||
["failedJobs", "Jobs failed", "count", sum("failed")],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractNextcloud = (r: Pick<Row<typeof nextcloudState>, "users" | "files" | "active24h" | "freeBytes">): Figure[] =>
|
||||
figures([
|
||||
["users", "Users", "count", r.users],
|
||||
["active24h", "Active in the last day", "count", r.active24h],
|
||||
["files", "Files", "count", r.files],
|
||||
["freeBytes", "Free space", "bytes", r.freeBytes],
|
||||
]);
|
||||
|
||||
export const extractNetbox = (r: Pick<Row<typeof netboxState>, "devices" | "virtualMachines" | "ipAddresses">): Figure[] =>
|
||||
figures([
|
||||
["devices", "Devices", "count", r.devices],
|
||||
["virtualMachines", "Virtual machines", "count", r.virtualMachines],
|
||||
["ipAddresses", "IP addresses", "count", r.ipAddresses],
|
||||
]);
|
||||
|
||||
export const extractN8n = (r: Pick<Row<typeof n8nState>, "workflows" | "executionsKnown">): Figure[] => {
|
||||
const w = parse<{ active?: boolean; failed24h?: number }[]>(r.workflows);
|
||||
if (!Array.isArray(w)) return [];
|
||||
return figures([
|
||||
["active", "Active workflows", "count", w.filter((x) => x?.active).length],
|
||||
["failed24h", "Failed runs in the last day", "count", r.executionsKnown ? w.reduce((n, x) => n + (isNum(x?.failed24h) ? (x.failed24h as number) : 0), 0) : null],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractGrafana = (r: Pick<Row<typeof grafanaState>, "rules">): Figure[] => {
|
||||
const rules = parse<{ total?: number; items?: { state?: string; health?: string }[] }>(r.rules);
|
||||
if (!rules) return [];
|
||||
const items = Array.isArray(rules.items) ? rules.items : [];
|
||||
return figures([
|
||||
["rules", "Alert rules", "count", rules.total],
|
||||
["firing", "Firing", "count", items.filter((i) => i?.state === "firing").length],
|
||||
["broken", "Cannot be evaluated", "count", items.filter((i) => i?.health === "error").length],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractPrometheus = (r: Pick<Row<typeof prometheusState>, "jobs" | "down" | "alerts">): Figure[] => {
|
||||
const jobs = parse<{ up?: number; down?: number }[]>(r.jobs);
|
||||
const alerts = parse<{ state?: string }[]>(r.alerts);
|
||||
return figures([
|
||||
["targetsUp", "Targets up", "count", Array.isArray(jobs) ? jobs.reduce((n, j) => n + (isNum(j?.up) ? (j.up as number) : 0), 0) : null],
|
||||
["targetsDown", "Targets down", "count", Array.isArray(jobs) ? jobs.reduce((n, j) => n + (isNum(j?.down) ? (j.down as number) : 0), 0) : null],
|
||||
["firing", "Alerts firing", "count", Array.isArray(alerts) ? alerts.filter((a) => a?.state === "firing").length : null],
|
||||
]);
|
||||
};
|
||||
|
||||
export const extractAuthentik = (r: Pick<Row<typeof authentikState>, "failedLogins24h" | "configErrors24h">): Figure[] =>
|
||||
figures([
|
||||
["failedLogins24h", "Failed sign-ins in the last day", "count", r.failedLogins24h],
|
||||
["configErrors24h", "Configuration errors in the last day", "count", r.configErrors24h],
|
||||
]);
|
||||
|
||||
export const extractJellyfin = (r: Pick<Row<typeof jellyfinState>, "movies" | "series" | "episodes" | "songs" | "failedLogins24h">): Figure[] =>
|
||||
figures([
|
||||
["movies", "Films", "count", r.movies],
|
||||
["series", "Series", "count", r.series],
|
||||
["episodes", "Episodes", "count", r.episodes],
|
||||
["songs", "Songs", "count", r.songs],
|
||||
["failedLogins24h", "Failed sign-ins in the last day", "count", r.failedLogins24h],
|
||||
]);
|
||||
|
||||
export const extractHetzner = (r: Pick<Row<typeof hetznerState>, "servers" | "cost">): Figure[] => {
|
||||
const cost = parse<{ currency?: string; total?: number }>(r.cost);
|
||||
const servers = parse<unknown[]>(r.servers);
|
||||
return figures([
|
||||
["servers", "Servers", "count", Array.isArray(servers) ? servers.length : null],
|
||||
["cost", "Monthly cost, estimated", `money:${typeof cost?.currency === "string" && /^[A-Za-z]{3}$/.test(cost.currency) ? cost.currency.toUpperCase() : "EUR"}` as FigureUnit, cost?.total],
|
||||
]);
|
||||
};
|
||||
|
||||
interface Source {
|
||||
/** The stored reading of one integration (null when it has none), with when it was read and its figures. */
|
||||
load: (integrationId: number) => Promise<{ at: string; figures: Figure[] } | null>;
|
||||
}
|
||||
|
||||
/** A source that reads one table's row for the integration and runs an extractor over it. */
|
||||
// (`extract` is checked where it is written, against the columns it reads; here it is handed the whole row.)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function source<R extends { updatedAt: string }>(read: (integrationId: number) => Promise<R | undefined>, extract: (row: any) => Figure[]): Source {
|
||||
return {
|
||||
load: async (id) => {
|
||||
const row = await read(id);
|
||||
return row ? { at: row.updatedAt, figures: extract(row) } : null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const SOURCES: Partial<Record<IntegrationType, Source>> = {
|
||||
adguard: source(async (id) => (await db.select().from(adguardState).where(eq(adguardState.integrationId, id)).limit(1))[0], extractAdguard),
|
||||
technitium: source(async (id) => (await db.select().from(technitiumState).where(eq(technitiumState.integrationId, id)).limit(1))[0], extractTechnitium),
|
||||
paperless: source(async (id) => (await db.select().from(paperlessState).where(eq(paperlessState.integrationId, id)).limit(1))[0], extractPaperless),
|
||||
immich: source(async (id) => (await db.select().from(immichState).where(eq(immichState.integrationId, id)).limit(1))[0], extractImmich),
|
||||
nextcloud: source(async (id) => (await db.select().from(nextcloudState).where(eq(nextcloudState.integrationId, id)).limit(1))[0], extractNextcloud),
|
||||
netbox: source(async (id) => (await db.select().from(netboxState).where(eq(netboxState.integrationId, id)).limit(1))[0], extractNetbox),
|
||||
n8n: source(async (id) => (await db.select().from(n8nState).where(eq(n8nState.integrationId, id)).limit(1))[0], extractN8n),
|
||||
grafana: source(async (id) => (await db.select().from(grafanaState).where(eq(grafanaState.integrationId, id)).limit(1))[0], extractGrafana),
|
||||
prometheus: source(async (id) => (await db.select().from(prometheusState).where(eq(prometheusState.integrationId, id)).limit(1))[0], extractPrometheus),
|
||||
authentik: source(async (id) => (await db.select().from(authentikState).where(eq(authentikState.integrationId, id)).limit(1))[0], extractAuthentik),
|
||||
jellyfin: source(async (id) => (await db.select().from(jellyfinState).where(eq(jellyfinState.integrationId, id)).limit(1))[0], extractJellyfin),
|
||||
hetzner: source(async (id) => (await db.select().from(hetznerState).where(eq(hetznerState.integrationId, id)).limit(1))[0], extractHetzner),
|
||||
};
|
||||
for (const k of ARR_KINDS) SOURCES[k] = source(async (id) => (await db.select().from(arrState).where(eq(arrState.integrationId, id)).limit(1))[0], extractArr);
|
||||
for (const k of DOWNLOAD_KINDS) SOURCES[k] = source(async (id) => (await db.select().from(downloadState).where(eq(downloadState.integrationId, id)).limit(1))[0], extractDownload);
|
||||
|
||||
/** Whether figures are kept for this kind of integration. */
|
||||
export const hasFigures = (type: IntegrationType): boolean => SOURCES[type] !== undefined;
|
||||
|
||||
// ─── Recording ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Adds a reading to its hour and day. Never allowed to break recording the reading itself. */
|
||||
async function addToRollups(figureId: number, value: number, at: Date): Promise<void> {
|
||||
try {
|
||||
for (const grain of ["h", "d"] as const) {
|
||||
const bucket = grain === "h" ? hourStart(at.getTime()) : dayStart(at.getTime());
|
||||
await db
|
||||
.insert(integrationFigureRollup)
|
||||
.values({ figureId, grain, at: bucket, sum: value, n: 1, min: value, max: value })
|
||||
.onConflictDoUpdate({
|
||||
target: [integrationFigureRollup.figureId, integrationFigureRollup.grain, integrationFigureRollup.at],
|
||||
set: {
|
||||
sum: sql`${integrationFigureRollup.sum} + ${value}`,
|
||||
n: sql`${integrationFigureRollup.n} + 1`,
|
||||
min: sql`min(${integrationFigureRollup.min}, ${value})`,
|
||||
max: sql`max(${integrationFigureRollup.max}, ${value})`,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[figures] couldn't add to the hourly and daily figures:", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Keeps one reading of each figure of one integration, as of `at` (when it was read). Returns how many were stored (those too close to the last one are not). */
|
||||
export async function recordFigures(integrationId: number, list: Figure[], at: Date): Promise<number> {
|
||||
let stored = 0;
|
||||
const iso = at.toISOString();
|
||||
for (const f of list) {
|
||||
const [row] = await db
|
||||
.insert(integrationFigures)
|
||||
.values({ integrationId, key: f.key, label: f.label, unit: f.unit, latest: f.value, updatedAt: iso })
|
||||
.onConflictDoUpdate({ target: [integrationFigures.integrationId, integrationFigures.key], set: { label: f.label, unit: f.unit, latest: f.value, updatedAt: iso } })
|
||||
.returning({ id: integrationFigures.id });
|
||||
const [last] = await db
|
||||
.select({ at: sql<string>`max(${integrationFigureSamples.at})` })
|
||||
.from(integrationFigureSamples)
|
||||
.where(eq(integrationFigureSamples.figureId, row.id));
|
||||
if (last?.at && Math.abs(at.getTime() - Date.parse(last.at)) < MIN_GAP_MS) continue;
|
||||
await db.insert(integrationFigureSamples).values({ figureId: row.id, at: iso, value: f.value });
|
||||
await addToRollups(row.id, f.value, at);
|
||||
stored++;
|
||||
}
|
||||
return stored;
|
||||
}
|
||||
|
||||
/** Throws away what is older than it is kept: readings after five weeks, hours after four months, days after about two years and a bit. */
|
||||
export async function pruneFigures(now: Date = new Date()): Promise<void> {
|
||||
const before = (days: number) => new Date(now.getTime() - days * DAY_MS).toISOString();
|
||||
await db.delete(integrationFigureSamples).where(lt(integrationFigureSamples.at, before(RETENTION_DAYS)));
|
||||
await db.delete(integrationFigureRollup).where(and(eq(integrationFigureRollup.grain, "h"), lt(integrationFigureRollup.at, before(HOURLY_KEEP_DAYS))));
|
||||
await db.delete(integrationFigureRollup).where(and(eq(integrationFigureRollup.grain, "d"), lt(integrationFigureRollup.at, before(DAILY_KEEP_DAYS))));
|
||||
}
|
||||
|
||||
/** One pass: every enabled integration with a stored reading that is recent has its figures kept. Returns how many readings were stored. */
|
||||
export async function recordAllFigures(now: Date = new Date()): Promise<number> {
|
||||
let stored = 0;
|
||||
for (const i of await db.select({ id: integrations.id, type: integrations.type }).from(integrations).where(eq(integrations.enabled, true))) {
|
||||
const src = SOURCES[i.type];
|
||||
if (!src) continue;
|
||||
try {
|
||||
const reading = await src.load(i.id);
|
||||
if (!reading) continue;
|
||||
const readAt = Date.parse(reading.at);
|
||||
if (!Number.isFinite(readAt) || now.getTime() - readAt > STALE_MS || readAt > now.getTime() + 60_000) continue;
|
||||
stored += await recordFigures(i.id, reading.figures, new Date(readAt));
|
||||
} catch (err) {
|
||||
console.error(`[figures] integration ${i.id}:`, err);
|
||||
}
|
||||
}
|
||||
await pruneFigures(now);
|
||||
return stored;
|
||||
}
|
||||
|
||||
// ─── Reading ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface FigureSeries {
|
||||
key: string;
|
||||
label: string;
|
||||
unit: FigureUnit;
|
||||
/** The latest reading, whatever stretch is asked for. */
|
||||
latest: number;
|
||||
/** ISO — when it was read. */
|
||||
readAt: string;
|
||||
points: Point[];
|
||||
/** The first point of the stretch, the lowest and the highest figure in it (null with no points). */
|
||||
first: number | null;
|
||||
low: number | null;
|
||||
high: number | null;
|
||||
}
|
||||
|
||||
/** How many points a chart over `days` is drawn from at most. */
|
||||
const pointsFor = (days: number) => (days <= RETENTION_DAYS ? 96 : days <= HOURLY_KEEP_DAYS ? 200 : MAX_RANGE_DAYS + 5);
|
||||
|
||||
/** Every figure kept for an integration over the last `days` (up to a year), ready to draw; the ones with no point in the stretch are left out. */
|
||||
export async function getIntegrationFigures(integrationId: number, days: number, now: Date = new Date()): Promise<FigureSeries[]> {
|
||||
days = Math.min(MAX_RANGE_DAYS, Math.max(1, Math.round(days)));
|
||||
const to = now.getTime();
|
||||
const from = to - days * DAY_MS;
|
||||
const kind = sourceFor(days);
|
||||
const series = await db.select().from(integrationFigures).where(eq(integrationFigures.integrationId, integrationId)).orderBy(integrationFigures.id);
|
||||
const out: FigureSeries[] = [];
|
||||
for (const f of series) {
|
||||
let points: Point[];
|
||||
if (kind === "samples") {
|
||||
const rows = await db
|
||||
.select({ at: integrationFigureSamples.at, value: integrationFigureSamples.value })
|
||||
.from(integrationFigureSamples)
|
||||
.where(and(eq(integrationFigureSamples.figureId, f.id), gte(integrationFigureSamples.at, new Date(from).toISOString())))
|
||||
.orderBy(integrationFigureSamples.at);
|
||||
points = bucketPoints(rows.map((r) => ({ t: Date.parse(r.at), v: r.value })), from, to, pointsFor(days));
|
||||
} else {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(integrationFigureRollup)
|
||||
.where(and(eq(integrationFigureRollup.figureId, f.id), eq(integrationFigureRollup.grain, kind === "hourly" ? "h" : "d"), gte(integrationFigureRollup.at, new Date(from).toISOString())))
|
||||
.orderBy(integrationFigureRollup.at);
|
||||
const list: RollupRow[] = rows.map((r) => ({ t: Date.parse(r.at), sum: r.sum, n: r.n, min: r.min, max: r.max }));
|
||||
points = bucketRollups(list, from, to, pointsFor(days));
|
||||
}
|
||||
if (points.length === 0) continue;
|
||||
out.push({
|
||||
key: f.key,
|
||||
label: f.label,
|
||||
unit: f.unit as FigureUnit,
|
||||
latest: f.latest,
|
||||
readAt: f.updatedAt,
|
||||
points,
|
||||
first: points[0].v,
|
||||
low: Math.min(...points.map((p) => p.lo ?? p.v)),
|
||||
high: Math.max(...points.map((p) => p.hi ?? p.v)),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── Running ───────────────────────────────────────────────────────────────
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initIntegrationFigures(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void recordAllFigures().catch((err) => console.error("[figures] couldn't keep the history:", err));
|
||||
timer = setInterval(pass, PASS_INTERVAL_MS);
|
||||
timer.unref();
|
||||
setTimeout(pass, 75_000).unref();
|
||||
console.log("[figures] keeping the history of integration figures");
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* How the figures of an integration are written on the page — numbers, sizes, speeds, money — and how a change over a stretch is put into words.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { axisFigure, changeWords, formatBytes, formatFigure } from "../../web/src/utils/figureUnits.js";
|
||||
|
||||
describe("writing a figure", () => {
|
||||
test("sizes count in thousands, with a decimal only while it is small", () => {
|
||||
assert.equal(formatBytes(0), "0 B");
|
||||
assert.equal(formatBytes(999), "999 B");
|
||||
assert.equal(formatBytes(1_536), "1.5 kB");
|
||||
assert.equal(formatBytes(15_360_000), "15 MB");
|
||||
assert.equal(formatBytes(5e9), "5 GB");
|
||||
assert.equal(formatBytes(2.5e12), "2.5 TB");
|
||||
assert.equal(formatBytes(-2_000), "-2 kB");
|
||||
assert.equal(formatBytes(Number.NaN), "—");
|
||||
});
|
||||
|
||||
test("each unit reads the way it should", () => {
|
||||
assert.equal(formatFigure("count", 12345), "12,345");
|
||||
assert.equal(formatFigure("count", 7.04), "7");
|
||||
assert.equal(formatFigure("percent", 41.56), "41.6%");
|
||||
assert.equal(formatFigure("ms", 11.5), "11.5 ms");
|
||||
assert.equal(formatFigure("bytes", 5e9), "5 GB");
|
||||
assert.equal(formatFigure("bytes_per_s", 5_000_000), "5 MB/s");
|
||||
assert.equal(formatFigure("money:EUR", 42.4), "€42");
|
||||
assert.equal(formatFigure("money:XXXX", 42.4), "42 XXXX", "an unknown currency is still a number");
|
||||
assert.equal(formatFigure("count", Number.POSITIVE_INFINITY), "—");
|
||||
});
|
||||
});
|
||||
|
||||
describe("a figure in a chart's margin", () => {
|
||||
test("short, so it fits", () => {
|
||||
assert.equal(axisFigure("count", 12_500), "12.5K");
|
||||
assert.equal(axisFigure("count", 950), "950");
|
||||
assert.equal(axisFigure("percent", 41.6), "42%");
|
||||
assert.equal(axisFigure("bytes", 340e9), "340 GB");
|
||||
assert.equal(axisFigure("money:EUR", 1200), "EUR 1.2K");
|
||||
assert.equal(axisFigure("count", Number.NaN), "");
|
||||
});
|
||||
});
|
||||
|
||||
describe("how a figure moved", () => {
|
||||
test("up, down and unchanged, in the unit's own terms", () => {
|
||||
assert.equal(changeWords("count", 100, 220), "up 120");
|
||||
assert.equal(changeWords("count", 100, 99), "down 1");
|
||||
assert.equal(changeWords("count", 100, 100.2), "unchanged");
|
||||
assert.equal(changeWords("percent", 40, 43.5), "up 3.5 points");
|
||||
assert.equal(changeWords("percent", 40, 39.98), "unchanged");
|
||||
assert.equal(changeWords("bytes", 5e9, 5.5e9), "up 500 MB");
|
||||
assert.equal(changeWords("bytes", 5e9, 5.00001e9), "unchanged", "a rounding-sized change in a big number is not a change");
|
||||
assert.equal(changeWords("ms", 10, 7), "down 3 ms");
|
||||
assert.equal(changeWords("count", null, 5), "");
|
||||
assert.equal(changeWords("count", 5, null), "");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* The history of integration figures: what is taken out of each kind of integration's stored reading, how a reading is kept (and not kept twice, or when stale),
|
||||
* the hours and days beyond five weeks, what is thrown away, and how it is asked for.
|
||||
*
|
||||
* Run with: npm test --workspace server
|
||||
*/
|
||||
import { test, describe, before, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { Server } from "node:http";
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-figures-"));
|
||||
process.env.DATABASE_PATH = join(dir, "test.sqlite");
|
||||
process.env.SESSION_DIR = join(dir, "sessions");
|
||||
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
|
||||
process.env.HEALTH_STARTUP_GRACE_MS = "0";
|
||||
|
||||
const express = (await import("express")).default;
|
||||
const { db } = await import("../src/db/client.js");
|
||||
const { users, integrations, adguardState, integrationFigures, integrationFigureSamples, integrationFigureRollup } = await import("../src/db/schema.js");
|
||||
const { runMigrations } = await import("../src/db/migrate.js");
|
||||
const F = await import("../src/services/integrationFigures.js");
|
||||
const { integrationsRouter } = await import("../src/routes/integrations.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const NOW = new Date("2026-11-10T12:00:00.000Z");
|
||||
const ago = (minutes: number) => new Date(NOW.getTime() - minutes * 60_000);
|
||||
const byKey = (list: { key: string; value: number }[]) => Object.fromEntries(list.map((f) => [f.key, f.value]));
|
||||
|
||||
describe("what is taken from each kind of integration", () => {
|
||||
test("AdGuard and Technitium: the day's queries and blocked share, and how fast the test question was answered — only when it was", () => {
|
||||
const stats = JSON.stringify({ queries: 12000, blocked: 3000, blockedPercent: 25, avgMs: 11.5 });
|
||||
assert.deepEqual(byKey(F.extractAdguard({ stats, probe: JSON.stringify({ ok: true, ms: 8 }) })), { queries: 12000, blocked: 3000, blockedPercent: 25, avgMs: 11.5, probeMs: 8 });
|
||||
assert.deepEqual(byKey(F.extractAdguard({ stats, probe: JSON.stringify({ ok: false, ms: null }) })), { queries: 12000, blocked: 3000, blockedPercent: 25, avgMs: 11.5 }, "an unanswered question has no time");
|
||||
assert.deepEqual(F.extractAdguard({ stats: null, probe: null }), [], "a reading that could not be taken is nothing, not zeros");
|
||||
assert.deepEqual(byKey(F.extractTechnitium({ stats: JSON.stringify({ queries: 5, blocked: 1, blockedPercent: null, clients: 3, cachedEntries: 40 }), probe: null })), { queries: 5, blocked: 1, clients: 3, cachedEntries: 40 });
|
||||
});
|
||||
|
||||
test("Sonarr/Radarr/Prowlarr and the download clients", () => {
|
||||
assert.deepEqual(byKey(F.extractArr({ queue: JSON.stringify({ total: 7, items: [] }), missing: 120 })), { queue: 7, missing: 120 });
|
||||
assert.deepEqual(byKey(F.extractArr({ queue: null, missing: null })), {}, "Prowlarr has neither");
|
||||
const counts = JSON.stringify({ downloading: 2, queued: 1, seeding: 10, completed: 5, failed: 1, paused: 0 });
|
||||
assert.deepEqual(byKey(F.extractDownload({ speedDown: 5_000_000, speedUp: 0, counts, total: 19 })), { speedDown: 5_000_000, speedUp: 0, downloading: 2, pending: 3, failed: 1 });
|
||||
assert.deepEqual(byKey(F.extractDownload({ speedDown: null, speedUp: null, counts: "not json", total: 0 })), {}, "nothing is made up");
|
||||
});
|
||||
|
||||
test("Paperless, Immich, Nextcloud, NetBox", () => {
|
||||
assert.deepEqual(byKey(F.extractPaperless({ documents: 900, inbox: 4, storageTotal: 1000, storageAvailable: 250 })), { documents: 900, inbox: 4, storagePercent: 75 });
|
||||
assert.deepEqual(byKey(F.extractPaperless({ documents: 900, inbox: null, storageTotal: null, storageAvailable: null })), { documents: 900 });
|
||||
const queues = JSON.stringify([{ name: "thumbs", active: 1, waiting: 10, failed: 2, paused: false }, { name: "video", active: 0, waiting: 5, failed: 1, paused: false }]);
|
||||
assert.deepEqual(byKey(F.extractImmich({ photos: 100, videos: 7, usageBytes: 5e9, diskPercent: 41.5, queues })), { photos: 100, videos: 7, usageBytes: 5e9, diskPercent: 41.5, waiting: 15, failedJobs: 3 });
|
||||
assert.deepEqual(byKey(F.extractNextcloud({ users: 3, files: 5000, active24h: 2, freeBytes: 1e11 })), { users: 3, active24h: 2, files: 5000, freeBytes: 1e11 });
|
||||
assert.deepEqual(byKey(F.extractNetbox({ devices: 12, virtualMachines: 30, ipAddresses: 200 })), { devices: 12, virtualMachines: 30, ipAddresses: 200 });
|
||||
});
|
||||
|
||||
test("n8n, Grafana, Prometheus, Authentik, Jellyfin", () => {
|
||||
const workflows = JSON.stringify([{ active: true, failed24h: 2 }, { active: true, failed24h: 0 }, { active: false, failed24h: 1 }]);
|
||||
assert.deepEqual(byKey(F.extractN8n({ workflows, executionsKnown: true })), { active: 2, failed24h: 3 });
|
||||
assert.deepEqual(byKey(F.extractN8n({ workflows, executionsKnown: false })), { active: 2 }, "failed runs are not known when the runs could not be asked");
|
||||
assert.deepEqual(byKey(F.extractGrafana({ rules: JSON.stringify({ total: 20, items: [{ state: "firing", health: "ok" }, { state: "inactive", health: "error" }, { state: "pending", health: "ok" }] }) })), { rules: 20, firing: 1, broken: 1 });
|
||||
assert.deepEqual(F.extractGrafana({ rules: null }), []);
|
||||
assert.deepEqual(byKey(F.extractPrometheus({ jobs: JSON.stringify([{ job: "a", up: 5, down: 1 }, { job: "b", up: 3, down: 0 }]), down: "[]", alerts: JSON.stringify([{ state: "firing" }, { state: "pending" }]) })), { targetsUp: 8, targetsDown: 1, firing: 1 });
|
||||
assert.deepEqual(byKey(F.extractAuthentik({ failedLogins24h: 4, configErrors24h: 0 })), { failedLogins24h: 4, configErrors24h: 0 });
|
||||
assert.deepEqual(byKey(F.extractJellyfin({ movies: 300, series: 40, episodes: 900, songs: null, failedLogins24h: 1 })), { movies: 300, series: 40, episodes: 900, failedLogins24h: 1 });
|
||||
});
|
||||
|
||||
test("Hetzner: its servers and the monthly estimate in the currency it was worked out in", () => {
|
||||
const list = F.extractHetzner({ servers: JSON.stringify([{}, {}, {}]), cost: JSON.stringify({ currency: "eur", total: 42.5 }) });
|
||||
assert.deepEqual(byKey(list), { servers: 3, cost: 42.5 });
|
||||
assert.equal(list.find((f) => f.key === "cost")!.unit, "money:EUR");
|
||||
assert.deepEqual(byKey(F.extractHetzner({ servers: "[]", cost: null })), { servers: 0 }, "no price list, no cost");
|
||||
});
|
||||
|
||||
test("which kinds have figures", () => {
|
||||
for (const t of ["adguard", "technitium", "sonarr", "radarr", "prowlarr", "qbittorrent", "sabnzbd", "paperless", "immich", "nextcloud", "netbox", "n8n", "grafana", "prometheus", "authentik", "jellyfin", "hetzner"] as const) assert.equal(F.hasFigures(t), true, t);
|
||||
for (const t of ["proxmox", "synology", "tailscale", "ups", "homeassistant", "speedtest"] as const) assert.equal(F.hasFigures(t), false, `${t} has history of its own, or nothing stored`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("keeping the figures", () => {
|
||||
let adguard = 0;
|
||||
before(async () => {
|
||||
[{ id: adguard }] = await db.insert(integrations).values({ type: "adguard", name: "AdGuard", baseUrl: "http://x" }).returning({ id: integrations.id });
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await db.delete(integrationFigures);
|
||||
await db.delete(adguardState);
|
||||
await db.update(integrations).set({ enabled: true }).where(eq(integrations.id, adguard));
|
||||
});
|
||||
after(() => {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Windows keeps the database file locked until the process exits.
|
||||
}
|
||||
});
|
||||
|
||||
const figs = (queries: number): import("../src/services/integrationFigures.js").Figure[] => [{ key: "queries", label: "DNS queries", unit: "count", value: queries }];
|
||||
const samples = async () => db.select().from(integrationFigureSamples).orderBy(integrationFigureSamples.at);
|
||||
const state = (readAt: Date, queries = 100) => db.insert(adguardState).values({ integrationId: adguard, stats: JSON.stringify({ queries, blocked: 10, blockedPercent: 10, avgMs: 5 }), probe: null, updatedAt: readAt.toISOString() });
|
||||
|
||||
test("a reading is a sample, an hour and a day; the figure remembers its latest, label and unit", async () => {
|
||||
assert.equal(await F.recordFigures(adguard, figs(100), ago(60)), 1);
|
||||
assert.equal(await F.recordFigures(adguard, [{ key: "queries", label: "Queries a day", unit: "count", value: 150 }], ago(30)), 1);
|
||||
const [fig] = await db.select().from(integrationFigures);
|
||||
assert.deepEqual([fig.label, fig.unit, fig.latest, fig.updatedAt], ["Queries a day", "count", 150, ago(30).toISOString()]);
|
||||
assert.equal((await samples()).length, 2);
|
||||
const rollup = await db.select().from(integrationFigureRollup).orderBy(integrationFigureRollup.grain);
|
||||
assert.deepEqual(rollup.map((r) => r.grain), ["d", "h", "h"].slice(0, rollup.length), "a day, and an hour each (the two readings are in different hours)");
|
||||
const day = rollup.find((r) => r.grain === "d")!;
|
||||
assert.deepEqual([day.n, day.sum, day.min, day.max], [2, 250, 100, 150]);
|
||||
});
|
||||
|
||||
test("two readings close together are one, and the same reading seen twice is not kept twice", async () => {
|
||||
assert.equal(await F.recordFigures(adguard, figs(100), ago(30)), 1);
|
||||
assert.equal(await F.recordFigures(adguard, figs(101), ago(30)), 0, "the same moment");
|
||||
assert.equal(await F.recordFigures(adguard, figs(102), ago(25)), 0, "five minutes later");
|
||||
assert.equal(await F.recordFigures(adguard, figs(103), ago(10)), 1, "twenty minutes later");
|
||||
assert.equal((await samples()).length, 2);
|
||||
assert.equal((await db.select().from(integrationFigures))[0].latest, 103, "the latest is the last reading given");
|
||||
});
|
||||
|
||||
test("a pass keeps the figures of a reading that is recent (one for each, and none for a test question that was not asked)", async () => {
|
||||
await state(ago(3));
|
||||
assert.equal(await F.recordAllFigures(NOW), 4);
|
||||
assert.equal(await F.recordAllFigures(NOW), 0, "the same reading again is not kept twice");
|
||||
const keys = (await db.select().from(integrationFigures)).map((f) => f.key).sort();
|
||||
assert.deepEqual(keys, ["avgMs", "blocked", "blockedPercent", "queries"]);
|
||||
});
|
||||
|
||||
test("a stale reading is not recorded — a gap is a gap", async () => {
|
||||
await state(ago(F.STALE_MS / 60_000 + 5));
|
||||
assert.equal(await F.recordAllFigures(NOW), 0);
|
||||
assert.equal((await db.select().from(integrationFigures)).length, 0);
|
||||
});
|
||||
|
||||
test("a switched-off integration and a kind with no figures are left alone", async () => {
|
||||
await state(ago(3));
|
||||
await db.update(integrations).set({ enabled: false }).where(eq(integrations.id, adguard));
|
||||
assert.equal(await F.recordAllFigures(NOW), 0);
|
||||
await db.update(integrations).set({ enabled: true }).where(eq(integrations.id, adguard));
|
||||
const [pve] = await db.insert(integrations).values({ type: "proxmox", name: "pve", baseUrl: "http://p" }).returning({ id: integrations.id });
|
||||
await F.recordAllFigures(NOW);
|
||||
assert.equal((await db.select().from(integrationFigures).where(eq(integrationFigures.integrationId, pve.id))).length, 0);
|
||||
await db.delete(integrations).where(eq(integrations.id, pve.id));
|
||||
});
|
||||
|
||||
test("an integration that has not been read yet has nothing to keep", async () => {
|
||||
assert.equal(await F.recordAllFigures(NOW), 0);
|
||||
});
|
||||
|
||||
test("the figures go when the integration does", async () => {
|
||||
const [other] = await db.insert(integrations).values({ type: "adguard", name: "Other", baseUrl: "http://y" }).returning({ id: integrations.id });
|
||||
await F.recordFigures(other.id, figs(1), ago(10));
|
||||
assert.equal((await db.select().from(integrationFigureSamples)).length, 1);
|
||||
await db.delete(integrations).where(eq(integrations.id, other.id));
|
||||
assert.equal((await db.select().from(integrationFigures).where(eq(integrationFigures.integrationId, other.id))).length, 0);
|
||||
assert.equal((await db.select().from(integrationFigureSamples)).length, 0, "its readings and rollups with it");
|
||||
assert.equal((await db.select().from(integrationFigureRollup)).length, 0);
|
||||
});
|
||||
|
||||
test("what is older than it is kept is thrown away: readings after five weeks, hours after four months, days after about two years", async () => {
|
||||
const day = (n: number) => new Date(NOW.getTime() - n * 86_400_000);
|
||||
for (const n of [1, 30, 40, 100, 150, 500, 900]) await F.recordFigures(adguard, figs(n), day(n));
|
||||
await F.pruneFigures(NOW);
|
||||
assert.deepEqual((await samples()).map((s) => s.value).sort((a, b) => a - b), [1, 30], "five weeks of readings");
|
||||
const rollup = await db.select().from(integrationFigureRollup);
|
||||
assert.deepEqual(rollup.filter((r) => r.grain === "h").map((r) => r.sum).sort((a, b) => a - b), [1, 30, 40, 100], "four months of hours");
|
||||
assert.deepEqual(rollup.filter((r) => r.grain === "d").map((r) => r.sum).sort((a, b) => a - b), [1, 30, 40, 100, 150, 500], "two years and a bit of days");
|
||||
});
|
||||
});
|
||||
|
||||
describe("asking for a stretch", () => {
|
||||
let id = 0;
|
||||
before(async () => {
|
||||
[{ id }] = await db.insert(integrations).values({ type: "immich", name: "Photos", baseUrl: "http://i" }).returning({ id: integrations.id });
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await db.delete(integrationFigures).where(eq(integrationFigures.integrationId, id));
|
||||
});
|
||||
const day = (n: number) => new Date(NOW.getTime() - n * 86_400_000);
|
||||
|
||||
test("up to five weeks it is the readings; the lowest, highest and first are those of the stretch, the latest is the latest of all", async () => {
|
||||
for (let d = 20; d >= 0; d--) await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: 1000 + (20 - d) * 10 }], day(d));
|
||||
const week = await F.getIntegrationFigures(id, 7, NOW);
|
||||
assert.equal(week.length, 1);
|
||||
const s = week[0];
|
||||
assert.deepEqual([s.key, s.label, s.unit, s.latest], ["photos", "Photos", "count", 1200]);
|
||||
assert.deepEqual([s.first, s.low, s.high], [1130, 1130, 1200]);
|
||||
assert.ok(s.points.length >= 7 && s.points.length <= 96);
|
||||
assert.ok(s.points.every((p) => Date.parse(p.t) >= day(7).getTime()), "only the last week");
|
||||
const month = (await F.getIntegrationFigures(id, 30, NOW))[0];
|
||||
assert.equal(month.first, 1000);
|
||||
});
|
||||
|
||||
test("beyond five weeks it is the hours, and beyond four months the days, each point with its lowest and highest", async () => {
|
||||
for (const n of [200, 150, 100, 80, 60]) {
|
||||
await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: n }], new Date(day(n).getTime() - 3600_000));
|
||||
await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: n + 10 }], day(n));
|
||||
}
|
||||
const sixty = (await F.getIntegrationFigures(id, 90, NOW))[0];
|
||||
assert.ok(sixty.points.every((p) => p.lo !== undefined && p.hi !== undefined), "from the hours");
|
||||
assert.ok(sixty.points.length >= 2, "the 80 and 60 day ones (the 100 day one is outside ninety days)");
|
||||
const year = (await F.getIntegrationFigures(id, 365, NOW))[0];
|
||||
assert.ok(year.points.length >= 5, "the days reach back further");
|
||||
assert.equal(year.low, 60);
|
||||
});
|
||||
|
||||
test("a figure with nothing in the stretch is left out; the stretch is limited to a year", async () => {
|
||||
await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: 5 }], day(200));
|
||||
assert.deepEqual(await F.getIntegrationFigures(id, 7, NOW), []);
|
||||
assert.equal((await F.getIntegrationFigures(id, 5000, NOW)).length, 1, "asked for more than a year it is a year, and the 200 day one is in it");
|
||||
await F.recordFigures(id, [{ key: "photos", label: "Photos", unit: "count", value: 6 }], day(400));
|
||||
assert.equal((await F.getIntegrationFigures(id, 5000, NOW))[0].points.length, 1, "but not the 400 day one");
|
||||
assert.equal((await F.getIntegrationFigures(id, 0, NOW)).length, 0, "at least a day: the last day has nothing");
|
||||
});
|
||||
});
|
||||
|
||||
describe("the route", () => {
|
||||
let app: Server;
|
||||
let base = "";
|
||||
let id = 0;
|
||||
let proxmox = 0;
|
||||
before(async () => {
|
||||
await db.insert(users).values({ oidcSub: "viewer", name: "Vera", role: "viewer" });
|
||||
[{ id }] = await db.insert(integrations).values({ type: "adguard", name: "AdGuard route", baseUrl: "http://r" }).returning({ id: integrations.id });
|
||||
[{ id: proxmox }] = await db.insert(integrations).values({ type: "proxmox", name: "pve route", baseUrl: "http://pp" }).returning({ id: integrations.id });
|
||||
await F.recordFigures(id, [{ key: "queries", label: "DNS queries", unit: "count", value: 321 }], new Date(Date.now() - 3600_000));
|
||||
const a = express();
|
||||
a.use(express.json());
|
||||
a.use((req, _res, next) => {
|
||||
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
|
||||
next();
|
||||
});
|
||||
a.use("/api/integrations", integrationsRouter);
|
||||
app = await new Promise<Server>((resolve) => {
|
||||
const x = a.listen(0, "127.0.0.1", () => resolve(x));
|
||||
});
|
||||
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/integrations`;
|
||||
});
|
||||
after(() => app.close());
|
||||
|
||||
test("anyone signed in can ask; the days are limited; a kind with no figures says so; a missing integration is a 404", async () => {
|
||||
const res = await fetch(`${base}/${id}/figures?days=7`);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.equal(body.kept, true);
|
||||
assert.equal(body.days, 7);
|
||||
assert.equal(body.figures.length, 1);
|
||||
assert.deepEqual([body.figures[0].key, body.figures[0].latest, body.figures[0].unit], ["queries", 321, "count"]);
|
||||
|
||||
assert.equal((await (await fetch(`${base}/${id}/figures?days=99999`)).json()).days, 365);
|
||||
assert.equal((await (await fetch(`${base}/${id}/figures?days=abc`)).json()).days, 7);
|
||||
const none = await (await fetch(`${base}/${proxmox}/figures`)).json();
|
||||
assert.deepEqual([none.kept, none.figures], [false, []]);
|
||||
assert.equal((await fetch(`${base}/9999/figures`)).status, 404);
|
||||
assert.equal((await fetch(`${base}/abc/figures`)).status, 400);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user