Speedtest Tracker, Costs, "Having trouble" on status pages, grouped Integrations and Dashboard

- Speedtest Tracker integration: charts, alerts for a slow line (median of the
  last three tests under a share of what is paid for) and for failing tests.
- Costs page (Records): things paid for by hand plus Hetzner's estimate, totals
  per currency, bills rolled forward into Coming up as Renewals.
- Status pages: a "Having trouble" state for things that answer but have an open
  alert; Speedtest reads as the internet.
- Integrations page and kind picker grouped like the sidebar (one shared types
  file); Notifications "What to send" shown one group at a time; Dashboard
  integration sections follow the menu groups, with tiles for integrations that
  have no widget.
- Docs, tests, and ROLES.md menu table brought up to date.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-26 23:23:46 +01:00
1 parent 78822346b1
commit f86661f6c9
49 files changed
+18156 -238

No files matched your search

@@ -0,0 +1,14 @@
CREATE TABLE `speedtest_results` (
`integration_id` integer NOT NULL,
`result_id` integer NOT NULL,
`at` text NOT NULL,
`status` text NOT NULL,
`download_bps` real,
`upload_bps` real,
`ping_ms` real,
`healthy` integer,
PRIMARY KEY(`integration_id`, `result_id`),
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `speedtest_results_at_idx` ON `speedtest_results` (`integration_id`,`at`);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE `cost_items` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text NOT NULL,
`amount` real NOT NULL,
`currency` text NOT NULL,
`period` text NOT NULL,
`category` text DEFAULT 'Other' NOT NULL,
`renews_on` text,
`note` text,
`created_at` text NOT NULL,
`updated_at` text NOT NULL
);
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+14
View File
@@ -484,6 +484,20 @@
"when": 1792964323718,
"tag": "0068_glorious_steel_serpent",
"breakpoints": true
},
{
"idx": 69,
"version": "6",
"when": 1793051897261,
"tag": "0069_overconfident_cloak",
"breakpoints": true
},
{
"idx": 70,
"version": "6",
"when": 1793052224023,
"tag": "0070_absurd_toad",
"breakpoints": true
}
]
}
+4
View File
@@ -67,6 +67,8 @@ import { paperlessRouter } from "./routes/paperless.js";
import { n8nRouter } from "./routes/n8n.js";
import { netboxRouter } from "./routes/netbox.js";
import { hetznerRouter } from "./routes/hetzner.js";
import { speedtestRouter } from "./routes/speedtest.js";
import { costsRouter } from "./routes/costs.js";
import { nextcloudRouter } from "./routes/nextcloud.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
import { publicApiRouter } from "./routes/publicApi.js";
@@ -211,6 +213,8 @@ export function createApp() {
app.use("/api/n8n", n8nRouter);
app.use("/api/netbox", netboxRouter);
app.use("/api/hetzner", hetznerRouter);
app.use("/api/speedtest", speedtestRouter);
app.use("/api/costs", costsRouter);
app.use("/api/nextcloud", nextcloudRouter);
app.use("/api/service-levels", serviceLevelsRouter);
app.use("/api/todos", todosRouter);
+39
View File
@@ -562,6 +562,7 @@ export const integrationTypes = [
"netbox",
"n8n",
"hetzner",
"speedtest",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1219,6 +1220,44 @@ export const nextcloudState = sqliteTable("nextcloud_state", {
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Costs ──────────────────────────────────────────────────────────────────
// Things paid for by hand (a domain, a VPN, a licence, a subscription), one row each: what it costs per month or per year, in which currency, and optionally the day it is billed on —
// an anchor from which the next renewal is worked out, not a date to keep up to date. What can be read (a Hetzner project's estimate) is not here. Read by services/costs.ts.
export const costItems = sqliteTable("cost_items", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
amount: real("amount").notNull(), // per period, including VAT
currency: text("currency").notNull(), // three capital letters: EUR, SEK …
period: text("period").notNull(), // month | year
category: text("category").notNull().default("Other"),
renewsOn: text("renews_on"), // YYYY-MM-DD: a day it is (or will first be) billed; null: not said
note: text("note"),
createdAt: text("created_at").notNull(), // ISO
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Speedtest Tracker ──────────────────────────────────────────────────────
// The speed tests a Speedtest Tracker (an integration of type "speedtest") has made, one row for each, kept for 90 days: when, how fast down and up (bits per second) and the ping (ms).
// Stored once under Speedtest Tracker's own id and never changed. A failed or skipped test has no speeds. Read by services/speedtestSync.ts.
export const speedtestResults = sqliteTable(
"speedtest_results",
{
integrationId: integer("integration_id")
.notNull()
.references(() => integrations.id, { onDelete: "cascade" }),
resultId: integer("result_id").notNull(), // Speedtest Tracker's own id
at: text("at").notNull(), // ISO: when the test was made
status: text("status").notNull(), // completed, failed, skipped …
downloadBps: real("download_bps"),
uploadBps: real("upload_bps"),
pingMs: real("ping_ms"),
healthy: integer("healthy", { mode: "boolean" }), // Speedtest Tracker's own judgement by its thresholds; null: it did not say
},
(t) => [primaryKey({ columns: [t.integrationId, t.resultId] }), index("speedtest_results_at_idx").on(t.integrationId, t.at)],
);
// ─── Hetzner Cloud ──────────────────────────────────────────────────────────
// How a Hetzner Cloud project (an integration of type "hetzner"; a token belongs to one project) is right now, one row each, replaced at every read: its servers, volumes, snapshots and
+2
View File
@@ -39,6 +39,7 @@ import { initNetBoxSync } from "./services/netboxSync.js";
import { initNextcloudSync } from "./services/nextcloudSync.js";
import { initDockerVulnSync } from "./services/dockerVulnSync.js";
import { initHetznerSync } from "./services/hetznerSync.js";
import { initSpeedtestSync } from "./services/speedtestSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -99,6 +100,7 @@ initNetBoxSync();
initNextcloudSync();
initDockerVulnSync();
initHetznerSync();
initSpeedtestSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+8
View File
@@ -105,6 +105,14 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "appPassword", label: "App password (optional)", secret: true, optional: true, type: "password", placeholder: "Personal settings → Security → Devices & sessions" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
speedtest: [
{ key: "url", label: "Speedtest Tracker URL", secret: false, placeholder: "http://speedtest.example.lan" },
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "profile menu → API tokens, with the results:read ability" },
{ key: "expectedDownload", label: "Download you pay for, Mbit/s (optional)", secret: false, optional: true, placeholder: "300 — alerts only judge what you give" },
{ key: "expectedUpload", label: "Upload you pay for, Mbit/s (optional)", secret: false, optional: true, placeholder: "30" },
{ key: "minPercent", label: "Warn below this share of it, % (optional)", secret: false, optional: true, placeholder: "60" },
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
hetzner: [{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Hetzner Console → the project → Security → API tokens; Read is enough" }],
netbox: [
{ key: "url", label: "NetBox URL", secret: false, placeholder: "https://netbox.example.lan" },
+3
View File
@@ -22,6 +22,7 @@ import { createPaperlessAdapter } from "./paperless/adapter.js";
import { createN8nAdapter } from "./n8n/adapter.js";
import { createNetBoxAdapter } from "./netbox/adapter.js";
import { createHetznerAdapter } from "./hetzner/adapter.js";
import { createSpeedtestAdapter } from "./speedtest/adapter.js";
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
export interface PingableAdapter {
@@ -77,6 +78,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createPaperlessAdapter(config as any);
case "n8n":
return createN8nAdapter(config as any);
case "speedtest":
return createSpeedtestAdapter(config as any);
case "hetzner":
return createHetznerAdapter(config as any);
case "netbox":
@@ -0,0 +1,140 @@
/**
* Speedtest Tracker adapter (https://github.com/alexjustesen/speedtest-tracker). Config: url, token (an API token made in Speedtest Tracker under the profile menu → API tokens, with the
* **results:read** ability — that is all this needs); optional: expectedDownload, expectedUpload (Mbit/s — what the line is sold at) and minPercent, which the alerts use (see services/speedtestSync.ts),
* and insecure.
*
* Reading only: the newest results (`GET /api/v1/results`, newest first, a page of 30 at a time) and, to see that it is there, `GET /api/healthcheck`, which needs no login. Nothing is sent to it
* but GET requests — in particular no speed test is started (that is `POST /api/v1/speedtests/run`, which is never called).
*
* Endpoints and field names are from Speedtest Tracker's own source (routes/api/v1/routes.php, ResultsController, ResultResource): a result has `download` and `upload` in **bytes** per second,
* `download_bits` and `upload_bits` as the same in bits, `ping` in ms, `healthy` (true, false, or null when it has no thresholds to judge by), a `status` (completed, failed, skipped …) and
* `created_at`. They have not been checked against a live Speedtest Tracker here; every field is read defensively.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export interface SpeedtestConfig {
url: string;
token: string;
insecure?: boolean;
}
export interface SpeedtestResult {
/** Speedtest Tracker's own id for the result. */
id: number;
/** completed, failed, skipped, running … (lower case). */
status: string;
/** ISO time the test was made. */
at: string;
/** Bits per second; null for a result that has none (a failed or skipped test). */
download: number | null;
upload: number | null;
/** Milliseconds. */
ping: number | null;
/** Whether Speedtest Tracker judged it fine by its own thresholds; null when it does not say. */
healthy: boolean | null;
}
export interface SpeedtestRead {
results: SpeedtestResult[];
}
export interface SpeedtestAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
/** The newest results, newest first: `pages` pages of 30. */
read(pages?: number): Promise<SpeedtestRead>;
}
export const PAGE_SIZE = 30;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
/** A number, whether it came as one or as a numeric string (a Laravel decimal column is sent as a string). */
const num = (v: unknown): number | null => {
const n = typeof v === "string" && v.trim() !== "" ? Number(v) : v;
return typeof n === "number" && Number.isFinite(n) ? n : null;
};
/** Pure: when a result was made — the time Ookla gives in the result's own data when there is one, else `created_at` (which has no zone; Speedtest Tracker's own is UTC unless it was told otherwise). */
function timeOf(r: Record<string, unknown>): string | null {
const ookla = str(obj(r.data).timestamp);
if (ookla && Number.isFinite(Date.parse(ookla))) return new Date(ookla).toISOString();
const created = str(r.created_at);
if (!created) return null;
const parsed = Date.parse(/[zZ]|[+-]\d\d:?\d\d$/.test(created) ? created : created.replace(" ", "T") + "Z");
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
}
/** Pure: one result as Speedtest Tracker sends it; null when it has no id or time. */
export function parseResult(raw: unknown): SpeedtestResult | null {
const r = obj(raw);
const id = num(r.id);
const at = timeOf(r);
if (id === null || at === null) return null;
const bits = (bitsField: unknown, bytesField: unknown): number | null => {
const b = num(bitsField);
if (b !== null) return b;
const bytes = num(bytesField);
return bytes === null ? null : bytes * 8;
};
return {
id,
status: (str(r.status) ?? "unknown").toLowerCase(),
at,
download: bits(r.download_bits, r.download),
upload: bits(r.upload_bits, r.upload),
ping: num(r.ping),
healthy: typeof r.healthy === "boolean" ? r.healthy : null,
};
}
/** Pure: the results of one page of /api/v1/results (`data` is the list), newest first as sent. */
export function parseResults(raw: unknown): SpeedtestResult[] {
const list = obj(raw).data;
return (Array.isArray(list) ? list : []).map(parseResult).filter((r): r is SpeedtestResult => r !== null);
}
export function createSpeedtestAdapter(config: SpeedtestConfig): SpeedtestAdapter {
const base = String(config.url ?? "").replace(/\/+$/, "");
const insecure = config.insecure === true;
const headers = { Authorization: `Bearer ${config.token}`, Accept: "application/json" };
async function get(path: string, auth = true): Promise<{ status: number; json: unknown }> {
const res = await requestJson(`${base}${path}`, { headers: auth ? headers : { Accept: "application/json" }, insecure });
if (auth && res.status === 401) throw new Error("Speedtest Tracker did not accept the token (HTTP 401) — make a new one under the profile menu → API tokens, and check it has not been deleted");
if (auth && res.status === 403) throw new Error("Speedtest Tracker says this token may not read results (HTTP 403) — give it the results:read ability");
if (res.status === 404 && path.startsWith("/api/healthcheck")) throw new Error("Speedtest Tracker has no API at this address — check the address (and that it is Speedtest Tracker's own, not a proxy's)");
return { status: res.status, json: res.json };
}
async function read(pages = 1): Promise<SpeedtestRead> {
const results: SpeedtestResult[] = [];
for (let page = 1; page <= pages; page++) {
let res = await get(`/api/v1/results?sort=-created_at&page[size]=${PAGE_SIZE}&page[number]=${page}`);
// A Speedtest Tracker that allows fewer per page says so with a 422; ask without a size then.
if (res.status === 422) res = await get(`/api/v1/results?sort=-created_at&page[number]=${page}`);
if (res.status < 200 || res.status >= 300) throw new Error(`Speedtest Tracker error: HTTP ${res.status}`);
if (res.json === null) throw new Error("Speedtest Tracker answered with something that isn't JSON — is the address its own?");
const got = parseResults(res.json);
results.push(...got);
if (got.length < PAGE_SIZE) break;
}
return { results };
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
const health = await get("/api/healthcheck", false);
if (health.status < 200 || health.status >= 300) throw new Error(`Speedtest Tracker error: HTTP ${health.status} — check the address`);
// The token is only tried on something that needs it (a page of one result).
const res = await get(`/api/v1/results?page[size]=1`);
if (res.status < 200 || res.status >= 300) throw new Error(`Speedtest Tracker error: HTTP ${res.status}`);
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("speedtest", { ping, read });
}
+84
View File
@@ -0,0 +1,84 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { z } from "zod";
import { db } from "../db/client.js";
import { costItems } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { recordAudit } from "../services/audit.js";
import { automaticCosts, costTotals, DEFAULT_CATEGORIES, isYmd, listCostItems, monthlyOf, nextRenewal, PERIODS, type CostRow } from "../services/costs.js";
import { localDate } from "../services/agenda.js";
import { getSettings } from "../services/settingsStore.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const costsRouter = Router();
costsRouter.use(requireAuth);
const ymd = z
.string()
.regex(/^\d{4}-\d{2}-\d{2}$/)
.refine((v) => isYmd(v), "not a date");
const body = z.object({
name: z.string().trim().min(1).max(100),
amount: z.number().positive().max(10_000_000),
currency: z.string().trim().regex(/^[A-Za-z]{3}$/).transform((c) => c.toUpperCase()),
period: z.enum(PERIODS),
category: z.string().trim().min(1).max(40).default("Other"),
renewsOn: ymd.nullable().optional(),
note: z.string().trim().max(300).nullable().optional(),
});
const shape = (r: CostRow, today: string) => ({
id: r.id,
name: r.name,
amount: r.amount,
currency: r.currency,
period: r.period,
category: r.category,
renewsOn: r.renewsOn,
/** The next day it is billed, worked out from the day given. */
nextRenewal: nextRenewal(r.renewsOn, r.period, today),
monthly: Math.round(monthlyOf(r.amount, r.period) * 100) / 100,
note: r.note,
});
/** What it costs: the items typed in, the estimates that can be read, and the totals per currency. Everyone signed in can see it. */
costsRouter.get("/", asyncHandler(async (_req, res) => {
const [items, automatic, settings] = await Promise.all([listCostItems(), automaticCosts(), getSettings()]);
const today = localDate(new Date(), settings.notifications.timezone || "UTC");
res.json({
items: items.map((r) => shape(r, today)),
automatic,
totals: costTotals(items, automatic),
categories: [...new Set([...DEFAULT_CATEGORIES, ...items.map((i) => i.category)])],
});
}));
costsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res) => {
const parsed = body.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name, an amount above zero, a three-letter currency (EUR, SEK …) and say whether it is per month or per year." });
const now = new Date().toISOString();
const [created] = await db.insert(costItems).values({ ...parsed.data, renewsOn: parsed.data.renewsOn ?? null, note: parsed.data.note ?? null, createdAt: now, updatedAt: now }).returning();
await recordAudit({ actor: req.currentUser!, category: "cost", action: "create", targetType: "cost", targetId: created.id, detail: { name: created.name, amount: created.amount, currency: created.currency, period: created.period } });
res.status(201).json({ id: created.id });
}));
costsRouter.put("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const parsed = body.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: "invalid_body", message: "Give it a name, an amount above zero, a three-letter currency (EUR, SEK …) and say whether it is per month or per year." });
const [updated] = await db.update(costItems).set({ ...parsed.data, renewsOn: parsed.data.renewsOn ?? null, note: parsed.data.note ?? null, updatedAt: new Date().toISOString() }).where(eq(costItems.id, id)).returning();
if (!updated) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "cost", action: "update", targetType: "cost", targetId: id, detail: { name: updated.name, amount: updated.amount, currency: updated.currency, period: updated.period } });
res.json({ id });
}));
costsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [removed] = await db.delete(costItems).where(eq(costItems.id, id)).returning();
if (!removed) return res.status(404).json({ error: "not_found" });
await recordAudit({ actor: req.currentUser!, category: "cost", action: "delete", targetType: "cost", targetId: id, detail: { name: removed.name } });
res.status(204).end();
}));
+4
View File
@@ -37,6 +37,7 @@ import { forgetPaperless, syncPaperless } from "../services/paperlessSync.js";
import { forgetN8n, syncN8n } from "../services/n8nSync.js";
import { forgetNetBox, syncNetBox } from "../services/netboxSync.js";
import { forgetHetzner, syncHetzner } from "../services/hetznerSync.js";
import { forgetSpeedtest, syncSpeedtest } from "../services/speedtestSync.js";
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
@@ -149,6 +150,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "n8n") void syncN8n(created.id).then(() => invalidateAlertsCache());
if (type === "netbox") void syncNetBox(created.id).then(() => invalidateAlertsCache());
if (type === "hetzner") void syncHetzner(created.id).then(() => invalidateAlertsCache());
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
if (type === "dockhand") void syncDockerVulns(created.id).then(() => invalidateAlertsCache());
@@ -262,6 +264,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "n8n" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncN8n(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "netbox" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNetBox(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "hetzner" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHetzner(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "dockhand" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDockerVulns(updated.id).then(() => invalidateAlertsCache());
@@ -351,6 +354,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "n8n") await forgetN8n(id);
if (existing.type === "netbox") await forgetNetBox(id);
if (existing.type === "hetzner") await forgetHetzner(id);
if (existing.type === "speedtest") await forgetSpeedtest(id);
if (existing.type === "nextcloud") await forgetNextcloud(id);
if (existing.type === "dockhand") await forgetDockerVulns(id);
await db.delete(integrations).where(eq(integrations.id, id));
+71
View File
@@ -0,0 +1,71 @@
import { Router } from "express";
import { and, asc, eq, gte } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, speedtestResults } from "../db/schema.js";
import { requireAuth } from "../auth/middleware.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { expectedOf, isSpeedtestFresh, median, recentResults, speedtestProblems, speedtestSyncStatuses, WEIGH } from "../services/speedtestSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const speedtestRouter = Router();
speedtestRouter.use(requireAuth);
/** Every Speedtest Tracker, how it is, from what the last reads stored: the newest test, the middle of the last few, what the line is sold at and what is wrong. Everyone signed in can see it. */
speedtestRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "speedtest")).orderBy(integrations.name),
speedtestSyncStatuses(),
]);
const now = Date.now();
const speedtests = [];
for (const r of rows) {
const status = statuses[r.id] ?? null;
const fresh = isSpeedtestFresh(status ?? undefined, now);
const results = await recentResults(r.id, 60);
let expected = expectedOf(null);
try {
expected = expectedOf((await loadIntegrationConfig(r.id))?.config);
} catch {
// a config that cannot be read costs only the numbers it holds
}
const completed = results.filter((x) => x.status === "completed");
const latest = completed[0] ?? null;
const lastFew = completed.slice(0, WEIGH);
const middle = (pick: (x: (typeof lastFew)[number]) => number | null) => median(lastFew.map(pick).filter((v): v is number => v !== null));
speedtests.push({
id: r.id,
name: r.name,
enabled: r.enabled,
status,
fresh,
expected,
state:
results.length === 0
? null
: {
latest: latest && { at: latest.at, download: latest.downloadBps, upload: latest.uploadBps, ping: latest.pingMs, healthy: latest.healthy },
/** The middle of the newest few completed tests — what the alerts judge. */
middle: { count: lastFew.length, download: middle((x) => x.downloadBps), upload: middle((x) => x.uploadBps), ping: middle((x) => x.pingMs) },
lastTest: results[0] && { at: results[0].at, status: results[0].status },
stored: results.length,
},
problems: results.length > 0 && fresh ? speedtestProblems(r.id, r.name, results, expected) : [],
});
}
res.json({ speedtests });
}));
/** The tests of one Speedtest Tracker over the last `days` (1–90), oldest first, for the charts: Mbit/s and milliseconds. */
speedtestRouter.get("/:id/history", asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.id, id), eq(integrations.type, "speedtest"))).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
const days = Math.min(90, Math.max(1, Math.floor(Number(req.query.days)) || 7));
const since = new Date(Date.now() - days * 86_400_000).toISOString();
const rows = await db.select().from(speedtestResults).where(and(eq(speedtestResults.integrationId, id), eq(speedtestResults.status, "completed"), gte(speedtestResults.at, since))).orderBy(asc(speedtestResults.at));
res.json({
days,
points: rows.map((r) => ({ t: r.at, download: r.downloadBps === null ? null : r.downloadBps / 1e6, upload: r.uploadBps === null ? null : r.uploadBps / 1e6, ping: r.pingMs })),
});
}));
+16 -1
View File
@@ -19,8 +19,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
import { latestTests } from "./restoreTests.js";
import { getSettings } from "./settingsStore.js";
import { env } from "../env.js";
import { listCostItems, nextRenewal } from "./costs.js";
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates", "todo"] as const;
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates", "todo", "renewal"] as const;
export type AgendaKind = (typeof AGENDA_KINDS)[number];
export const KIND_LABELS: Record<AgendaKind, string> = {
@@ -34,6 +35,7 @@ export const KIND_LABELS: Record<AgendaKind, string> = {
restore: "Restore tests",
updates: "Updates waiting",
todo: "To do",
renewal: "Renewals",
};
export interface AgendaItem {
@@ -100,6 +102,8 @@ export interface AgendaData {
updates: { name: string; count: number; security: number | null; since: string | null }[];
/** Open jobs that have a day they are due. */
todos: { id: number; title: string; dueDate: string | null; assigneeName: string | null; serverId: number | null }[];
/** Things paid for that have a day they are billed on (Costs): the next day it falls, worked out from the day given. */
renewals?: { id: number; name: string; amount: number; currency: string; period: string; nextRenewal: string }[];
notes: string[];
}
@@ -180,6 +184,10 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
restoreLimitDays: settings.healthChecks.restoreTestDays,
updates,
renewals: (await listCostItems()).flatMap((c) => {
const next = nextRenewal(c.renewsOn, c.period, localDate(now, settings.notifications.timezone || "UTC"));
return next ? [{ id: c.id, name: c.name, amount: c.amount, currency: c.currency, period: c.period, nextRenewal: next }] : [];
}),
todos: todoRows.map((t) => ({ id: t.id, title: t.title, dueDate: t.dueDate, assigneeName: t.assigneeName, serverId: t.targetType === "server" ? t.targetId : null })),
notes,
};
@@ -225,6 +233,12 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
day(`todo:${t.id}`, "todo", t.dueDate, (past) => ({ title: `${past ? "Overdue" : "To do"}: ${t.title}`, detail: t.assigneeName ? `for ${t.assigneeName}` : null, link: "/todo", serverId: t.serverId }));
}
// Things paid for, on the day they are billed (Costs): the next time it falls; the day is rolled forward by the cost service, so a monthly one is always the coming one.
for (const r of data.renewals ?? []) {
const money = `${r.amount.toFixed(2).replace(/\.00$/, "")} ${r.currency} a ${r.period === "year" ? "year" : "month"}`;
day(`renewal:${r.id}`, "renewal", r.nextRenewal, () => ({ title: `${r.name} is billed — ${money}`, link: "/costs" }));
}
// A restore test is due a set time after the last good one — only if a time limit is set (Settings → Notifications).
if (data.restoreLimitDays > 0) {
for (const t of data.restoreTests) {
@@ -330,6 +344,7 @@ const GENERIC: Record<AgendaKind, string> = {
restore: "A restore test is due",
updates: "Updates are waiting",
todo: "A job is due",
renewal: "A bill is due",
};
/**
+5 -2
View File
@@ -84,6 +84,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
netbox: { label: "NetBox", link: "/netbox" },
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
hetzner: { label: "Hetzner", link: "/hetzner" },
speedtest: { label: "Speedtest", link: "/speedtest" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -122,6 +123,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^docker-vuln:(\d+):/.exec(key);
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
@@ -264,7 +267,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -274,7 +277,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+111
View File
@@ -0,0 +1,111 @@
/**
* What the homelab costs: the things you pay for by hand (a domain, a VPN, a licence, a subscription — entered on the Costs page) and the things that can be read (a Hetzner project's
* estimate), added up per currency as a monthly and a yearly figure. No currency is converted into another: a total in EUR and one in SEK are two totals, because a rate is a guess
* made on the day it is read and a bill is not.
*
* Each item may carry the day it is billed. That day is an anchor, not a date that has to be kept up to date: the next time it falls is worked out from it (the same day next month, or the
* same day next year), and that is what "Coming up" shows.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { costItems, hetznerState, integrations } from "../db/schema.js";
import { costOf } from "./hetznerSync.js";
export const PERIODS = ["month", "year"] as const;
export type Period = (typeof PERIODS)[number];
export type CostRow = typeof costItems.$inferSelect;
export const DEFAULT_CATEGORIES = ["Hosting", "Domains", "Subscriptions", "Licences", "Hardware", "Other"];
const round2 = (n: number) => Math.round(n * 100) / 100;
export const monthlyOf = (amount: number, period: Period | string): number => (period === "year" ? amount / 12 : amount);
/** Whether it is a real calendar day, written YYYY-MM-DD (not 2026-02-31, which a Date would quietly move to March). */
export const isYmd = (v: string) => /^\d{4}-\d{2}-\d{2}$/.test(v) && !Number.isNaN(Date.parse(`${v}T00:00:00Z`)) && new Date(`${v}T00:00:00Z`).toISOString().startsWith(v);
const daysInMonth = (year: number, month: number) => new Date(Date.UTC(year, month, 0)).getUTCDate(); // month is 1–12
const ymd = (y: number, m: number, d: number) => `${String(y).padStart(4, "0")}-${String(m).padStart(2, "0")}-${String(Math.min(d, daysInMonth(y, m))).padStart(2, "0")}`;
/**
* Pure: the next day an item is billed on or after `today` (YYYY-MM-DD), from the day it was billed (or will first be). A day that does not exist in a month (the 31st, 29 February) falls on
* the month's last day. Null when there is no usable day.
*/
export function nextRenewal(anchor: string | null | undefined, period: Period | string, today: string): string | null {
if (!anchor || !isYmd(anchor) || !isYmd(today)) return null;
if (anchor >= today) return anchor;
const [ay, am, ad] = anchor.split("-").map(Number);
const [ty, tm] = today.split("-").map(Number);
if (period === "year") {
const thisYear = ymd(ty, am, ad);
return thisYear >= today ? thisYear : ymd(ty + 1, am, ad);
}
void ay;
const thisMonth = ymd(ty, tm, ad);
if (thisMonth >= today) return thisMonth;
return tm === 12 ? ymd(ty + 1, 1, ad) : ymd(ty, tm + 1, ad);
}
export interface AutomaticCost {
key: string;
source: string;
name: string;
currency: string;
/** Per month, including VAT, at list prices. */
monthly: number;
detail: string;
link: string;
/** Whether it was read lately; a stale figure is still shown (and counted), with that said. */
stale: boolean;
}
export interface CurrencyTotal {
currency: string;
monthly: number;
yearly: number;
items: number;
/** The monthly figure split by category, largest first. */
categories: { category: string; monthly: number }[];
}
/** Pure: the totals per currency of manual items and automatic ones. Hetzner's estimate is filed under "Hosting". */
export function costTotals(items: Pick<CostRow, "amount" | "currency" | "period" | "category">[], automatic: Pick<AutomaticCost, "currency" | "monthly">[]): CurrencyTotal[] {
const byCurrency = new Map<string, { monthly: number; items: number; categories: Map<string, number> }>();
const add = (currency: string, category: string, monthly: number) => {
const e = byCurrency.get(currency) ?? { monthly: 0, items: 0, categories: new Map<string, number>() };
e.monthly += monthly;
e.items++;
e.categories.set(category, (e.categories.get(category) ?? 0) + monthly);
byCurrency.set(currency, e);
};
for (const i of items) add(i.currency, i.category, monthlyOf(i.amount, i.period));
for (const a of automatic) add(a.currency, "Hosting", a.monthly);
return [...byCurrency.entries()]
.map(([currency, e]) => ({
currency,
monthly: round2(e.monthly),
yearly: round2(e.monthly * 12),
items: e.items,
categories: [...e.categories.entries()].map(([category, monthly]) => ({ category, monthly: round2(monthly) })).sort((a, b) => b.monthly - a.monthly || a.category.localeCompare(b.category)),
}))
.sort((a, b) => b.monthly - a.monthly || a.currency.localeCompare(b.currency));
}
export async function listCostItems(): Promise<CostRow[]> {
return (await db.select().from(costItems)).sort((a, b) => a.name.localeCompare(b.name));
}
/** What can be read rather than typed: each Hetzner project's estimate. */
export async function automaticCosts(nowMs: number = Date.now(), staleAfterMs = 20 * 60_000): Promise<AutomaticCost[]> {
const rows = await db
.select({ id: integrations.id, name: integrations.name, state: hetznerState })
.from(integrations)
.innerJoin(hetznerState, eq(hetznerState.integrationId, integrations.id))
.where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
const out: AutomaticCost[] = [];
for (const r of rows) {
const cost = costOf(r.state);
if (!cost) continue;
const parts = [`servers ${cost.servers.toFixed(2)}`, `backups ${cost.backups.toFixed(2)}`, `volumes ${cost.volumes.toFixed(2)}`, `snapshots ${cost.snapshots.toFixed(2)}`, `IPs ${cost.ips.toFixed(2)}`].join(" · ");
out.push({ key: `hetzner:${r.id}`, source: "Hetzner Cloud", name: r.name, currency: cost.currency, monthly: cost.total, detail: `${parts} — an estimate at list prices, including VAT`, link: "/hetzner", stale: nowMs - Date.parse(r.state.updatedAt) > staleAfterMs });
}
return out;
}
+25
View File
@@ -29,6 +29,7 @@ import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8n
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
@@ -122,6 +123,8 @@ export interface HealthSnapshot {
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
speedtest?: { integrationId: number; problems: SpeedtestProblem[] }[];
/** Containers to leave alone. */
dockerIgnore?: string[];
/** The clock the container judgements are made on (the pass's, so a test can set it). */
@@ -260,6 +263,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
}
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
// Hetzner Cloud: servers that went off, traffic quotas, rescue mode left on, backups that are late.
for (const h of snapshot.hetzner ?? []) for (const p of h.problems) out.push({ key: p.key, source: `hetzner:${h.integrationId}`, message: p.message, severity: p.severity });
@@ -506,6 +512,25 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Nextcloud:", err instanceof Error ? err.message : err);
}
// Speedtest Tracker: read by its own pass every ten minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsS = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)));
if (rowsS.length > 0) {
const statuses = await speedtestSyncStatuses();
for (const r of rowsS) {
const results = await recentSpeedtestResults(r.id, 30);
if (results.length === 0 || !isSpeedtestFresh(statuses[r.id], Date.now())) {
held.add(`speedtest:${r.id}`);
continue;
}
const loaded = await loadIntegrationConfig(r.id);
(snapshot.speedtest ??= []).push({ integrationId: r.id, problems: speedtestProblems(r.id, r.name, results, speedtestExpected(loaded?.config)) });
}
}
} catch (err) {
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
}
// Hetzner Cloud: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
+2
View File
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^netbox-(?:workers):(\d+)/.exec(key);
+16
View File
@@ -12,6 +12,7 @@ import { backupState, listAll as listDuplicati } from "./duplicati.js";
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
@@ -85,6 +86,20 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
}
}
/** Each Speedtest Tracker, for /metrics: its newest completed test — download and upload in bits per second, ping in seconds. Only while the reading is recent. */
async function addSpeedtestMetrics(out: Exposition, now: Date) {
const statuses = await speedtestSyncStatuses();
for (const row of await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)))) {
if (!isSpeedtestFresh(statuses[row.id], now.getTime())) continue;
const latest = (await recentSpeedtestResults(row.id, 20)).find((r) => r.status === "completed");
if (!latest) continue;
const labels = { name: row.name };
if (latest.downloadBps !== null) out.add("homelab_speedtest_download_bits_per_second", "gauge", "Download speed of the newest completed speed test, in bits per second.", latest.downloadBps, labels);
if (latest.uploadBps !== null) out.add("homelab_speedtest_upload_bits_per_second", "gauge", "Upload speed of the newest completed speed test, in bits per second.", latest.uploadBps, labels);
if (latest.pingMs !== null) out.add("homelab_speedtest_ping_seconds", "gauge", "Ping of the newest completed speed test, in seconds.", latest.pingMs / 1000, labels);
}
}
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
async function addHetznerMetrics(out: Exposition, now: Date) {
const statuses = await hetznerSyncStatuses();
@@ -426,6 +441,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addNextcloudMetrics(out, now);
await addDockerVulnMetrics(out, now);
await addHetznerMetrics(out, now);
await addSpeedtestMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+184
View File
@@ -0,0 +1,184 @@
/**
* Keeps what Speedtest Tracker has measured: every ten minutes each Speedtest Tracker integration is asked for its newest results (a page of 30 — three pages the first time, so the chart
* has a past) and each is stored, once, under Speedtest Tracker's own id. Pages and alerts are built from what is stored, so none of them talks to Speedtest Tracker.
*
* What is raised: **the internet is slow** — the median of the newest three completed tests is under a share (60% unless you say otherwise) of what the line is sold at, which you tell the
* integration (*download you pay for*, *upload you pay for*); nothing is judged without those numbers, because a number alone says nothing — and **the tests are failing** — the newest three all
* failed (the internet is down, or Speedtest Tracker cannot reach Ookla). One slow test is not a problem: the median of three is what is judged. This app never starts a test.
*/
import { and, desc, eq, lt } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, speedtestResults } from "../db/schema.js";
import { createSpeedtestAdapter, type SpeedtestRead, type SpeedtestResult } from "../integrations/speedtest/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 10 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 45 * 60_000;
/** The first read goes this many pages back, so there is a past to chart. */
export const BACKFILL_PAGES = 3;
export const RETENTION_DAYS = 90;
export const DEFAULT_MIN_PERCENT = 60;
/** How many of the newest tests are weighed, and how many of them must be there to judge. */
export const WEIGH = 3;
const STATUS_FLAG = "speedtestSync";
export interface SpeedtestSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, SpeedtestSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, SpeedtestSyncStatus>) : {};
} catch {
return {};
}
}
export async function speedtestSyncStatuses(): Promise<Record<number, SpeedtestSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isSpeedtestFresh = (status: SpeedtestSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type ResultRow = typeof speedtestResults.$inferSelect;
/** What the line is sold at, from the integration's own config (Mbit/s; empty or nonsense is "not said"), and the share below which it is called slow. */
export interface Expected {
downloadMbps: number | null;
uploadMbps: number | null;
minPercent: number;
}
const positive = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v.trim().replace(",", ".")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
export function expectedOf(config: Record<string, unknown> | null | undefined): Expected {
const percent = positive(config?.minPercent);
return { downloadMbps: positive(config?.expectedDownload), uploadMbps: positive(config?.expectedUpload), minPercent: percent === null ? DEFAULT_MIN_PERCENT : Math.min(100, Math.max(1, percent)) };
}
/** Writes down each result once (a result already stored is left as it was — a test does not change afterwards) and forgets what is older than the retention. */
export async function storeResults(integrationId: number, results: SpeedtestResult[], now: Date = new Date()): Promise<number> {
let added = 0;
for (const r of results) {
const inserted = await db
.insert(speedtestResults)
.values({ integrationId, resultId: r.id, at: r.at, status: r.status, downloadBps: r.download, uploadBps: r.upload, pingMs: r.ping, healthy: r.healthy })
.onConflictDoNothing()
.returning({ id: speedtestResults.resultId });
added += inserted.length;
}
await db.delete(speedtestResults).where(and(eq(speedtestResults.integrationId, integrationId), lt(speedtestResults.at, new Date(now.getTime() - RETENTION_DAYS * 86_400_000).toISOString())));
return added;
}
/** The stored results of one integration, newest first. */
export async function recentResults(integrationId: number, limit = 60): Promise<ResultRow[]> {
return db.select().from(speedtestResults).where(eq(speedtestResults.integrationId, integrationId)).orderBy(desc(speedtestResults.at)).limit(limit);
}
/** Reads one Speedtest Tracker and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncSpeedtest(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>, pages: number) => Promise<SpeedtestRead>): Promise<SpeedtestSyncStatus> {
let status: SpeedtestSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "speedtest") throw new Error("not a Speedtest Tracker integration");
const [any] = await db.select({ id: speedtestResults.resultId }).from(speedtestResults).where(eq(speedtestResults.integrationId, integrationId)).limit(1);
const pages = any ? 1 : BACKFILL_PAGES;
const result = await (read ? read(loaded.config, pages) : createSpeedtestAdapter(loaded.config as never).read(pages));
await storeResults(integrationId, result.results, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[speedtest] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[speedtest] couldn't record the result:", err);
}
return status;
}
export async function syncAllSpeedtest(now: Date = new Date()): Promise<Record<number, SpeedtestSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)));
const out: Record<number, SpeedtestSyncStatus> = {};
for (const r of rows) out[r.id] = await syncSpeedtest(r.id, now);
return out;
}
export async function forgetSpeedtest(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
export interface SpeedtestProblem {
key: string;
message: string;
severity: "warning";
}
export const median = (xs: number[]): number | null => {
if (xs.length === 0) return null;
const s = [...xs].sort((a, b) => a - b);
const mid = Math.floor(s.length / 2);
return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
};
export const mbit = (bps: number): string => {
const m = bps / 1e6;
return `${m >= 100 ? Math.round(m) : m.toFixed(1)} Mbit/s`;
};
/**
* Pure: what is wrong with one Speedtest Tracker, from its stored results (newest first). Slow: the median of the newest three *completed* tests under `minPercent` of what the line is
* sold at — judged only for a direction whose number was given, and only with at least two completed tests to take the median of. Failing: the newest three tests that either completed or
* failed (skipped and unfinished ones say nothing) all failed.
*/
export function speedtestProblems(id: number, name: string, results: Pick<ResultRow, "status" | "downloadBps" | "uploadBps">[], expected: Expected): SpeedtestProblem[] {
const out: SpeedtestProblem[] = [];
const finished = results.filter((r) => r.status === "completed" || r.status === "failed");
if (finished.length >= WEIGH && finished.slice(0, WEIGH).every((r) => r.status === "failed")) {
out.push({ key: `speedtest-failing:${id}`, severity: "warning", message: `${name}: the last ${WEIGH} speed tests failed — the internet may be down, or Speedtest Tracker cannot reach Ookla` });
}
const completed = results.filter((r) => r.status === "completed").slice(0, WEIGH);
if (completed.length >= 2) {
const limit = expected.minPercent / 100;
const slow: string[] = [];
for (const [label, expectedMbps, values] of [["download", expected.downloadMbps, completed.map((r) => r.downloadBps)], ["upload", expected.uploadMbps, completed.map((r) => r.uploadBps)]] as const) {
if (expectedMbps === null) continue;
const m = median(values.filter((v): v is number => v !== null));
if (m === null) continue;
const share = m / (expectedMbps * 1e6);
if (share < limit) slow.push(`${label} ${mbit(m)}, ${Math.round(share * 100)}% of the ${expectedMbps} Mbit/s it is sold at`);
}
if (slow.length > 0) out.push({ key: `speedtest-slow:${id}`, severity: "warning", message: `${name}: the internet is slow — ${slow.join("; ")} (the middle of the last ${completed.length} tests)` });
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initSpeedtestSync(): void {
if (timer) return;
const pass = () => void syncAllSpeedtest().catch((err) => console.error("[speedtest] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 31_000);
console.log(`[speedtest] reading Speedtest Tracker every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+38 -8
View File
@@ -4,22 +4,27 @@
* ("Movies", "Photos") instead of what the machine is called. Each is a server (reporting, or not), an integration (its latest calls work,
* or don't) or an Uptime Kuma monitor (up, down, or in maintenance).
*
* What the public gets back is only: the page's title and message, and for each item its name, whether it is up, down, being worked on
* or can't be told, and — where known — how it has done over 30 days. No addresses, no ids, no alert text.
* What the public gets back is only: the page's title and message, and for each item its name, whether it is up, having trouble, down, being
* worked on or can't be told, and — where known — how it has done over 30 days. No addresses, no ids, no alert text.
*
* "Having trouble" is a thing that answers but has an alert open that nobody has silenced, snoozed or closed — a disk filling up, a queue that
* is stuck, an internet line slower than it is sold at. The alert itself is never shown; the family is only told that it is not quite right.
*/
import { and, asc, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { integrations, servers, statusItems, type StatusItemKind } from "../db/schema.js";
import { createUptimeKumaAdapter, type UptimeKumaMonitor } from "../integrations/uptimekuma/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getAlerts } from "./alerts.js";
import type { Alert } from "./alertTypes.js";
import { getSourceStatus } from "./integrationHealthMonitor.js";
import { activeSubjects } from "./maintenance.js";
import { effectiveThresholds, parseThresholds } from "./serverThresholds.js";
import { getSettings } from "./settingsStore.js";
import { getUptime } from "./uptimeHistory.js";
export type ItemState = "up" | "down" | "maintenance" | "unknown";
export type PageLevel = "ok" | "issue" | "maintenance" | "unknown";
export type ItemState = "up" | "trouble" | "down" | "maintenance" | "unknown";
export type PageLevel = "ok" | "trouble" | "issue" | "maintenance" | "unknown";
export const MAX_ITEMS = 30;
export const MAX_LABEL = 60;
@@ -111,14 +116,22 @@ export interface PublicStatus {
display: { dateFormat: string; timeFormat: string };
}
/** How the whole page reads: anything down is an issue; otherwise anything being worked on says so; otherwise fine — unless nothing could be told at all. */
/** How the whole page reads: anything down is an issue; then anything having trouble; then anything being worked on; otherwise fine — unless nothing could be told at all. */
export function pageLevel(counts: Record<ItemState, number>): PageLevel {
if (counts.down > 0) return "issue";
if (counts.trouble > 0) return "trouble";
if (counts.maintenance > 0) return "maintenance";
if (counts.up === 0 && counts.unknown > 0) return "unknown";
return "ok";
}
let alertsLoader: (() => Promise<Pick<Alert, "id" | "severity" | "silenced" | "target">[]>) | null = null;
/** For tests: use these alerts instead of collecting the real ones (which asks every integration). */
export function setAlertsLoaderForTests(loader: typeof alertsLoader): void {
alertsLoader = loader;
}
// Uptime Kuma is asked at most this often per instance, however many people have the page open.
const KUMA_CACHE_MS = 30_000;
const kumaCache = new Map<number, { at: number; monitors: Map<string, UptimeKumaMonitor> | null }>();
@@ -157,6 +170,23 @@ export async function buildPublicStatus(link: { id: number; label: string; title
const integrationRows = new Map((await db.select({ id: integrations.id, type: integrations.type, enabled: integrations.enabled }).from(integrations)).map((i) => [i.id, i]));
const uptime = items.some((i) => i.kind !== "monitor") ? await getUptime(now, 30).catch(() => null) : null;
// What has an alert open that nobody has put aside: a server or an integration that answers but is not quite right. If the alerts cannot be read nothing is said to be in trouble.
const trouble = { server: new Set<number>(), integration: new Set<number>() };
const failingTests = new Set<number>();
try {
for (const a of alertsLoader ? await alertsLoader() : (await getAlerts(false)).alerts) {
if (a.silenced || a.severity === "info" || !a.target) continue;
if (a.target.type === "server") trouble.server.add(a.target.id);
else if (a.target.type === "integration") {
trouble.integration.add(a.target.id);
// Speed tests that keep failing mean the internet is down, not slow.
if (a.id.includes(`speedtest-failing:${a.target.id}`)) failingTests.add(a.target.id);
}
}
} catch {
// no alerts, no "trouble"
}
const out: PublicStatusItem[] = [];
for (const item of items) {
let state: ItemState = "unknown";
@@ -167,14 +197,14 @@ export async function buildPublicStatus(link: { id: number; label: string; title
if (s?.lastSeenAt) {
const limit = effectiveThresholds({ ...settings.healthChecks }, parseThresholds(s.thresholds)).serverOfflineMinutes * 60_000;
const quiet = now.getTime() - Date.parse(s.lastSeenAt) > limit;
state = subjects.has(`server:${s.id}`) ? "maintenance" : quiet ? "down" : "up";
state = subjects.has(`server:${s.id}`) ? "maintenance" : quiet ? "down" : trouble.server.has(s.id) ? "trouble" : "up";
}
uptime30d = uptime?.servers[item.targetId]?.percent ?? null;
} else if (item.kind === "integration") {
const i = integrationRows.get(item.targetId);
if (i?.enabled) {
const status = getSourceStatus(i.type).status;
state = subjects.has(`integration:${i.id}`) ? "maintenance" : status === "ok" ? "up" : status === "failing" ? "down" : "unknown";
state = subjects.has(`integration:${i.id}`) ? "maintenance" : status === "failing" || failingTests.has(i.id) ? "down" : status === "ok" ? (trouble.integration.has(i.id) ? "trouble" : "up") : "unknown";
}
uptime30d = uptime?.integrations[item.targetId]?.percent ?? null;
} else {
@@ -187,7 +217,7 @@ export async function buildPublicStatus(link: { id: number; label: string; title
out.push({ id: item.id, label: item.label, state, uptime30d: uptime30d === null ? null : Math.round(uptime30d * 10) / 10 });
}
const counts: Record<ItemState, number> = { up: 0, down: 0, maintenance: 0, unknown: 0 };
const counts: Record<ItemState, number> = { up: 0, trouble: 0, down: 0, maintenance: 0, unknown: 0 };
for (const o of out) counts[o.state]++;
return {
title: link.title?.trim() || link.label,
+189
View File
@@ -0,0 +1,189 @@
/**
* Costs: the next day a bill falls (rolled forward from the day given), the monthly and yearly figures per currency (never converted), what is read from Hetzner, the routes
* (everyone signed in sees, an administrator changes) and how a bill's day reaches "Coming up".
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import type { Server } from "node:http";
const dir = mkdtempSync(join(tmpdir(), "hm-costs-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const costs = await import("../src/services/costs.js");
const hz = await import("../src/services/hetznerSync.js");
const { costsRouter } = await import("../src/routes/costs.js");
const { agendaFromData, loadAgendaData, AGENDA_KINDS, buildIcs } = await import("../src/services/agenda.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
describe("the next day a bill falls", () => {
const next = costs.nextRenewal;
test("a day still ahead is that day; a day gone by is rolled forward by the period", () => {
assert.equal(next("2026-12-24", "month", "2026-10-26"), "2026-12-24", "the first bill has not come yet");
assert.equal(next("2026-01-15", "month", "2026-10-26"), "2026-11-15");
assert.equal(next("2026-01-15", "month", "2026-10-15"), "2026-10-15", "today counts");
assert.equal(next("2026-01-15", "month", "2026-10-10"), "2026-10-15");
assert.equal(next("2024-03-20", "year", "2026-10-26"), "2027-03-20");
assert.equal(next("2024-11-20", "year", "2026-10-26"), "2026-11-20");
});
test("December rolls into the next year, and a day a month does not have falls on its last day", () => {
assert.equal(next("2026-01-05", "month", "2026-12-20"), "2027-01-05");
assert.equal(next("2026-01-31", "month", "2026-02-10"), "2026-02-28");
assert.equal(next("2026-01-31", "month", "2026-04-10"), "2026-04-30");
assert.equal(next("2024-02-29", "year", "2026-03-01"), "2027-02-28");
assert.equal(next("2024-02-29", "year", "2027-01-01"), "2027-02-28");
});
test("no day, or a day that is not one, is no renewal", () => {
assert.equal(next(null, "month", "2026-10-26"), null);
assert.equal(next("2026-13-40", "month", "2026-10-26"), null);
assert.equal(next("soon", "year", "2026-10-26"), null);
});
});
describe("the totals", () => {
const item = (amount: number, currency: string, period: string, category: string) => ({ amount, currency, period, category });
test("a yearly cost is a twelfth of it each month; each currency is its own total, never converted; categories add up, largest first", () => {
const t = costs.costTotals([item(12, "EUR", "year", "Domains"), item(5, "EUR", "month", "Subscriptions"), item(100, "SEK", "month", "Subscriptions"), item(24, "EUR", "year", "Domains")], []);
assert.deepEqual(t.map((x) => [x.currency, x.monthly, x.yearly, x.items]), [["SEK", 100, 1200, 1], ["EUR", 8, 96, 3]]);
assert.deepEqual(t.find((x) => x.currency === "EUR")!.categories, [{ category: "Subscriptions", monthly: 5 }, { category: "Domains", monthly: 3 }]);
});
test("an estimate that was read is added to its currency under Hosting", () => {
const t = costs.costTotals([item(5, "EUR", "month", "Subscriptions")], [{ currency: "EUR", monthly: 19.53 }]);
assert.deepEqual([t[0].monthly, t[0].yearly, t[0].items], [24.53, 294.36, 2]);
assert.deepEqual(t[0].categories, [{ category: "Hosting", monthly: 19.53 }, { category: "Subscriptions", monthly: 5 }]);
assert.deepEqual(costs.costTotals([], []), []);
});
});
describe("through the routes", () => {
let app: Server;
let base = "";
let who = "admin";
const call = async (method: string, path: string, body?: unknown) => {
const res = await fetch(`${base}${path}`, { method, headers: { "content-type": "application/json", connection: "close" }, body: body === undefined ? undefined : JSON.stringify(body) });
return { status: res.status, body: (await res.json().catch(() => null)) as any };
};
before(async () => {
await db.insert(users).values([
{ oidcSub: "admin", name: "Ada", role: "admin" },
{ oidcSub: "operator", name: "Olga", role: "operator" },
{ oidcSub: "viewer", name: "Vic", role: "viewer" },
]);
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: who } };
next();
});
a.use("/api/costs", costsRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/costs`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("an administrator adds, changes and removes an item, each in the audit log; the currency is made capitals and a bad body is refused with a sentence", async () => {
const created = await call("POST", "", { name: " tooweb.se ", amount: 12, currency: "eur", period: "year", category: "Domains", renewsOn: "2026-03-20", note: "at the registrar" });
assert.equal(created.status, 201);
const id = created.body.id as number;
let list = await call("GET", "");
const item = list.body.items.find((i: { id: number }) => i.id === id);
assert.deepEqual([item.name, item.currency, item.period, item.monthly, item.renewsOn, item.note], ["tooweb.se", "EUR", "year", 1, "2026-03-20", "at the registrar"]);
assert.match(item.nextRenewal, /^\d{4}-03-20$/, "worked out from the day given");
assert.ok(item.nextRenewal >= new Date().toISOString().slice(0, 10));
assert.equal((await call("PUT", `/${id}`, { name: "tooweb.se", amount: 15, currency: "EUR", period: "year", category: "Domains" })).status, 200);
list = await call("GET", "");
assert.deepEqual([list.body.items.find((i: { id: number }) => i.id === id).amount, list.body.items.find((i: { id: number }) => i.id === id).renewsOn], [15, null]);
for (const bad of [{ name: "", amount: 1, currency: "EUR", period: "month" }, { name: "x", amount: 0, currency: "EUR", period: "month" }, { name: "x", amount: -4, currency: "EUR", period: "month" }, { name: "x", amount: 1, currency: "EURO", period: "month" }, { name: "x", amount: 1, currency: "EUR", period: "week" }, { name: "x", amount: 1, currency: "EUR", period: "month", renewsOn: "2026-02-31" }]) {
const r = await call("POST", "", bad);
assert.equal(r.status, 400, JSON.stringify(bad));
assert.match(r.body.message, /three-letter currency/);
}
assert.equal((await call("DELETE", `/${id}`)).status, 204);
assert.equal((await call("DELETE", `/${id}`)).status, 404);
assert.equal((await call("PUT", "/99999", { name: "x", amount: 1, currency: "EUR", period: "month" })).status, 404);
const actions = (await db.select().from(auditLog).where(eq(auditLog.category, "cost"))).map((e) => e.action);
assert.deepEqual(actions, ["create", "update", "delete"]);
});
test("anyone signed in sees the costs; an operator and a viewer cannot change them", async () => {
await call("POST", "", { name: "VPN", amount: 5, currency: "EUR", period: "month", category: "Subscriptions" });
for (const sub of ["operator", "viewer"]) {
who = sub;
assert.equal((await call("GET", "")).status, 200);
assert.equal((await call("POST", "", { name: "x", amount: 1, currency: "EUR", period: "month" })).status, 403);
assert.equal((await call("PUT", "/1", { name: "x", amount: 1, currency: "EUR", period: "month" })).status, 403);
assert.equal((await call("DELETE", "/1")).status, 403);
}
who = "admin";
});
test("a Hetzner project's estimate is in the totals under Hosting, saying it is an estimate, and is marked when it was not read lately", async () => {
const id = (await db.insert(integrations).values({ type: "hetzner", name: "Prod", baseUrl: "https://api.hetzner.cloud", config: "{}" }).returning({ id: integrations.id }))[0].id;
const server = { id: 1, name: "web", status: "running", created: null, type: "cx22", cores: 2, memoryGb: 4, diskGb: 40, location: "fsn1", city: null, os: null, ipv4: null, ipv6: null, privateIps: [], locked: false, rescueEnabled: false, backupWindow: null, includedTraffic: null, outgoingTraffic: null, ingoingTraffic: null, monthlyPrice: 10 };
await hz.storeRead(id, { servers: [server], volumes: [], images: [], primaryIps: [], prices: { currency: "EUR", imagePerGb: 0.01, volumePerGb: 0.05, backupPercent: 20, primaryIps: [] } }, new Date());
let report = await call("GET", "");
assert.deepEqual(report.body.automatic.map((a: { name: string; source: string; monthly: number; stale: boolean }) => [a.name, a.source, a.monthly, a.stale]), [["Prod", "Hetzner Cloud", 10, false]]);
assert.match(report.body.automatic[0].detail, /an estimate at list prices, including VAT/);
const eur = report.body.totals.find((t: { currency: string }) => t.currency === "EUR");
assert.equal(eur.categories.find((c: { category: string }) => c.category === "Hosting").monthly, 10);
const old = await costs.automaticCosts(Date.now() + 3_600_000);
assert.equal(old[0].stale, true);
await db.delete(integrations).where(eq(integrations.id, id));
report = await call("GET", "");
assert.deepEqual(report.body.automatic, []);
});
});
describe("a bill's day in Coming up", () => {
const NOW = new Date("2026-10-26T12:00:00Z");
const empty = { secrets: [], domains: [], endpoints: [], tailscale: [], assets: [], windows: [], schedules: [], patchPlans: [], restoreTests: [], restoreLimitDays: 0, updates: [], todos: [], notes: [] };
test("it is a kind of its own, and the bill is on the coming day with what it costs and a link to Costs", () => {
assert.ok((AGENDA_KINDS as readonly string[]).includes("renewal"));
const a = agendaFromData({ ...empty, renewals: [{ id: 3, name: "tooweb.se domain", amount: 12, currency: "EUR", period: "year", nextRenewal: "2026-11-20" }, { id: 4, name: "VPN", amount: 5.5, currency: "EUR", period: "month", nextRenewal: "2026-11-02" }] }, NOW, "UTC", 90);
assert.deepEqual(a.items.map((i) => [i.kind, i.at, i.title, i.link]), [["renewal", "2026-11-02", "VPN is billed — 5.50 EUR a month", "/costs"], ["renewal", "2026-11-20", "tooweb.se domain is billed — 12 EUR a year", "/costs"]]);
});
test("one beyond the horizon is not listed; data with no renewals at all (older callers) still works", () => {
const far = agendaFromData({ ...empty, renewals: [{ id: 3, name: "X", amount: 1, currency: "EUR", period: "year", nextRenewal: "2027-09-01" }] }, NOW, "UTC", 90);
assert.deepEqual(far.items, []);
assert.deepEqual(agendaFromData(empty, NOW, "UTC", 90).items, []);
});
test("read from what is stored, a monthly bill whose day has gone by comes up on its next day; the feed with no names says only that a bill is due", async () => {
await db.insert((await import("../src/db/schema.js")).costItems).values({ name: "Plex pass", amount: 5, currency: "EUR", period: "month", category: "Subscriptions", renewsOn: "2026-01-15", note: null, createdAt: NOW.toISOString(), updatedAt: NOW.toISOString() });
const data = await loadAgendaData(NOW);
const r = data.renewals!.find((x) => x.name === "Plex pass")!;
assert.equal(r.nextRenewal, "2026-11-15");
const agenda = agendaFromData(data, NOW, "UTC", 90);
const item = agenda.items.find((i) => i.kind === "renewal" && /Plex pass/.test(i.title))!;
assert.ok(item);
const feed = (detail: "summary" | "names") => buildIcs({ ...agenda, items: agenda.items.filter((i) => i.kind === "renewal") }, { name: "Coming up", detail });
assert.match(feed("summary"), /SUMMARY:A bill is due/);
assert.doesNotMatch(feed("summary"), /Plex/);
assert.match(feed("names"), /SUMMARY:Plex pass is billed/);
});
});
+373
View File
@@ -0,0 +1,373 @@
/**
* Speedtest Tracker: reading what it says (read-only, paged, newest first), what is stored once and for how long, the expected-speed numbers, what is wrong (slow, failing), the health
* pass, the alerts, the page's routes and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-speedtest-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, speedtestResults } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/speedtest/adapter.js");
const st = await import("../src/services/speedtestSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { speedtestRouter } = await import("../src/routes/speedtest.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const MBIT = 1e6;
const iso = (ms: number) => new Date(ms).toISOString();
// What Speedtest Tracker sends: bytes per second (download/upload), the same in bits, a ping in ms, a status and a time.
const apiResult = (id: number, extra: Record<string, unknown> = {}) => ({
id,
service: "faker",
ping: 12.5,
download: 31_250_000, // 250 Mbit/s in bytes per second
upload: 3_125_000, // 25 Mbit/s
download_bits: 250_000_000,
upload_bits: 25_000_000,
healthy: true,
status: "completed",
scheduled: true,
data: { timestamp: "2026-06-01T10:00:00Z" },
created_at: "2026-06-01 10:00:05",
...extra,
});
describe("what Speedtest Tracker says", () => {
test("a result: speeds in bits per second, the ping, the status in lower case, healthy true, false or not said", () => {
const r = adapter.parseResult(apiResult(7, { status: "Completed", healthy: null }))!;
assert.deepEqual([r.id, r.status, r.download, r.upload, r.ping, r.healthy], [7, "completed", 250_000_000, 25_000_000, 12.5, null]);
assert.equal(adapter.parseResult(apiResult(8, { healthy: false }))!.healthy, false);
});
test("without the bits fields the bytes are turned into bits; numbers sent as strings are read; a failed test has no speeds", () => {
const r = adapter.parseResult(apiResult(1, { download_bits: undefined, upload_bits: undefined, download: "12500000", upload: 1_250_000 }))!;
assert.deepEqual([r.download, r.upload], [100_000_000, 10_000_000]);
const failed = adapter.parseResult({ id: 2, status: "failed", created_at: "2026-06-01 10:00:05", download: null, upload: null, ping: null })!;
assert.deepEqual([failed.status, failed.download, failed.upload, failed.ping], ["failed", null, null, null]);
});
test("when it was made: the time Ookla gives, else created_at taken as UTC (it has no zone); a result with no id or no time is skipped", () => {
assert.equal(adapter.parseResult(apiResult(1))!.at, "2026-06-01T10:00:00.000Z");
assert.equal(adapter.parseResult(apiResult(1, { data: null }))!.at, "2026-06-01T10:00:05.000Z");
assert.equal(adapter.parseResult(apiResult(1, { data: null, created_at: "2026-06-01T10:00:05+02:00" }))!.at, "2026-06-01T08:00:05.000Z");
assert.equal(adapter.parseResult({ status: "completed", created_at: "2026-06-01 10:00:05" }), null);
assert.equal(adapter.parseResult({ id: 3, status: "completed" }), null);
assert.deepEqual(adapter.parseResults({ data: [apiResult(1), "junk", { id: 9 }] }).map((r) => r.id), [1]);
assert.deepEqual(adapter.parseResults(null), []);
});
describe("the adapter against a Speedtest Tracker that answers", () => {
let fake: Server;
let url = "";
let tokenStatus = 200;
let sizeStatus = 200;
let total = 70;
const seen: { method: string | undefined; path: string; auth: string | undefined }[] = [];
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
seen.push({ method: req.method, path: u.pathname + u.search, auth: req.headers.authorization });
const send = (code: number, body: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(body));
};
if (u.pathname === "/api/healthcheck") return send(200, { message: "Speedtest Tracker is running!" });
if (tokenStatus !== 200) return send(tokenStatus, { message: "no" });
if (u.pathname === "/api/v1/results") {
if (u.searchParams.has("page[size]") && sizeStatus === 422) return send(422, { message: "Validation failed." });
const size = Number(u.searchParams.get("page[size]") ?? 15);
const page = Number(u.searchParams.get("page[number]") ?? 1);
const ids = Array.from({ length: size }, (_, i) => total - (page - 1) * size - i).filter((id) => id > 0);
return send(200, { data: ids.map((id) => apiResult(id)), links: {}, meta: {} });
}
send(404, {});
});
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
});
after(() => fake.close());
test("pages of 30 are read, newest first, until a short page; with the token as a Bearer and only ever GET — never the run endpoint", async () => {
seen.length = 0;
const r = await adapter.createSpeedtestAdapter({ url, token: "tok" }).read(5);
assert.deepEqual([r.results.length, r.results[0].id, r.results.at(-1)!.id], [70, 70, 1]);
assert.deepEqual(seen.map((s) => s.path.replace(/&page\[number\]=\d+|&?sort=-created_at/g, "")).length, 3, "three pages: 30, 30 and a short 10");
assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer tok"));
assert.ok(!seen.some((s) => s.path.includes("/speedtests/run")));
});
test("one page by default; a Speedtest Tracker that allows fewer per page (422) is asked again without a size", async () => {
assert.equal((await adapter.createSpeedtestAdapter({ url, token: "tok" }).read()).results.length, 30);
sizeStatus = 422;
const r = await adapter.createSpeedtestAdapter({ url, token: "tok" }).read();
assert.equal(r.results.length, 15);
sizeStatus = 200;
});
test("a token that is refused, one without the ability, and a dead address are said; ping tries the address without a login and the token on a page of one", async () => {
tokenStatus = 401;
let ping = await adapter.createSpeedtestAdapter({ url, token: "bad" }).ping();
assert.match(ping.error ?? "", /did not accept the token/);
tokenStatus = 403;
ping = await adapter.createSpeedtestAdapter({ url, token: "weak" }).ping();
assert.match(ping.error ?? "", /results:read/);
tokenStatus = 200;
seen.length = 0;
ping = await adapter.createSpeedtestAdapter({ url, token: "tok" }).ping();
assert.equal(ping.ok, true);
assert.deepEqual(seen.map((s) => [s.path.split("?")[0], s.auth === undefined]), [["/api/healthcheck", true], ["/api/v1/results", false]]);
assert.equal((await adapter.createSpeedtestAdapter({ url: "http://127.0.0.1:1", token: "t" }).ping()).ok, false);
});
});
});
describe("what you tell it the line is sold at", () => {
test("numbers in Mbit/s (a comma is accepted), nothing or nonsense is 'not said', and the share defaults to 60%", () => {
assert.deepEqual(st.expectedOf({ expectedDownload: "300", expectedUpload: "27,5", minPercent: "" }), { downloadMbps: 300, uploadMbps: 27.5, minPercent: 60 });
assert.deepEqual(st.expectedOf({ expectedDownload: "fast", expectedUpload: "-3" }), { downloadMbps: null, uploadMbps: null, minPercent: 60 });
assert.deepEqual(st.expectedOf(null), { downloadMbps: null, uploadMbps: null, minPercent: 60 });
assert.equal(st.expectedOf({ minPercent: "75" }).minPercent, 75);
assert.equal(st.expectedOf({ minPercent: "500" }).minPercent, 100, "kept within 1 to 100");
});
});
describe("what is wrong", () => {
const row = (status: string, down: number | null, up: number | null = 25) => ({ status, downloadBps: down === null ? null : down * MBIT, uploadBps: up === null ? null : up * MBIT });
const completed = (...downs: number[]) => downs.map((d) => row("completed", d));
const expected = (downloadMbps: number | null, uploadMbps: number | null = null, minPercent = 60) => ({ downloadMbps, uploadMbps, minPercent });
const keys = (list: ReturnType<typeof st.speedtestProblems>) => list.map((p) => p.key);
test("nothing is judged without what the line is sold at", () => {
assert.deepEqual(st.speedtestProblems(1, "Home", completed(10, 10, 10), expected(null)), []);
});
test("the middle of the newest three under the share is slow, saying what it was and what it should be; at or over the share is fine", () => {
const p = st.speedtestProblems(1, "Home", completed(120, 130, 125), expected(300));
assert.deepEqual(keys(p), ["speedtest-slow:1"]);
assert.equal(p[0].message, "Home: the internet is slow — download 125 Mbit/s, 42% of the 300 Mbit/s it is sold at (the middle of the last 3 tests)");
assert.deepEqual(st.speedtestProblems(1, "Home", completed(190, 200, 195), expected(300)), [], "65% is over 60%");
assert.deepEqual(keys(st.speedtestProblems(1, "Home", completed(190, 200, 195), expected(300, null, 70))), ["speedtest-slow:1"], "the share can be raised");
});
test("one slow test among three is not slow (the middle is judged), and only the newest three count", () => {
assert.deepEqual(st.speedtestProblems(1, "Home", completed(20, 290, 300), expected(300)), []);
assert.deepEqual(st.speedtestProblems(1, "Home", completed(290, 295, 300, 10, 10, 10), expected(300)), [], "older tests do not weigh");
});
test("a direction is judged only when its number is given; both can be slow in one alert; fewer than two completed tests are not judged", () => {
const up = st.speedtestProblems(1, "Home", [row("completed", 300, 5), row("completed", 300, 6), row("completed", 300, 4)], expected(300, 30));
assert.match(up[0].message, /^Home: the internet is slow — upload 5\.0 Mbit\/s, 17% of the 30 Mbit\/s it is sold at/);
assert.doesNotMatch(up[0].message, /download/);
const both = st.speedtestProblems(1, "Home", [row("completed", 100, 5), row("completed", 100, 5)], expected(300, 30));
assert.match(both[0].message, /download 100 Mbit\/s, 33% of the 300 Mbit\/s it is sold at; upload 5\.0 Mbit\/s/);
assert.deepEqual(st.speedtestProblems(1, "Home", completed(10), expected(300)), []);
});
test("failed, skipped and unfinished tests are not speeds", () => {
const mixed = [row("failed", null, null), row("skipped", null, null), ...completed(290, 300), row("running", null, null)];
assert.deepEqual(st.speedtestProblems(1, "Home", mixed, expected(300)), []);
});
test("the newest three tests all failing is a warning; a completed one among them, or fewer than three, is not; skipped ones say nothing", () => {
const failed = row("failed", null, null);
assert.deepEqual(keys(st.speedtestProblems(1, "Home", [failed, failed, failed, ...completed(300)], expected(null))), ["speedtest-failing:1"]);
assert.match(st.speedtestProblems(1, "Home", [failed, failed, failed], expected(null))[0].message, /the last 3 speed tests failed — the internet may be down, or Speedtest Tracker cannot reach Ookla/);
assert.deepEqual(st.speedtestProblems(1, "Home", [failed, row("completed", 300), failed, failed], expected(null)), []);
assert.deepEqual(st.speedtestProblems(1, "Home", [failed, failed], expected(null)), []);
assert.deepEqual(keys(st.speedtestProblems(1, "Home", [failed, row("skipped", null, null), failed, failed], expected(null))), ["speedtest-failing:1"]);
});
});
describe("what is stored", () => {
let id = 0;
let fake: Server;
let url = "";
const reads: number[] = [];
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
res.writeHead(200, { "content-type": "application/json" });
if (u.pathname !== "/api/v1/results") return void res.end("{}");
const page = Number(u.searchParams.get("page[number]") ?? 1);
reads.push(page);
const now = Date.now();
// 100 results, one every 20 minutes, newest first; ids 100 .. 1
const ids = Array.from({ length: 30 }, (_, i) => 100 - (page - 1) * 30 - i).filter((n) => n > 0);
res.end(JSON.stringify({ data: ids.map((n) => apiResult(n, { data: { timestamp: iso(now - (100 - n) * 20 * MIN) } })) }));
});
await new Promise<void>((r) => fake.listen(0, "127.0.0.1", r));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
id = (await db.insert(integrations).values({ type: "speedtest", name: "Tracker", baseUrl: url, config: JSON.stringify({ url, token: "t", expectedDownload: "300" }) }).returning({ id: integrations.id }))[0].id;
});
after(() => fake.close());
const count = async () => (await db.select().from(speedtestResults).where(eq(speedtestResults.integrationId, id))).length;
test("the first read goes three pages back; later reads one page, and a result already stored is not stored again", async () => {
reads.length = 0;
assert.equal((await st.syncSpeedtest(id)).ok, true);
assert.deepEqual([reads.length, await count()], [3, 90]);
reads.length = 0;
await st.syncSpeedtest(id);
assert.deepEqual([reads.length, await count()], [1, 90], "the same results again add nothing");
});
test("results older than 90 days are forgotten, and each is kept as it was first read", async () => {
const old = Date.now() - 91 * 24 * HOUR;
await db.insert(speedtestResults).values({ integrationId: id, resultId: 5000, at: iso(old), status: "completed", downloadBps: 1, uploadBps: 1, pingMs: 1, healthy: true });
await st.storeResults(id, [adapter.parseResult(apiResult(5001, { download_bits: 999, data: { timestamp: iso(Date.now()) } }))!], new Date());
const rows = await db.select().from(speedtestResults).where(eq(speedtestResults.integrationId, id));
assert.ok(!rows.some((r) => r.resultId === 5000), "the old one is gone");
await st.storeResults(id, [adapter.parseResult(apiResult(5001, { download_bits: 123, data: { timestamp: iso(Date.now()) } }))!], new Date());
assert.equal((await db.select().from(speedtestResults).where(eq(speedtestResults.resultId, 5001)))[0].downloadBps, 999, "a test does not change afterwards");
});
test("a read that failed is said, and what was stored is left alone", async () => {
const before_ = await count();
const failed = await st.syncSpeedtest(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error, await count()], [false, "connection refused", before_]);
assert.equal(st.isSpeedtestFresh((await st.speedtestSyncStatuses())[id], Date.now() + 2 * HOUR), false);
await st.syncSpeedtest(id);
});
test("a row of another type is refused, not read", async () => {
const other = (await db.insert(integrations).values({ type: "hetzner", name: "H", baseUrl: "", config: "{}" }).returning({ id: integrations.id }))[0].id;
const r = await st.syncSpeedtest(other);
assert.deepEqual([r.ok, r.error], [false, "not a Speedtest Tracker integration"]);
});
});
describe("the health pass, the page and /metrics", () => {
let stId = 0;
let app: Server;
let base = "";
const fresh = () => setInternalFlag("speedtestSync", JSON.stringify({ [stId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("speedtestSync", JSON.stringify({ [stId]: { at: new Date(Date.now() - 2 * HOUR).toISOString(), ok: true, error: null } }));
const put = async (rows: { n: number; status?: string; down?: number | null; agoMin: number }[]) => {
await db.delete(speedtestResults).where(eq(speedtestResults.integrationId, stId));
for (const r of rows) await db.insert(speedtestResults).values({ integrationId: stId, resultId: r.n, at: iso(Date.now() - r.agoMin * MIN), status: r.status ?? "completed", downloadBps: r.down === undefined ? 100 * MBIT : r.down === null ? null : r.down * MBIT, uploadBps: 20 * MBIT, pingMs: 14, healthy: null });
};
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }]);
stId = (await db.insert(integrations).values({ type: "speedtest", name: "Home line", baseUrl: "http://x", config: JSON.stringify({ url: "http://x", expectedDownload: "300", expectedUpload: "30" }) }).returning({ id: integrations.id }))[0].id;
await put([{ n: 1, agoMin: 10 }, { n: 2, agoMin: 130 }, { n: 3, agoMin: 250 }, { n: 4, agoMin: 4 * 24 * 60 }, { n: 5, status: "failed", down: null, agoMin: 300 }]);
await fresh();
const a = express();
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: "viewer" } };
next();
});
a.use("/api/speedtest", speedtestRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/speedtest`;
});
after(() => {
app.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored with the numbers from the integration, and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.speedtest?.map((s) => s.integrationId), [stId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const c = list.find((x) => x.key === `speedtest-slow:${stId}`)!;
assert.deepEqual([c.severity, c.source], ["warning", `speedtest:${stId}`]);
assert.match(c.message, /^Home line: the internet is slow — download 100 Mbit\/s, 33% of the 300 Mbit\/s it is sold at \(the middle of the last 3 tests\)$/);
assert.doesNotMatch(c.message, /upload/, "67% of what is paid for upload is over the 60% share");
for (const k of ["slow", "failing"]) assert.equal(subjectOfConditionKey(`speedtest-${k}:${stId}`), `integration:${stId}`, "maintenance on it silences the alert");
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.speedtest ?? [], held.has(`speedtest:${stId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows it under Monitoring with a link to the page", async () => {
invalidateAlertsCache();
const a = (await getAlerts(true)).alerts.find((x) => x.id === `monitoring:speedtest-slow:${stId}`)!;
assert.ok(a);
assert.deepEqual([a.severity, a.category, a.source, a.link, a.target?.type, a.target?.id], ["warning", "monitoring", "Speedtest", "/speedtest", "integration", stId]);
});
test("a line that is as fast as it is sold raises nothing, and tests that keep failing do", async () => {
await put([{ n: 1, down: 290, agoMin: 10 }, { n: 2, down: 300, agoMin: 130 }, { n: 3, down: 280, agoMin: 250 }]);
invalidateAlertsCache();
assert.ok(!(await getAlerts(true)).alerts.some((x) => x.id.includes("speedtest-")));
await put([{ n: 1, status: "failed", down: null, agoMin: 10 }, { n: 2, status: "failed", down: null, agoMin: 130 }, { n: 3, status: "failed", down: null, agoMin: 250 }]);
invalidateAlertsCache();
assert.ok((await getAlerts(true)).alerts.some((x) => x.id === `monitoring:speedtest-failing:${stId}`));
await put([{ n: 1, agoMin: 10 }, { n: 2, agoMin: 130 }, { n: 3, agoMin: 250 }, { n: 4, agoMin: 4 * 24 * 60 }, { n: 5, status: "failed", down: null, agoMin: 300 }]);
});
test("the page: the newest completed test, the middle of the last few, what it is sold at, and what is wrong; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { speedtests: { id: number; fresh: boolean; expected: { downloadMbps: number; uploadMbps: number; minPercent: number }; state: { latest: { download: number; ping: number }; middle: { count: number; download: number }; lastTest: { status: string }; stored: number }; problems: { key: string }[] }[] };
const s = body.speedtests.find((x) => x.id === stId)!;
assert.deepEqual([s.fresh, s.expected.downloadMbps, s.expected.uploadMbps, s.expected.minPercent], [true, 300, 30, 60]);
assert.deepEqual([s.state.latest.download, s.state.latest.ping, s.state.middle.count, s.state.middle.download, s.state.lastTest.status, s.state.stored], [100 * MBIT, 14, 3, 100 * MBIT, "completed", 5]);
assert.deepEqual(s.problems.map((p) => p.key), [`speedtest-slow:${stId}`]);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.speedtests.find((x) => x.id === stId)!.problems, []);
await fresh();
});
test("the history for the charts is Mbit/s and ms, oldest first, completed tests only, within the days asked", async () => {
const day = (await (await fetch(`${base}/${stId}/history?days=1`)).json()) as { days: number; points: { t: string; download: number; upload: number; ping: number }[] };
assert.equal(day.days, 1);
assert.deepEqual(day.points.map((p) => [p.download, p.upload, p.ping]), [[100, 20, 14], [100, 20, 14], [100, 20, 14]], "three completed tests in the last day; the failed one is left out");
assert.ok(day.points[0].t < day.points[2].t);
const week = (await (await fetch(`${base}/${stId}/history?days=7`)).json()) as typeof day;
assert.equal(week.points.length, 4);
assert.equal(((await (await fetch(`${base}/${stId}/history?days=900`)).json()) as typeof day).days, 90, "no more than the 90 days kept");
assert.equal((await fetch(`${base}/99999/history`)).status, 404);
assert.equal((await fetch(`${base}/abc/history`)).status, 400);
});
test("/metrics has the newest completed test — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_speedtest_download_bits_per_second\{name="Home line"\} 100000000|homelab_speedtest_download_bits_per_second\{name="Home line"\} 1e\+8/);
assert.match(text, /homelab_speedtest_ping_seconds\{name="Home line"\} 0\.014/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_speedtest/);
await fresh();
});
test("taking the integration away takes what was read with it", async () => {
await st.forgetSpeedtest(stId);
assert.equal((await st.speedtestSyncStatuses())[stId], undefined);
await db.delete(integrations).where(eq(integrations.id, stId));
assert.deepEqual(await db.select().from(speedtestResults).where(eq(speedtestResults.integrationId, stId)), []);
});
});
+78 -3
View File
@@ -32,14 +32,22 @@ const { eq } = await import("drizzle-orm");
await runMigrations();
// The alerts the page sees: none, unless a test says so (the real collector asks every integration, which is not what these tests are about).
type TestAlert = { id: string; severity: "critical" | "warning" | "info"; silenced: boolean; target: { type: "server" | "integration"; id: number; name: string } | null };
let testAlerts: TestAlert[] = [];
sp.setAlertsLoaderForTests(async () => testAlerts as never);
const NOW = new Date("2026-10-17T12:00:00.000Z");
const minutesAgo = (m: number) => new Date(NOW.getTime() - m * 60_000).toISOString();
describe("how the whole page reads", () => {
const c = (up: number, down: number, maintenance: number, unknown: number) => ({ up, down, maintenance, unknown });
test("anything down is a problem; otherwise anything being worked on says so; nothing known at all is unknown", () => {
const c = (up: number, down: number, maintenance: number, unknown: number, trouble = 0) => ({ up, trouble, down, maintenance, unknown });
test("anything down is a problem; then anything having trouble; then anything being worked on; nothing known at all is unknown", () => {
assert.equal(sp.pageLevel(c(5, 0, 0, 0)), "ok");
assert.equal(sp.pageLevel(c(4, 1, 1, 1)), "issue");
assert.equal(sp.pageLevel(c(4, 1, 0, 0, 2)), "issue", "down outranks trouble");
assert.equal(sp.pageLevel(c(4, 0, 1, 0, 1)), "trouble", "trouble outranks being worked on");
assert.equal(sp.pageLevel(c(0, 0, 0, 0, 2)), "trouble");
assert.equal(sp.pageLevel(c(4, 0, 1, 1)), "maintenance");
assert.equal(sp.pageLevel(c(4, 0, 0, 1)), "ok", "one thing that can't be told doesn't spoil the rest");
assert.equal(sp.pageLevel(c(0, 0, 0, 3)), "unknown");
@@ -193,7 +201,7 @@ describe("how each thing reads", () => {
const page = await sp.buildPublicStatus(link, NOW);
assert.equal(page.title, "Our home");
assert.equal(page.message, "Movies are being updated tonight.");
assert.deepEqual(page.counts, { up: 4, down: 2, maintenance: 2, unknown: 3 });
assert.deepEqual(page.counts, { up: 4, trouble: 0, down: 2, maintenance: 2, unknown: 3 });
assert.equal(page.level, "issue");
assert.deepEqual(page.items.map((i) => i.label), ["Files", "Backups", "Printer", "Photos", "Laptop", "Automation", "Movies", "Music", "Podcasts", "Books", "Gone"], "in the order they were put");
assert.equal(page.items.find((i) => i.label === "Music")!.uptime30d, 80, "30-day uptime from Uptime Kuma, to one decimal");
@@ -204,6 +212,73 @@ describe("how each thing reads", () => {
assert.equal(bare.message, null);
});
describe("having trouble: it answers, but something is not right", () => {
const alert = (target: TestAlert["target"], extra: Partial<TestAlert> = {}): TestAlert => ({ id: `services:x:${Math.random()}`, severity: "warning", silenced: false, target, ...extra });
const server = (id: number) => ({ type: "server" as const, id, name: "private server name" });
const integration = (id: number) => ({ type: "integration" as const, id, name: "private integration name" });
const read = async () => stateOf(await sp.buildPublicStatus(link, NOW));
after(() => {
testAlerts = [];
});
test("a server that reports, with a warning or critical alert nobody has put aside, is having trouble; a server that is down stays down", async () => {
testAlerts = [alert(server(ids.up), { severity: "critical" }), alert(server(ids.down))];
const s = await read();
assert.equal(s.Files, "trouble");
assert.equal(s.Backups, "down", "down outranks trouble");
testAlerts = [alert(server(ids.up))];
assert.equal((await read()).Files, "trouble");
});
test("an alert that is silenced (maintenance, snoozed, closed), only information, or about something else says nothing", async () => {
testAlerts = [alert(server(ids.up), { silenced: true }), alert(server(ids.up), { severity: "info" }), alert(server(ids.never)), alert(null), alert({ type: "integration", id: ids.up, name: "x" })];
assert.equal((await read()).Files, "up", "an integration with the server's number is not the server");
});
test("an integration that works, with an alert open, is having trouble; one that fails is down; one under maintenance is being worked on", async () => {
testAlerts = [alert(integration(sem))];
assert.equal((await read()).Automation, "trouble");
await trackIntegrationHealth("semaphore", false);
assert.equal((await read()).Automation, "down");
await trackIntegrationHealth("semaphore", true);
await db.insert(maintenanceWindows).values({ targetType: "integration", targetId: sem, reason: "x", startedAt: minutesAgo(10), endsAt: new Date(NOW.getTime() + 3600_000).toISOString() });
assert.equal((await read()).Automation, "maintenance");
await db.delete(maintenanceWindows).where(eq(maintenanceWindows.targetId, sem));
});
test("the internet, through Speedtest Tracker: slow is trouble, speed tests that keep failing is down, fine is up", async () => {
const st = (await db.insert(integrations).values({ type: "speedtest", name: "Line", baseUrl: "http://x", config: "{}", enabled: true }).returning())[0].id;
await trackIntegrationHealth("speedtest", true);
await sp.setStatusItems(linkId, [...(await sp.getStatusItems(linkId)).map((i) => ({ label: i.label, kind: i.kind, targetId: i.targetId, monitorId: i.monitorId })), { label: "Internet", kind: "integration", targetId: st }]);
testAlerts = [];
assert.equal((await read()).Internet, "up");
testAlerts = [alert(integration(st), { id: `monitoring:speedtest-slow:${st}` })];
assert.equal((await read()).Internet, "trouble");
testAlerts = [alert(integration(st), { id: `monitoring:speedtest-failing:${st}` })];
assert.equal((await read()).Internet, "down");
testAlerts = [];
});
test("the page says how many are in trouble and reads as trouble when nothing is down; and no alert text, name or id reaches the public", async () => {
await sp.setStatusItems(linkId, (await sp.getStatusItems(linkId)).filter((i) => ["Files", "Automation", "Movies"].includes(i.label)).map((i) => ({ label: i.label, kind: i.kind, targetId: i.targetId, monitorId: i.monitorId })));
testAlerts = [alert(server(ids.up), { id: "services:secret-alert-id" })];
const page = await sp.buildPublicStatus(link, NOW);
assert.deepEqual(page.counts, { up: 2, trouble: 1, down: 0, maintenance: 0, unknown: 0 });
assert.equal(page.level, "trouble");
const text = JSON.stringify(page);
for (const secret of ["private server name", "secret-alert-id", "private integration name"]) assert.ok(!text.includes(secret), secret);
testAlerts = [];
});
test("alerts that cannot be read say nothing is in trouble, and the page still works", async () => {
sp.setAlertsLoaderForTests(async () => {
throw new Error("alerts are down");
});
assert.equal((await read()).Files, "up");
sp.setAlertsLoaderForTests(async () => testAlerts as never);
});
});
test("the public is given names and states and nothing else", async () => {
const text = JSON.stringify(await sp.buildPublicStatus(link, NOW));
for (const secret of ["10.1.2.3", "kuma.lab", "sem.lab", "private name", "m-up", "apiToken", "elsa", "tok"]) assert.ok(!text.includes(secret), `the page leaks "${secret}"`);