Speedtest Tracker, Costs, "Having trouble" on status pages, grouped Integrations and Dashboard
- Speedtest Tracker integration: charts, alerts for a slow line (median of the last three tests under a share of what is paid for) and for failing tests. - Costs page (Records): things paid for by hand plus Hetzner's estimate, totals per currency, bills rolled forward into Coming up as Renewals. - Status pages: a "Having trouble" state for things that answer but have an open alert; Speedtest reads as the internet. - Integrations page and kind picker grouped like the sidebar (one shared types file); Notifications "What to send" shown one group at a time; Dashboard integration sections follow the menu groups, with tiles for integrations that have no widget. - Docs, tests, and ROLES.md menu table brought up to date. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
78822346b1
commit
f86661f6c9
49 files changed
+18156
-238
No files matched your search
@@ -19,8 +19,9 @@ import { parsePendingUpdates } from "./pendingUpdates.js";
|
||||
import { latestTests } from "./restoreTests.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { env } from "../env.js";
|
||||
import { listCostItems, nextRenewal } from "./costs.js";
|
||||
|
||||
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates", "todo"] as const;
|
||||
export const AGENDA_KINDS = ["secret", "certificate", "domain", "tailscale", "warranty", "support", "maintenance", "restore", "updates", "todo", "renewal"] as const;
|
||||
export type AgendaKind = (typeof AGENDA_KINDS)[number];
|
||||
|
||||
export const KIND_LABELS: Record<AgendaKind, string> = {
|
||||
@@ -34,6 +35,7 @@ export const KIND_LABELS: Record<AgendaKind, string> = {
|
||||
restore: "Restore tests",
|
||||
updates: "Updates waiting",
|
||||
todo: "To do",
|
||||
renewal: "Renewals",
|
||||
};
|
||||
|
||||
export interface AgendaItem {
|
||||
@@ -100,6 +102,8 @@ export interface AgendaData {
|
||||
updates: { name: string; count: number; security: number | null; since: string | null }[];
|
||||
/** Open jobs that have a day they are due. */
|
||||
todos: { id: number; title: string; dueDate: string | null; assigneeName: string | null; serverId: number | null }[];
|
||||
/** Things paid for that have a day they are billed on (Costs): the next day it falls, worked out from the day given. */
|
||||
renewals?: { id: number; name: string; amount: number; currency: string; period: string; nextRenewal: string }[];
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
@@ -180,6 +184,10 @@ export async function loadAgendaData(now: Date = new Date()): Promise<AgendaData
|
||||
restoreTests: [...tests.values()].map((t) => ({ subjectKey: t.subjectKey, label: t.label, testedAt: t.testedAt, result: t.result })),
|
||||
restoreLimitDays: settings.healthChecks.restoreTestDays,
|
||||
updates,
|
||||
renewals: (await listCostItems()).flatMap((c) => {
|
||||
const next = nextRenewal(c.renewsOn, c.period, localDate(now, settings.notifications.timezone || "UTC"));
|
||||
return next ? [{ id: c.id, name: c.name, amount: c.amount, currency: c.currency, period: c.period, nextRenewal: next }] : [];
|
||||
}),
|
||||
todos: todoRows.map((t) => ({ id: t.id, title: t.title, dueDate: t.dueDate, assigneeName: t.assigneeName, serverId: t.targetType === "server" ? t.targetId : null })),
|
||||
notes,
|
||||
};
|
||||
@@ -225,6 +233,12 @@ export function agendaFromData(data: AgendaData, now: Date, tz: string, days: nu
|
||||
day(`todo:${t.id}`, "todo", t.dueDate, (past) => ({ title: `${past ? "Overdue" : "To do"}: ${t.title}`, detail: t.assigneeName ? `for ${t.assigneeName}` : null, link: "/todo", serverId: t.serverId }));
|
||||
}
|
||||
|
||||
// Things paid for, on the day they are billed (Costs): the next time it falls; the day is rolled forward by the cost service, so a monthly one is always the coming one.
|
||||
for (const r of data.renewals ?? []) {
|
||||
const money = `${r.amount.toFixed(2).replace(/\.00$/, "")} ${r.currency} a ${r.period === "year" ? "year" : "month"}`;
|
||||
day(`renewal:${r.id}`, "renewal", r.nextRenewal, () => ({ title: `${r.name} is billed — ${money}`, link: "/costs" }));
|
||||
}
|
||||
|
||||
// A restore test is due a set time after the last good one — only if a time limit is set (Settings → Notifications).
|
||||
if (data.restoreLimitDays > 0) {
|
||||
for (const t of data.restoreTests) {
|
||||
@@ -330,6 +344,7 @@ const GENERIC: Record<AgendaKind, string> = {
|
||||
restore: "A restore test is due",
|
||||
updates: "Updates are waiting",
|
||||
todo: "A job is due",
|
||||
renewal: "A bill is due",
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -84,6 +84,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
netbox: { label: "NetBox", link: "/netbox" },
|
||||
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
|
||||
hetzner: { label: "Hetzner", link: "/hetzner" },
|
||||
speedtest: { label: "Speedtest", link: "/speedtest" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -122,6 +123,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^docker-vuln:(\d+):/.exec(key);
|
||||
if (m) return { type: "dockhand", id: Number(m[1]), category: "services" };
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
@@ -264,7 +267,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -274,7 +277,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* What the homelab costs: the things you pay for by hand (a domain, a VPN, a licence, a subscription — entered on the Costs page) and the things that can be read (a Hetzner project's
|
||||
* estimate), added up per currency as a monthly and a yearly figure. No currency is converted into another: a total in EUR and one in SEK are two totals, because a rate is a guess
|
||||
* made on the day it is read and a bill is not.
|
||||
*
|
||||
* Each item may carry the day it is billed. That day is an anchor, not a date that has to be kept up to date: the next time it falls is worked out from it (the same day next month, or the
|
||||
* same day next year), and that is what "Coming up" shows.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { costItems, hetznerState, integrations } from "../db/schema.js";
|
||||
import { costOf } from "./hetznerSync.js";
|
||||
|
||||
export const PERIODS = ["month", "year"] as const;
|
||||
export type Period = (typeof PERIODS)[number];
|
||||
export type CostRow = typeof costItems.$inferSelect;
|
||||
|
||||
export const DEFAULT_CATEGORIES = ["Hosting", "Domains", "Subscriptions", "Licences", "Hardware", "Other"];
|
||||
|
||||
const round2 = (n: number) => Math.round(n * 100) / 100;
|
||||
export const monthlyOf = (amount: number, period: Period | string): number => (period === "year" ? amount / 12 : amount);
|
||||
|
||||
/** Whether it is a real calendar day, written YYYY-MM-DD (not 2026-02-31, which a Date would quietly move to March). */
|
||||
export const isYmd = (v: string) => /^\d{4}-\d{2}-\d{2}$/.test(v) && !Number.isNaN(Date.parse(`${v}T00:00:00Z`)) && new Date(`${v}T00:00:00Z`).toISOString().startsWith(v);
|
||||
const daysInMonth = (year: number, month: number) => new Date(Date.UTC(year, month, 0)).getUTCDate(); // month is 1–12
|
||||
const ymd = (y: number, m: number, d: number) => `${String(y).padStart(4, "0")}-${String(m).padStart(2, "0")}-${String(Math.min(d, daysInMonth(y, m))).padStart(2, "0")}`;
|
||||
|
||||
/**
|
||||
* Pure: the next day an item is billed on or after `today` (YYYY-MM-DD), from the day it was billed (or will first be). A day that does not exist in a month (the 31st, 29 February) falls on
|
||||
* the month's last day. Null when there is no usable day.
|
||||
*/
|
||||
export function nextRenewal(anchor: string | null | undefined, period: Period | string, today: string): string | null {
|
||||
if (!anchor || !isYmd(anchor) || !isYmd(today)) return null;
|
||||
if (anchor >= today) return anchor;
|
||||
const [ay, am, ad] = anchor.split("-").map(Number);
|
||||
const [ty, tm] = today.split("-").map(Number);
|
||||
if (period === "year") {
|
||||
const thisYear = ymd(ty, am, ad);
|
||||
return thisYear >= today ? thisYear : ymd(ty + 1, am, ad);
|
||||
}
|
||||
void ay;
|
||||
const thisMonth = ymd(ty, tm, ad);
|
||||
if (thisMonth >= today) return thisMonth;
|
||||
return tm === 12 ? ymd(ty + 1, 1, ad) : ymd(ty, tm + 1, ad);
|
||||
}
|
||||
|
||||
export interface AutomaticCost {
|
||||
key: string;
|
||||
source: string;
|
||||
name: string;
|
||||
currency: string;
|
||||
/** Per month, including VAT, at list prices. */
|
||||
monthly: number;
|
||||
detail: string;
|
||||
link: string;
|
||||
/** Whether it was read lately; a stale figure is still shown (and counted), with that said. */
|
||||
stale: boolean;
|
||||
}
|
||||
|
||||
export interface CurrencyTotal {
|
||||
currency: string;
|
||||
monthly: number;
|
||||
yearly: number;
|
||||
items: number;
|
||||
/** The monthly figure split by category, largest first. */
|
||||
categories: { category: string; monthly: number }[];
|
||||
}
|
||||
|
||||
/** Pure: the totals per currency of manual items and automatic ones. Hetzner's estimate is filed under "Hosting". */
|
||||
export function costTotals(items: Pick<CostRow, "amount" | "currency" | "period" | "category">[], automatic: Pick<AutomaticCost, "currency" | "monthly">[]): CurrencyTotal[] {
|
||||
const byCurrency = new Map<string, { monthly: number; items: number; categories: Map<string, number> }>();
|
||||
const add = (currency: string, category: string, monthly: number) => {
|
||||
const e = byCurrency.get(currency) ?? { monthly: 0, items: 0, categories: new Map<string, number>() };
|
||||
e.monthly += monthly;
|
||||
e.items++;
|
||||
e.categories.set(category, (e.categories.get(category) ?? 0) + monthly);
|
||||
byCurrency.set(currency, e);
|
||||
};
|
||||
for (const i of items) add(i.currency, i.category, monthlyOf(i.amount, i.period));
|
||||
for (const a of automatic) add(a.currency, "Hosting", a.monthly);
|
||||
return [...byCurrency.entries()]
|
||||
.map(([currency, e]) => ({
|
||||
currency,
|
||||
monthly: round2(e.monthly),
|
||||
yearly: round2(e.monthly * 12),
|
||||
items: e.items,
|
||||
categories: [...e.categories.entries()].map(([category, monthly]) => ({ category, monthly: round2(monthly) })).sort((a, b) => b.monthly - a.monthly || a.category.localeCompare(b.category)),
|
||||
}))
|
||||
.sort((a, b) => b.monthly - a.monthly || a.currency.localeCompare(b.currency));
|
||||
}
|
||||
|
||||
export async function listCostItems(): Promise<CostRow[]> {
|
||||
return (await db.select().from(costItems)).sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/** What can be read rather than typed: each Hetzner project's estimate. */
|
||||
export async function automaticCosts(nowMs: number = Date.now(), staleAfterMs = 20 * 60_000): Promise<AutomaticCost[]> {
|
||||
const rows = await db
|
||||
.select({ id: integrations.id, name: integrations.name, state: hetznerState })
|
||||
.from(integrations)
|
||||
.innerJoin(hetznerState, eq(hetznerState.integrationId, integrations.id))
|
||||
.where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
const out: AutomaticCost[] = [];
|
||||
for (const r of rows) {
|
||||
const cost = costOf(r.state);
|
||||
if (!cost) continue;
|
||||
const parts = [`servers ${cost.servers.toFixed(2)}`, `backups ${cost.backups.toFixed(2)}`, `volumes ${cost.volumes.toFixed(2)}`, `snapshots ${cost.snapshots.toFixed(2)}`, `IPs ${cost.ips.toFixed(2)}`].join(" · ");
|
||||
out.push({ key: `hetzner:${r.id}`, source: "Hetzner Cloud", name: r.name, currency: cost.currency, monthly: cost.total, detail: `${parts} — an estimate at list prices, including VAT`, link: "/hetzner", stale: nowMs - Date.parse(r.state.updatedAt) > staleAfterMs });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import { isN8nFresh, n8nProblems, n8nSyncStatuses, type N8nProblem } from "./n8n
|
||||
import { isNetBoxFresh, netboxProblems, netboxSyncStatuses, type NetBoxProblem } from "./netboxSync.js";
|
||||
import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type NextcloudProblem } from "./nextcloudSync.js";
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
@@ -122,6 +123,8 @@ export interface HealthSnapshot {
|
||||
dockerVulns?: { integrationId: number; problems: DockerVulnProblem[] }[];
|
||||
/** What is wrong with each Hetzner Cloud project (a server that went off, a traffic quota nearly used, rescue mode left on, a missing backup), only while that reading is recent (services/hetznerSync.ts). */
|
||||
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
|
||||
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
|
||||
speedtest?: { integrationId: number; problems: SpeedtestProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -260,6 +263,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
for (const c of evaluateContainers(snapshot.docker, snapshot.dockerMemory ?? {}, snapshot.dockerNow ?? now, snapshot.dockerIgnore ?? []).conditions) out.push(c);
|
||||
}
|
||||
|
||||
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
|
||||
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Hetzner Cloud: servers that went off, traffic quotas, rescue mode left on, backups that are late.
|
||||
for (const h of snapshot.hetzner ?? []) for (const p of h.problems) out.push({ key: p.key, source: `hetzner:${h.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -506,6 +512,25 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Nextcloud:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Speedtest Tracker: read by its own pass every ten minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsS = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)));
|
||||
if (rowsS.length > 0) {
|
||||
const statuses = await speedtestSyncStatuses();
|
||||
for (const r of rowsS) {
|
||||
const results = await recentSpeedtestResults(r.id, 30);
|
||||
if (results.length === 0 || !isSpeedtestFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`speedtest:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
const loaded = await loadIntegrationConfig(r.id);
|
||||
(snapshot.speedtest ??= []).push({ integrationId: r.id, problems: speedtestProblems(r.id, r.name, results, speedtestExpected(loaded?.config)) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Hetzner Cloud: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -37,6 +37,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^speedtest-(?:slow|failing):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
|
||||
@@ -12,6 +12,7 @@ import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
import { isJellyfinFresh, jellyfinSyncStatuses, streamsOf } from "./jellyfinSync.js";
|
||||
import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
||||
@@ -85,6 +86,20 @@ async function addNextcloudMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Speedtest Tracker, for /metrics: its newest completed test — download and upload in bits per second, ping in seconds. Only while the reading is recent. */
|
||||
async function addSpeedtestMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await speedtestSyncStatuses();
|
||||
for (const row of await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)))) {
|
||||
if (!isSpeedtestFresh(statuses[row.id], now.getTime())) continue;
|
||||
const latest = (await recentSpeedtestResults(row.id, 20)).find((r) => r.status === "completed");
|
||||
if (!latest) continue;
|
||||
const labels = { name: row.name };
|
||||
if (latest.downloadBps !== null) out.add("homelab_speedtest_download_bits_per_second", "gauge", "Download speed of the newest completed speed test, in bits per second.", latest.downloadBps, labels);
|
||||
if (latest.uploadBps !== null) out.add("homelab_speedtest_upload_bits_per_second", "gauge", "Upload speed of the newest completed speed test, in bits per second.", latest.uploadBps, labels);
|
||||
if (latest.pingMs !== null) out.add("homelab_speedtest_ping_seconds", "gauge", "Ping of the newest completed speed test, in seconds.", latest.pingMs / 1000, labels);
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
|
||||
async function addHetznerMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
@@ -426,6 +441,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addNextcloudMetrics(out, now);
|
||||
await addDockerVulnMetrics(out, now);
|
||||
await addHetznerMetrics(out, now);
|
||||
await addSpeedtestMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Keeps what Speedtest Tracker has measured: every ten minutes each Speedtest Tracker integration is asked for its newest results (a page of 30 — three pages the first time, so the chart
|
||||
* has a past) and each is stored, once, under Speedtest Tracker's own id. Pages and alerts are built from what is stored, so none of them talks to Speedtest Tracker.
|
||||
*
|
||||
* What is raised: **the internet is slow** — the median of the newest three completed tests is under a share (60% unless you say otherwise) of what the line is sold at, which you tell the
|
||||
* integration (*download you pay for*, *upload you pay for*); nothing is judged without those numbers, because a number alone says nothing — and **the tests are failing** — the newest three all
|
||||
* failed (the internet is down, or Speedtest Tracker cannot reach Ookla). One slow test is not a problem: the median of three is what is judged. This app never starts a test.
|
||||
*/
|
||||
import { and, desc, eq, lt } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, speedtestResults } from "../db/schema.js";
|
||||
import { createSpeedtestAdapter, type SpeedtestRead, type SpeedtestResult } from "../integrations/speedtest/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 10 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 45 * 60_000;
|
||||
/** The first read goes this many pages back, so there is a past to chart. */
|
||||
export const BACKFILL_PAGES = 3;
|
||||
export const RETENTION_DAYS = 90;
|
||||
export const DEFAULT_MIN_PERCENT = 60;
|
||||
/** How many of the newest tests are weighed, and how many of them must be there to judge. */
|
||||
export const WEIGH = 3;
|
||||
const STATUS_FLAG = "speedtestSync";
|
||||
|
||||
export interface SpeedtestSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, SpeedtestSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, SpeedtestSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function speedtestSyncStatuses(): Promise<Record<number, SpeedtestSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isSpeedtestFresh = (status: SpeedtestSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type ResultRow = typeof speedtestResults.$inferSelect;
|
||||
|
||||
/** What the line is sold at, from the integration's own config (Mbit/s; empty or nonsense is "not said"), and the share below which it is called slow. */
|
||||
export interface Expected {
|
||||
downloadMbps: number | null;
|
||||
uploadMbps: number | null;
|
||||
minPercent: number;
|
||||
}
|
||||
|
||||
const positive = (v: unknown): number | null => {
|
||||
const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v.trim().replace(",", ".")) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
};
|
||||
|
||||
export function expectedOf(config: Record<string, unknown> | null | undefined): Expected {
|
||||
const percent = positive(config?.minPercent);
|
||||
return { downloadMbps: positive(config?.expectedDownload), uploadMbps: positive(config?.expectedUpload), minPercent: percent === null ? DEFAULT_MIN_PERCENT : Math.min(100, Math.max(1, percent)) };
|
||||
}
|
||||
|
||||
/** Writes down each result once (a result already stored is left as it was — a test does not change afterwards) and forgets what is older than the retention. */
|
||||
export async function storeResults(integrationId: number, results: SpeedtestResult[], now: Date = new Date()): Promise<number> {
|
||||
let added = 0;
|
||||
for (const r of results) {
|
||||
const inserted = await db
|
||||
.insert(speedtestResults)
|
||||
.values({ integrationId, resultId: r.id, at: r.at, status: r.status, downloadBps: r.download, uploadBps: r.upload, pingMs: r.ping, healthy: r.healthy })
|
||||
.onConflictDoNothing()
|
||||
.returning({ id: speedtestResults.resultId });
|
||||
added += inserted.length;
|
||||
}
|
||||
await db.delete(speedtestResults).where(and(eq(speedtestResults.integrationId, integrationId), lt(speedtestResults.at, new Date(now.getTime() - RETENTION_DAYS * 86_400_000).toISOString())));
|
||||
return added;
|
||||
}
|
||||
|
||||
/** The stored results of one integration, newest first. */
|
||||
export async function recentResults(integrationId: number, limit = 60): Promise<ResultRow[]> {
|
||||
return db.select().from(speedtestResults).where(eq(speedtestResults.integrationId, integrationId)).orderBy(desc(speedtestResults.at)).limit(limit);
|
||||
}
|
||||
|
||||
/** Reads one Speedtest Tracker and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncSpeedtest(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>, pages: number) => Promise<SpeedtestRead>): Promise<SpeedtestSyncStatus> {
|
||||
let status: SpeedtestSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "speedtest") throw new Error("not a Speedtest Tracker integration");
|
||||
const [any] = await db.select({ id: speedtestResults.resultId }).from(speedtestResults).where(eq(speedtestResults.integrationId, integrationId)).limit(1);
|
||||
const pages = any ? 1 : BACKFILL_PAGES;
|
||||
const result = await (read ? read(loaded.config, pages) : createSpeedtestAdapter(loaded.config as never).read(pages));
|
||||
await storeResults(integrationId, result.results, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[speedtest] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[speedtest] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllSpeedtest(now: Date = new Date()): Promise<Record<number, SpeedtestSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "speedtest"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, SpeedtestSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncSpeedtest(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetSpeedtest(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface SpeedtestProblem {
|
||||
key: string;
|
||||
message: string;
|
||||
severity: "warning";
|
||||
}
|
||||
|
||||
export const median = (xs: number[]): number | null => {
|
||||
if (xs.length === 0) return null;
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
const mid = Math.floor(s.length / 2);
|
||||
return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
||||
};
|
||||
|
||||
export const mbit = (bps: number): string => {
|
||||
const m = bps / 1e6;
|
||||
return `${m >= 100 ? Math.round(m) : m.toFixed(1)} Mbit/s`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pure: what is wrong with one Speedtest Tracker, from its stored results (newest first). Slow: the median of the newest three *completed* tests under `minPercent` of what the line is
|
||||
* sold at — judged only for a direction whose number was given, and only with at least two completed tests to take the median of. Failing: the newest three tests that either completed or
|
||||
* failed (skipped and unfinished ones say nothing) all failed.
|
||||
*/
|
||||
export function speedtestProblems(id: number, name: string, results: Pick<ResultRow, "status" | "downloadBps" | "uploadBps">[], expected: Expected): SpeedtestProblem[] {
|
||||
const out: SpeedtestProblem[] = [];
|
||||
const finished = results.filter((r) => r.status === "completed" || r.status === "failed");
|
||||
if (finished.length >= WEIGH && finished.slice(0, WEIGH).every((r) => r.status === "failed")) {
|
||||
out.push({ key: `speedtest-failing:${id}`, severity: "warning", message: `${name}: the last ${WEIGH} speed tests failed — the internet may be down, or Speedtest Tracker cannot reach Ookla` });
|
||||
}
|
||||
const completed = results.filter((r) => r.status === "completed").slice(0, WEIGH);
|
||||
if (completed.length >= 2) {
|
||||
const limit = expected.minPercent / 100;
|
||||
const slow: string[] = [];
|
||||
for (const [label, expectedMbps, values] of [["download", expected.downloadMbps, completed.map((r) => r.downloadBps)], ["upload", expected.uploadMbps, completed.map((r) => r.uploadBps)]] as const) {
|
||||
if (expectedMbps === null) continue;
|
||||
const m = median(values.filter((v): v is number => v !== null));
|
||||
if (m === null) continue;
|
||||
const share = m / (expectedMbps * 1e6);
|
||||
if (share < limit) slow.push(`${label} ${mbit(m)}, ${Math.round(share * 100)}% of the ${expectedMbps} Mbit/s it is sold at`);
|
||||
}
|
||||
if (slow.length > 0) out.push({ key: `speedtest-slow:${id}`, severity: "warning", message: `${name}: the internet is slow — ${slow.join("; ")} (the middle of the last ${completed.length} tests)` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initSpeedtestSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllSpeedtest().catch((err) => console.error("[speedtest] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 31_000);
|
||||
console.log(`[speedtest] reading Speedtest Tracker every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -4,22 +4,27 @@
|
||||
* ("Movies", "Photos") instead of what the machine is called. Each is a server (reporting, or not), an integration (its latest calls work,
|
||||
* or don't) or an Uptime Kuma monitor (up, down, or in maintenance).
|
||||
*
|
||||
* What the public gets back is only: the page's title and message, and for each item its name, whether it is up, down, being worked on
|
||||
* or can't be told, and — where known — how it has done over 30 days. No addresses, no ids, no alert text.
|
||||
* What the public gets back is only: the page's title and message, and for each item its name, whether it is up, having trouble, down, being
|
||||
* worked on or can't be told, and — where known — how it has done over 30 days. No addresses, no ids, no alert text.
|
||||
*
|
||||
* "Having trouble" is a thing that answers but has an alert open that nobody has silenced, snoozed or closed — a disk filling up, a queue that
|
||||
* is stuck, an internet line slower than it is sold at. The alert itself is never shown; the family is only told that it is not quite right.
|
||||
*/
|
||||
import { and, asc, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { integrations, servers, statusItems, type StatusItemKind } from "../db/schema.js";
|
||||
import { createUptimeKumaAdapter, type UptimeKumaMonitor } from "../integrations/uptimekuma/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getAlerts } from "./alerts.js";
|
||||
import type { Alert } from "./alertTypes.js";
|
||||
import { getSourceStatus } from "./integrationHealthMonitor.js";
|
||||
import { activeSubjects } from "./maintenance.js";
|
||||
import { effectiveThresholds, parseThresholds } from "./serverThresholds.js";
|
||||
import { getSettings } from "./settingsStore.js";
|
||||
import { getUptime } from "./uptimeHistory.js";
|
||||
|
||||
export type ItemState = "up" | "down" | "maintenance" | "unknown";
|
||||
export type PageLevel = "ok" | "issue" | "maintenance" | "unknown";
|
||||
export type ItemState = "up" | "trouble" | "down" | "maintenance" | "unknown";
|
||||
export type PageLevel = "ok" | "trouble" | "issue" | "maintenance" | "unknown";
|
||||
|
||||
export const MAX_ITEMS = 30;
|
||||
export const MAX_LABEL = 60;
|
||||
@@ -111,14 +116,22 @@ export interface PublicStatus {
|
||||
display: { dateFormat: string; timeFormat: string };
|
||||
}
|
||||
|
||||
/** How the whole page reads: anything down is an issue; otherwise anything being worked on says so; otherwise fine — unless nothing could be told at all. */
|
||||
/** How the whole page reads: anything down is an issue; then anything having trouble; then anything being worked on; otherwise fine — unless nothing could be told at all. */
|
||||
export function pageLevel(counts: Record<ItemState, number>): PageLevel {
|
||||
if (counts.down > 0) return "issue";
|
||||
if (counts.trouble > 0) return "trouble";
|
||||
if (counts.maintenance > 0) return "maintenance";
|
||||
if (counts.up === 0 && counts.unknown > 0) return "unknown";
|
||||
return "ok";
|
||||
}
|
||||
|
||||
let alertsLoader: (() => Promise<Pick<Alert, "id" | "severity" | "silenced" | "target">[]>) | null = null;
|
||||
|
||||
/** For tests: use these alerts instead of collecting the real ones (which asks every integration). */
|
||||
export function setAlertsLoaderForTests(loader: typeof alertsLoader): void {
|
||||
alertsLoader = loader;
|
||||
}
|
||||
|
||||
// Uptime Kuma is asked at most this often per instance, however many people have the page open.
|
||||
const KUMA_CACHE_MS = 30_000;
|
||||
const kumaCache = new Map<number, { at: number; monitors: Map<string, UptimeKumaMonitor> | null }>();
|
||||
@@ -157,6 +170,23 @@ export async function buildPublicStatus(link: { id: number; label: string; title
|
||||
const integrationRows = new Map((await db.select({ id: integrations.id, type: integrations.type, enabled: integrations.enabled }).from(integrations)).map((i) => [i.id, i]));
|
||||
const uptime = items.some((i) => i.kind !== "monitor") ? await getUptime(now, 30).catch(() => null) : null;
|
||||
|
||||
// What has an alert open that nobody has put aside: a server or an integration that answers but is not quite right. If the alerts cannot be read nothing is said to be in trouble.
|
||||
const trouble = { server: new Set<number>(), integration: new Set<number>() };
|
||||
const failingTests = new Set<number>();
|
||||
try {
|
||||
for (const a of alertsLoader ? await alertsLoader() : (await getAlerts(false)).alerts) {
|
||||
if (a.silenced || a.severity === "info" || !a.target) continue;
|
||||
if (a.target.type === "server") trouble.server.add(a.target.id);
|
||||
else if (a.target.type === "integration") {
|
||||
trouble.integration.add(a.target.id);
|
||||
// Speed tests that keep failing mean the internet is down, not slow.
|
||||
if (a.id.includes(`speedtest-failing:${a.target.id}`)) failingTests.add(a.target.id);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// no alerts, no "trouble"
|
||||
}
|
||||
|
||||
const out: PublicStatusItem[] = [];
|
||||
for (const item of items) {
|
||||
let state: ItemState = "unknown";
|
||||
@@ -167,14 +197,14 @@ export async function buildPublicStatus(link: { id: number; label: string; title
|
||||
if (s?.lastSeenAt) {
|
||||
const limit = effectiveThresholds({ ...settings.healthChecks }, parseThresholds(s.thresholds)).serverOfflineMinutes * 60_000;
|
||||
const quiet = now.getTime() - Date.parse(s.lastSeenAt) > limit;
|
||||
state = subjects.has(`server:${s.id}`) ? "maintenance" : quiet ? "down" : "up";
|
||||
state = subjects.has(`server:${s.id}`) ? "maintenance" : quiet ? "down" : trouble.server.has(s.id) ? "trouble" : "up";
|
||||
}
|
||||
uptime30d = uptime?.servers[item.targetId]?.percent ?? null;
|
||||
} else if (item.kind === "integration") {
|
||||
const i = integrationRows.get(item.targetId);
|
||||
if (i?.enabled) {
|
||||
const status = getSourceStatus(i.type).status;
|
||||
state = subjects.has(`integration:${i.id}`) ? "maintenance" : status === "ok" ? "up" : status === "failing" ? "down" : "unknown";
|
||||
state = subjects.has(`integration:${i.id}`) ? "maintenance" : status === "failing" || failingTests.has(i.id) ? "down" : status === "ok" ? (trouble.integration.has(i.id) ? "trouble" : "up") : "unknown";
|
||||
}
|
||||
uptime30d = uptime?.integrations[item.targetId]?.percent ?? null;
|
||||
} else {
|
||||
@@ -187,7 +217,7 @@ export async function buildPublicStatus(link: { id: number; label: string; title
|
||||
out.push({ id: item.id, label: item.label, state, uptime30d: uptime30d === null ? null : Math.round(uptime30d * 10) / 10 });
|
||||
}
|
||||
|
||||
const counts: Record<ItemState, number> = { up: 0, down: 0, maintenance: 0, unknown: 0 };
|
||||
const counts: Record<ItemState, number> = { up: 0, trouble: 0, down: 0, maintenance: 0, unknown: 0 };
|
||||
for (const o of out) counts[o.state]++;
|
||||
return {
|
||||
title: link.title?.trim() || link.label,
|
||||
|
||||
Reference in new issue
Block a user