Fragnet game servers: games, addresses, backups, power actions

New integration kind "fragnet" reading Fragnet's Pterodactyl-based panel through the
client API (account key). Game servers page under Infrastructure with game, IP:port,
state, memory/disk, backups, schedules and databases; alerts for late backups and
nearly full disk/memory; operators can start/stop/restart/kill (audited). Metrics,
docs and tests included; migration 0071 adds fragnet_state.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
bobbanandClaude Sonnet 5.5 committed 2026-10-27 19:01:36 +01:00
1 parent f86661f6c9
commit 9a3d5c466a
29 files changed
+9836 -8

No files matched your search

@@ -0,0 +1,6 @@
CREATE TABLE `fragnet_state` (
`integration_id` integer PRIMARY KEY NOT NULL,
`servers` text DEFAULT '[]' NOT NULL,
`updated_at` text NOT NULL,
FOREIGN KEY (`integration_id`) REFERENCES `integrations`(`id`) ON UPDATE no action ON DELETE cascade
);
File diff suppressed because it is too large. Load diff
+7
View File
@@ -498,6 +498,13 @@
"when": 1793052224023,
"tag": "0070_absurd_toad",
"breakpoints": true
},
{
"idx": 71,
"version": "6",
"when": 1793122879270,
"tag": "0071_cuddly_strong_guy",
"breakpoints": true
}
]
}
+2
View File
@@ -68,6 +68,7 @@ import { n8nRouter } from "./routes/n8n.js";
import { netboxRouter } from "./routes/netbox.js";
import { hetznerRouter } from "./routes/hetzner.js";
import { speedtestRouter } from "./routes/speedtest.js";
import { fragnetRouter } from "./routes/fragnet.js";
import { costsRouter } from "./routes/costs.js";
import { nextcloudRouter } from "./routes/nextcloud.js";
import { serviceLevelsRouter } from "./routes/serviceLevels.js";
@@ -214,6 +215,7 @@ export function createApp() {
app.use("/api/netbox", netboxRouter);
app.use("/api/hetzner", hetznerRouter);
app.use("/api/speedtest", speedtestRouter);
app.use("/api/fragnet", fragnetRouter);
app.use("/api/costs", costsRouter);
app.use("/api/nextcloud", nextcloudRouter);
app.use("/api/service-levels", serviceLevelsRouter);
+13
View File
@@ -563,6 +563,7 @@ export const integrationTypes = [
"n8n",
"hetzner",
"speedtest",
"fragnet",
] as const;
export type IntegrationType = (typeof integrationTypes)[number];
@@ -1273,6 +1274,18 @@ export const hetznerState = sqliteTable("hetzner_state", {
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── Fragnet game servers ───────────────────────────────────────────────────
// How a Fragnet game panel account (an integration of type "fragnet"; a key belongs to one account) is right now, one row each, replaced at every read: its game servers as JSON — the game,
// the addresses, the limits, what each uses, its backups, schedules and databases — plus when this app first saw each and since when it has had its present state. Read by services/fragnetSync.ts.
export const fragnetState = sqliteTable("fragnet_state", {
integrationId: integer("integration_id")
.primaryKey()
.references(() => integrations.id, { onDelete: "cascade" }),
servers: text("servers").notNull().default("[]"),
updatedAt: text("updated_at").notNull(), // ISO
});
// ─── NetBox ─────────────────────────────────────────────────────────────────
// How a NetBox (an integration of type "netbox") is right now, one row each, replaced at every read: its version, how many background workers are running, the counts and the IP
+2
View File
@@ -40,6 +40,7 @@ import { initNextcloudSync } from "./services/nextcloudSync.js";
import { initDockerVulnSync } from "./services/dockerVulnSync.js";
import { initHetznerSync } from "./services/hetznerSync.js";
import { initSpeedtestSync } from "./services/speedtestSync.js";
import { initFragnetSync } from "./services/fragnetSync.js";
import { backfillMetricRollups } from "./services/serverMetrics.js";
import { backfillEndpointHourly } from "./services/endpointChecks.js";
import { initSelfBackupScheduler } from "./services/selfBackupScheduler.js";
@@ -101,6 +102,7 @@ initNextcloudSync();
initDockerVulnSync();
initHetznerSync();
initSpeedtestSync();
initFragnetSync();
// The hourly and daily figures are built from what is already kept, once, the first time after they were added (in the background: nothing waits for it).
void Promise.all([backfillMetricRollups(), backfillEndpointHourly()]).catch((err) => console.error("[history] couldn't build the hourly and daily figures:", err));
+8
View File
@@ -114,6 +114,13 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
{ key: "insecure", label: "Allow self-signed certificate", secret: false, type: "checkbox" },
],
hetzner: [{ key: "token", label: "API token", secret: true, type: "password", placeholder: "Hetzner Console → the project → Security → API tokens; Read is enough" }],
fragnet: [
{ key: "token", label: "API key", secret: true, type: "password", placeholder: "ptlc_… — the game panel: profile → API Credentials (not an Application key, ptla_)" },
{ key: "url", label: "Panel address (optional)", secret: false, optional: true, placeholder: "https://panel.fragnet.net — only if yours is somewhere else" },
{ key: "backupDays", label: "Warn when the newest backup is older than, days (optional)", secret: false, optional: true, placeholder: "7 — 0 to not judge backups" },
{ key: "fullPercent", label: "Warn when disk or memory is fuller than, % (optional)", secret: false, optional: true, placeholder: "90" },
{ key: "ignoreBackups", label: "Servers whose backups are not judged (optional)", secret: false, optional: true, placeholder: "names, separated by commas" },
],
netbox: [
{ key: "url", label: "NetBox URL", secret: false, placeholder: "https://netbox.example.lan" },
{ key: "token", label: "API token", secret: true, type: "password", placeholder: "your profile → API Tokens; read-only is enough" },
@@ -189,6 +196,7 @@ export const INTEGRATION_FIELDS: Partial<Record<IntegrationType, IntegrationFiel
export const INTEGRATION_BASE_URLS: Partial<Record<IntegrationType, string>> = {
tailscale: "https://api.tailscale.com",
hetzner: "https://api.hetzner.cloud",
fragnet: "https://panel.fragnet.net",
};
/**
+359
View File
@@ -0,0 +1,359 @@
/**
* Fragnet game server adapter. Fragnet's game panel (https://panel.fragnet.net) is a fork of Pterodactyl, so this speaks Pterodactyl's *client* API
* (https://dashflo.net/docs/api/pterodactyl/v1/#req_…, base `/api/client`). Config: token (the account's API key — the panel's profile menu → API Credentials; it starts with **ptlc_**, an
* Application key, ptla_, does not work), optionally url (the panel; https://panel.fragnet.net when left out) and the three that the alerts use (see services/fragnetSync.ts): backupDays,
* fullPercent, ignoreBackups.
*
* Reading: the servers of the account (`GET /api/client`, paged, with the game — the egg — and the addresses) and, for each, `…/resources` (running or not, memory, CPU, disk, uptime), `…/backups`,
* `…/schedules` and `…/databases`. A part that cannot be read (a sub-user without that permission gets a 403) is null for that server, which is not the same as none. The only thing ever
* written is a **power signal** (`POST …/power`: start, stop, restart or kill), and only when someone asks for it on the page.
*
* Endpoints and field names are from Pterodactyl's source (ServerTransformer, StatsTransformer, BackupTransformer, ScheduleTransformer) and the panel's own documentation (docs.fragnet.net),
* not from a live account: Fragnet's fork may differ, so every field is read defensively. The panel allows a limited number of requests a minute (429); a pass makes four per server.
*/
import { requestJson } from "../httpJson.js";
import { withDiagLogging } from "../../services/diagLog.js";
export const DEFAULT_URL = "https://panel.fragnet.net";
export interface FragnetConfig {
token: string;
url?: string;
}
export interface FragnetAllocation {
ip: string;
/** A name the host gave the address (ip_alias), when there is one. */
alias: string | null;
port: number;
/** The server's main address. */
isDefault: boolean;
notes: string | null;
}
export interface FragnetBackup {
uuid: string;
name: string | null;
bytes: number;
createdAt: string | null;
/** Null while it is still being made. */
completedAt: string | null;
successful: boolean;
locked: boolean;
}
export interface FragnetSchedule {
id: number;
name: string;
/** The cron fields as one text: "minute hour day-of-month month day-of-week". */
cron: string;
active: boolean;
processing: boolean;
onlyWhenOnline: boolean;
lastRunAt: string | null;
nextRunAt: string | null;
}
export interface FragnetUsage {
memoryBytes: number;
/** Percent of one core (200 = two cores busy). */
cpu: number;
diskBytes: number;
rxBytes: number;
txBytes: number;
/** Milliseconds since it was started; 0 when it is not running. */
uptimeMs: number;
}
export interface FragnetServer {
/** The short id (8 characters) the panel uses in its addresses and API. */
id: string;
uuid: string;
name: string;
description: string | null;
node: string | null;
/** The game, as the panel names its egg ("Minecraft: Paper", "Rust"); null when the panel did not say. */
game: string | null;
dockerImage: string | null;
/** The panel's own status: installing, install_failed, reinstall_failed, suspended, restoring_backup; null when it is none of them. */
panelStatus: string | null;
suspended: boolean;
installing: boolean;
transferring: boolean;
/** Whether the account owns it (false: it was shared with the account as a sub-user). */
owner: boolean;
/** running, starting, stopping, offline — or unknown when the resources could not be read. */
state: string;
allocations: FragnetAllocation[];
/** Where SFTP is, as "host:port"; null when not said. */
sftp: string | null;
/** MB; 0 means no limit. */
memoryLimitMb: number | null;
diskLimitMb: number | null;
/** Percent of one core; 0 means no limit. */
cpuLimit: number | null;
databaseLimit: number | null;
backupLimit: number | null;
allocationLimit: number | null;
/** Null when the resources could not be read. */
usage: FragnetUsage | null;
/** Newest first; null when they could not be listed. */
backups: FragnetBackup[] | null;
schedules: FragnetSchedule[] | null;
databases: number | null;
}
export type PowerSignal = "start" | "stop" | "restart" | "kill";
export const POWER_SIGNALS: PowerSignal[] = ["start", "stop", "restart", "kill"];
export interface FragnetRead {
servers: FragnetServer[];
}
export interface FragnetAdapter {
ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }>;
read(): Promise<FragnetRead>;
power(serverId: string, signal: PowerSignal): Promise<void>;
}
const PER_PAGE = 50;
export const MAX_PAGES = 10;
/** The newest backups are what matters; a server rarely has more than a few. */
const BACKUP_PAGES = 3;
export const SERVER_ID = /^[A-Za-z0-9]{8}$/;
const obj = (v: unknown): Record<string, unknown> => (v !== null && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {});
const arr = (v: unknown): unknown[] => (Array.isArray(v) ? v : []);
const str = (v: unknown): string | null => (typeof v === "string" && v.trim() !== "" ? v.trim() : null);
const num = (v: unknown): number | null => {
const n = typeof v === "string" && v.trim() !== "" ? Number(v) : v;
return typeof n === "number" && Number.isFinite(n) ? n : null;
};
const iso = (v: unknown): string | null => {
const s = str(v);
return s !== null && Number.isFinite(Date.parse(s)) ? new Date(s).toISOString() : null;
};
/** What a Pterodactyl list wraps each item in: `{ object, attributes }`. */
const attrs = (item: unknown): Record<string, unknown> => obj(obj(item).attributes);
/** Pure: the page after this one, or null when this is the last. */
export function nextPage(raw: unknown): number | null {
const p = obj(obj(obj(raw).meta).pagination);
const current = num(p.current_page);
const total = num(p.total_pages);
return current !== null && total !== null && current < total ? current + 1 : null;
}
/** Pure: the addresses of a server (its `relationships.allocations`). */
export function parseAllocations(relationships: unknown): FragnetAllocation[] {
const out: FragnetAllocation[] = [];
for (const item of arr(obj(obj(relationships).allocations).data)) {
const a = attrs(item);
const ip = str(a.ip);
const port = num(a.port);
if (!ip || port === null) continue;
out.push({ ip, alias: str(a.ip_alias), port, isDefault: a.is_default === true, notes: str(a.notes) });
}
// The main address first, then by port.
return out.sort((x, y) => Number(y.isDefault) - Number(x.isDefault) || x.port - y.port);
}
/** Pure: the servers of one page of `GET /api/client` — what the list says of each, before its resources, backups and so on are asked for. */
export function parseServers(raw: unknown): FragnetServer[] {
const out: FragnetServer[] = [];
for (const item of arr(obj(raw).data)) {
const s = attrs(item);
const id = str(s.identifier);
const name = str(s.name);
if (!id || !SERVER_ID.test(id) || !name) continue;
const limits = obj(s.limits);
const feature = obj(s.feature_limits);
const sftp = obj(s.sftp_details);
const sftpHost = str(sftp.ip);
const rel = obj(s.relationships);
const egg = attrs(obj(rel.egg));
const panelStatus = str(s.status);
out.push({
id,
uuid: str(s.uuid) ?? id,
name,
description: str(s.description),
node: str(s.node),
game: str(egg.name),
dockerImage: str(s.docker_image),
panelStatus,
suspended: s.is_suspended === true || panelStatus === "suspended",
installing: s.is_installing === true || panelStatus === "installing",
transferring: s.is_transferring === true,
owner: s.server_owner !== false,
state: "unknown",
allocations: parseAllocations(rel),
sftp: sftpHost ? `${sftpHost}${num(sftp.port) !== null ? `:${num(sftp.port)}` : ""}` : null,
memoryLimitMb: num(limits.memory),
diskLimitMb: num(limits.disk),
cpuLimit: num(limits.cpu),
databaseLimit: num(feature.databases),
backupLimit: num(feature.backups),
allocationLimit: num(feature.allocations),
usage: null,
backups: null,
schedules: null,
databases: null,
});
}
return out;
}
/** Pure: `…/resources` — whether it runs and what it uses. Null when the answer is not that. */
export function parseResources(raw: unknown): { state: string; suspended: boolean; usage: FragnetUsage } | null {
const a = obj(obj(raw).attributes);
const state = str(a.current_state);
if (!state) return null;
const r = obj(a.resources);
return {
state: state.toLowerCase(),
suspended: a.is_suspended === true,
usage: { memoryBytes: num(r.memory_bytes) ?? 0, cpu: num(r.cpu_absolute) ?? 0, diskBytes: num(r.disk_bytes) ?? 0, rxBytes: num(r.network_rx_bytes) ?? 0, txBytes: num(r.network_tx_bytes) ?? 0, uptimeMs: num(r.uptime) ?? 0 },
};
}
/** Pure: the backups of one page of `…/backups`, newest first. */
export function parseBackups(raw: unknown): FragnetBackup[] {
const out: FragnetBackup[] = [];
for (const item of arr(obj(raw).data)) {
const b = attrs(item);
const uuid = str(b.uuid);
if (!uuid) continue;
out.push({ uuid, name: str(b.name), bytes: num(b.bytes) ?? 0, createdAt: iso(b.created_at), completedAt: iso(b.completed_at), successful: b.is_successful === true, locked: b.is_locked === true });
}
return out.sort((x, y) => (y.createdAt ?? "").localeCompare(x.createdAt ?? ""));
}
/** Pure: `…/schedules` — the tasks that run on their own (a restart every night, a backup every Sunday). */
export function parseSchedules(raw: unknown): FragnetSchedule[] {
const out: FragnetSchedule[] = [];
for (const item of arr(obj(raw).data)) {
const s = attrs(item);
const id = num(s.id);
const name = str(s.name);
if (id === null || !name) continue;
const c = obj(s.cron);
const cron = [c.minute, c.hour, c.day_of_month, c.month, c.day_of_week].map((x) => (typeof x === "string" || typeof x === "number" ? String(x) : "*")).join(" ");
out.push({ id, name, cron, active: s.is_active !== false, processing: s.is_processing === true, onlyWhenOnline: s.only_when_online === true, lastRunAt: iso(s.last_run_at), nextRunAt: iso(s.next_run_at) });
}
return out;
}
export function createFragnetAdapter(config: FragnetConfig): FragnetAdapter {
const base = `${String(config.url ?? "").trim().replace(/\/+$/, "") || DEFAULT_URL}/api/client`;
const token = String(config.token ?? "").trim();
const headers = { Authorization: `Bearer ${token}`, Accept: "application/json" };
/** `forbiddenIsFine`: a 403 on one part of one server is that part's permission (a sub-user), not a key the panel refuses. */
function refusal(status: number, forbiddenIsFine = false): Error | null {
if (status === 401 || (status === 403 && !forbiddenIsFine)) {
return new Error(
token.startsWith("ptla_")
? `The panel did not accept the key (HTTP ${status}) — that is an Application key (ptla_); this needs the account's own API key, which starts with ptlc_`
: `The panel did not accept the key (HTTP ${status}) — make a new one under the profile menu → API Credentials (it starts with ptlc_), and check it has not been deleted or limited to other addresses`,
);
}
if (status === 429) return new Error("The panel says too many requests (HTTP 429) — it will be tried again at the next read");
return null;
}
async function get(path: string, forbiddenIsFine = false): Promise<{ status: number; json: unknown }> {
const res = await requestJson(`${base}${path}`, { headers });
const refused = refusal(res.status, forbiddenIsFine);
if (refused) throw refused;
return { status: res.status, json: res.json };
}
/** A part of a server that may be missing for it: null if it could not be read (a 403 for a sub-user without that permission, a 404, a server that is being installed). */
async function part<T>(path: string, parse: (json: unknown) => T): Promise<T | null> {
try {
const res = await get(path, true);
if (res.status < 200 || res.status >= 300 || res.json === null) return null;
return parse(res.json);
} catch (err) {
// A key the panel has stopped accepting, or a panel that is rate-limiting, is not "this part is missing": the whole read has failed.
if (err instanceof Error && /^(The panel did not accept|The panel says too many)/.test(err.message)) throw err;
return null;
}
}
async function listServers(): Promise<FragnetServer[]> {
const all: FragnetServer[] = [];
let page: number | null = 1;
for (let i = 0; i < MAX_PAGES && page !== null; i++) {
let res = await get(`?include=egg,allocations&per_page=${PER_PAGE}&page=${page}`);
// A panel that refuses what it does not know of the request is asked again plainly (the addresses come with a server anyway).
if (res.status >= 400 && res.status < 500) res = await get(`?per_page=${PER_PAGE}&page=${page}`);
if (res.status < 200 || res.status >= 300) throw new Error(`The panel answered with an error: HTTP ${res.status}`);
if (res.json === null) throw new Error("The panel answered with something that isn't JSON — is the address its own?");
all.push(...parseServers(res.json));
page = nextPage(res.json);
}
return all;
}
async function backupsOf(id: string): Promise<FragnetBackup[] | null> {
const all: FragnetBackup[] = [];
let page: number | null = 1;
for (let i = 0; i < BACKUP_PAGES && page !== null; i++) {
const res: { backups: FragnetBackup[]; next: number | null } | null = await part(`/servers/${id}/backups?per_page=50&page=${page}`, (json) => ({ backups: parseBackups(json), next: nextPage(json) }));
if (res === null) return i === 0 ? null : all;
all.push(...res.backups);
page = res.next;
}
return all.sort((x, y) => (y.createdAt ?? "").localeCompare(x.createdAt ?? ""));
}
async function read(): Promise<FragnetRead> {
const servers = await listServers();
const out: FragnetServer[] = [];
for (const s of servers) {
const [resources, backups, schedules, databases] = await Promise.all([
part(`/servers/${s.id}/resources`, parseResources),
backupsOf(s.id),
part(`/servers/${s.id}/schedules`, parseSchedules),
part(`/servers/${s.id}/databases`, (json) => arr(obj(json).data).length),
]);
out.push({
...s,
state: resources?.state ?? "unknown",
suspended: s.suspended || resources?.suspended === true,
usage: resources?.usage ?? null,
backups,
schedules,
databases,
});
}
return { servers: out };
}
async function power(serverId: string, signal: PowerSignal): Promise<void> {
if (!SERVER_ID.test(serverId)) throw new Error("That isn't a server id of the panel");
if (!POWER_SIGNALS.includes(signal)) throw new Error("That isn't a power signal");
const res = await requestJson(`${base}/servers/${serverId}/power`, { method: "POST", headers, body: { signal } });
const refused = refusal(res.status);
if (refused) throw refused;
if (res.status === 409) throw new Error("The panel says the server is busy (HTTP 409) — it is being installed, transferred or restored, or is suspended");
if (res.status === 404) throw new Error("The panel does not know that server (HTTP 404)");
if (res.status < 200 || res.status >= 300) throw new Error(`The panel answered with an error: HTTP ${res.status}`);
}
async function ping(): Promise<{ ok: boolean; latencyMs?: number; error?: string }> {
const start = Date.now();
try {
const res = await get("?per_page=1");
if (res.status < 200 || res.status >= 300) throw new Error(`The panel answered with an error: HTTP ${res.status} — check the address`);
if (res.json === null) throw new Error("The panel answered with something that isn't JSON — is the address its own?");
return { ok: true, latencyMs: Date.now() - start };
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) };
}
}
return withDiagLogging("fragnet", { ping, read, power });
}
+3
View File
@@ -23,6 +23,7 @@ import { createN8nAdapter } from "./n8n/adapter.js";
import { createNetBoxAdapter } from "./netbox/adapter.js";
import { createHetznerAdapter } from "./hetzner/adapter.js";
import { createSpeedtestAdapter } from "./speedtest/adapter.js";
import { createFragnetAdapter } from "./fragnet/adapter.js";
import { createNextcloudAdapter } from "./nextcloud/adapter.js";
export interface PingableAdapter {
@@ -82,6 +83,8 @@ export function createIntegrationAdapter(type: IntegrationType, config: Integrat
return createSpeedtestAdapter(config as any);
case "hetzner":
return createHetznerAdapter(config as any);
case "fragnet":
return createFragnetAdapter(config as any);
case "netbox":
return createNetBoxAdapter(config as any);
case "nextcloud":
+94
View File
@@ -0,0 +1,94 @@
import { Router } from "express";
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { fragnetState, integrations } from "../db/schema.js";
import { requireAuth, requireRole } from "../auth/middleware.js";
import { createFragnetAdapter, POWER_SIGNALS, SERVER_ID, type PowerSignal } from "../integrations/fragnet/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { recordAudit } from "../services/audit.js";
import { invalidateAlertsCache } from "../services/alerts.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, newestBackup, optionsOf, serversOf, syncFragnet } from "../services/fragnetSync.js";
import { asyncHandler } from "../utils/asyncHandler.js";
export const fragnetRouter = Router();
fragnetRouter.use(requireAuth);
/** Every Fragnet account, how it is, from what the last read stored: its game servers (game, addresses, use, backups, schedules) and what is wrong. Everyone signed in can see it. */
fragnetRouter.get("/", asyncHandler(async (_req, res) => {
const [rows, states, statuses] = await Promise.all([
db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled, baseUrl: integrations.baseUrl }).from(integrations).where(eq(integrations.type, "fragnet")).orderBy(integrations.name),
db.select().from(fragnetState),
fragnetSyncStatuses(),
]);
const byId = new Map(states.map((s) => [s.integrationId, s]));
const now = Date.now();
const accounts = [];
for (const r of rows) {
const state = byId.get(r.id) ?? null;
const status = statuses[r.id] ?? null;
const fresh = isFragnetFresh(status ?? undefined, now);
const stored = state ? serversOf(state) : [];
let options = optionsOf(null);
try {
options = optionsOf((await loadIntegrationConfig(r.id))?.config);
} catch {
// a config that cannot be read costs only the settings it holds
}
accounts.push({
id: r.id,
name: r.name,
enabled: r.enabled,
panelUrl: r.baseUrl,
status,
fresh,
options,
state: state ? { servers: stored.map((s) => ({ ...s, newestBackup: newestBackup(s) })), updatedAt: state.updatedAt } : null,
problems: state && fresh ? fragnetProblems(r.id, r.name, stored, now, options) : [],
});
}
res.json({ accounts });
}));
/** Reads one account now, instead of at the next pass (what the page does a few seconds after a power signal). */
fragnetRouter.post("/:id/refresh", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const [row] = await db.select({ type: integrations.type, enabled: integrations.enabled }).from(integrations).where(eq(integrations.id, id)).limit(1);
if (!row) return res.status(404).json({ error: "not_found" });
if (row.type !== "fragnet") return res.status(400).json({ error: "wrong_type" });
if (!row.enabled) return res.status(400).json({ error: "integration_disabled" });
const status = await syncFragnet(id);
invalidateAlertsCache();
res.json({ status });
}));
/** Sends a power signal (start, stop, restart or kill) to one game server. Only a server the last read found can be signalled — what is typed in the address is never put into a call to the panel. */
fragnetRouter.post("/:id/servers/:server/power", requireRole("operator"), asyncHandler(async (req, res) => {
const id = Number(req.params.id);
if (!Number.isInteger(id)) return res.status(400).json({ error: "invalid_id" });
const signal = req.body?.signal as PowerSignal;
if (!POWER_SIGNALS.includes(signal)) return res.status(400).json({ error: "invalid_signal" });
const serverId = String(req.params.server);
if (!SERVER_ID.test(serverId)) return res.status(400).json({ error: "invalid_server" });
const loaded = await loadIntegrationConfig(id);
if (!loaded) return res.status(404).json({ error: "not_found" });
if (loaded.integration.type !== "fragnet") return res.status(400).json({ error: "wrong_type" });
if (!loaded.integration.enabled) return res.status(400).json({ error: "integration_disabled" });
const [state] = await db.select().from(fragnetState).where(eq(fragnetState.integrationId, id)).limit(1);
const server = state ? serversOf(state).find((s) => s.id === serverId) : undefined;
if (!server) return res.status(404).json({ error: "server_not_found" });
try {
await createFragnetAdapter(loaded.config as never).power(serverId, signal);
} catch (err) {
return res.status(502).json({ error: err instanceof Error ? err.message : String(err) });
}
await recordAudit({
actor: req.currentUser!,
category: "integration",
action: `${signal}_game_server`,
targetType: "fragnet_server",
targetId: serverId,
detail: { integrationId: id, name: server.name, game: server.game },
});
res.status(204).end();
}));
+4
View File
@@ -38,6 +38,7 @@ import { forgetN8n, syncN8n } from "../services/n8nSync.js";
import { forgetNetBox, syncNetBox } from "../services/netboxSync.js";
import { forgetHetzner, syncHetzner } from "../services/hetznerSync.js";
import { forgetSpeedtest, syncSpeedtest } from "../services/speedtestSync.js";
import { forgetFragnet, syncFragnet } from "../services/fragnetSync.js";
import { forgetNextcloud, syncNextcloud } from "../services/nextcloudSync.js";
import { dockerVulnSyncStatuses, findingsOf, forgetDockerVulns, isDockerVulnFresh, syncDockerVulns } from "../services/dockerVulnSync.js";
import { invalidateAlertsCache } from "../services/alerts.js";
@@ -151,6 +152,7 @@ integrationsRouter.post("/", requireRole("admin"), asyncHandler(async (req, res)
if (type === "netbox") void syncNetBox(created.id).then(() => invalidateAlertsCache());
if (type === "hetzner") void syncHetzner(created.id).then(() => invalidateAlertsCache());
if (type === "speedtest") void syncSpeedtest(created.id).then(() => invalidateAlertsCache());
if (type === "fragnet") void syncFragnet(created.id).then(() => invalidateAlertsCache());
if (type === "nextcloud") void syncNextcloud(created.id).then(() => invalidateAlertsCache());
if (type === "dockhand") void syncDockerVulns(created.id).then(() => invalidateAlertsCache());
@@ -265,6 +267,7 @@ integrationsRouter.patch("/:id", requireRole("admin"), asyncHandler(async (req,
if (updated.type === "netbox" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNetBox(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "hetzner" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncHetzner(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "speedtest" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncSpeedtest(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "fragnet" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncFragnet(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "nextcloud" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncNextcloud(updated.id).then(() => invalidateAlertsCache());
if (updated.type === "dockhand" && updated.enabled && (parsed.data.config || parsed.data.enabled === true)) void syncDockerVulns(updated.id).then(() => invalidateAlertsCache());
@@ -355,6 +358,7 @@ integrationsRouter.delete("/:id", requireRole("admin"), asyncHandler(async (req,
if (existing.type === "netbox") await forgetNetBox(id);
if (existing.type === "hetzner") await forgetHetzner(id);
if (existing.type === "speedtest") await forgetSpeedtest(id);
if (existing.type === "fragnet") await forgetFragnet(id);
if (existing.type === "nextcloud") await forgetNextcloud(id);
if (existing.type === "dockhand") await forgetDockerVulns(id);
await db.delete(integrations).where(eq(integrations.id, id));
+5 -2
View File
@@ -85,6 +85,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
hetzner: { label: "Hetzner", link: "/hetzner" },
speedtest: { label: "Speedtest", link: "/speedtest" },
fragnet: { label: "Fragnet", link: "/game-servers" },
secrets: { label: "Secrets", link: "/secrets" },
domains: { label: "Domains", link: "/dns?view=domains" },
endpoint: { label: "Endpoint", link: "/endpoints" },
@@ -127,6 +128,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
@@ -267,7 +270,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
type: origin.type,
message: c.message,
key: c.key,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : undefined,
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : undefined,
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
});
}
@@ -277,7 +280,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
for (const h of held) {
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest):(\d+)(?::(.+))?$/.exec(h);
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet):(\d+)(?::(.+))?$/.exec(h);
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
const row = intById.get(Number(m[2]));
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
+244
View File
@@ -0,0 +1,244 @@
/**
* Keeps what Fragnet's game panel says up to date: every five minutes each Fragnet integration (one account) is read — its game servers, their addresses, how much they use, their backups,
* schedules and databases — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to the panel (which limits how many requests are made a minute).
*
* What is raised: **a backup that is late** — a server that may have backups (its plan allows some) whose newest successful one is older than `backupDays` (7 unless you say otherwise; 0 =
* not judged), or that has none and has been known for longer than that; a server can be left out by name (`ignoreBackups`). **Disk or memory nearly full** — over `fullPercent` (90 unless you
* say otherwise) of its limit; memory is judged only while it runs. A server that is stopped is not an alert: game servers are often stopped on purpose. A panel that cannot be reached shows
* as the integration being down, like any other. "Since when" a server has been in its state is what this app saw: the time of the first read that showed it.
*/
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { fragnetState, integrations } from "../db/schema.js";
import { createFragnetAdapter, type FragnetRead, type FragnetServer } from "../integrations/fragnet/adapter.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
export const SYNC_INTERVAL_MS = 5 * 60_000;
/** After this long without a good read, what is stored is no longer believed. */
export const STALE_AFTER_MS = 20 * 60_000;
const STATUS_FLAG = "fragnetSync";
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
export const DEFAULT_BACKUP_DAYS = 7;
export const DEFAULT_FULL_PERCENT = 90;
/** A disk this full is critical, whatever the limit for a warning is. */
export const CRITICAL_PERCENT = 98;
export interface FragnetSyncStatus {
/** The last read that worked (kept when a later one failed). */
at: string;
ok: boolean;
error: string | null;
}
async function readStatuses(): Promise<Record<string, FragnetSyncStatus>> {
try {
const raw = await getInternalFlag(STATUS_FLAG);
return raw ? (JSON.parse(raw) as Record<string, FragnetSyncStatus>) : {};
} catch {
return {};
}
}
export async function fragnetSyncStatuses(): Promise<Record<number, FragnetSyncStatus>> {
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
}
export const isFragnetFresh = (status: FragnetSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
export type StateRow = typeof fragnetState.$inferSelect;
/** A server as stored: as read, plus what the app has seen of it between reads. */
export interface StoredServer extends FragnetServer {
/** When the first read that showed its present state was made. */
stateSince: string;
/** When the first read that showed the server was made. */
firstSeenAt: string;
}
export const serversOf = (row: Pick<StateRow, "servers">): StoredServer[] => {
try {
const v = JSON.parse(row.servers);
return Array.isArray(v) ? (v as StoredServer[]) : [];
} catch {
return [];
}
};
/** Pure: what the app has seen between reads, carried from the last read of the same server. A server first seen starts its clocks now. */
export function carryTimes(servers: FragnetServer[], previous: StoredServer[], now: Date): StoredServer[] {
const before = new Map(previous.map((p) => [p.id, p]));
const at = now.toISOString();
return servers.map((s) => {
const p = before.get(s.id);
return { ...s, stateSince: p && p.state === s.state ? p.stateSince : at, firstSeenAt: p?.firstSeenAt ?? at };
});
}
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
export async function storeRead(integrationId: number, r: FragnetRead, now: Date = new Date()): Promise<void> {
const [previous] = await db.select({ servers: fragnetState.servers }).from(fragnetState).where(eq(fragnetState.integrationId, integrationId)).limit(1);
const values = { servers: JSON.stringify(carryTimes(r.servers, previous ? serversOf(previous) : [], now)), updatedAt: now.toISOString() };
await db.insert(fragnetState).values({ integrationId, ...values }).onConflictDoUpdate({ target: fragnetState.integrationId, set: values });
}
/** Reads one account and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
export async function syncFragnet(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<FragnetRead>): Promise<FragnetSyncStatus> {
let status: FragnetSyncStatus;
try {
const loaded = await loadIntegrationConfig(integrationId);
if (!loaded || loaded.integration.type !== "fragnet") throw new Error("not a Fragnet integration");
const result = await (read ? read(loaded.config) : createFragnetAdapter(loaded.config as never).read());
await storeRead(integrationId, result, now);
status = { at: now.toISOString(), ok: true, error: null };
} catch (err) {
const previous = (await readStatuses())[String(integrationId)];
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
console.error(`[fragnet] couldn't read integration ${integrationId}:`, status.error);
}
try {
const all = await readStatuses();
all[String(integrationId)] = status;
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
} catch (err) {
console.error("[fragnet] couldn't record the result:", err);
}
return status;
}
export async function syncAllFragnet(now: Date = new Date()): Promise<Record<number, FragnetSyncStatus>> {
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
const out: Record<number, FragnetSyncStatus> = {};
for (const r of rows) out[r.id] = await syncFragnet(r.id, now);
return out;
}
export async function forgetFragnet(integrationId: number): Promise<void> {
const all = await readStatuses();
if (String(integrationId) in all) {
delete all[String(integrationId)];
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
}
}
// ─── What is wrong ──────────────────────────────────────────────────────────
/** What the integration's own settings say about how strict to be. */
export interface FragnetOptions {
/** A backup older than this many days is late; 0: backups are not judged. */
backupDays: number;
/** Percent of a limit above which disk and memory are called nearly full. */
fullPercent: number;
/** Names (lower case) of the servers whose backups are not judged. */
ignoreBackups: string[];
}
const positive = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v.trim().replace(",", ".")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
export function optionsOf(config: Record<string, unknown> | null | undefined): FragnetOptions {
const days = positive(config?.backupDays);
const percent = positive(config?.fullPercent);
const ignore = typeof config?.ignoreBackups === "string" ? config.ignoreBackups : "";
return {
backupDays: days === null ? DEFAULT_BACKUP_DAYS : days,
fullPercent: percent === null || percent < 1 ? DEFAULT_FULL_PERCENT : Math.min(100, percent),
ignoreBackups: ignore.split(/[,\n]/).map((s) => s.trim().toLowerCase()).filter(Boolean),
};
}
export interface FragnetProblem {
key: string;
/** The panel's short id of the server it is about. */
serverId: string;
message: string;
severity: "warning" | "critical";
}
/** In the units the page shows (binary, written MB, GB or TB), so an alert and the tile beside it agree. */
export const sizeText = (bytes: number): string => {
const mb = bytes / 1024 ** 2;
if (mb < 1024) return `${Math.round(mb)} MB`;
const gb = mb / 1024;
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
};
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
/** Pure: the newest backup that finished and worked (its time), and whether a newer one failed. */
export function newestBackup(s: Pick<FragnetServer, "backups">): { at: string | null; failedSince: boolean } | null {
if (s.backups === null) return null;
const done = s.backups.filter((b) => b.completedAt !== null);
const good = done.filter((b) => b.successful).map((b) => b.completedAt as string).sort().pop() ?? null;
const failedSince = done.some((b) => !b.successful && (good === null || (b.completedAt as string) > good));
return { at: good, failedSince };
}
/** Pure: whether a server is one the panel lets make backups (its plan has room for some). */
const mayHaveBackups = (s: FragnetServer) => s.backupLimit !== null && s.backupLimit > 0;
/** Pure: what is wrong with one account, from what is stored for it. */
export function fragnetProblems(id: number, name: string, servers: StoredServer[], nowMs: number, options: FragnetOptions): FragnetProblem[] {
const out: FragnetProblem[] = [];
const limit = options.fullPercent / 100;
for (const s of [...servers].sort((a, b) => a.name.localeCompare(b.name))) {
// A server being installed or suspended is in no state to judge.
if (s.suspended || s.installing) continue;
if (options.backupDays > 0 && mayHaveBackups(s) && !options.ignoreBackups.includes(s.name.toLowerCase())) {
const newest = newestBackup(s);
if (newest !== null) {
const window = options.backupDays * DAY;
const age = newest.at === null ? null : nowMs - Date.parse(newest.at);
const young = nowMs - Date.parse(s.firstSeenAt) < window; // a server this app has only just met has had no time to be backed up
if (age === null ? !young : age > window) {
out.push({
key: `fragnet-backup:${id}:${s.id}`,
serverId: s.id,
severity: "warning",
message: `${name}: ${s.name} ${age === null ? "has no backup" : `has no backup newer than ${ago(age)}`}${newest.failedSince ? " (the newest attempt failed)" : ""} — it should have one at least every ${plural(options.backupDays, "day")}`,
});
}
}
}
if (s.usage !== null && s.diskLimitMb !== null && s.diskLimitMb > 0) {
const share = s.usage.diskBytes / (s.diskLimitMb * 1024 ** 2);
if (share >= limit) {
out.push({
key: `fragnet-disk:${id}:${s.id}`,
serverId: s.id,
severity: share >= CRITICAL_PERCENT / 100 ? "critical" : "warning",
message: `${name}: ${s.name} has used ${sizeText(s.usage.diskBytes)} of its ${sizeText(s.diskLimitMb * 1024 ** 2)} disk (${Math.floor(share * 100)}%)`,
});
}
}
if (s.state === "running" && s.usage !== null && s.memoryLimitMb !== null && s.memoryLimitMb > 0) {
const share = s.usage.memoryBytes / (s.memoryLimitMb * 1024 ** 2);
if (share >= limit) {
out.push({
key: `fragnet-memory:${id}:${s.id}`,
serverId: s.id,
severity: "warning",
message: `${name}: ${s.name} is using ${sizeText(s.usage.memoryBytes)} of its ${sizeText(s.memoryLimitMb * 1024 ** 2)} memory (${Math.floor(share * 100)}%) — past the limit the game is stopped by the host`,
});
}
}
}
return out;
}
let timer: ReturnType<typeof setInterval> | null = null;
export function initFragnetSync(): void {
if (timer) return;
const pass = () => void syncAllFragnet().catch((err) => console.error("[fragnet] pass failed:", err));
timer = setInterval(pass, SYNC_INTERVAL_MS);
setTimeout(pass, 33_000);
console.log(`[fragnet] reading Fragnet every ${SYNC_INTERVAL_MS / 60000} minutes`);
}
+26 -1
View File
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState } from "../db/schema.js";
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState } from "../db/schema.js";
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
@@ -31,6 +31,7 @@ import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type Nextcl
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
import { recordStorageSamples } from "./storageHistory.js";
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
@@ -125,6 +126,8 @@ export interface HealthSnapshot {
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
speedtest?: { integrationId: number; problems: SpeedtestProblem[] }[];
/** What is wrong with each Fragnet game panel account (a backup that is late, a disk or memory nearly full), only while that reading is recent (services/fragnetSync.ts). */
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
/** Containers to leave alone. */
dockerIgnore?: string[];
/** The clock the container judgements are made on (the pass's, so a test can set it). */
@@ -266,6 +269,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
// Fragnet game servers: backups that are late, disks and memory nearly full.
for (const f of snapshot.fragnet ?? []) for (const p of f.problems) out.push({ key: p.key, source: `fragnet:${f.integrationId}`, message: p.message, severity: p.severity });
// Hetzner Cloud: servers that went off, traffic quotas, rescue mode left on, backups that are late.
for (const h of snapshot.hetzner ?? []) for (const p of h.problems) out.push({ key: p.key, source: `hetzner:${h.integrationId}`, message: p.message, severity: p.severity });
@@ -531,6 +537,25 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
}
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
if (rowsF.length > 0) {
const statuses = await fragnetSyncStatuses();
for (const r of rowsF) {
const [state] = await db.select().from(fragnetState).where(eq(fragnetState.integrationId, r.id)).limit(1);
if (!state || !isFragnetFresh(statuses[r.id], Date.now())) {
held.add(`fragnet:${r.id}`);
continue;
}
const loaded = await loadIntegrationConfig(r.id);
(snapshot.fragnet ??= []).push({ integrationId: r.id, problems: fragnetProblems(r.id, r.name, fragnetServersOf(state), Date.now(), fragnetOptions(loaded?.config)) });
}
}
} catch (err) {
console.error("[health] couldn't judge Fragnet:", err instanceof Error ? err.message : err);
}
// Hetzner Cloud: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
try {
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
+2
View File
@@ -41,6 +41,8 @@ export function subjectOfConditionKey(key: string): string | null {
if (m) return `integration:${m[1]}`;
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^fragnet-(?:disk|memory|backup):(\d+):/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^netbox-(?:workers):(\d+)/.exec(key);
if (m) return `integration:${m[1]}`;
m = /^n8n-(?:failed):(\d+)/.exec(key);
+25 -1
View File
@@ -5,7 +5,7 @@
*/
import { eq } from "drizzle-orm";
import { db } from "../db/client.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState } from "../db/schema.js";
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState } from "../db/schema.js";
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
import { levelStatuses } from "./serviceLevels.js";
import { backupState, listAll as listDuplicati } from "./duplicati.js";
@@ -14,6 +14,7 @@ import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
@@ -100,6 +101,28 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
}
}
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
async function addFragnetMetrics(out: Exposition, now: Date) {
const statuses = await fragnetSyncStatuses();
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "fragnet"))).map((r) => [r.id, r]));
for (const s of await db.select().from(fragnetState)) {
const row = names.get(s.integrationId);
if (!row || !row.enabled || !isFragnetFresh(statuses[s.integrationId], now.getTime())) continue;
for (const server of fragnetServersOf(s)) {
const labels = { account: row.name, server: server.name, game: server.game ?? "" };
out.add("homelab_fragnet_server_running", "gauge", "1 if the Fragnet game server is running, else 0.", server.state === "running" ? 1 : 0, labels);
if (server.usage) {
out.add("homelab_fragnet_disk_used_bytes", "gauge", "Disk a Fragnet game server uses, in bytes.", server.usage.diskBytes, labels);
out.add("homelab_fragnet_memory_used_bytes", "gauge", "Memory a Fragnet game server uses, in bytes.", server.usage.memoryBytes, labels);
}
if (server.diskLimitMb) out.add("homelab_fragnet_disk_limit_bytes", "gauge", "The disk a Fragnet game server may use, in bytes.", server.diskLimitMb * 1024 ** 2, labels);
if (server.memoryLimitMb) out.add("homelab_fragnet_memory_limit_bytes", "gauge", "The memory a Fragnet game server may use, in bytes.", server.memoryLimitMb * 1024 ** 2, labels);
const backedUp = fragnetNewestBackup(server)?.at;
if (backedUp) out.add("homelab_fragnet_newest_backup_timestamp_seconds", "gauge", "When the newest successful backup of a Fragnet game server finished, in seconds since 1970.", Date.parse(backedUp) / 1000, labels);
}
}
}
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
async function addHetznerMetrics(out: Exposition, now: Date) {
const statuses = await hetznerSyncStatuses();
@@ -442,6 +465,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
await addDockerVulnMetrics(out, now);
await addHetznerMetrics(out, now);
await addSpeedtestMetrics(out, now);
await addFragnetMetrics(out, now);
await addHeartbeatMetrics(out, now);
return out.toString();
}
+505
View File
@@ -0,0 +1,505 @@
/**
* Fragnet game servers (a Pterodactyl panel): reading what the client API says, what is stored and carried between reads, what is wrong (late backups, a disk or memory nearly full), the
* health pass, the alerts, the page's routes, the power signals and /metrics.
*
* Run with: npm test --workspace server
*/
import { test, describe, before, after } from "node:test";
import assert from "node:assert/strict";
import { createServer as createHttpServer, type Server } from "node:http";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
const dir = mkdtempSync(join(tmpdir(), "hm-fragnet-"));
process.env.DATABASE_PATH = join(dir, "test.sqlite");
process.env.SESSION_DIR = join(dir, "sessions");
process.env.CREDENTIALS_ENCRYPTION_KEY = "ab".repeat(32);
process.env.HEALTH_STARTUP_GRACE_MS = "0";
const express = (await import("express")).default;
const { db } = await import("../src/db/client.js");
const { users, integrations, fragnetState, auditLog } = await import("../src/db/schema.js");
const { runMigrations } = await import("../src/db/migrate.js");
const adapter = await import("../src/integrations/fragnet/adapter.js");
const fn = await import("../src/services/fragnetSync.js");
const { evaluateHealth, collectSnapshot } = await import("../src/services/healthMonitor.js");
const { subjectOfConditionKey } = await import("../src/services/maintenance.js");
const { getAlerts, invalidateAlertsCache } = await import("../src/services/alerts.js");
const { fragnetRouter } = await import("../src/routes/fragnet.js");
const { buildMetrics } = await import("../src/services/metrics.js");
const { setInternalFlag } = await import("../src/services/settingsStore.js");
const { eq } = await import("drizzle-orm");
await runMigrations();
const MIN = 60_000;
const HOUR = 60 * MIN;
const DAY = 24 * HOUR;
const MB = 1024 ** 2;
const GB = 1024 ** 3;
const iso = (ms: number) => new Date(ms).toISOString();
type Srv = import("../src/integrations/fragnet/adapter.js").FragnetServer;
type Read = import("../src/integrations/fragnet/adapter.js").FragnetRead;
type Backup = import("../src/integrations/fragnet/adapter.js").FragnetBackup;
const server = (extra: Partial<Srv> = {}): Srv => ({
id: "abcd1234",
uuid: "abcd1234-0000-0000-0000-000000000000",
name: "survival",
description: null,
node: "stockholm-1",
game: "Minecraft: Paper",
dockerImage: "ghcr.io/pterodactyl/yolks:java_21",
panelStatus: null,
suspended: false,
installing: false,
transferring: false,
owner: true,
state: "running",
allocations: [{ ip: "203.0.113.20", alias: null, port: 25565, isDefault: true, notes: null }],
sftp: "sftp.fragnet.net:2022",
memoryLimitMb: 4096,
diskLimitMb: 20_480,
cpuLimit: 200,
databaseLimit: 1,
backupLimit: 3,
allocationLimit: 1,
usage: { memoryBytes: 2 * GB, cpu: 35, diskBytes: 5 * GB, rxBytes: 1000, txBytes: 2000, uptimeMs: 3 * HOUR },
backups: [],
schedules: [],
databases: 0,
...extra,
});
const read = (...s: Srv[]): Read => ({ servers: s });
const okBackup = (ageMs: number, extra: Partial<Backup> = {}, now = Date.now()): Backup => ({ uuid: `b-${ageMs}-${extra.successful}`, name: "auto", bytes: 5 * MB, createdAt: iso(now - ageMs - MIN), completedAt: iso(now - ageMs), successful: true, locked: false, ...extra });
// What Pterodactyl's client API sends ({ object, attributes } wrappers).
const apiServer = (identifier: string, name: string, extra: Record<string, unknown> = {}) => ({
object: "server",
attributes: {
server_owner: true,
identifier,
uuid: `${identifier}-1111-2222-3333-444444444444`,
name,
node: "stockholm-1",
sftp_details: { ip: "sftp.fragnet.net", port: 2022 },
description: "",
limits: { memory: 4096, swap: 0, disk: 20_480, io: 500, cpu: 200 },
feature_limits: { databases: 1, allocations: 1, backups: 3 },
is_suspended: false,
is_installing: false,
is_transferring: false,
docker_image: "ghcr.io/pterodactyl/yolks:java_21",
relationships: {
allocations: { object: "list", data: [{ object: "allocation", attributes: { id: 2, ip: "203.0.113.20", ip_alias: null, port: 25566, is_default: false, notes: "query" } }, { object: "allocation", attributes: { id: 1, ip: "203.0.113.20", ip_alias: null, port: 25565, is_default: true, notes: null } }] },
egg: { object: "egg", attributes: { id: 3, name: "Minecraft: Paper" } },
},
...extra,
},
});
const apiResources = (state: string, extra: Record<string, unknown> = {}) => ({ object: "stats", attributes: { current_state: state, is_suspended: false, resources: { memory_bytes: 2 * GB, cpu_absolute: 35.5, disk_bytes: 5 * GB, network_rx_bytes: 10, network_tx_bytes: 20, uptime: 60_000, ...extra } } });
const apiBackup = (uuid: string, createdAt: string, completedAt: string | null, ok = true) => ({ object: "backup", attributes: { uuid, name: uuid, ignored_files: [], checksum: "sha1:x", bytes: 1048576, created_at: createdAt, completed_at: completedAt, is_successful: ok, is_locked: false } });
describe("what the panel's API says", () => {
test("servers: the id the panel uses, the game, the addresses (the main one first), the limits and the plan's room for backups", () => {
const [s] = adapter.parseServers({ data: [apiServer("abcd1234", "survival")] });
assert.deepEqual([s.id, s.name, s.game, s.node, s.sftp], ["abcd1234", "survival", "Minecraft: Paper", "stockholm-1", "sftp.fragnet.net:2022"]);
assert.deepEqual(s.allocations.map((a) => `${a.ip}:${a.port}${a.isDefault ? "*" : ""}`), ["203.0.113.20:25565*", "203.0.113.20:25566"]);
assert.deepEqual([s.memoryLimitMb, s.diskLimitMb, s.cpuLimit, s.backupLimit, s.databaseLimit], [4096, 20_480, 200, 3, 1]);
assert.deepEqual([s.suspended, s.installing, s.owner, s.state, s.usage, s.backups], [false, false, true, "unknown", null, null]);
});
test("a server without a proper id or a name is skipped; a panel that did not say the game leaves it null; suspended and installing show from the status too", () => {
const list = adapter.parseServers({ data: [apiServer("short", "x"), apiServer("abcd1234", ""), apiServer("eeee5555", "bare", { relationships: {}, status: "suspended" }), apiServer("ffff6666", "setup", { status: "installing" }), "junk"] });
assert.deepEqual(list.map((s) => s.name), ["bare", "setup"]);
assert.deepEqual([list[0].game, list[0].allocations, list[0].suspended, list[1].installing], [null, [], true, true]);
assert.deepEqual(adapter.parseServers(null), []);
});
test("resources: whether it runs, and what it uses; something that is not that is not read", () => {
const r = adapter.parseResources(apiResources("running"))!;
assert.deepEqual([r.state, r.usage.memoryBytes, r.usage.cpu, r.usage.diskBytes, r.usage.uptimeMs], ["running", 2 * GB, 35.5, 5 * GB, 60_000]);
assert.equal(adapter.parseResources({ errors: [] }), null);
});
test("backups come newest first, and one with no finish time is still being made", () => {
const list = adapter.parseBackups({ data: [apiBackup("old", "2026-05-01T00:00:00Z", "2026-05-01T00:05:00Z"), apiBackup("new", "2026-06-01T00:00:00Z", null, false), apiBackup("failed", "2026-05-20T00:00:00Z", "2026-05-20T00:01:00Z", false), { attributes: {} }] });
assert.deepEqual(list.map((b) => [b.uuid, b.completedAt === null, b.successful]), [["new", true, false], ["failed", false, false], ["old", false, true]]);
});
test("schedules: the name, when it runs, whether it is on", () => {
const [s] = adapter.parseSchedules({ data: [{ attributes: { id: 4, name: "Nightly restart", cron: { day_of_week: "*", day_of_month: "*", month: "*", hour: "4", minute: "0" }, is_active: true, is_processing: false, only_when_online: true, last_run_at: "2026-06-01T04:00:00+00:00", next_run_at: "2026-06-02T04:00:00+00:00" } }, { attributes: { name: "no id" } }] });
assert.deepEqual([s.id, s.name, s.cron, s.active, s.onlyWhenOnline, s.nextRunAt], [4, "Nightly restart", "0 4 * * *", true, true, "2026-06-02T04:00:00.000Z"]);
});
test("the next page is the one after this, up to the last", () => {
assert.equal(adapter.nextPage({ meta: { pagination: { current_page: 1, total_pages: 2 } } }), 2);
assert.equal(adapter.nextPage({ meta: { pagination: { current_page: 2, total_pages: 2 } } }), null);
assert.equal(adapter.nextPage({}), null);
});
describe("the adapter against a panel that answers", () => {
let fake: Server;
let url = "";
let keyOk = true;
let rateLimited = false;
let databasesStatus = 200;
let rejectIncludes = false;
let powerStatus = 204;
const seen: { method: string | undefined; path: string; auth: string | undefined; body: string }[] = [];
before(async () => {
fake = createHttpServer((req, res) => {
const u = new URL(req.url ?? "", "http://x");
let body = "";
req.on("data", (c) => (body += c));
req.on("end", () => {
seen.push({ method: req.method, path: u.pathname + u.search, auth: req.headers.authorization, body });
const send = (code: number, json: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(code === 204 ? "" : JSON.stringify(json));
};
if (!keyOk) return send(401, { errors: [{ code: "AuthenticationException" }] });
if (rateLimited) return send(429, { errors: [{ code: "TooManyRequestsHttpException" }] });
const page = Number(u.searchParams.get("page") ?? 1);
if (u.pathname === "/api/client") {
if (rejectIncludes && u.searchParams.has("include")) return send(400, { errors: [{ code: "BadRequest" }] });
return send(200, { object: "list", data: [apiServer(page === 1 ? "abcd1234" : "eeee5555", page === 1 ? "survival" : "rust")], meta: { pagination: { total: 2, count: 1, per_page: 1, current_page: page, total_pages: 2 } } });
}
const m = /^\/api\/client\/servers\/([a-z0-9]{8})\/(resources|backups|schedules|databases|power)$/.exec(u.pathname);
if (!m) return send(404, {});
if (m[2] === "power") return req.method === "POST" ? send(powerStatus, {}) : send(405, {});
if (m[2] === "resources") return send(200, apiResources(m[1] === "abcd1234" ? "running" : "offline"));
if (m[2] === "backups") return send(200, { object: "list", data: [apiBackup("b1", "2026-06-01T00:00:00Z", "2026-06-01T00:03:00Z")], meta: { pagination: { current_page: 1, total_pages: 1 } } });
if (m[2] === "schedules") return send(200, { object: "list", data: [{ attributes: { id: 1, name: "Restart", cron: { minute: "0", hour: "4", day_of_month: "*", month: "*", day_of_week: "*" }, is_active: true } }] });
return databasesStatus === 200 ? send(200, { object: "list", data: [{ attributes: { id: "x" } }, { attributes: { id: "y" } }] }) : send(databasesStatus, {});
});
});
await new Promise<void>((resolve) => fake.listen(0, "127.0.0.1", resolve));
url = `http://127.0.0.1:${(fake.address() as AddressInfo).port}`;
});
after(() => fake.close());
test("every page of the servers is read, and for each its resources, backups, schedules and databases; only GET is ever sent by a read", async () => {
seen.length = 0;
const r = await adapter.createFragnetAdapter({ token: "ptlc_x", url }).read();
assert.deepEqual(r.servers.map((s) => [s.id, s.name, s.state]), [["abcd1234", "survival", "running"], ["eeee5555", "rust", "offline"]]);
const [a] = r.servers;
assert.deepEqual([a.usage?.diskBytes, a.backups?.length, a.schedules?.[0].name, a.databases, a.game], [5 * GB, 1, "Restart", 2, "Minecraft: Paper"]);
assert.ok(seen.every((s) => s.method === "GET" && s.auth === "Bearer ptlc_x"));
assert.ok(seen.some((s) => s.path.startsWith("/api/client?include=egg,allocations&per_page=50&page=2")));
});
test("a part a server will not give (a sub-user without that permission) is null, not none; the servers themselves must work", async () => {
databasesStatus = 403;
const r = await adapter.createFragnetAdapter({ token: "ptlc_x", url }).read();
assert.deepEqual([r.servers[0].databases, r.servers[0].backups?.length], [null, 1]);
databasesStatus = 200;
});
test("a panel that refuses the extras is asked again plainly", async () => {
rejectIncludes = true;
seen.length = 0;
const r = await adapter.createFragnetAdapter({ token: "ptlc_x", url }).read();
assert.equal(r.servers.length, 2);
assert.ok(seen.some((s) => s.path.startsWith("/api/client?per_page=50")));
rejectIncludes = false;
});
test("a key that is refused, an Application key and a panel over its limit are said", async () => {
keyOk = false;
const ping = await adapter.createFragnetAdapter({ token: "bad", url }).ping();
assert.equal(ping.ok, false);
assert.match(ping.error ?? "", /did not accept the key \(HTTP 401\).*ptlc_/);
assert.match((await adapter.createFragnetAdapter({ token: "ptla_app", url }).ping()).error ?? "", /Application key \(ptla_\)/);
keyOk = true;
rateLimited = true;
await assert.rejects(() => adapter.createFragnetAdapter({ token: "t", url }).read(), /too many requests/);
rateLimited = false;
assert.equal((await adapter.createFragnetAdapter({ token: "t", url }).ping()).ok, true);
assert.equal((await adapter.createFragnetAdapter({ token: "t", url: "http://127.0.0.1:1" }).ping()).ok, false);
});
test("a power signal is a POST of exactly that signal to that server; a busy server and an unknown one are said; nothing else is accepted", async () => {
seen.length = 0;
await adapter.createFragnetAdapter({ token: "t", url }).power("abcd1234", "restart");
assert.deepEqual(seen.map((s) => [s.method, s.path, s.body]), [["POST", "/api/client/servers/abcd1234/power", JSON.stringify({ signal: "restart" })]]);
powerStatus = 409;
await assert.rejects(() => adapter.createFragnetAdapter({ token: "t", url }).power("abcd1234", "start"), /busy/);
powerStatus = 204;
await assert.rejects(() => adapter.createFragnetAdapter({ token: "t", url }).power("../../x", "start"), /isn't a server id/);
await assert.rejects(() => adapter.createFragnetAdapter({ token: "t", url }).power("abcd1234", "format" as never), /isn't a power signal/);
});
test("the panel address is optional: without it the panel is Fragnet's", () => {
assert.equal(adapter.DEFAULT_URL, "https://panel.fragnet.net");
});
});
});
describe("what is carried between reads", () => {
const t0 = new Date("2026-06-01T10:00:00Z");
test("a server first seen starts its clocks now; the state's clock restarts only when the state changes", () => {
const t1 = new Date(t0.getTime() + 5 * MIN);
const t2 = new Date(t0.getTime() + 10 * MIN);
const a = fn.carryTimes([server()], [], t0);
assert.deepEqual([a[0].stateSince, a[0].firstSeenAt], [t0.toISOString(), t0.toISOString()]);
const off = fn.carryTimes([server({ state: "offline" })], a, t1);
assert.deepEqual([off[0].stateSince, off[0].firstSeenAt], [t1.toISOString(), t0.toISOString()]);
const still = fn.carryTimes([server({ state: "offline" })], off, t2);
assert.deepEqual([still[0].stateSince, still[0].firstSeenAt], [t1.toISOString(), t0.toISOString()]);
});
});
describe("what is wrong", () => {
const NOW = Date.parse("2026-06-10T12:00:00Z");
const opts = (extra: Partial<import("../src/services/fragnetSync.js").FragnetOptions> = {}) => ({ ...fn.optionsOf(null), ...extra });
const stored = (extra: Partial<Srv> = {}, times: Partial<import("../src/services/fragnetSync.js").StoredServer> = {}) => ({ ...server(extra), stateSince: iso(NOW - HOUR), firstSeenAt: iso(NOW - 30 * DAY), ...times });
const keys = (list: ReturnType<typeof fn.fragnetProblems>) => list.map((p) => `${p.key}=${p.severity}`);
const problems = (s: ReturnType<typeof stored>, o = opts()) => fn.fragnetProblems(1, "Fragnet", [s], NOW, o);
test("a server that is well says nothing, and one that is stopped is not an alert", () => {
assert.deepEqual(problems(stored({ backups: [okBackup(DAY, {}, NOW)] })), []);
assert.deepEqual(problems(stored({ state: "offline", usage: { memoryBytes: 0, cpu: 0, diskBytes: 5 * GB, rxBytes: 0, txBytes: 0, uptimeMs: 0 }, backups: [okBackup(DAY, {}, NOW)] })), []);
});
describe("backups", () => {
test("recent is fine; older than the limit is a warning that says how old; none for a server known for long is a warning", () => {
assert.deepEqual(problems(stored({ backups: [okBackup(6 * DAY, {}, NOW)] })), []);
const old = problems(stored({ backups: [okBackup(9 * DAY, {}, NOW)] }));
assert.deepEqual(keys(old), ["fragnet-backup:1:abcd1234=warning"]);
assert.match(old[0].message, /Fragnet: survival has no backup newer than 9 days — it should have one at least every 7 days/);
assert.match(problems(stored({ backups: [] }))[0].message, /survival has no backup — it should/);
});
test("a backup that failed does not count, and the message says the newest attempt failed", () => {
const p = problems(stored({ backups: [okBackup(HOUR, { successful: false }, NOW), okBackup(9 * DAY, {}, NOW)] }));
assert.match(p[0].message, /no backup newer than 9 days \(the newest attempt failed\)/);
assert.deepEqual(problems(stored({ backups: [okBackup(HOUR, { successful: false }, NOW), okBackup(2 * DAY, {}, NOW)] })), []);
});
test("a backup still being made does not count either way", () => {
const making = okBackup(0, { completedAt: null, successful: false }, NOW);
assert.deepEqual(problems(stored({ backups: [making, okBackup(DAY, {}, NOW)] })), []);
assert.match(problems(stored({ backups: [making] }))[0].message, /has no backup/);
});
test("a server this app has only just met has had no time; the limit is the setting, 0 turns it off; a list that could not be read, a plan without backups, a name left out, and a server in no state to judge say nothing", () => {
assert.deepEqual(problems(stored({ backups: [] }, { firstSeenAt: iso(NOW - 2 * DAY) })), []);
assert.deepEqual(problems(stored({ backups: [okBackup(9 * DAY, {}, NOW)] }), opts({ backupDays: 14 })), []);
assert.deepEqual(problems(stored({ backups: [] }), opts({ backupDays: 0 })), []);
assert.deepEqual(problems(stored({ backups: null })), []);
assert.deepEqual(problems(stored({ backups: [], backupLimit: 0 })), []);
assert.deepEqual(problems(stored({ backups: [] }), opts({ ignoreBackups: ["survival"] })), []);
assert.deepEqual(problems(stored({ backups: [], suspended: true })), []);
assert.deepEqual(problems(stored({ backups: [], installing: true })), []);
});
});
describe("disk and memory", () => {
const use = (diskGb: number, memGb: number) => ({ memoryBytes: memGb * GB, cpu: 10, diskBytes: diskGb * GB, rxBytes: 0, txBytes: 0, uptimeMs: HOUR });
const fine = [okBackup(DAY, {}, NOW)];
test("disk: a warning from 90% of its limit, critical from 98%, in the units the page shows", () => {
assert.deepEqual(problems(stored({ backups: fine, usage: use(17, 1) })), []);
const warn = problems(stored({ backups: fine, usage: use(18.5, 1) }));
assert.deepEqual(keys(warn), ["fragnet-disk:1:abcd1234=warning"]);
assert.match(warn[0].message, /survival has used 18\.5 GB of its 20\.0 GB disk \(92%\)/);
assert.deepEqual(keys(problems(stored({ backups: fine, usage: use(19.8, 1) }))), ["fragnet-disk:1:abcd1234=critical"]);
});
test("memory: judged only while it runs; no limit (0) and no reading say nothing", () => {
assert.deepEqual(keys(problems(stored({ backups: fine, usage: use(1, 3.7) }))), ["fragnet-memory:1:abcd1234=warning"]);
assert.match(problems(stored({ backups: fine, usage: use(1, 3.7) }))[0].message, /using 3.7 GB of its 4\.0 GB memory \(92%\)/);
assert.deepEqual(problems(stored({ backups: fine, state: "offline", usage: use(1, 3.7) })), []);
assert.deepEqual(problems(stored({ backups: fine, memoryLimitMb: 0, diskLimitMb: 0, usage: use(500, 500) })), []);
assert.deepEqual(problems(stored({ backups: fine, usage: null })), []);
});
test("the limit is the setting", () => {
assert.deepEqual(problems(stored({ backups: fine, usage: use(15, 1) }), opts({ fullPercent: 70 })).map((p) => p.key), ["fragnet-disk:1:abcd1234"]);
});
});
test("the settings: what is given is used, nonsense falls back to the defaults, names are compared without regard to case", () => {
assert.deepEqual({ ...fn.optionsOf(null) }, { backupDays: 7, fullPercent: 90, ignoreBackups: [] });
assert.deepEqual({ ...fn.optionsOf({ backupDays: "3", fullPercent: "85,5", ignoreBackups: "Test, Rust\nBuild" }) }, { backupDays: 3, fullPercent: 85.5, ignoreBackups: ["test", "rust", "build"] });
assert.deepEqual({ ...fn.optionsOf({ backupDays: "0", fullPercent: "x" }) }, { backupDays: 0, fullPercent: 90, ignoreBackups: [] });
assert.deepEqual({ ...fn.optionsOf({ backupDays: "-4" }) }, { backupDays: 7, fullPercent: 90, ignoreBackups: [] });
});
});
describe("what is stored", () => {
let id = 0;
before(async () => {
id = (await db.insert(integrations).values({ type: "fragnet", name: "Account", baseUrl: "https://panel.fragnet.net", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
});
const row = async () => (await db.select().from(fragnetState).where(eq(fragnetState.integrationId, id)))[0];
after(async () => {
await fn.forgetFragnet(id);
await db.delete(integrations).where(eq(integrations.id, id));
});
test("the state is replaced at each read, carrying what was seen between them", async () => {
const t0 = new Date();
await fn.storeRead(id, read(server()), t0);
await fn.storeRead(id, read(server({ state: "offline" })), new Date(t0.getTime() + 5 * MIN));
const [s] = fn.serversOf(await row());
assert.deepEqual([s.state, s.firstSeenAt, s.stateSince], ["offline", t0.toISOString(), new Date(t0.getTime() + 5 * MIN).toISOString()]);
});
test("a read that failed is said, and what was stored is left alone", async () => {
const ok = await fn.syncFragnet(id, new Date(), async () => read(server({ name: "kept" })));
assert.equal(ok.ok, true);
const failed = await fn.syncFragnet(id, new Date(), async () => {
throw new Error("connection refused");
});
assert.deepEqual([failed.ok, failed.error], [false, "connection refused"]);
assert.equal(failed.at, ok.at);
assert.equal(fn.serversOf(await row())[0].name, "kept");
assert.equal(fn.isFragnetFresh((await fn.fragnetSyncStatuses())[id], Date.now()), false);
});
});
describe("the health pass, the page, the power signals and /metrics", () => {
let fgId = 0;
let app: Server;
let panel: Server;
let base = "";
const signals: string[] = [];
const fresh = () => setInternalFlag("fragnetSync", JSON.stringify({ [fgId]: { at: new Date().toISOString(), ok: true, error: null } }));
const stale = () => setInternalFlag("fragnetSync", JSON.stringify({ [fgId]: { at: new Date(Date.now() - 30 * MIN).toISOString(), ok: true, error: null } }));
const as = (sub: string) => ({ headers: { "x-sub": sub } });
before(async () => {
await db.insert(users).values([{ oidcSub: "viewer", name: "Vic", role: "viewer" }, { oidcSub: "operator", name: "Olle", role: "operator" }]);
// A panel that takes power signals.
panel = createHttpServer((req, res) => {
let body = "";
req.on("data", (c) => (body += c));
req.on("end", () => {
signals.push(`${req.method} ${req.url} ${body}`);
res.writeHead(204);
res.end();
});
});
await new Promise<void>((resolve) => panel.listen(0, "127.0.0.1", resolve));
const panelUrl = `http://127.0.0.1:${(panel.address() as AddressInfo).port}`;
fgId = (await db.insert(integrations).values({ type: "fragnet", name: "Fragnet account", baseUrl: panelUrl, config: JSON.stringify({ token: "ptlc_x", url: panelUrl, ignoreBackups: "scratch" }) }).returning({ id: integrations.id }))[0].id;
const t0 = new Date(Date.now() - 3 * HOUR);
const servers = read(
// survival: running, disk nearly full, newest backup is old; rust: stopped, no backup but its name is left out; modded: fine.
server({ backups: [okBackup(10 * DAY)], usage: { memoryBytes: GB, cpu: 5, diskBytes: 19.9 * GB, rxBytes: 0, txBytes: 0, uptimeMs: HOUR } }),
server({ id: "eeee5555", name: "scratch", game: "Rust", state: "offline", backups: [], allocations: [{ ip: "203.0.113.21", alias: "rust.example", port: 28015, isDefault: true, notes: null }] }),
server({ id: "ffff6666", name: "modded", game: "Valheim", backups: [okBackup(HOUR)] }),
);
await fn.storeRead(fgId, servers, t0);
await fn.storeRead(fgId, servers, new Date());
await fresh();
const a = express();
a.use(express.json());
a.use((req, _res, next) => {
(req as unknown as { session: unknown }).session = { user: { sub: String(req.headers["x-sub"] ?? "viewer") } };
next();
});
a.use("/api/fragnet", fragnetRouter);
app = await new Promise<Server>((resolve) => {
const s = a.listen(0, "127.0.0.1", () => resolve(s));
});
base = `http://127.0.0.1:${(app.address() as AddressInfo).port}/api/fragnet`;
});
after(() => {
app.close();
panel.close();
try {
rmSync(dir, { recursive: true, force: true });
} catch {
// Windows keeps the database file locked until the process exits.
}
});
test("the health pass judges it from what is stored (with the integration's own settings), and holds it when the last read is old", async () => {
let { snapshot, held } = await collectSnapshot();
assert.deepEqual(snapshot.fragnet?.map((f) => f.integrationId), [fgId]);
const list = evaluateHealth(snapshot, { serverOfflineMinutes: 60, diskUsagePercent: 90 }, Date.now(), { skipServers: true });
const mine = list.filter((x) => x.key.startsWith("fragnet-")).map((x) => `${x.key}=${x.severity}`).sort();
assert.deepEqual(mine, [`fragnet-backup:${fgId}:abcd1234=warning`, `fragnet-disk:${fgId}:abcd1234=critical`].sort(), "scratch is left out of the backups by name; modded is fine");
assert.equal(list.find((x) => x.key === `fragnet-disk:${fgId}:abcd1234`)!.source, `fragnet:${fgId}`);
for (const k of ["disk", "memory", "backup"]) assert.equal(subjectOfConditionKey(`fragnet-${k}:${fgId}:abcd1234`), `integration:${fgId}`, "maintenance on the integration silences its alerts");
await stale();
({ snapshot, held } = await collectSnapshot());
assert.deepEqual([snapshot.fragnet ?? [], held.has(`fragnet:${fgId}`)], [[], true]);
await fresh();
});
test("the Alerts page shows them with where to look: the disk under Disk, the backup under Backups", async () => {
invalidateAlertsCache();
const report = await getAlerts(true);
const find = (kind: string) => report.alerts.find((x) => x.id.endsWith(`fragnet-${kind}:${fgId}:abcd1234`))!;
assert.deepEqual([find("disk").category, find("disk").source, find("disk").link, find("disk").target?.type], ["disk", "Fragnet", "/game-servers", "integration"]);
assert.equal(find("backup").category, "backup");
});
test("the page: each game server with its game, addresses, use, backups and what is wrong; an old reading says nothing is wrong", async () => {
const body = (await (await fetch(base)).json()) as { accounts: { id: number; fresh: boolean; problems: { key: string }[]; options: { ignoreBackups: string[] }; state: { servers: { name: string; game: string; allocations: { ip: string; port: number }[]; newestBackup: { at: string | null } | null }[] } }[] };
const a = body.accounts.find((x) => x.id === fgId)!;
assert.equal(a.fresh, true);
assert.deepEqual(a.state.servers.map((s) => [s.name, s.game, `${s.allocations[0].ip}:${s.allocations[0].port}`]), [["survival", "Minecraft: Paper", "203.0.113.20:25565"], ["scratch", "Rust", "203.0.113.21:28015"], ["modded", "Valheim", "203.0.113.20:25565"]]);
assert.equal(a.state.servers[0].newestBackup?.at !== null, true);
assert.equal(a.state.servers[1].newestBackup?.at, null);
assert.deepEqual(a.options.ignoreBackups, ["scratch"]);
assert.equal(a.problems.length, 2);
await stale();
const old = (await (await fetch(base)).json()) as typeof body;
assert.deepEqual(old.accounts.find((x) => x.id === fgId)!.problems, []);
await fresh();
});
test("a power signal: an operator may, a viewer may not; it goes to the panel as that signal for that server, and is written in the audit log", async () => {
const post = (sub: string, server: string, signal: unknown) => fetch(`${base}/${fgId}/servers/${server}/power`, { method: "POST", headers: { "content-type": "application/json", "x-sub": sub }, body: JSON.stringify({ signal }) });
assert.equal((await post("viewer", "abcd1234", "restart")).status, 403);
assert.deepEqual(signals, []);
assert.equal((await post("operator", "abcd1234", "restart")).status, 204);
assert.deepEqual(signals, [`POST /api/client/servers/abcd1234/power ${JSON.stringify({ signal: "restart" })}`]);
const [entry] = (await db.select().from(auditLog)).filter((e) => e.action === "restart_game_server");
assert.deepEqual([entry.category, entry.targetType, entry.targetId], ["integration", "fragnet_server", "abcd1234"]);
assert.match(entry.detail ?? "", /survival/);
});
test("only a server the last read found, and only a real signal, are accepted — nothing typed in the address reaches the panel", async () => {
const post = (server: string, signal: unknown) => fetch(`${base}/${fgId}/servers/${server}/power`, { method: "POST", headers: { "content-type": "application/json", "x-sub": "operator" }, body: JSON.stringify({ signal }) });
signals.length = 0;
assert.equal((await post("zzzz9999", "start")).status, 404, "a well-formed id the panel account does not have");
assert.equal((await post("..%2F..%2Fx", "start")).status, 400);
assert.equal((await post("abcd1234", "format")).status, 400);
assert.equal((await post("abcd1234", undefined)).status, 400);
assert.equal((await fetch(`${base}/999999/servers/abcd1234/power`, { method: "POST", headers: { "content-type": "application/json", "x-sub": "operator" }, body: JSON.stringify({ signal: "start" }) })).status, 404);
assert.deepEqual(signals, []);
});
test("a row of another kind does not crash the server", async () => {
const other = (await db.insert(integrations).values({ type: "hetzner", name: "Other", baseUrl: "https://api.hetzner.cloud", config: JSON.stringify({}) }).returning({ id: integrations.id }))[0].id;
const res = await fetch(`${base}/${other}/servers/abcd1234/power`, { method: "POST", headers: { "content-type": "application/json", "x-sub": "operator" }, body: JSON.stringify({ signal: "start" }) });
assert.equal(res.status, 400);
assert.equal((await fetch(`${base}/${other}/refresh`, { method: "POST", ...as("operator") })).status, 400);
await db.delete(integrations).where(eq(integrations.id, other));
});
test("/metrics has whether each server runs, its disk and memory against the limits and when it was last backed up — only while the reading is recent", async () => {
let text = await buildMetrics(new Date());
assert.match(text, /homelab_fragnet_server_running\{account="Fragnet account",server="survival",game="Minecraft: Paper"\} 1/);
assert.match(text, /homelab_fragnet_server_running\{account="Fragnet account",server="scratch",game="Rust"\} 0/);
assert.match(text, /homelab_fragnet_disk_limit_bytes\{[^}]*server="survival"[^}]*\} 21474836480/);
assert.match(text, /homelab_fragnet_newest_backup_timestamp_seconds\{[^}]*server="modded"[^}]*\} \d/);
await stale();
text = await buildMetrics(new Date());
assert.doesNotMatch(text, /homelab_fragnet/);
await fresh();
});
test("taking the integration away takes what was read with it", async () => {
await fn.forgetFragnet(fgId);
assert.equal((await fn.fragnetSyncStatuses())[fgId], undefined);
await db.delete(integrations).where(eq(integrations.id, fgId));
assert.deepEqual(await db.select().from(fragnetState).where(eq(fragnetState.integrationId, fgId)), []);
});
});