Fragnet game servers: games, addresses, backups, power actions
New integration kind "fragnet" reading Fragnet's Pterodactyl-based panel through the client API (account key). Game servers page under Infrastructure with game, IP:port, state, memory/disk, backups, schedules and databases; alerts for late backups and nearly full disk/memory; operators can start/stop/restart/kill (audited). Metrics, docs and tests included; migration 0071 adds fragnet_state. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f86661f6c9
commit
9a3d5c466a
29 files changed
+9836
-8
No files matched your search
@@ -85,6 +85,7 @@ const SOURCES: Record<string, { label: string; link: string }> = {
|
||||
nextcloud: { label: "Nextcloud", link: "/nextcloud" },
|
||||
hetzner: { label: "Hetzner", link: "/hetzner" },
|
||||
speedtest: { label: "Speedtest", link: "/speedtest" },
|
||||
fragnet: { label: "Fragnet", link: "/game-servers" },
|
||||
secrets: { label: "Secrets", link: "/secrets" },
|
||||
domains: { label: "Domains", link: "/dns?view=domains" },
|
||||
endpoint: { label: "Endpoint", link: "/endpoints" },
|
||||
@@ -127,6 +128,8 @@ function originOfConditionKey(key: string): { type: string; id: number | null; c
|
||||
if (m) return { type: "speedtest", id: Number(m[1]), category: "monitoring" };
|
||||
m = /^hetzner-(off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "hetzner", id: Number(m[2]), category: m[1] === "traffic" ? "monitoring" : m[1] === "backup" ? "backup" : "services" };
|
||||
m = /^fragnet-(disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return { type: "fragnet", id: Number(m[2]), category: m[1] === "disk" ? "disk" : m[1] === "backup" ? "backup" : "monitoring" };
|
||||
m = /^duplicati-(?:failed|stale):(\d+)/.exec(key);
|
||||
if (m) return { type: "duplicati", id: Number(m[1]), category: "backup" };
|
||||
m = /^nextcloud-(?:maintenance|upgrade|disk|cron|checks):(\d+)/.exec(key);
|
||||
@@ -267,7 +270,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
type: origin.type,
|
||||
message: c.message,
|
||||
key: c.key,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : undefined,
|
||||
link: origin.type === "server" && origin.id !== null ? `/servers/${origin.id}${/^(failed-units|smart|zfs):/.test(c.key) ? "?view=health" : c.key.startsWith("new-port:") ? "?view=network" : c.key.startsWith("rule:") || c.key.startsWith("baseline:") ? "?view=history" : ""}` : origin.type === "unifi" ? "/network/devices" : origin.type === "ups" ? "/ups" : origin.type === "slo" ? "/service-levels" : origin.type === "dockhand" ? "/docker" : origin.type === "duplicati" ? "/duplicati" : origin.type === "jellyfin" ? "/jellyfin" : origin.type === "authentik" ? "/authentik" : origin.type === "prometheus" ? "/prometheus" : origin.type === "immich" ? "/immich" : origin.type === "paperless" ? "/paperless" : origin.type === "nextcloud" ? "/nextcloud" : origin.type === "netbox" ? "/netbox" : origin.type === "n8n" ? "/n8n" : origin.type === "hetzner" ? "/hetzner" : origin.type === "speedtest" ? "/speedtest" : origin.type === "fragnet" ? "/game-servers" : undefined,
|
||||
silenced: subject !== null && subjects.has(subject), target: targetOfSubject(subject),
|
||||
});
|
||||
}
|
||||
@@ -277,7 +280,7 @@ export async function collectAlerts(): Promise<AlertsReport> {
|
||||
if (held.has("slo")) cannotCheck("Service-level targets", "couldn't be judged — see the server log", "Server and storage health");
|
||||
if (held.has("server:baseline")) cannotCheck("What is usual for the time", "couldn't be judged — see the server log", "Server and storage health");
|
||||
for (const h of held) {
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest):(\d+)(?::(.+))?$/.exec(h);
|
||||
const m = /^(proxmox|synology|unifi|ups|dockhand|jellyfin|authentik|prometheus|immich|paperless|nextcloud|netbox|n8n|hetzner|speedtest|fragnet):(\d+)(?::(.+))?$/.exec(h);
|
||||
if (!m || (m[3] && held.has(`${m[1]}:${m[2]}`))) continue;
|
||||
const row = intById.get(Number(m[2]));
|
||||
cannotCheck(`${SOURCES[m[1]].label} “${row?.name ?? `#${m[2]}`}”${m[3] === "vuln" ? " (vulnerability scans)" : m[3] ? ` (node ${m[3]})` : ""}`, "couldn't be read — see the Diagnostic Log", "Server and storage health");
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Keeps what Fragnet's game panel says up to date: every five minutes each Fragnet integration (one account) is read — its game servers, their addresses, how much they use, their backups,
|
||||
* schedules and databases — and the state is stored. Pages and alerts are built from what is stored, so none of them talks to the panel (which limits how many requests are made a minute).
|
||||
*
|
||||
* What is raised: **a backup that is late** — a server that may have backups (its plan allows some) whose newest successful one is older than `backupDays` (7 unless you say otherwise; 0 =
|
||||
* not judged), or that has none and has been known for longer than that; a server can be left out by name (`ignoreBackups`). **Disk or memory nearly full** — over `fullPercent` (90 unless you
|
||||
* say otherwise) of its limit; memory is judged only while it runs. A server that is stopped is not an alert: game servers are often stopped on purpose. A panel that cannot be reached shows
|
||||
* as the integration being down, like any other. "Since when" a server has been in its state is what this app saw: the time of the first read that showed it.
|
||||
*/
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { fragnetState, integrations } from "../db/schema.js";
|
||||
import { createFragnetAdapter, type FragnetRead, type FragnetServer } from "../integrations/fragnet/adapter.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { getInternalFlag, setInternalFlag } from "./settingsStore.js";
|
||||
|
||||
export const SYNC_INTERVAL_MS = 5 * 60_000;
|
||||
/** After this long without a good read, what is stored is no longer believed. */
|
||||
export const STALE_AFTER_MS = 20 * 60_000;
|
||||
const STATUS_FLAG = "fragnetSync";
|
||||
|
||||
const MIN = 60_000;
|
||||
const HOUR = 60 * MIN;
|
||||
const DAY = 24 * HOUR;
|
||||
|
||||
export const DEFAULT_BACKUP_DAYS = 7;
|
||||
export const DEFAULT_FULL_PERCENT = 90;
|
||||
/** A disk this full is critical, whatever the limit for a warning is. */
|
||||
export const CRITICAL_PERCENT = 98;
|
||||
|
||||
export interface FragnetSyncStatus {
|
||||
/** The last read that worked (kept when a later one failed). */
|
||||
at: string;
|
||||
ok: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
async function readStatuses(): Promise<Record<string, FragnetSyncStatus>> {
|
||||
try {
|
||||
const raw = await getInternalFlag(STATUS_FLAG);
|
||||
return raw ? (JSON.parse(raw) as Record<string, FragnetSyncStatus>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fragnetSyncStatuses(): Promise<Record<number, FragnetSyncStatus>> {
|
||||
return Object.fromEntries(Object.entries(await readStatuses()).map(([k, v]) => [Number(k), v]));
|
||||
}
|
||||
|
||||
export const isFragnetFresh = (status: FragnetSyncStatus | undefined, nowMs: number) => !!status && status.ok && nowMs - Date.parse(status.at) <= STALE_AFTER_MS;
|
||||
|
||||
export type StateRow = typeof fragnetState.$inferSelect;
|
||||
|
||||
/** A server as stored: as read, plus what the app has seen of it between reads. */
|
||||
export interface StoredServer extends FragnetServer {
|
||||
/** When the first read that showed its present state was made. */
|
||||
stateSince: string;
|
||||
/** When the first read that showed the server was made. */
|
||||
firstSeenAt: string;
|
||||
}
|
||||
|
||||
export const serversOf = (row: Pick<StateRow, "servers">): StoredServer[] => {
|
||||
try {
|
||||
const v = JSON.parse(row.servers);
|
||||
return Array.isArray(v) ? (v as StoredServer[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
/** Pure: what the app has seen between reads, carried from the last read of the same server. A server first seen starts its clocks now. */
|
||||
export function carryTimes(servers: FragnetServer[], previous: StoredServer[], now: Date): StoredServer[] {
|
||||
const before = new Map(previous.map((p) => [p.id, p]));
|
||||
const at = now.toISOString();
|
||||
return servers.map((s) => {
|
||||
const p = before.get(s.id);
|
||||
return { ...s, stateSince: p && p.state === s.state ? p.stateSince : at, firstSeenAt: p?.firstSeenAt ?? at };
|
||||
});
|
||||
}
|
||||
|
||||
/** Writes down one read, replacing the last (and carrying what was seen between reads). */
|
||||
export async function storeRead(integrationId: number, r: FragnetRead, now: Date = new Date()): Promise<void> {
|
||||
const [previous] = await db.select({ servers: fragnetState.servers }).from(fragnetState).where(eq(fragnetState.integrationId, integrationId)).limit(1);
|
||||
const values = { servers: JSON.stringify(carryTimes(r.servers, previous ? serversOf(previous) : [], now)), updatedAt: now.toISOString() };
|
||||
await db.insert(fragnetState).values({ integrationId, ...values }).onConflictDoUpdate({ target: fragnetState.integrationId, set: values });
|
||||
}
|
||||
|
||||
/** Reads one account and stores it. Never throws: how it went is the answer, and what was stored is left alone when it failed. */
|
||||
export async function syncFragnet(integrationId: number, now: Date = new Date(), read?: (config: Record<string, unknown>) => Promise<FragnetRead>): Promise<FragnetSyncStatus> {
|
||||
let status: FragnetSyncStatus;
|
||||
try {
|
||||
const loaded = await loadIntegrationConfig(integrationId);
|
||||
if (!loaded || loaded.integration.type !== "fragnet") throw new Error("not a Fragnet integration");
|
||||
const result = await (read ? read(loaded.config) : createFragnetAdapter(loaded.config as never).read());
|
||||
await storeRead(integrationId, result, now);
|
||||
status = { at: now.toISOString(), ok: true, error: null };
|
||||
} catch (err) {
|
||||
const previous = (await readStatuses())[String(integrationId)];
|
||||
status = { at: previous?.at ?? now.toISOString(), ok: false, error: err instanceof Error ? err.message : String(err) };
|
||||
console.error(`[fragnet] couldn't read integration ${integrationId}:`, status.error);
|
||||
}
|
||||
try {
|
||||
const all = await readStatuses();
|
||||
all[String(integrationId)] = status;
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
} catch (err) {
|
||||
console.error("[fragnet] couldn't record the result:", err);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function syncAllFragnet(now: Date = new Date()): Promise<Record<number, FragnetSyncStatus>> {
|
||||
const rows = await db.select({ id: integrations.id }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
|
||||
const out: Record<number, FragnetSyncStatus> = {};
|
||||
for (const r of rows) out[r.id] = await syncFragnet(r.id, now);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function forgetFragnet(integrationId: number): Promise<void> {
|
||||
const all = await readStatuses();
|
||||
if (String(integrationId) in all) {
|
||||
delete all[String(integrationId)];
|
||||
await setInternalFlag(STATUS_FLAG, JSON.stringify(all));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── What is wrong ──────────────────────────────────────────────────────────
|
||||
|
||||
/** What the integration's own settings say about how strict to be. */
|
||||
export interface FragnetOptions {
|
||||
/** A backup older than this many days is late; 0: backups are not judged. */
|
||||
backupDays: number;
|
||||
/** Percent of a limit above which disk and memory are called nearly full. */
|
||||
fullPercent: number;
|
||||
/** Names (lower case) of the servers whose backups are not judged. */
|
||||
ignoreBackups: string[];
|
||||
}
|
||||
|
||||
const positive = (v: unknown): number | null => {
|
||||
const n = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" ? Number(v.trim().replace(",", ".")) : NaN;
|
||||
return Number.isFinite(n) && n >= 0 ? n : null;
|
||||
};
|
||||
|
||||
export function optionsOf(config: Record<string, unknown> | null | undefined): FragnetOptions {
|
||||
const days = positive(config?.backupDays);
|
||||
const percent = positive(config?.fullPercent);
|
||||
const ignore = typeof config?.ignoreBackups === "string" ? config.ignoreBackups : "";
|
||||
return {
|
||||
backupDays: days === null ? DEFAULT_BACKUP_DAYS : days,
|
||||
fullPercent: percent === null || percent < 1 ? DEFAULT_FULL_PERCENT : Math.min(100, percent),
|
||||
ignoreBackups: ignore.split(/[,\n]/).map((s) => s.trim().toLowerCase()).filter(Boolean),
|
||||
};
|
||||
}
|
||||
|
||||
export interface FragnetProblem {
|
||||
key: string;
|
||||
/** The panel's short id of the server it is about. */
|
||||
serverId: string;
|
||||
message: string;
|
||||
severity: "warning" | "critical";
|
||||
}
|
||||
|
||||
/** In the units the page shows (binary, written MB, GB or TB), so an alert and the tile beside it agree. */
|
||||
export const sizeText = (bytes: number): string => {
|
||||
const mb = bytes / 1024 ** 2;
|
||||
if (mb < 1024) return `${Math.round(mb)} MB`;
|
||||
const gb = mb / 1024;
|
||||
return gb >= 1024 ? `${(gb / 1024).toFixed(1)} TB` : `${gb.toFixed(gb >= 100 ? 0 : 1)} GB`;
|
||||
};
|
||||
|
||||
const plural = (n: number, unit: string) => `${n} ${unit}${n === 1 ? "" : "s"}`;
|
||||
const ago = (ms: number): string => (ms >= 48 * HOUR ? plural(Math.round(ms / DAY), "day") : ms >= HOUR ? plural(Math.round(ms / HOUR), "hour") : plural(Math.max(1, Math.round(ms / MIN)), "minute"));
|
||||
|
||||
/** Pure: the newest backup that finished and worked (its time), and whether a newer one failed. */
|
||||
export function newestBackup(s: Pick<FragnetServer, "backups">): { at: string | null; failedSince: boolean } | null {
|
||||
if (s.backups === null) return null;
|
||||
const done = s.backups.filter((b) => b.completedAt !== null);
|
||||
const good = done.filter((b) => b.successful).map((b) => b.completedAt as string).sort().pop() ?? null;
|
||||
const failedSince = done.some((b) => !b.successful && (good === null || (b.completedAt as string) > good));
|
||||
return { at: good, failedSince };
|
||||
}
|
||||
|
||||
/** Pure: whether a server is one the panel lets make backups (its plan has room for some). */
|
||||
const mayHaveBackups = (s: FragnetServer) => s.backupLimit !== null && s.backupLimit > 0;
|
||||
|
||||
/** Pure: what is wrong with one account, from what is stored for it. */
|
||||
export function fragnetProblems(id: number, name: string, servers: StoredServer[], nowMs: number, options: FragnetOptions): FragnetProblem[] {
|
||||
const out: FragnetProblem[] = [];
|
||||
const limit = options.fullPercent / 100;
|
||||
for (const s of [...servers].sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
// A server being installed or suspended is in no state to judge.
|
||||
if (s.suspended || s.installing) continue;
|
||||
if (options.backupDays > 0 && mayHaveBackups(s) && !options.ignoreBackups.includes(s.name.toLowerCase())) {
|
||||
const newest = newestBackup(s);
|
||||
if (newest !== null) {
|
||||
const window = options.backupDays * DAY;
|
||||
const age = newest.at === null ? null : nowMs - Date.parse(newest.at);
|
||||
const young = nowMs - Date.parse(s.firstSeenAt) < window; // a server this app has only just met has had no time to be backed up
|
||||
if (age === null ? !young : age > window) {
|
||||
out.push({
|
||||
key: `fragnet-backup:${id}:${s.id}`,
|
||||
serverId: s.id,
|
||||
severity: "warning",
|
||||
message: `${name}: ${s.name} ${age === null ? "has no backup" : `has no backup newer than ${ago(age)}`}${newest.failedSince ? " (the newest attempt failed)" : ""} — it should have one at least every ${plural(options.backupDays, "day")}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (s.usage !== null && s.diskLimitMb !== null && s.diskLimitMb > 0) {
|
||||
const share = s.usage.diskBytes / (s.diskLimitMb * 1024 ** 2);
|
||||
if (share >= limit) {
|
||||
out.push({
|
||||
key: `fragnet-disk:${id}:${s.id}`,
|
||||
serverId: s.id,
|
||||
severity: share >= CRITICAL_PERCENT / 100 ? "critical" : "warning",
|
||||
message: `${name}: ${s.name} has used ${sizeText(s.usage.diskBytes)} of its ${sizeText(s.diskLimitMb * 1024 ** 2)} disk (${Math.floor(share * 100)}%)`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (s.state === "running" && s.usage !== null && s.memoryLimitMb !== null && s.memoryLimitMb > 0) {
|
||||
const share = s.usage.memoryBytes / (s.memoryLimitMb * 1024 ** 2);
|
||||
if (share >= limit) {
|
||||
out.push({
|
||||
key: `fragnet-memory:${id}:${s.id}`,
|
||||
serverId: s.id,
|
||||
severity: "warning",
|
||||
message: `${name}: ${s.name} is using ${sizeText(s.usage.memoryBytes)} of its ${sizeText(s.memoryLimitMb * 1024 ** 2)} memory (${Math.floor(share * 100)}%) — past the limit the game is stopped by the host`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function initFragnetSync(): void {
|
||||
if (timer) return;
|
||||
const pass = () => void syncAllFragnet().catch((err) => console.error("[fragnet] pass failed:", err));
|
||||
timer = setInterval(pass, SYNC_INTERVAL_MS);
|
||||
setTimeout(pass, 33_000);
|
||||
console.log(`[fragnet] reading Fragnet every ${SYNC_INTERVAL_MS / 60000} minutes`);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState } from "../db/schema.js";
|
||||
import { authentikState, n8nState, netboxState, nextcloudState, paperlessState, immichState, prometheusState, integrations, jellyfinState, networkDevices, serverPorts, servers, upsState , dockhandVulnerabilities , hetznerState , fragnetState } from "../db/schema.js";
|
||||
import { loadIntegrationConfig } from "../integrations/loadIntegration.js";
|
||||
import { createProxmoxAdapter, type ProxmoxNodeStats } from "../integrations/proxmox/adapter.js";
|
||||
import { createSynologyAdapter, type SynologyStorageInfo } from "../integrations/synology/adapter.js";
|
||||
@@ -31,6 +31,7 @@ import { isNextcloudFresh, nextcloudProblems, nextcloudSyncStatuses, type Nextcl
|
||||
import { dockerVulnProblems, dockerVulnSyncStatuses, isDockerVulnFresh, type DockerVulnProblem } from "./dockerVulnSync.js";
|
||||
import { expectedOf as speedtestExpected, isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestProblems, speedtestSyncStatuses, type SpeedtestProblem } from "./speedtestSync.js";
|
||||
import { hetznerProblems, hetznerSyncStatuses, imagesOf as hetznerImagesOf, isHetznerFresh, serversOf as hetznerServersOf, type HetznerProblem } from "./hetznerSync.js";
|
||||
import { fragnetProblems, fragnetSyncStatuses, isFragnetFresh, optionsOf as fragnetOptions, serversOf as fragnetServersOf, type FragnetProblem } from "./fragnetSync.js";
|
||||
import { evaluateContainers, loadContainerMemory, saveContainerMemory, type ContainerMemory, type DockerSnapshot } from "./containerHealth.js";
|
||||
import { recordStorageSamples } from "./storageHistory.js";
|
||||
import { isUpsFresh, upsProblems, upsSyncStatuses, type UpsStateRow } from "./upsSync.js";
|
||||
@@ -125,6 +126,8 @@ export interface HealthSnapshot {
|
||||
hetzner?: { integrationId: number; problems: HetznerProblem[] }[];
|
||||
/** What is wrong with each Speedtest Tracker (the internet slower than it is sold at, speed tests that keep failing), only while that reading is recent (services/speedtestSync.ts). */
|
||||
speedtest?: { integrationId: number; problems: SpeedtestProblem[] }[];
|
||||
/** What is wrong with each Fragnet game panel account (a backup that is late, a disk or memory nearly full), only while that reading is recent (services/fragnetSync.ts). */
|
||||
fragnet?: { integrationId: number; problems: FragnetProblem[] }[];
|
||||
/** Containers to leave alone. */
|
||||
dockerIgnore?: string[];
|
||||
/** The clock the container judgements are made on (the pass's, so a test can set it). */
|
||||
@@ -266,6 +269,9 @@ export function evaluateHealth(snapshot: HealthSnapshot, thresholds: Thresholds,
|
||||
// Speedtest Tracker: the internet slower than it is sold at, speed tests that keep failing.
|
||||
for (const s of snapshot.speedtest ?? []) for (const p of s.problems) out.push({ key: p.key, source: `speedtest:${s.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Fragnet game servers: backups that are late, disks and memory nearly full.
|
||||
for (const f of snapshot.fragnet ?? []) for (const p of f.problems) out.push({ key: p.key, source: `fragnet:${f.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
// Hetzner Cloud: servers that went off, traffic quotas, rescue mode left on, backups that are late.
|
||||
for (const h of snapshot.hetzner ?? []) for (const p of h.problems) out.push({ key: p.key, source: `hetzner:${h.integrationId}`, message: p.message, severity: p.severity });
|
||||
|
||||
@@ -531,6 +537,25 @@ export async function collectSnapshot(): Promise<{ snapshot: HealthSnapshot; hel
|
||||
console.error("[health] couldn't judge Speedtest Tracker:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Fragnet game servers: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsF = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "fragnet"), eq(integrations.enabled, true)));
|
||||
if (rowsF.length > 0) {
|
||||
const statuses = await fragnetSyncStatuses();
|
||||
for (const r of rowsF) {
|
||||
const [state] = await db.select().from(fragnetState).where(eq(fragnetState.integrationId, r.id)).limit(1);
|
||||
if (!state || !isFragnetFresh(statuses[r.id], Date.now())) {
|
||||
held.add(`fragnet:${r.id}`);
|
||||
continue;
|
||||
}
|
||||
const loaded = await loadIntegrationConfig(r.id);
|
||||
(snapshot.fragnet ??= []).push({ integrationId: r.id, problems: fragnetProblems(r.id, r.name, fragnetServersOf(state), Date.now(), fragnetOptions(loaded?.config)) });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[health] couldn't judge Fragnet:", err instanceof Error ? err.message : err);
|
||||
}
|
||||
|
||||
// Hetzner Cloud: read by its own pass every five minutes; what is judged is what that pass stored, and only while it is recent.
|
||||
try {
|
||||
const rowsH = await db.select({ id: integrations.id, name: integrations.name }).from(integrations).where(and(eq(integrations.type, "hetzner"), eq(integrations.enabled, true)));
|
||||
|
||||
@@ -41,6 +41,8 @@ export function subjectOfConditionKey(key: string): string | null {
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^hetzner-(?:off|stuck|rescue|traffic|backup):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^fragnet-(?:disk|memory|backup):(\d+):/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^netbox-(?:workers):(\d+)/.exec(key);
|
||||
if (m) return `integration:${m[1]}`;
|
||||
m = /^n8n-(?:failed):(\d+)/.exec(key);
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../db/client.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState } from "../db/schema.js";
|
||||
import { assets, endpointChecks, heartbeats, integrations, maintenanceWindows, secrets, servers, upsState, jellyfinState, authentikState, immichState, paperlessState, nextcloudState, netboxState, n8nState, dockhandVulnerabilities, hetznerState, fragnetState } from "../db/schema.js";
|
||||
import { isUpsFresh, upsSyncStatuses } from "./upsSync.js";
|
||||
import { levelStatuses } from "./serviceLevels.js";
|
||||
import { backupState, listAll as listDuplicati } from "./duplicati.js";
|
||||
@@ -14,6 +14,7 @@ import { isNextcloudFresh, nextcloudSyncStatuses } from "./nextcloudSync.js";
|
||||
import { dockerVulnSyncStatuses, isDockerVulnFresh } from "./dockerVulnSync.js";
|
||||
import { isSpeedtestFresh, recentResults as recentSpeedtestResults, speedtestSyncStatuses } from "./speedtestSync.js";
|
||||
import { costOf as hetznerCostOf, hetznerSyncStatuses, isHetznerFresh, serversOf as hetznerServersOf } from "./hetznerSync.js";
|
||||
import { fragnetSyncStatuses, isFragnetFresh, newestBackup as fragnetNewestBackup, serversOf as fragnetServersOf } from "./fragnetSync.js";
|
||||
import { isNetBoxFresh, netboxSyncStatuses } from "./netboxSync.js";
|
||||
import { isN8nFresh, n8nSyncStatuses, workflowsOf as n8nWorkflowsOf } from "./n8nSync.js";
|
||||
import { componentsOf as paperlessComponentsOf, failedOf as paperlessFailedOf, isPaperlessFresh, paperlessSyncStatuses } from "./paperlessSync.js";
|
||||
@@ -100,6 +101,28 @@ async function addSpeedtestMetrics(out: Exposition, now: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Fragnet game server, for /metrics: whether it is running, its disk and memory against their limits and when it was last backed up. Only while the reading is recent. */
|
||||
async function addFragnetMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await fragnetSyncStatuses();
|
||||
const names = new Map((await db.select({ id: integrations.id, name: integrations.name, enabled: integrations.enabled }).from(integrations).where(eq(integrations.type, "fragnet"))).map((r) => [r.id, r]));
|
||||
for (const s of await db.select().from(fragnetState)) {
|
||||
const row = names.get(s.integrationId);
|
||||
if (!row || !row.enabled || !isFragnetFresh(statuses[s.integrationId], now.getTime())) continue;
|
||||
for (const server of fragnetServersOf(s)) {
|
||||
const labels = { account: row.name, server: server.name, game: server.game ?? "" };
|
||||
out.add("homelab_fragnet_server_running", "gauge", "1 if the Fragnet game server is running, else 0.", server.state === "running" ? 1 : 0, labels);
|
||||
if (server.usage) {
|
||||
out.add("homelab_fragnet_disk_used_bytes", "gauge", "Disk a Fragnet game server uses, in bytes.", server.usage.diskBytes, labels);
|
||||
out.add("homelab_fragnet_memory_used_bytes", "gauge", "Memory a Fragnet game server uses, in bytes.", server.usage.memoryBytes, labels);
|
||||
}
|
||||
if (server.diskLimitMb) out.add("homelab_fragnet_disk_limit_bytes", "gauge", "The disk a Fragnet game server may use, in bytes.", server.diskLimitMb * 1024 ** 2, labels);
|
||||
if (server.memoryLimitMb) out.add("homelab_fragnet_memory_limit_bytes", "gauge", "The memory a Fragnet game server may use, in bytes.", server.memoryLimitMb * 1024 ** 2, labels);
|
||||
const backedUp = fragnetNewestBackup(server)?.at;
|
||||
if (backedUp) out.add("homelab_fragnet_newest_backup_timestamp_seconds", "gauge", "When the newest successful backup of a Fragnet game server finished, in seconds since 1970.", Date.parse(backedUp) / 1000, labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Each Hetzner Cloud server, for /metrics: whether it is running, its traffic this month against the quota, and the project's monthly cost estimate. Only while the reading is recent. */
|
||||
async function addHetznerMetrics(out: Exposition, now: Date) {
|
||||
const statuses = await hetznerSyncStatuses();
|
||||
@@ -442,6 +465,7 @@ export async function buildMetrics(now: Date = new Date()): Promise<string> {
|
||||
await addDockerVulnMetrics(out, now);
|
||||
await addHetznerMetrics(out, now);
|
||||
await addSpeedtestMetrics(out, now);
|
||||
await addFragnetMetrics(out, now);
|
||||
await addHeartbeatMetrics(out, now);
|
||||
return out.toString();
|
||||
}
|
||||
Reference in new issue
Block a user