training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:07.496976Z",
     "iopub.status.busy": "2023-04-25T20:45:07.496697Z",
     "iopub.status.idle": "2023-04-25T20:45:08.612440Z",
     "shell.execute_reply": "2023-04-25T20:45:08.611784Z",
     "shell.execute_reply.started": "2023-04-25T20:45:07.496952Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'rainbow'...\n",
      "remote: Enumerating objects: 121, done.\u001b[K\n",
      "remote: Counting objects: 100% (121/121), done.\u001b[K\n",
      "remote: Compressing objects: 100% (87/87), done.\u001b[K\n",
      "remote: Total 121 (delta 79), reused 69 (delta 33), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (121/121), 43.74 KiB | 2.30 MiB/s, done.\n",
      "Resolving deltas: 100% (79/79), done.\n"
     ]
    }
   ],
   "source": [
    "import shutil, os\n",
    "if os.path.isdir('rainbow') : shutil.rmtree(\"rainbow\")\n",
    "!git clone https://github.com/ClementPerroud/Rainbow-Agent rainbow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:08.613828Z",
     "iopub.status.busy": "2023-04-25T20:45:08.613611Z",
     "iopub.status.idle": "2023-04-25T20:45:24.690072Z",
     "shell.execute_reply": "2023-04-25T20:45:24.689575Z",
     "shell.execute_reply.started": "2023-04-25T20:45:08.613801Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gym-trading-env\n",
      "  Downloading gym_trading_env-0.2.3-py3-none-any.whl (16 kB)\n",
      "Collecting tensorflow-addons\n",
      "  Downloading tensorflow_addons-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (591 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m591.0/591.0 kB\u001b[0m \u001b[31m54.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.9/dist-packages (1.1.2)\n",
      "Collecting scikit-learn\n",
      "  Downloading scikit_learn-1.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m102.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting pandas>=1.5.3\n",
      "  Downloading pandas-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m87.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: nest-asyncio in /usr/local/lib/python3.9/dist-packages (from gym-trading-env) (1.5.6)\n",
      "Collecting numpy>=1.24.2\n",
      "  Downloading numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m85.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting gymnasium>=0.28.1\n",
      "  Downloading gymnasium-0.28.1-py3-none-any.whl (925 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m925.5/925.5 kB\u001b[0m \u001b[31m104.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting flask>=2.2.3\n",
      "  Downloading Flask-2.3.0-py3-none-any.whl (96 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.0/97.0 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pyecharts>=2.0.2\n",
      "  Downloading pyecharts-2.0.3-py3-none-any.whl (147 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.1/147.1 kB\u001b[0m \u001b[31m47.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting ccxt==3.0.59\n",
      "  Downloading ccxt-3.0.59-py2.py3-none-any.whl (3.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m123.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting cryptography>=2.6.1\n",
      "  Downloading cryptography-40.0.2-cp36-abi3-manylinux_2_28_x86_64.whl (3.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.7/3.7 MB\u001b[0m \u001b[31m126.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: yarl>=1.7.2 in /usr/local/lib/python3.9/dist-packages (from ccxt==3.0.59->gym-trading-env) (1.8.2)\n",
      "Requirement already satisfied: setuptools>=60.9.0 in /usr/local/lib/python3.9/dist-packages (from ccxt==3.0.59->gym-trading-env) (66.1.1)\n",
      "Requirement already satisfied: certifi>=2018.1.18 in /usr/lib/python3/dist-packages (from ccxt==3.0.59->gym-trading-env) (2019.11.28)\n",
      "Collecting aiodns>=1.1.1\n",
      "  Downloading aiodns-3.0.0-py3-none-any.whl (5.0 kB)\n",
      "Requirement already satisfied: requests>=2.18.4 in /usr/local/lib/python3.9/dist-packages (from ccxt==3.0.59->gym-trading-env) (2.28.2)\n",
      "Requirement already satisfied: aiohttp>=3.8 in /usr/local/lib/python3.9/dist-packages (from ccxt==3.0.59->gym-trading-env) (3.8.3)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from tensorflow-addons) (23.0)\n",
      "Collecting typeguard<3.0.0,>=2.7\n",
      "  Downloading typeguard-2.13.3-py3-none-any.whl (17 kB)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (1.2.0)\n",
      "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (1.9.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (3.1.0)\n",
      "Requirement already satisfied: importlib-metadata>=3.6.0 in /usr/local/lib/python3.9/dist-packages (from flask>=2.2.3->gym-trading-env) (6.0.0)\n",
      "Requirement already satisfied: click>=8.1.3 in /usr/local/lib/python3.9/dist-packages (from flask>=2.2.3->gym-trading-env) (8.1.3)\n",
      "Collecting Werkzeug>=2.3.0\n",
      "  Downloading Werkzeug-2.3.0-py3-none-any.whl (233 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.7/233.7 kB\u001b[0m \u001b[31m66.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting blinker>=1.6.2\n",
      "  Downloading blinker-1.6.2-py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: Jinja2>=3.1.2 in /usr/local/lib/python3.9/dist-packages (from flask>=2.2.3->gym-trading-env) (3.1.2)\n",
      "Collecting itsdangerous>=2.1.2\n",
      "  Downloading itsdangerous-2.1.2-py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.9/dist-packages (from gymnasium>=0.28.1->gym-trading-env) (2.2.0)\n",
      "Collecting farama-notifications>=0.0.1\n",
      "  Downloading Farama_Notifications-0.0.4-py3-none-any.whl (2.5 kB)\n",
      "Collecting jax-jumpy>=1.0.0\n",
      "  Downloading jax_jumpy-1.0.0-py3-none-any.whl (20 kB)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in /usr/local/lib/python3.9/dist-packages (from gymnasium>=0.28.1->gym-trading-env) (4.4.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas>=1.5.3->gym-trading-env) (2022.7.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.9/dist-packages (from pandas>=1.5.3->gym-trading-env) (2.8.2)\n",
      "Collecting tzdata>=2022.1\n",
      "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m76.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting simplejson\n",
      "  Downloading simplejson-3.19.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (137 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.4/137.4 kB\u001b[0m \u001b[31m41.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting prettytable\n",
      "  Downloading prettytable-3.7.0-py3-none-any.whl (27 kB)\n",
      "Collecting pycares>=4.0.0\n",
      "  Downloading pycares-4.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (288 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m288.7/288.7 kB\u001b[0m \u001b[31m65.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp>=3.8->ccxt==3.0.59->gym-trading-env) (4.0.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp>=3.8->ccxt==3.0.59->gym-trading-env) (1.3.3)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp>=3.8->ccxt==3.0.59->gym-trading-env) (6.0.4)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp>=3.8->ccxt==3.0.59->gym-trading-env) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp>=3.8->ccxt==3.0.59->gym-trading-env) (18.2.0)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp>=3.8->ccxt==3.0.59->gym-trading-env) (2.1.1)\n",
      "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.9/dist-packages (from cryptography>=2.6.1->ccxt==3.0.59->gym-trading-env) (1.15.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata>=3.6.0->flask>=2.2.3->gym-trading-env) (3.11.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from Jinja2>=3.1.2->flask>=2.2.3->gym-trading-env) (2.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas>=1.5.3->gym-trading-env) (1.14.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.18.4->ccxt==3.0.59->gym-trading-env) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.18.4->ccxt==3.0.59->gym-trading-env) (1.26.14)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.9/dist-packages (from prettytable->pyecharts>=2.0.2->gym-trading-env) (0.2.6)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.9/dist-packages (from cffi>=1.12->cryptography>=2.6.1->ccxt==3.0.59->gym-trading-env) (2.21)\n",
      "Installing collected packages: farama-notifications, Werkzeug, tzdata, typeguard, simplejson, prettytable, numpy, itsdangerous, blinker, tensorflow-addons, pyecharts, pycares, pandas, jax-jumpy, flask, cryptography, scikit-learn, gymnasium, aiodns, ccxt, gym-trading-env\n",
      "  Attempting uninstall: Werkzeug\n",
      "    Found existing installation: Werkzeug 2.2.2\n",
      "    Uninstalling Werkzeug-2.2.2:\n",
      "      Successfully uninstalled Werkzeug-2.2.2\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.23.4\n",
      "    Uninstalling numpy-1.23.4:\n",
      "      Successfully uninstalled numpy-1.23.4\n",
      "  Attempting uninstall: pandas\n",
      "    Found existing installation: pandas 1.5.0\n",
      "    Uninstalling pandas-1.5.0:\n",
      "      Successfully uninstalled pandas-1.5.0\n",
      "  Attempting uninstall: scikit-learn\n",
      "    Found existing installation: scikit-learn 1.1.2\n",
      "    Uninstalling scikit-learn-1.1.2:\n",
      "      Successfully uninstalled scikit-learn-1.1.2\n",
      "Successfully installed Werkzeug-2.3.0 aiodns-3.0.0 blinker-1.6.2 ccxt-3.0.59 cryptography-40.0.2 farama-notifications-0.0.4 flask-2.3.0 gym-trading-env-0.2.3 gymnasium-0.28.1 itsdangerous-2.1.2 jax-jumpy-1.0.0 numpy-1.24.3 pandas-2.0.1 prettytable-3.7.0 pycares-4.3.0 pyecharts-2.0.3 scikit-learn-1.2.2 simplejson-3.19.1 tensorflow-addons-0.20.0 typeguard-2.13.3 tzdata-2023.3\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade gym-trading-env tensorflow-addons scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:24.690958Z",
     "iopub.status.busy": "2023-04-25T20:45:24.690789Z",
     "iopub.status.idle": "2023-04-25T20:45:27.647431Z",
     "shell.execute_reply": "2023-04-25T20:45:27.646692Z",
     "shell.execute_reply.started": "2023-04-25T20:45:24.690939Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
      "\n",
      "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
      "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
      "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
      "\n",
      "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
      "\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.9/dist-packages/tensorflow_addons/utils/ensure_tf_install.py:53: UserWarning: Tensorflow Addons supports using Python ops for all Tensorflow versions above or equal to 2.10.0 and strictly below 2.13.0 (nightly versions are not supported). \n",
      " The versions of TensorFlow you are currently using is 2.9.2 and is not supported. \n",
      "Some things might work, some things might not.\n",
      "If you were to encounter a bug, do not file an issue.\n",
      "If you want to make sure you're using a tested and supported configuration, either change the TensorFlow version or the TensorFlow Addons's version. \n",
      "You can find the compatibility matrix in TensorFlow Addon's readme:\n",
      "https://github.com/tensorflow/addons\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import gymnasium as gym\n",
    "from sklearn.preprocessing import robust_scale\n",
    "\n",
    "from rainbow.agent import Rainbow\n",
    "\n",
    "import sys\n",
    "import gym_trading_env\n",
    "import nest_asyncio\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create environments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:27.649230Z",
     "iopub.status.busy": "2023-04-25T20:45:27.648870Z",
     "iopub.status.idle": "2023-04-25T20:45:28.576145Z",
     "shell.execute_reply": "2023-04-25T20:45:28.575522Z",
     "shell.execute_reply.started": "2023-04-25T20:45:27.649211Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_features(df):\n",
    "    df[\"feature_close\"] = robust_scale(df[\"close\"].pct_change())\n",
    "    df[\"feature_open\"] = robust_scale(df[\"open\"]/df[\"close\"])\n",
    "    df[\"feature_high\"] = robust_scale(df[\"high\"]/df[\"close\"])\n",
    "    df[\"feature_low\"] = robust_scale(df[\"low\"]/df[\"close\"])\n",
    "    df[\"feature_volume\"] = robust_scale(df[\"volume\"] / df[\"volume\"].rolling(7*24).max())\n",
    "    df.dropna(inplace= True) # Clean your data !\n",
    "    return df\n",
    "\n",
    "\n",
    "def reward_function(history):\n",
    "    return 800*np.log(history[\"portfolio_valuation\", -1] / history[\"portfolio_valuation\", -2]) #log (p_t / p_t-1 )\n",
    "\n",
    "def max_drawdown(history):\n",
    "    networth_array = history['portfolio_valuation']\n",
    "    _max_networth = networth_array[0]\n",
    "    _max_drawdown = 0\n",
    "    for networth in networth_array:\n",
    "        if networth > _max_networth:\n",
    "            _max_networth = networth\n",
    "        drawdown = ( networth - _max_networth ) / _max_networth\n",
    "        if drawdown < _max_drawdown:\n",
    "            _max_drawdown = drawdown\n",
    "    return f\"{_max_drawdown*100:5.2f}%\"\n",
    "\n",
    "def make_env(dir):\n",
    "    env = gym.make(\n",
    "        \"MultiDatasetTradingEnv\",\n",
    "        \n",
    "        \n",
    "        dataset_dir= dir,\n",
    "        preprocess= add_features,\n",
    "        windows= 15,\n",
    "        positions = [ -1, -0.5, 0, 1, 2], # From -1 (=SHORT), to +1 (=LONG)\n",
    "        initial_position = 0,\n",
    "        trading_fees = 0.01/100, # 0.01% per stock buy / sell (Binance fees)\n",
    "        borrow_interest_rate= 0.0003/100, # 0.0003% per timestep (= 1h here)\n",
    "        reward_function = reward_function,\n",
    "        portfolio_initial_value = 1000, # here, in USDT\n",
    "        \n",
    "        verbose= 1,\n",
    "    )\n",
    "    env.add_metric('Position Changes', lambda history : f\"{ 100*np.sum(np.diff(history['position']) != 0)/len(history['position']):5.2f}%\" )\n",
    "    env.add_metric('Max Drawdown', max_drawdown)\n",
    "    return env\n",
    "\n",
    "\n",
    "training_envs = gym.vector.SyncVectorEnv([lambda: make_env(\"/notebooks/data/processed/training/*.pkl\") for _ in range(5)])\n",
    "validation_envs = gym.vector.SyncVectorEnv([lambda: make_env(\"/notebooks/data/processed/validation/*.pkl\") for _ in range(5)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:28.577112Z",
     "iopub.status.busy": "2023-04-25T20:45:28.576930Z",
     "iopub.status.idle": "2023-04-25T20:45:31.067018Z",
     "shell.execute_reply": "2023-04-25T20:45:31.066482Z",
     "shell.execute_reply.started": "2023-04-25T20:45:28.577095Z"
    }
   },
   "outputs": [],
   "source": [
    "agent = Rainbow(\n",
    "    simultaneous_training_env = 5,\n",
    "    \n",
    "    #Distributional\n",
    "    distributional= True,\n",
    "    v_min= -200,\n",
    "    v_max = 250,\n",
    "    nb_atoms= 51, \n",
    "    # Prioritized Replay\n",
    "    prioritized_replay = False,\n",
    "    prioritized_replay_alpha= 0.5,\n",
    "    prioritized_replay_beta_function = lambda episode, step : min(1, 0.5 + 0.5*step/150_000),\n",
    "    \n",
    "    # General\n",
    "    multi_steps = 3,\n",
    "    nb_states = 6,\n",
    "    nb_actions = 4,\n",
    "    gamma = 0.99,\n",
    "    replay_capacity = 1E8,\n",
    "    tau = 2000,\n",
    "    \n",
    "    # Model\n",
    "    window= 15,\n",
    "    units = [16,16, 16],\n",
    "    dropout= 0.2,\n",
    "    adversarial= True,\n",
    "    noisy= False,\n",
    "    learning_rate = 3*2.5E-4,\n",
    "\n",
    "    batch_size= 128,\n",
    "    train_every = 10,\n",
    "    epsilon_function = lambda episode, step : max(0.001, (1 - 5E-5)** step),\n",
    "    name = \"Rainbow\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:31.068198Z",
     "iopub.status.busy": "2023-04-25T20:45:31.067808Z",
     "iopub.status.idle": "2023-04-25T20:45:31.072860Z",
     "shell.execute_reply": "2023-04-25T20:45:31.072377Z",
     "shell.execute_reply.started": "2023-04-25T20:45:31.068180Z"
    }
   },
   "outputs": [],
   "source": [
    "def train(steps = 100_000):\n",
    "    print(\"___________________________________________ TRAINING ___________________________________________\")\n",
    "    if 'obs' not in globals():\n",
    "        global obs\n",
    "        obs, info = training_envs.reset()\n",
    "    for _ in range(steps):\n",
    "        actions = agent.e_greedy_pick_actions_or_random(obs)\n",
    "        next_obs, rewards, dones, truncateds, infos = training_envs.step(actions)\n",
    "\n",
    "        agent.store_replays(obs, actions, rewards, next_obs, dones, truncateds)\n",
    "        agent.train()\n",
    "\n",
    "        obs = next_obs\n",
    "\n",
    "def evaluation():\n",
    "    print(\"___________________________________________ VALIDATION ___________________________________________\")\n",
    "    val_obs, info = validation_envs.reset()\n",
    "    check = np.array([False for _ in range(val_obs.shape[0])])\n",
    "    while not np.all(check):\n",
    "        actions = agent.e_greedy_pick_actions_or_random(val_obs)\n",
    "        next_obs, rewards, dones, truncateds, infos = validation_envs.step(actions)\n",
    "        val_obs = next_obs\n",
    "        check += dones + truncateds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T20:45:31.073670Z",
     "iopub.status.busy": "2023-04-25T20:45:31.073517Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : -29.94%   |   Position Changes : 32.31%   |   Max Drawdown : -53.06%\n",
      "Market Return : -24.83%   |   Portfolio Return : -37.88%   |   Position Changes : 32.38%   |   Max Drawdown : -48.07%\n",
      "Market Return : -24.83%   |   Portfolio Return : -27.37%   |   Position Changes : 32.37%   |   Max Drawdown : -45.06%\n",
      "Market Return : -36.93%   |   Portfolio Return : -46.70%   |   Position Changes : 32.15%   |   Max Drawdown : -59.95%\n",
      "Market Return : -24.98%   |   Portfolio Return : -42.31%   |   Position Changes : 31.96%   |   Max Drawdown : -50.17%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1896.93%   |   Portfolio Return : -73.74%   |   Position Changes : 39.31%   |   Max Drawdown : -88.44%\n",
      "↳ Env 0 : 000 :    52314   |   00:03:10   |   Epsilon :  7.31%   |   Mean Loss (last 10k) : 3.6348E+00   |   Tot. Rewards : -1069.59   |   Rewards (/1000 steps) :   -20.45   |   Length :  52314\n",
      "Market Return : 1753.28%   |   Portfolio Return : -97.36%   |   Position Changes : 36.80%   |   Max Drawdown : -98.20%\n",
      "Market Return : 1753.28%   |   Portfolio Return : -96.68%   |   Position Changes : 36.52%   |   Max Drawdown : -97.29%\n",
      "↳ Env 1 : 000 :    58071   |   00:03:32   |   Epsilon :  5.48%   |   Mean Loss (last 10k) : 3.6003E+00   |   Tot. Rewards : -2908.65   |   Rewards (/1000 steps) :   -50.09   |   Length :  58071\n",
      "↳ Env 4 : 000 :    58071   |   00:03:32   |   Epsilon :  5.48%   |   Mean Loss (last 10k) : 3.6003E+00   |   Tot. Rewards : -2724.69   |   Rewards (/1000 steps) :   -46.92   |   Length :  58071\n",
      "Market Return : 1755.22%   |   Portfolio Return : -96.53%   |   Position Changes : 36.50%   |   Max Drawdown : -96.69%\n",
      "Market Return : 908.31%   |   Portfolio Return : -97.30%   |   Position Changes : 36.83%   |   Max Drawdown : -97.56%\n",
      "↳ Env 2 : 000 :    58186   |   00:03:33   |   Epsilon :  5.45%   |   Mean Loss (last 10k) : 3.5996E+00   |   Tot. Rewards : -2689.72   |   Rewards (/1000 steps) :   -46.23   |   Length :  58186\n",
      "↳ Env 3 : 000 :    58186   |   00:03:33   |   Epsilon :  5.45%   |   Mean Loss (last 10k) : 3.5996E+00   |   Tot. Rewards : -2888.17   |   Rewards (/1000 steps) :   -49.64   |   Length :  58186\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : -20.06%   |   Position Changes :  7.60%   |   Max Drawdown : -25.14%\n",
      "Market Return : -24.98%   |   Portfolio Return : -11.05%   |   Position Changes :  7.84%   |   Max Drawdown : -17.96%\n",
      "Market Return : -24.95%   |   Portfolio Return : -16.40%   |   Position Changes :  7.74%   |   Max Drawdown : -20.90%\n",
      "Market Return : -24.98%   |   Portfolio Return : -16.94%   |   Position Changes :  7.81%   |   Max Drawdown : -21.51%\n",
      "Market Return : -36.88%   |   Portfolio Return :  1.37%   |   Position Changes :  7.38%   |   Max Drawdown : -8.38%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.99%   |   Portfolio Return : -21.81%   |   Position Changes :  3.88%   |   Max Drawdown : -57.75%\n",
      "Market Return : -36.93%   |   Portfolio Return : -52.67%   |   Position Changes :  3.74%   |   Max Drawdown : -70.25%\n",
      "Market Return : -24.85%   |   Portfolio Return : -3.92%   |   Position Changes :  3.88%   |   Max Drawdown : -48.94%\n",
      "Market Return : -24.98%   |   Portfolio Return : -4.94%   |   Position Changes :  4.00%   |   Max Drawdown : -51.21%\n",
      "Market Return : -24.98%   |   Portfolio Return : -11.92%   |   Position Changes :  3.89%   |   Max Drawdown : -51.88%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 835.58%   |   Portfolio Return : -4.09%   |   Position Changes :  5.56%   |   Max Drawdown : -62.34%\n",
      "↳ Env 0 : 001 :   105489   |   00:07:59   |   Epsilon :  0.51%   |   Mean Loss (last 10k) : 3.2821E+00   |   Tot. Rewards :   -33.41   |   Rewards (/1000 steps) :    -0.63   |   Length :  53175\n",
      "Market Return : 1907.87%   |   Portfolio Return : 177.62%   |   Position Changes :  5.12%   |   Max Drawdown : -50.57%\n",
      "↳ Env 2 : 001 :   110541   |   00:08:21   |   Epsilon :  0.40%   |   Mean Loss (last 10k) : 3.2237E+00   |   Tot. Rewards :   816.87   |   Rewards (/1000 steps) :    15.60   |   Length :  52355\n",
      "Market Return : 832.99%   |   Portfolio Return : -21.41%   |   Position Changes :  4.55%   |   Max Drawdown : -50.38%\n",
      "↳ Env 3 : 001 :   111379   |   00:08:25   |   Epsilon :  0.38%   |   Mean Loss (last 10k) : 3.2141E+00   |   Tot. Rewards :  -192.79   |   Rewards (/1000 steps) :    -3.62   |   Length :  53193\n",
      "Market Return : 1754.87%   |   Portfolio Return : 27.74%   |   Position Changes :  4.58%   |   Max Drawdown : -59.58%\n",
      "Market Return : 1754.87%   |   Portfolio Return : 73.03%   |   Position Changes :  4.60%   |   Max Drawdown : -61.22%\n",
      "↳ Env 1 : 001 :   116129   |   00:08:46   |   Epsilon :  0.30%   |   Mean Loss (last 10k) : 3.1597E+00   |   Tot. Rewards :   195.87   |   Rewards (/1000 steps) :     3.37   |   Length :  58058\n",
      "↳ Env 4 : 001 :   116129   |   00:08:46   |   Epsilon :  0.30%   |   Mean Loss (last 10k) : 3.1597E+00   |   Tot. Rewards :   438.62   |   Rewards (/1000 steps) :     7.55   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.87%   |   Portfolio Return : -10.95%   |   Position Changes :  6.47%   |   Max Drawdown : -37.08%\n",
      "Market Return : -24.98%   |   Portfolio Return :  2.58%   |   Position Changes :  6.63%   |   Max Drawdown : -22.86%\n",
      "Market Return : -24.95%   |   Portfolio Return : -7.30%   |   Position Changes :  6.31%   |   Max Drawdown : -24.99%\n",
      "Market Return : -36.96%   |   Portfolio Return : -26.81%   |   Position Changes :  6.22%   |   Max Drawdown : -42.39%\n",
      "Market Return : -36.96%   |   Portfolio Return : -23.33%   |   Position Changes :  6.25%   |   Max Drawdown : -39.53%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : -42.13%   |   Position Changes :  3.66%   |   Max Drawdown : -69.03%\n",
      "Market Return : -24.99%   |   Portfolio Return :  1.77%   |   Position Changes :  3.03%   |   Max Drawdown : -43.43%\n",
      "Market Return : -36.94%   |   Portfolio Return : -41.01%   |   Position Changes :  3.53%   |   Max Drawdown : -66.58%\n",
      "Market Return : -36.96%   |   Portfolio Return : -30.21%   |   Position Changes :  3.39%   |   Max Drawdown : -60.61%\n",
      "Market Return : -24.98%   |   Portfolio Return : -29.40%   |   Position Changes :  3.27%   |   Max Drawdown : -56.76%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 909.84%   |   Portfolio Return : 371.43%   |   Position Changes :  2.33%   |   Max Drawdown : -50.22%\n",
      "↳ Env 0 : 002 :   163562   |   00:13:46   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.7428E+00   |   Tot. Rewards :  1240.48   |   Rewards (/1000 steps) :    21.36   |   Length :  58073\n",
      "Market Return : 1883.15%   |   Portfolio Return : 898.06%   |   Position Changes :  2.34%   |   Max Drawdown : -59.50%\n",
      "↳ Env 3 : 002 :   163753   |   00:13:47   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.7417E+00   |   Tot. Rewards :  1840.52   |   Rewards (/1000 steps) :    35.14   |   Length :  52374\n",
      "Market Return : 908.42%   |   Portfolio Return : 204.13%   |   Position Changes :  2.21%   |   Max Drawdown : -60.35%\n",
      "↳ Env 2 : 002 :   168613   |   00:14:12   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.7159E+00   |   Tot. Rewards :   889.83   |   Rewards (/1000 steps) :    15.32   |   Length :  58072\n",
      "Market Return : 834.53%   |   Portfolio Return : 80.42%   |   Position Changes :  2.10%   |   Max Drawdown : -71.60%\n",
      "↳ Env 1 : 002 :   169340   |   00:14:16   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.7121E+00   |   Tot. Rewards :   472.09   |   Rewards (/1000 steps) :     8.87   |   Length :  53211\n",
      "Market Return : 1751.40%   |   Portfolio Return : 642.72%   |   Position Changes :  1.84%   |   Max Drawdown : -80.92%\n",
      "↳ Env 4 : 002 :   174200   |   00:14:41   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6880E+00   |   Tot. Rewards :  1604.12   |   Rewards (/1000 steps) :    27.62   |   Length :  58071\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.83%   |   Portfolio Return : -9.69%   |   Position Changes :  4.94%   |   Max Drawdown : -47.04%\n",
      "Market Return : -24.97%   |   Portfolio Return : 11.38%   |   Position Changes :  5.12%   |   Max Drawdown : -39.57%\n",
      "Market Return : -36.93%   |   Portfolio Return : -27.21%   |   Position Changes :  5.43%   |   Max Drawdown : -63.37%\n",
      "Market Return : -36.78%   |   Portfolio Return : -17.87%   |   Position Changes :  5.73%   |   Max Drawdown : -58.84%\n",
      "Market Return : -24.98%   |   Portfolio Return : -13.94%   |   Position Changes :  4.97%   |   Max Drawdown : -51.03%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : -39.38%   |   Position Changes :  0.15%   |   Max Drawdown : -70.52%\n",
      "Market Return : -36.87%   |   Portfolio Return : -36.40%   |   Position Changes :  0.14%   |   Max Drawdown : -69.46%\n",
      "Market Return : -24.95%   |   Portfolio Return : -23.32%   |   Position Changes :  0.16%   |   Max Drawdown : -59.79%\n",
      "Market Return : -24.98%   |   Portfolio Return : -24.41%   |   Position Changes :  0.15%   |   Max Drawdown : -60.20%\n",
      "Market Return : -36.94%   |   Portfolio Return : -38.33%   |   Position Changes :  0.19%   |   Max Drawdown : -69.97%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1907.87%   |   Portfolio Return : 4958.19%   |   Position Changes :  2.11%   |   Max Drawdown : -68.18%\n",
      "↳ Env 0 : 003 :   215917   |   00:19:35   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5816E+00   |   Tot. Rewards :  3138.87   |   Rewards (/1000 steps) :    59.95   |   Length :  52355\n",
      "Market Return : 833.62%   |   Portfolio Return : 705.48%   |   Position Changes :  2.39%   |   Max Drawdown : -52.33%\n",
      "↳ Env 2 : 003 :   221776   |   00:20:07   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5788E+00   |   Tot. Rewards :  1669.01   |   Rewards (/1000 steps) :    31.39   |   Length :  53163\n",
      "Market Return : 1754.87%   |   Portfolio Return : 1715.36%   |   Position Changes :  2.18%   |   Max Drawdown : -65.96%\n",
      "↳ Env 3 : 003 :   221811   |   00:20:08   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5788E+00   |   Tot. Rewards :  2319.10   |   Rewards (/1000 steps) :    39.94   |   Length :  58058\n",
      "Market Return : 1755.22%   |   Portfolio Return : 878.54%   |   Position Changes :  2.13%   |   Max Drawdown : -69.77%\n",
      "↳ Env 1 : 003 :   227527   |   00:20:40   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5763E+00   |   Tot. Rewards :  1824.72   |   Rewards (/1000 steps) :    31.36   |   Length :  58187\n",
      "Market Return : 1754.87%   |   Portfolio Return : 1481.13%   |   Position Changes :  2.15%   |   Max Drawdown : -69.66%\n",
      "↳ Env 4 : 003 :   232258   |   00:21:08   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5767E+00   |   Tot. Rewards :  2208.58   |   Rewards (/1000 steps) :    38.04   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : -36.22%   |   Position Changes :  0.09%   |   Max Drawdown : -69.46%\n",
      "Market Return : -24.83%   |   Portfolio Return : -24.29%   |   Position Changes :  0.13%   |   Max Drawdown : -60.35%\n",
      "Market Return : -36.78%   |   Portfolio Return : -38.35%   |   Position Changes :  0.14%   |   Max Drawdown : -69.68%\n",
      "Market Return : -24.85%   |   Portfolio Return : -25.49%   |   Position Changes :  0.13%   |   Max Drawdown : -59.84%\n",
      "Market Return : -24.95%   |   Portfolio Return : -27.44%   |   Position Changes :  0.13%   |   Max Drawdown : -61.34%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.94%   |   Portfolio Return : -24.35%   |   Position Changes :  0.20%   |   Max Drawdown : -60.28%\n",
      "Market Return : -24.95%   |   Portfolio Return : -24.24%   |   Position Changes :  0.14%   |   Max Drawdown : -60.43%\n",
      "Market Return : -24.98%   |   Portfolio Return : -24.69%   |   Position Changes :  0.21%   |   Max Drawdown : -60.03%\n",
      "Market Return : -24.85%   |   Portfolio Return : -24.44%   |   Position Changes :  0.20%   |   Max Drawdown : -61.31%\n",
      "Market Return : -36.94%   |   Portfolio Return : -42.07%   |   Position Changes :  0.21%   |   Max Drawdown : -69.65%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1752.50%   |   Portfolio Return : 4151.97%   |   Position Changes :  3.13%   |   Max Drawdown : -59.53%\n",
      "↳ Env 0 : 004 :   273989   |   00:26:39   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5764E+00   |   Tot. Rewards :  2999.97   |   Rewards (/1000 steps) :    51.66   |   Length :  58072\n",
      "Market Return : 835.01%   |   Portfolio Return : 1610.61%   |   Position Changes :  3.01%   |   Max Drawdown : -71.26%\n",
      "↳ Env 2 : 004 :   274960   |   00:26:45   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5773E+00   |   Tot. Rewards :  2271.55   |   Rewards (/1000 steps) :    42.71   |   Length :  53184\n",
      "Market Return : 908.42%   |   Portfolio Return : 1363.50%   |   Position Changes :  2.81%   |   Max Drawdown : -57.81%\n",
      "↳ Env 3 : 004 :   279883   |   00:27:16   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5813E+00   |   Tot. Rewards :  2146.73   |   Rewards (/1000 steps) :    36.97   |   Length :  58072\n",
      "Market Return : 1883.15%   |   Portfolio Return : 2147.21%   |   Position Changes :  2.81%   |   Max Drawdown : -73.48%\n",
      "↳ Env 1 : 004 :   279901   |   00:27:17   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5813E+00   |   Tot. Rewards :  2489.82   |   Rewards (/1000 steps) :    47.54   |   Length :  52374\n",
      "Market Return : 1896.93%   |   Portfolio Return : 3589.10%   |   Position Changes :  2.93%   |   Max Drawdown : -67.42%\n",
      "↳ Env 4 : 004 :   284573   |   00:27:48   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.5862E+00   |   Tot. Rewards :  2886.37   |   Rewards (/1000 steps) :    55.17   |   Length :  52315\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return :  0.77%   |   Position Changes :  2.06%   |   Max Drawdown : -53.45%\n",
      "Market Return : -36.96%   |   Portfolio Return : -11.12%   |   Position Changes :  2.15%   |   Max Drawdown : -57.55%\n",
      "Market Return : -24.98%   |   Portfolio Return :  4.40%   |   Position Changes :  2.01%   |   Max Drawdown : -43.55%\n",
      "Market Return : -36.94%   |   Portfolio Return : -2.39%   |   Position Changes :  2.16%   |   Max Drawdown : -56.64%\n",
      "Market Return : -24.86%   |   Portfolio Return : 14.23%   |   Position Changes :  2.20%   |   Max Drawdown : -44.11%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1883.15%   |   Portfolio Return : 4173.94%   |   Position Changes :  2.08%   |   Max Drawdown : -68.21%\n",
      "↳ Env 0 : 005 :   326363   |   00:33:20   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6292E+00   |   Tot. Rewards :  3004.10   |   Rewards (/1000 steps) :    57.36   |   Length :  52374\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : -38.44%   |   Position Changes :  3.42%   |   Max Drawdown : -64.24%\n",
      "Market Return : -36.93%   |   Portfolio Return : -22.15%   |   Position Changes :  2.94%   |   Max Drawdown : -67.83%\n",
      "Market Return : -36.78%   |   Portfolio Return : -21.99%   |   Position Changes :  3.83%   |   Max Drawdown : -61.94%\n",
      "Market Return : -24.95%   |   Portfolio Return : -15.42%   |   Position Changes :  3.55%   |   Max Drawdown : -51.39%\n",
      "Market Return : -36.93%   |   Portfolio Return : -20.32%   |   Position Changes :  3.63%   |   Max Drawdown : -67.21%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1751.40%   |   Portfolio Return : 2721.10%   |   Position Changes :  2.25%   |   Max Drawdown : -68.63%\n",
      "↳ Env 2 : 005 :   333031   |   00:34:44   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6373E+00   |   Tot. Rewards :  2671.77   |   Rewards (/1000 steps) :    46.01   |   Length :  58071\n",
      "Market Return : 834.53%   |   Portfolio Return : 1104.04%   |   Position Changes :  2.58%   |   Max Drawdown : -71.95%\n",
      "↳ Env 4 : 005 :   337784   |   00:35:20   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6423E+00   |   Tot. Rewards :  1990.62   |   Rewards (/1000 steps) :    37.41   |   Length :  53211\n",
      "Market Return : 909.84%   |   Portfolio Return : 1597.38%   |   Position Changes :  2.71%   |   Max Drawdown : -66.40%\n",
      "↳ Env 3 : 005 :   337956   |   00:35:21   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6425E+00   |   Tot. Rewards :  2265.34   |   Rewards (/1000 steps) :    39.01   |   Length :  58073\n",
      "Market Return : 909.11%   |   Portfolio Return : 1218.53%   |   Position Changes :  2.66%   |   Max Drawdown : -68.44%\n",
      "↳ Env 1 : 005 :   338088   |   00:35:22   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6426E+00   |   Tot. Rewards :  2063.28   |   Rewards (/1000 steps) :    35.46   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.95%   |   Portfolio Return : -17.74%   |   Position Changes :  2.43%   |   Max Drawdown : -56.34%\n",
      "Market Return : -36.93%   |   Portfolio Return : -31.94%   |   Position Changes :  2.48%   |   Max Drawdown : -69.16%\n",
      "Market Return : -36.93%   |   Portfolio Return : -32.91%   |   Position Changes :  2.50%   |   Max Drawdown : -69.42%\n",
      "Market Return : -36.93%   |   Portfolio Return : -31.77%   |   Position Changes :  2.48%   |   Max Drawdown : -68.94%\n",
      "Market Return : -24.96%   |   Portfolio Return : -23.99%   |   Position Changes :  2.24%   |   Max Drawdown : -58.98%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.76%   |   Portfolio Return : 1137.78%   |   Position Changes :  4.01%   |   Max Drawdown : -66.01%\n",
      "↳ Env 0 : 006 :   384550   |   00:41:60   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6697E+00   |   Tot. Rewards :  2012.72   |   Rewards (/1000 steps) :    34.59   |   Length :  58187\n",
      "Market Return : 1896.93%   |   Portfolio Return : 9692.91%   |   Position Changes :  3.50%   |   Max Drawdown : -63.50%\n",
      "↳ Env 2 : 006 :   385346   |   00:42:06   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6698E+00   |   Tot. Rewards :  3667.39   |   Rewards (/1000 steps) :    70.10   |   Length :  52315\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : -31.48%   |   Position Changes :  1.26%   |   Max Drawdown : -65.76%\n",
      "Market Return : -24.95%   |   Portfolio Return :  1.15%   |   Position Changes :  1.48%   |   Max Drawdown : -48.78%\n",
      "Market Return : -36.87%   |   Portfolio Return : -3.81%   |   Position Changes :  1.33%   |   Max Drawdown : -59.34%\n",
      "Market Return : -36.78%   |   Portfolio Return : -12.37%   |   Position Changes :  1.26%   |   Max Drawdown : -62.50%\n",
      "Market Return : -36.93%   |   Portfolio Return : -28.80%   |   Position Changes :  1.20%   |   Max Drawdown : -65.26%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 833.62%   |   Portfolio Return : 1949.47%   |   Position Changes :  3.55%   |   Max Drawdown : -56.37%\n",
      "↳ Env 3 : 006 :   391119   |   00:43:31   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6709E+00   |   Tot. Rewards :  2416.13   |   Rewards (/1000 steps) :    45.45   |   Length :  53163\n",
      "Market Return : 907.99%   |   Portfolio Return : 2547.89%   |   Position Changes :  3.71%   |   Max Drawdown : -67.29%\n",
      "↳ Env 4 : 006 :   395856   |   00:44:10   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6717E+00   |   Tot. Rewards :  2621.08   |   Rewards (/1000 steps) :    45.13   |   Length :  58072\n",
      "Market Return : 908.42%   |   Portfolio Return : 2623.62%   |   Position Changes :  3.71%   |   Max Drawdown : -62.40%\n",
      "↳ Env 1 : 006 :   396160   |   00:44:13   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6717E+00   |   Tot. Rewards :  2643.64   |   Rewards (/1000 steps) :    45.52   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 15.16%   |   Position Changes :  4.43%   |   Max Drawdown : -50.32%\n",
      "Market Return : -36.96%   |   Portfolio Return : -0.53%   |   Position Changes :  4.92%   |   Max Drawdown : -51.23%\n",
      "Market Return : -24.95%   |   Portfolio Return : -9.01%   |   Position Changes :  4.86%   |   Max Drawdown : -42.00%\n",
      "Market Return : -36.93%   |   Portfolio Return :  9.96%   |   Position Changes :  4.88%   |   Max Drawdown : -48.19%\n",
      "Market Return : -24.85%   |   Portfolio Return :  4.67%   |   Position Changes :  4.58%   |   Max Drawdown : -42.39%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1753.28%   |   Portfolio Return : 3236.60%   |   Position Changes :  3.82%   |   Max Drawdown : -69.98%\n",
      "↳ Env 0 : 007 :   442622   |   00:51:31   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6572E+00   |   Tot. Rewards :  2806.03   |   Rewards (/1000 steps) :    48.32   |   Length :  58072\n",
      "Market Return : 1751.91%   |   Portfolio Return : 2004.59%   |   Position Changes :  3.86%   |   Max Drawdown : -64.83%\n",
      "↳ Env 2 : 007 :   443533   |   00:51:40   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6572E+00   |   Tot. Rewards :  2437.36   |   Rewards (/1000 steps) :    41.89   |   Length :  58187\n",
      "Market Return : 909.61%   |   Portfolio Return : 2055.41%   |   Position Changes :  4.06%   |   Max Drawdown : -59.34%\n",
      "↳ Env 3 : 007 :   449306   |   00:52:32   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6567E+00   |   Tot. Rewards :  2456.45   |   Rewards (/1000 steps) :    42.22   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.87%   |   Portfolio Return : -2.15%   |   Position Changes :  3.16%   |   Max Drawdown : -58.87%\n",
      "Market Return : -24.98%   |   Portfolio Return : -1.22%   |   Position Changes :  2.91%   |   Max Drawdown : -48.53%\n",
      "Market Return : -36.87%   |   Portfolio Return : -3.81%   |   Position Changes :  3.15%   |   Max Drawdown : -58.90%\n",
      "Market Return : -36.94%   |   Portfolio Return : -6.41%   |   Position Changes :  3.12%   |   Max Drawdown : -58.75%\n",
      "Market Return : -24.86%   |   Portfolio Return : 10.37%   |   Position Changes :  2.79%   |   Max Drawdown : -45.31%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.31%   |   Portfolio Return : 533.45%   |   Position Changes :  3.29%   |   Max Drawdown : -69.69%\n",
      "↳ Env 4 : 007 :   454043   |   00:53:53   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6569E+00   |   Tot. Rewards :  1476.81   |   Rewards (/1000 steps) :    25.38   |   Length :  58187\n",
      "Market Return : 1752.50%   |   Portfolio Return : 1431.78%   |   Position Changes :  3.09%   |   Max Drawdown : -58.49%\n",
      "↳ Env 1 : 007 :   454232   |   00:53:55   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6570E+00   |   Tot. Rewards :  2183.21   |   Rewards (/1000 steps) :    37.59   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.87%   |   Portfolio Return :  6.04%   |   Position Changes :  1.71%   |   Max Drawdown : -53.68%\n",
      "Market Return : -36.96%   |   Portfolio Return : -6.74%   |   Position Changes :  1.72%   |   Max Drawdown : -58.30%\n",
      "Market Return : -24.95%   |   Portfolio Return : 29.37%   |   Position Changes :  1.95%   |   Max Drawdown : -36.61%\n",
      "Market Return : -24.86%   |   Portfolio Return : 37.39%   |   Position Changes :  1.90%   |   Max Drawdown : -38.29%\n",
      "Market Return : -24.96%   |   Portfolio Return : 19.56%   |   Position Changes :  1.86%   |   Max Drawdown : -39.97%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1754.87%   |   Portfolio Return : 2307.26%   |   Position Changes :  3.62%   |   Max Drawdown : -71.20%\n",
      "↳ Env 0 : 008 :   500680   |   01:02:01   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6479E+00   |   Tot. Rewards :  2544.86   |   Rewards (/1000 steps) :    43.83   |   Length :  58058\n",
      "Market Return : 908.31%   |   Portfolio Return : 1643.09%   |   Position Changes :  3.74%   |   Max Drawdown : -69.71%\n",
      "↳ Env 2 : 008 :   501720   |   01:02:12   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6477E+00   |   Tot. Rewards :  2286.59   |   Rewards (/1000 steps) :    39.30   |   Length :  58187\n",
      "Market Return : 835.01%   |   Portfolio Return : 3192.49%   |   Position Changes :  4.12%   |   Max Drawdown : -62.04%\n",
      "↳ Env 3 : 008 :   502490   |   01:02:20   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6478E+00   |   Tot. Rewards :  2795.38   |   Rewards (/1000 steps) :    52.56   |   Length :  53184\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : -4.01%   |   Position Changes : 12.43%   |   Max Drawdown : -37.56%\n",
      "Market Return : -36.93%   |   Portfolio Return : -19.18%   |   Position Changes : 12.51%   |   Max Drawdown : -49.14%\n",
      "Market Return : -24.95%   |   Portfolio Return : -14.29%   |   Position Changes : 11.30%   |   Max Drawdown : -29.71%\n",
      "Market Return : -36.78%   |   Portfolio Return :  7.91%   |   Position Changes : 11.34%   |   Max Drawdown : -29.87%\n",
      "Market Return : -24.96%   |   Portfolio Return : -3.79%   |   Position Changes : 11.69%   |   Max Drawdown : -30.22%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1752.50%   |   Portfolio Return : 2540.90%   |   Position Changes :  4.02%   |   Max Drawdown : -76.13%\n",
      "↳ Env 4 : 008 :   512115   |   01:04:34   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6475E+00   |   Tot. Rewards :  2618.96   |   Rewards (/1000 steps) :    45.10   |   Length :  58072\n",
      "Market Return : 1760.22%   |   Portfolio Return : 1663.68%   |   Position Changes :  3.80%   |   Max Drawdown : -80.27%\n",
      "↳ Env 1 : 008 :   512305   |   01:04:37   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6475E+00   |   Tot. Rewards :  2295.99   |   Rewards (/1000 steps) :    39.54   |   Length :  58073\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : 14.92%   |   Position Changes :  4.35%   |   Max Drawdown : -57.91%\n",
      "Market Return : -36.99%   |   Portfolio Return : -11.51%   |   Position Changes :  4.38%   |   Max Drawdown : -59.08%\n",
      "Market Return : -36.87%   |   Portfolio Return :  9.02%   |   Position Changes :  4.78%   |   Max Drawdown : -53.87%\n",
      "Market Return : -36.78%   |   Portfolio Return : -6.11%   |   Position Changes :  4.96%   |   Max Drawdown : -54.52%\n",
      "Market Return : -24.95%   |   Portfolio Return :  2.53%   |   Position Changes :  3.96%   |   Max Drawdown : -42.66%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1883.15%   |   Portfolio Return : 3563.64%   |   Position Changes :  3.43%   |   Max Drawdown : -74.53%\n",
      "↳ Env 3 : 009 :   554864   |   01:12:52   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6516E+00   |   Tot. Rewards :  2880.83   |   Rewards (/1000 steps) :    55.01   |   Length :  52374\n",
      "Market Return : 1754.87%   |   Portfolio Return : 2532.07%   |   Position Changes :  3.55%   |   Max Drawdown : -63.42%\n",
      "↳ Env 0 : 009 :   558738   |   01:13:36   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6520E+00   |   Tot. Rewards :  2616.29   |   Rewards (/1000 steps) :    45.06   |   Length :  58058\n",
      "Market Return : 1751.42%   |   Portfolio Return : 6143.53%   |   Position Changes :  3.56%   |   Max Drawdown : -68.31%\n",
      "↳ Env 2 : 009 :   559907   |   01:13:49   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6521E+00   |   Tot. Rewards :  3307.31   |   Rewards (/1000 steps) :    56.84   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : 94.95%   |   Position Changes :  8.48%   |   Max Drawdown : -29.67%\n",
      "Market Return : -36.99%   |   Portfolio Return : 48.37%   |   Position Changes :  8.09%   |   Max Drawdown : -29.55%\n",
      "Market Return : -36.87%   |   Portfolio Return : 27.88%   |   Position Changes :  8.23%   |   Max Drawdown : -37.13%\n",
      "Market Return : -36.87%   |   Portfolio Return : 26.96%   |   Position Changes :  8.26%   |   Max Drawdown : -36.86%\n",
      "Market Return : -24.96%   |   Portfolio Return :  6.66%   |   Position Changes :  7.92%   |   Max Drawdown : -32.38%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 909.11%   |   Portfolio Return : 2194.32%   |   Position Changes :  4.30%   |   Max Drawdown : -62.77%\n",
      "↳ Env 4 : 009 :   570302   |   01:16:31   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6543E+00   |   Tot. Rewards :  2506.42   |   Rewards (/1000 steps) :    43.08   |   Length :  58187\n",
      "Market Return : 1751.42%   |   Portfolio Return : 21147.22%   |   Position Changes :  4.12%   |   Max Drawdown : -67.40%\n",
      "↳ Env 1 : 009 :   570492   |   01:16:33   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6544E+00   |   Tot. Rewards :  4287.05   |   Rewards (/1000 steps) :    73.68   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : -13.80%   |   Position Changes :  2.71%   |   Max Drawdown : -56.57%\n",
      "Market Return : -24.85%   |   Portfolio Return : 17.89%   |   Position Changes :  3.10%   |   Max Drawdown : -37.88%\n",
      "Market Return : -24.85%   |   Portfolio Return : 19.34%   |   Position Changes :  3.10%   |   Max Drawdown : -38.15%\n",
      "Market Return : -24.96%   |   Portfolio Return : 38.66%   |   Position Changes :  3.17%   |   Max Drawdown : -31.24%\n",
      "Market Return : -24.96%   |   Portfolio Return : 38.61%   |   Position Changes :  3.17%   |   Max Drawdown : -31.27%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1896.93%   |   Portfolio Return : 4773.95%   |   Position Changes :  5.52%   |   Max Drawdown : -62.99%\n",
      "↳ Env 0 : 010 :   611053   |   01:25:17   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6589E+00   |   Tot. Rewards :  3109.19   |   Rewards (/1000 steps) :    59.43   |   Length :  52315\n",
      "Market Return : 1760.39%   |   Portfolio Return : 2678.46%   |   Position Changes :  5.65%   |   Max Drawdown : -67.91%\n",
      "↳ Env 3 : 010 :   613049   |   01:25:42   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6596E+00   |   Tot. Rewards :  2659.59   |   Rewards (/1000 steps) :    45.71   |   Length :  58185\n",
      "Market Return : 909.11%   |   Portfolio Return : 4185.32%   |   Position Changes :  5.72%   |   Max Drawdown : -65.55%\n",
      "↳ Env 2 : 010 :   618094   |   01:26:46   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6604E+00   |   Tot. Rewards :  3006.22   |   Rewards (/1000 steps) :    51.66   |   Length :  58187\n",
      "Market Return : 908.42%   |   Portfolio Return : 1627.60%   |   Position Changes :  5.86%   |   Max Drawdown : -72.78%\n",
      "↳ Env 4 : 010 :   628374   |   01:28:55   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6602E+00   |   Tot. Rewards :  2279.45   |   Rewards (/1000 steps) :    39.25   |   Length :  58072\n",
      "Market Return : 1754.87%   |   Portfolio Return : 5962.77%   |   Position Changes :  5.32%   |   Max Drawdown : -63.97%\n",
      "↳ Env 1 : 010 :   628550   |   01:28:57   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6602E+00   |   Tot. Rewards :  3283.80   |   Rewards (/1000 steps) :    56.56   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 42.65%   |   Position Changes :  6.25%   |   Max Drawdown : -41.18%\n",
      "Market Return : -24.83%   |   Portfolio Return : 21.50%   |   Position Changes :  6.31%   |   Max Drawdown : -33.47%\n",
      "Market Return : -36.93%   |   Portfolio Return : -2.58%   |   Position Changes :  5.98%   |   Max Drawdown : -46.42%\n",
      "Market Return : -36.93%   |   Portfolio Return : 26.41%   |   Position Changes :  6.08%   |   Max Drawdown : -41.84%\n",
      "Market Return : -24.86%   |   Portfolio Return : 15.11%   |   Position Changes :  6.27%   |   Max Drawdown : -37.26%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.83%   |   Portfolio Return : 44.80%   |   Position Changes :  8.07%   |   Max Drawdown : -41.02%\n",
      "Market Return : -36.96%   |   Portfolio Return : 76.22%   |   Position Changes :  8.14%   |   Max Drawdown : -37.14%\n",
      "Market Return : -36.78%   |   Portfolio Return : 99.79%   |   Position Changes :  7.56%   |   Max Drawdown : -35.62%\n",
      "Market Return : -24.95%   |   Portfolio Return : 29.04%   |   Position Changes :  7.79%   |   Max Drawdown : -30.87%\n",
      "Market Return : -24.85%   |   Portfolio Return : 43.51%   |   Position Changes :  8.05%   |   Max Drawdown : -37.69%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 909.11%   |   Portfolio Return : 5352.64%   |   Position Changes :  5.97%   |   Max Drawdown : -69.99%\n",
      "↳ Env 0 : 011 :   669240   |   01:39:06   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6502E+00   |   Tot. Rewards :  3198.95   |   Rewards (/1000 steps) :    54.98   |   Length :  58187\n",
      "Market Return : 1753.28%   |   Portfolio Return : 3568.44%   |   Position Changes :  5.78%   |   Max Drawdown : -62.25%\n",
      "↳ Env 3 : 011 :   671121   |   01:39:32   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6502E+00   |   Tot. Rewards :  2881.88   |   Rewards (/1000 steps) :    49.63   |   Length :  58072\n",
      "Market Return : 1760.22%   |   Portfolio Return : 6000.99%   |   Position Changes :  5.75%   |   Max Drawdown : -78.80%\n",
      "↳ Env 2 : 011 :   676167   |   01:40:40   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6498E+00   |   Tot. Rewards :  3288.83   |   Rewards (/1000 steps) :    56.63   |   Length :  58073\n",
      "Market Return : 909.84%   |   Portfolio Return : 4126.37%   |   Position Changes :  5.88%   |   Max Drawdown : -60.90%\n",
      "↳ Env 4 : 011 :   686447   |   01:42:53   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6504E+00   |   Tot. Rewards :  2995.14   |   Rewards (/1000 steps) :    51.58   |   Length :  58073\n",
      "Market Return : 909.61%   |   Portfolio Return : 4529.34%   |   Position Changes :  6.20%   |   Max Drawdown : -60.06%\n",
      "↳ Env 1 : 011 :   686737   |   01:42:57   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6503E+00   |   Tot. Rewards :  3068.00   |   Rewards (/1000 steps) :    52.73   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 114.42%   |   Position Changes :  8.90%   |   Max Drawdown : -31.45%\n",
      "Market Return : -36.96%   |   Portfolio Return : 91.60%   |   Position Changes :  9.24%   |   Max Drawdown : -42.20%\n",
      "Market Return : -36.96%   |   Portfolio Return : 98.06%   |   Position Changes :  9.25%   |   Max Drawdown : -42.20%\n",
      "Market Return : -36.93%   |   Portfolio Return : 78.73%   |   Position Changes :  9.58%   |   Max Drawdown : -35.20%\n",
      "Market Return : -36.94%   |   Portfolio Return : 127.78%   |   Position Changes :  9.30%   |   Max Drawdown : -37.20%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : -40.55%   |   Position Changes :  4.88%   |   Max Drawdown : -57.88%\n",
      "Market Return : -24.98%   |   Portfolio Return :  9.35%   |   Position Changes :  4.92%   |   Max Drawdown : -44.71%\n",
      "Market Return : -36.94%   |   Portfolio Return : 26.47%   |   Position Changes :  4.87%   |   Max Drawdown : -51.20%\n",
      "Market Return : -24.86%   |   Portfolio Return : 24.99%   |   Position Changes :  4.74%   |   Max Drawdown : -42.63%\n",
      "Market Return : -24.86%   |   Portfolio Return : 20.02%   |   Position Changes :  4.73%   |   Max Drawdown : -42.64%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1754.87%   |   Portfolio Return : 27693.51%   |   Position Changes :  7.67%   |   Max Drawdown : -61.16%\n",
      "↳ Env 0 : 012 :   727298   |   01:53:30   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6312E+00   |   Tot. Rewards :  4501.91   |   Rewards (/1000 steps) :    77.54   |   Length :  58058\n",
      "Market Return : 1754.87%   |   Portfolio Return : 19000.35%   |   Position Changes :  7.46%   |   Max Drawdown : -58.35%\n",
      "↳ Env 3 : 012 :   729179   |   01:53:56   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6307E+00   |   Tot. Rewards :  4201.83   |   Rewards (/1000 steps) :    72.37   |   Length :  58058\n",
      "Market Return : 1751.42%   |   Portfolio Return : 51938.94%   |   Position Changes :  7.90%   |   Max Drawdown : -58.24%\n",
      "↳ Env 2 : 012 :   734354   |   01:55:09   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6289E+00   |   Tot. Rewards :  5003.66   |   Rewards (/1000 steps) :    85.99   |   Length :  58187\n",
      "Market Return : 1901.46%   |   Portfolio Return : 4669.50%   |   Position Changes :  7.92%   |   Max Drawdown : -69.74%\n",
      "↳ Env 1 : 012 :   739069   |   01:56:17   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6280E+00   |   Tot. Rewards :  3091.86   |   Rewards (/1000 steps) :    59.08   |   Length :  52332\n",
      "Market Return : 1751.40%   |   Portfolio Return : 22441.57%   |   Position Changes :  7.55%   |   Max Drawdown : -70.65%\n",
      "↳ Env 4 : 012 :   744518   |   01:57:35   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6267E+00   |   Tot. Rewards :  4334.36   |   Rewards (/1000 steps) :    74.64   |   Length :  58071\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 76.45%   |   Position Changes :  7.36%   |   Max Drawdown : -33.85%\n",
      "Market Return : -36.99%   |   Portfolio Return : 104.69%   |   Position Changes :  7.16%   |   Max Drawdown : -37.55%\n",
      "Market Return : -36.87%   |   Portfolio Return : 13.84%   |   Position Changes :  6.69%   |   Max Drawdown : -37.68%\n",
      "Market Return : -24.95%   |   Portfolio Return : -14.93%   |   Position Changes :  6.28%   |   Max Drawdown : -32.85%\n",
      "Market Return : -36.93%   |   Portfolio Return : 75.63%   |   Position Changes :  6.77%   |   Max Drawdown : -35.95%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1901.46%   |   Portfolio Return : 8998.52%   |   Position Changes :  6.86%   |   Max Drawdown : -70.41%\n",
      "↳ Env 0 : 013 :   779630   |   02:06:33   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6186E+00   |   Tot. Rewards :  3608.56   |   Rewards (/1000 steps) :    68.96   |   Length :  52332\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 15.24%   |   Position Changes :  2.36%   |   Max Drawdown : -50.24%\n",
      "Market Return : -24.98%   |   Portfolio Return : 36.72%   |   Position Changes :  2.29%   |   Max Drawdown : -42.29%\n",
      "Market Return : -36.96%   |   Portfolio Return : 47.26%   |   Position Changes :  2.26%   |   Max Drawdown : -49.40%\n",
      "Market Return : -24.85%   |   Portfolio Return : 35.71%   |   Position Changes :  2.23%   |   Max Drawdown : -36.13%\n",
      "Market Return : -24.96%   |   Portfolio Return : 59.96%   |   Position Changes :  2.23%   |   Max Drawdown : -36.11%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1754.87%   |   Portfolio Return : 5131.96%   |   Position Changes :  5.80%   |   Max Drawdown : -65.80%\n",
      "↳ Env 3 : 013 :   787237   |   02:09:02   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6179E+00   |   Tot. Rewards :  3165.90   |   Rewards (/1000 steps) :    54.53   |   Length :  58058\n",
      "Market Return : 832.99%   |   Portfolio Return : 1491.36%   |   Position Changes :  6.12%   |   Max Drawdown : -66.10%\n",
      "↳ Env 1 : 013 :   792262   |   02:10:15   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6179E+00   |   Tot. Rewards :  2213.74   |   Rewards (/1000 steps) :    41.62   |   Length :  53193\n",
      "Market Return : 1754.87%   |   Portfolio Return : 11047.48%   |   Position Changes :  5.88%   |   Max Drawdown : -71.07%\n",
      "↳ Env 2 : 013 :   792412   |   02:10:17   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6179E+00   |   Tot. Rewards :  3771.04   |   Rewards (/1000 steps) :    64.95   |   Length :  58058\n",
      "Market Return : 908.31%   |   Portfolio Return : 7445.60%   |   Position Changes :  7.08%   |   Max Drawdown : -58.00%\n",
      "↳ Env 4 : 013 :   802705   |   02:12:49   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6216E+00   |   Tot. Rewards :  3458.84   |   Rewards (/1000 steps) :    59.44   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : 44.29%   |   Position Changes :  4.84%   |   Max Drawdown : -44.71%\n",
      "Market Return : -24.83%   |   Portfolio Return : 34.26%   |   Position Changes :  4.71%   |   Max Drawdown : -28.45%\n",
      "Market Return : -36.78%   |   Portfolio Return : 18.84%   |   Position Changes :  4.51%   |   Max Drawdown : -45.05%\n",
      "Market Return : -24.85%   |   Portfolio Return : 58.99%   |   Position Changes :  4.54%   |   Max Drawdown : -25.51%\n",
      "Market Return : -24.86%   |   Portfolio Return : 28.49%   |   Position Changes :  4.59%   |   Max Drawdown : -32.87%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1752.50%   |   Portfolio Return : 5207.37%   |   Position Changes :  6.41%   |   Max Drawdown : -58.23%\n",
      "↳ Env 0 : 014 :   837702   |   02:22:25   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6310E+00   |   Tot. Rewards :  3177.35   |   Rewards (/1000 steps) :    54.71   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 49.27%   |   Position Changes :  5.14%   |   Max Drawdown : -43.00%\n",
      "Market Return : -36.93%   |   Portfolio Return : 11.88%   |   Position Changes :  4.84%   |   Max Drawdown : -49.38%\n",
      "Market Return : -36.93%   |   Portfolio Return : 27.65%   |   Position Changes :  5.17%   |   Max Drawdown : -49.67%\n",
      "Market Return : -24.86%   |   Portfolio Return : 36.17%   |   Position Changes :  5.06%   |   Max Drawdown : -31.75%\n",
      "Market Return : -24.96%   |   Portfolio Return : 20.08%   |   Position Changes :  4.72%   |   Max Drawdown : -36.95%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.92%   |   Portfolio Return : 2878.51%   |   Position Changes :  6.53%   |   Max Drawdown : -51.07%\n",
      "↳ Env 3 : 014 :   845295   |   02:25:05   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6322E+00   |   Tot. Rewards :  2715.21   |   Rewards (/1000 steps) :    46.77   |   Length :  58058\n",
      "Market Return : 1753.28%   |   Portfolio Return : 5417.48%   |   Position Changes :  6.94%   |   Max Drawdown : -62.78%\n",
      "↳ Env 1 : 014 :   850334   |   02:26:29   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6321E+00   |   Tot. Rewards :  3208.40   |   Rewards (/1000 steps) :    55.25   |   Length :  58072\n",
      "Market Return : 1754.87%   |   Portfolio Return : 23253.32%   |   Position Changes :  7.06%   |   Max Drawdown : -56.09%\n",
      "↳ Env 2 : 014 :   850470   |   02:26:32   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6321E+00   |   Tot. Rewards :  4362.66   |   Rewards (/1000 steps) :    75.14   |   Length :  58058\n",
      "Market Return : 1907.79%   |   Portfolio Return : 29673.74%   |   Position Changes :  6.39%   |   Max Drawdown : -72.29%\n",
      "↳ Env 4 : 014 :   855070   |   02:27:49   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6316E+00   |   Tot. Rewards :  4556.97   |   Rewards (/1000 steps) :    87.02   |   Length :  52365\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.96%   |   Portfolio Return : 55.09%   |   Position Changes :  4.80%   |   Max Drawdown : -48.57%\n",
      "Market Return : -24.95%   |   Portfolio Return : 48.90%   |   Position Changes :  4.68%   |   Max Drawdown : -44.20%\n",
      "Market Return : -24.95%   |   Portfolio Return : 48.78%   |   Position Changes :  4.63%   |   Max Drawdown : -44.03%\n",
      "Market Return : -24.96%   |   Portfolio Return : 39.53%   |   Position Changes :  4.65%   |   Max Drawdown : -43.50%\n",
      "Market Return : -24.86%   |   Portfolio Return :  7.07%   |   Position Changes :  4.98%   |   Max Drawdown : -43.68%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1753.28%   |   Portfolio Return : 22520.35%   |   Position Changes :  6.88%   |   Max Drawdown : -62.48%\n",
      "↳ Env 0 : 015 :   895774   |   02:40:22   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6361E+00   |   Tot. Rewards :  4337.15   |   Rewards (/1000 steps) :    74.69   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : 107.27%   |   Position Changes :  3.46%   |   Max Drawdown : -47.78%\n",
      "Market Return : -36.87%   |   Portfolio Return : 74.52%   |   Position Changes :  3.75%   |   Max Drawdown : -52.55%\n",
      "Market Return : -24.98%   |   Portfolio Return : 26.46%   |   Position Changes :  3.58%   |   Max Drawdown : -37.03%\n",
      "Market Return : -36.78%   |   Portfolio Return : 79.39%   |   Position Changes :  3.88%   |   Max Drawdown : -46.61%\n",
      "Market Return : -36.78%   |   Portfolio Return : 76.95%   |   Position Changes :  3.88%   |   Max Drawdown : -47.12%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1896.93%   |   Portfolio Return : 23119.55%   |   Position Changes :  6.90%   |   Max Drawdown : -65.50%\n",
      "↳ Env 2 : 015 :   902785   |   02:43:07   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6352E+00   |   Tot. Rewards :  4358.06   |   Rewards (/1000 steps) :    83.30   |   Length :  52315\n",
      "Market Return : 907.99%   |   Portfolio Return : 4947.66%   |   Position Changes :  7.40%   |   Max Drawdown : -60.76%\n",
      "↳ Env 3 : 015 :   903367   |   02:43:19   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6352E+00   |   Tot. Rewards :  3137.21   |   Rewards (/1000 steps) :    54.02   |   Length :  58072\n",
      "Market Return : 1751.91%   |   Portfolio Return : 25675.09%   |   Position Changes :  7.08%   |   Max Drawdown : -60.80%\n",
      "↳ Env 1 : 015 :   908521   |   02:44:52   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6350E+00   |   Tot. Rewards :  4441.59   |   Rewards (/1000 steps) :    76.33   |   Length :  58187\n",
      "Market Return : 908.42%   |   Portfolio Return : 2608.52%   |   Position Changes :  6.86%   |   Max Drawdown : -62.99%\n",
      "↳ Env 4 : 015 :   913142   |   02:46:15   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6352E+00   |   Tot. Rewards :  2639.19   |   Rewards (/1000 steps) :    45.45   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.87%   |   Portfolio Return : -4.11%   |   Position Changes :  7.96%   |   Max Drawdown : -45.93%\n",
      "Market Return : -36.87%   |   Portfolio Return : -5.28%   |   Position Changes :  7.97%   |   Max Drawdown : -46.42%\n",
      "Market Return : -24.98%   |   Portfolio Return : -12.55%   |   Position Changes :  7.50%   |   Max Drawdown : -40.08%\n",
      "Market Return : -36.93%   |   Portfolio Return : 79.21%   |   Position Changes :  7.78%   |   Max Drawdown : -39.66%\n",
      "Market Return : -24.96%   |   Portfolio Return : 12.65%   |   Position Changes :  6.92%   |   Max Drawdown : -28.58%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 833.62%   |   Portfolio Return : 2308.04%   |   Position Changes :  5.69%   |   Max Drawdown : -58.46%\n",
      "↳ Env 0 : 016 :   948937   |   02:57:57   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6399E+00   |   Tot. Rewards :  2545.12   |   Rewards (/1000 steps) :    47.87   |   Length :  53163\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 30.06%   |   Position Changes :  2.52%   |   Max Drawdown : -45.19%\n",
      "Market Return : -36.99%   |   Portfolio Return : 28.80%   |   Position Changes :  2.51%   |   Max Drawdown : -45.19%\n",
      "Market Return : -36.99%   |   Portfolio Return : 39.83%   |   Position Changes :  2.87%   |   Max Drawdown : -45.09%\n",
      "Market Return : -24.83%   |   Portfolio Return : 47.77%   |   Position Changes :  2.75%   |   Max Drawdown : -34.57%\n",
      "Market Return : -24.86%   |   Portfolio Return : 62.17%   |   Position Changes :  2.80%   |   Max Drawdown : -30.07%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 909.61%   |   Portfolio Return : 1260.00%   |   Position Changes :  5.93%   |   Max Drawdown : -69.59%\n",
      "↳ Env 2 : 016 :   960972   |   03:02:23   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6429E+00   |   Tot. Rewards :  2088.06   |   Rewards (/1000 steps) :    35.89   |   Length :  58187\n",
      "Market Return : 1751.42%   |   Portfolio Return : 20102.82%   |   Position Changes :  5.54%   |   Max Drawdown : -62.68%\n",
      "↳ Env 3 : 016 :   961554   |   03:02:34   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6430E+00   |   Tot. Rewards :  4246.73   |   Rewards (/1000 steps) :    72.98   |   Length :  58187\n",
      "Market Return : 834.53%   |   Portfolio Return : 5780.98%   |   Position Changes :  5.35%   |   Max Drawdown : -51.21%\n",
      "↳ Env 4 : 016 :   966353   |   03:04:07   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6442E+00   |   Tot. Rewards :  3259.45   |   Rewards (/1000 steps) :    61.26   |   Length :  53211\n",
      "Market Return : 908.92%   |   Portfolio Return : 2653.75%   |   Position Changes :  5.21%   |   Max Drawdown : -46.94%\n",
      "↳ Env 1 : 016 :   966579   |   03:04:11   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6442E+00   |   Tot. Rewards :  2652.44   |   Rewards (/1000 steps) :    45.69   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : 35.37%   |   Position Changes :  5.79%   |   Max Drawdown : -43.50%\n",
      "Market Return : -36.96%   |   Portfolio Return : 47.53%   |   Position Changes :  6.24%   |   Max Drawdown : -44.52%\n",
      "Market Return : -36.96%   |   Portfolio Return : 51.19%   |   Position Changes :  6.27%   |   Max Drawdown : -44.52%\n",
      "Market Return : -24.85%   |   Portfolio Return : 25.88%   |   Position Changes :  5.71%   |   Max Drawdown : -37.20%\n",
      "Market Return : -24.86%   |   Portfolio Return : 33.26%   |   Position Changes :  6.04%   |   Max Drawdown : -34.58%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.42%   |   Portfolio Return : 2902.14%   |   Position Changes :  5.95%   |   Max Drawdown : -60.19%\n",
      "↳ Env 0 : 017 :  1007009   |   03:18:13   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6472E+00   |   Tot. Rewards :  2721.53   |   Rewards (/1000 steps) :    46.86   |   Length :  58072\n",
      "Market Return : 1907.79%   |   Portfolio Return : 14042.78%   |   Position Changes :  6.36%   |   Max Drawdown : -72.31%\n",
      "↳ Env 2 : 017 :  1013337   |   03:20:24   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6462E+00   |   Tot. Rewards :  3961.43   |   Rewards (/1000 steps) :    75.65   |   Length :  52365\n",
      "Market Return : 832.99%   |   Portfolio Return : 6082.09%   |   Position Changes :  6.68%   |   Max Drawdown : -58.10%\n",
      "↳ Env 3 : 017 :  1014747   |   03:20:53   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6458E+00   |   Tot. Rewards :  3299.39   |   Rewards (/1000 steps) :    62.03   |   Length :  53193\n",
      "Market Return : 1901.46%   |   Portfolio Return : 14839.92%   |   Position Changes :  6.75%   |   Max Drawdown : -66.51%\n",
      "↳ Env 4 : 017 :  1018685   |   03:22:13   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6444E+00   |   Tot. Rewards :  4005.30   |   Rewards (/1000 steps) :    76.54   |   Length :  52332\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.83%   |   Portfolio Return : 12.82%   |   Position Changes : 10.36%   |   Max Drawdown : -36.14%\n",
      "Market Return : -24.98%   |   Portfolio Return : -1.11%   |   Position Changes : 10.84%   |   Max Drawdown : -40.54%\n",
      "Market Return : -36.96%   |   Portfolio Return : 92.09%   |   Position Changes : 11.01%   |   Max Drawdown : -45.42%\n",
      "Market Return : -24.85%   |   Portfolio Return :  1.70%   |   Position Changes : 10.23%   |   Max Drawdown : -40.63%\n",
      "Market Return : -24.96%   |   Portfolio Return :  8.48%   |   Position Changes : 10.44%   |   Max Drawdown : -38.65%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.31%   |   Portfolio Return : 10657.24%   |   Position Changes :  7.14%   |   Max Drawdown : -52.88%\n",
      "↳ Env 1 : 017 :  1024766   |   03:24:56   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6415E+00   |   Tot. Rewards :  3742.53   |   Rewards (/1000 steps) :    64.32   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 87.74%   |   Position Changes :  9.28%   |   Max Drawdown : -45.32%\n",
      "Market Return : -24.98%   |   Portfolio Return : -2.07%   |   Position Changes : 10.07%   |   Max Drawdown : -51.38%\n",
      "Market Return : -24.98%   |   Portfolio Return : -3.51%   |   Position Changes : 10.01%   |   Max Drawdown : -51.36%\n",
      "Market Return : -24.85%   |   Portfolio Return : -15.74%   |   Position Changes :  9.03%   |   Max Drawdown : -46.91%\n",
      "Market Return : -24.85%   |   Portfolio Return : -16.81%   |   Position Changes :  9.08%   |   Max Drawdown : -47.49%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1753.28%   |   Portfolio Return : 9499.32%   |   Position Changes :  7.97%   |   Max Drawdown : -54.95%\n",
      "↳ Env 0 : 018 :  1065081   |   03:39:35   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6248E+00   |   Tot. Rewards :  3651.42   |   Rewards (/1000 steps) :    62.88   |   Length :  58072\n",
      "Market Return : 907.99%   |   Portfolio Return : 2969.40%   |   Position Changes :  8.16%   |   Max Drawdown : -55.08%\n",
      "↳ Env 2 : 018 :  1071409   |   03:41:49   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6226E+00   |   Tot. Rewards :  2739.25   |   Rewards (/1000 steps) :    47.17   |   Length :  58072\n",
      "Market Return : 834.53%   |   Portfolio Return : 4229.33%   |   Position Changes :  7.79%   |   Max Drawdown : -46.82%\n",
      "↳ Env 4 : 018 :  1071896   |   03:41:59   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6224E+00   |   Tot. Rewards :  3014.40   |   Rewards (/1000 steps) :    56.65   |   Length :  53211\n",
      "Market Return : 908.23%   |   Portfolio Return : 4802.04%   |   Position Changes :  8.02%   |   Max Drawdown : -52.38%\n",
      "↳ Env 3 : 018 :  1072934   |   03:42:20   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6219E+00   |   Tot. Rewards :  3113.79   |   Rewards (/1000 steps) :    53.51   |   Length :  58187\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 138.27%   |   Position Changes :  8.99%   |   Max Drawdown : -44.83%\n",
      "Market Return : -36.87%   |   Portfolio Return : 22.17%   |   Position Changes :  9.62%   |   Max Drawdown : -42.55%\n",
      "Market Return : -36.96%   |   Portfolio Return : 134.19%   |   Position Changes :  9.42%   |   Max Drawdown : -39.36%\n",
      "Market Return : -36.94%   |   Portfolio Return : 132.29%   |   Position Changes :  9.23%   |   Max Drawdown : -38.94%\n",
      "Market Return : -36.94%   |   Portfolio Return : 134.64%   |   Position Changes :  9.20%   |   Max Drawdown : -38.57%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 907.99%   |   Portfolio Return : 7353.06%   |   Position Changes :  8.68%   |   Max Drawdown : -49.35%\n",
      "↳ Env 1 : 018 :  1082838   |   03:46:36   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6194E+00   |   Tot. Rewards :  3448.97   |   Rewards (/1000 steps) :    59.39   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return :  1.40%   |   Position Changes :  5.07%   |   Max Drawdown : -51.24%\n",
      "Market Return : -24.83%   |   Portfolio Return : 11.20%   |   Position Changes :  4.80%   |   Max Drawdown : -30.51%\n",
      "Market Return : -36.87%   |   Portfolio Return : -4.18%   |   Position Changes :  5.26%   |   Max Drawdown : -45.15%\n",
      "Market Return : -24.85%   |   Portfolio Return : 33.57%   |   Position Changes :  5.16%   |   Max Drawdown : -26.80%\n",
      "Market Return : -36.93%   |   Portfolio Return : 35.10%   |   Position Changes :  5.29%   |   Max Drawdown : -33.18%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1751.42%   |   Portfolio Return : 76032.16%   |   Position Changes :  8.59%   |   Max Drawdown : -63.08%\n",
      "↳ Env 0 : 019 :  1123268   |   04:02:14   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6142E+00   |   Tot. Rewards :  5308.04   |   Rewards (/1000 steps) :    91.22   |   Length :  58187\n",
      "Market Return : 908.42%   |   Portfolio Return : 9797.80%   |   Position Changes :  7.77%   |   Max Drawdown : -49.65%\n",
      "↳ Env 2 : 019 :  1129481   |   04:04:33   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6162E+00   |   Tot. Rewards :  3675.92   |   Rewards (/1000 steps) :    63.30   |   Length :  58072\n",
      "Market Return : 909.61%   |   Portfolio Return : 2990.45%   |   Position Changes :  7.63%   |   Max Drawdown : -55.07%\n",
      "↳ Env 4 : 019 :  1130083   |   04:04:47   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6164E+00   |   Tot. Rewards :  2744.72   |   Rewards (/1000 steps) :    47.17   |   Length :  58187\n",
      "Market Return : 1754.87%   |   Portfolio Return : 9637.31%   |   Position Changes :  7.62%   |   Max Drawdown : -63.68%\n",
      "↳ Env 3 : 019 :  1130992   |   04:05:07   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6167E+00   |   Tot. Rewards :  3662.84   |   Rewards (/1000 steps) :    63.09   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.93%   |   Portfolio Return : -13.80%   |   Position Changes : 13.61%   |   Max Drawdown : -35.67%\n",
      "Market Return : -24.95%   |   Portfolio Return : 10.91%   |   Position Changes : 13.17%   |   Max Drawdown : -29.35%\n",
      "Market Return : -36.78%   |   Portfolio Return : -8.76%   |   Position Changes : 14.17%   |   Max Drawdown : -44.03%\n",
      "Market Return : -24.85%   |   Portfolio Return : 18.94%   |   Position Changes : 13.43%   |   Max Drawdown : -33.23%\n",
      "Market Return : -24.96%   |   Portfolio Return : 29.52%   |   Position Changes : 13.65%   |   Max Drawdown : -30.90%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.92%   |   Portfolio Return : 7131.84%   |   Position Changes :  7.98%   |   Max Drawdown : -47.02%\n",
      "↳ Env 1 : 019 :  1140896   |   04:09:27   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6191E+00   |   Tot. Rewards :  3424.86   |   Rewards (/1000 steps) :    58.99   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.96%   |   Portfolio Return : 47.71%   |   Position Changes : 12.81%   |   Max Drawdown : -46.18%\n",
      "Market Return : -36.87%   |   Portfolio Return : 19.20%   |   Position Changes : 12.61%   |   Max Drawdown : -47.90%\n",
      "Market Return : -24.95%   |   Portfolio Return : -9.23%   |   Position Changes : 12.68%   |   Max Drawdown : -40.65%\n",
      "Market Return : -36.94%   |   Portfolio Return : 103.93%   |   Position Changes : 12.29%   |   Max Drawdown : -44.76%\n",
      "Market Return : -36.94%   |   Portfolio Return : 98.00%   |   Position Changes : 12.29%   |   Max Drawdown : -44.94%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 1907.79%   |   Portfolio Return : 34336.61%   |   Position Changes :  7.42%   |   Max Drawdown : -64.37%\n",
      "↳ Env 0 : 020 :  1175633   |   04:23:33   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6237E+00   |   Tot. Rewards :  4673.36   |   Rewards (/1000 steps) :    89.25   |   Length :  52365\n",
      "Market Return : 1751.91%   |   Portfolio Return : 30032.02%   |   Position Changes :  8.41%   |   Max Drawdown : -54.32%\n",
      "↳ Env 2 : 020 :  1187668   |   04:28:22   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6245E+00   |   Tot. Rewards :  4566.54   |   Rewards (/1000 steps) :    78.48   |   Length :  58187\n",
      "Market Return : 909.84%   |   Portfolio Return : 16254.03%   |   Position Changes :  8.72%   |   Max Drawdown : -44.26%\n",
      "↳ Env 4 : 020 :  1188156   |   04:28:34   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6246E+00   |   Tot. Rewards :  4077.65   |   Rewards (/1000 steps) :    70.22   |   Length :  58073\n",
      "Market Return : 1751.40%   |   Portfolio Return : 12504.88%   |   Position Changes :  8.60%   |   Max Drawdown : -64.52%\n",
      "↳ Env 3 : 020 :  1189063   |   04:28:57   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6247E+00   |   Tot. Rewards :  3869.34   |   Rewards (/1000 steps) :    66.63   |   Length :  58071\n",
      "Market Return : 1753.28%   |   Portfolio Return : 6642.17%   |   Position Changes :  8.84%   |   Max Drawdown : -66.00%\n",
      "↳ Env 1 : 020 :  1198968   |   04:32:54   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6261E+00   |   Tot. Rewards :  3368.77   |   Rewards (/1000 steps) :    58.01   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 22.89%   |   Position Changes : 12.67%   |   Max Drawdown : -43.35%\n",
      "Market Return : -36.99%   |   Portfolio Return : 25.32%   |   Position Changes : 12.69%   |   Max Drawdown : -43.12%\n",
      "Market Return : -36.96%   |   Portfolio Return : 28.84%   |   Position Changes : 14.30%   |   Max Drawdown : -50.03%\n",
      "Market Return : -36.78%   |   Portfolio Return : -13.96%   |   Position Changes : 15.15%   |   Max Drawdown : -52.26%\n",
      "Market Return : -36.94%   |   Portfolio Return : 45.05%   |   Position Changes : 13.97%   |   Max Drawdown : -41.45%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 36.25%   |   Position Changes :  7.23%   |   Max Drawdown : -54.74%\n",
      "Market Return : -24.83%   |   Portfolio Return : 31.08%   |   Position Changes :  7.58%   |   Max Drawdown : -28.21%\n",
      "Market Return : -36.93%   |   Portfolio Return : 27.15%   |   Position Changes :  6.79%   |   Max Drawdown : -46.78%\n",
      "Market Return : -36.93%   |   Portfolio Return : 48.26%   |   Position Changes :  7.37%   |   Max Drawdown : -44.66%\n",
      "Market Return : -36.78%   |   Portfolio Return : 29.40%   |   Position Changes :  7.35%   |   Max Drawdown : -45.51%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.23%   |   Portfolio Return : 18870.19%   |   Position Changes :  8.68%   |   Max Drawdown : -57.22%\n",
      "↳ Env 0 : 021 :  1233820   |   04:48:11   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6318E+00   |   Tot. Rewards :  4196.36   |   Rewards (/1000 steps) :    72.12   |   Length :  58187\n",
      "Market Return : 909.11%   |   Portfolio Return : 27145.57%   |   Position Changes :  8.78%   |   Max Drawdown : -46.23%\n",
      "↳ Env 2 : 021 :  1245855   |   04:53:14   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6334E+00   |   Tot. Rewards :  4485.98   |   Rewards (/1000 steps) :    77.10   |   Length :  58187\n",
      "Market Return : 907.99%   |   Portfolio Return : 19725.64%   |   Position Changes :  8.65%   |   Max Drawdown : -67.31%\n",
      "↳ Env 4 : 021 :  1246228   |   04:53:22   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6334E+00   |   Tot. Rewards :  4231.65   |   Rewards (/1000 steps) :    72.87   |   Length :  58072\n",
      "Market Return : 908.76%   |   Portfolio Return : 5747.18%   |   Position Changes :  8.27%   |   Max Drawdown : -65.31%\n",
      "↳ Env 3 : 021 :  1247250   |   04:53:49   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6335E+00   |   Tot. Rewards :  3254.84   |   Rewards (/1000 steps) :    55.94   |   Length :  58187\n",
      "Market Return : 1896.93%   |   Portfolio Return : 7343.72%   |   Position Changes :  7.89%   |   Max Drawdown : -65.29%\n",
      "↳ Env 1 : 021 :  1251283   |   04:55:31   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6340E+00   |   Tot. Rewards :  3447.96   |   Rewards (/1000 steps) :    65.91   |   Length :  52315\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.83%   |   Portfolio Return : -4.11%   |   Position Changes : 11.87%   |   Max Drawdown : -30.42%\n",
      "Market Return : -36.93%   |   Portfolio Return : 19.96%   |   Position Changes : 12.24%   |   Max Drawdown : -52.99%\n",
      "Market Return : -24.85%   |   Portfolio Return : -19.63%   |   Position Changes : 12.47%   |   Max Drawdown : -37.44%\n",
      "Market Return : -24.96%   |   Portfolio Return : -18.27%   |   Position Changes : 12.29%   |   Max Drawdown : -50.28%\n",
      "Market Return : -24.96%   |   Portfolio Return : -18.31%   |   Position Changes : 12.28%   |   Max Drawdown : -50.47%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.78%   |   Portfolio Return : 26.34%   |   Position Changes : 11.95%   |   Max Drawdown : -51.93%\n",
      "Market Return : -24.95%   |   Portfolio Return : 53.11%   |   Position Changes : 11.46%   |   Max Drawdown : -31.11%\n",
      "Market Return : -36.94%   |   Portfolio Return : 43.40%   |   Position Changes : 11.74%   |   Max Drawdown : -38.52%\n",
      "Market Return : -24.85%   |   Portfolio Return : 18.38%   |   Position Changes : 11.06%   |   Max Drawdown : -36.34%\n",
      "Market Return : -24.96%   |   Portfolio Return : 10.65%   |   Position Changes : 11.13%   |   Max Drawdown : -30.88%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 909.61%   |   Portfolio Return : 6905.18%   |   Position Changes : 10.08%   |   Max Drawdown : -45.04%\n",
      "↳ Env 0 : 022 :  1292007   |   05:13:38   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6302E+00   |   Tot. Rewards :  3399.39   |   Rewards (/1000 steps) :    58.42   |   Length :  58187\n",
      "Market Return : 908.76%   |   Portfolio Return : 7450.76%   |   Position Changes : 10.61%   |   Max Drawdown : -47.97%\n",
      "↳ Env 2 : 022 :  1304042   |   05:18:43   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6258E+00   |   Tot. Rewards :  3459.39   |   Rewards (/1000 steps) :    59.45   |   Length :  58187\n",
      "Market Return : 1753.28%   |   Portfolio Return : 16187.57%   |   Position Changes :  9.90%   |   Max Drawdown : -57.89%\n",
      "↳ Env 4 : 022 :  1304300   |   05:18:50   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6258E+00   |   Tot. Rewards :  4074.39   |   Rewards (/1000 steps) :    70.16   |   Length :  58072\n",
      "Market Return : 1751.40%   |   Portfolio Return : 40800.21%   |   Position Changes : 10.35%   |   Max Drawdown : -61.23%\n",
      "↳ Env 3 : 022 :  1305321   |   05:19:17   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6255E+00   |   Tot. Rewards :  4810.98   |   Rewards (/1000 steps) :    82.85   |   Length :  58071\n",
      "Market Return : 1753.28%   |   Portfolio Return : 7556.72%   |   Position Changes : 10.25%   |   Max Drawdown : -52.35%\n",
      "↳ Env 1 : 022 :  1309355   |   05:20:59   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6238E+00   |   Tot. Rewards :  3470.53   |   Rewards (/1000 steps) :    59.76   |   Length :  58072\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.99%   |   Portfolio Return : 85.54%   |   Position Changes : 17.30%   |   Max Drawdown : -49.88%\n",
      "Market Return : -36.99%   |   Portfolio Return : 81.74%   |   Position Changes : 17.29%   |   Max Drawdown : -49.88%\n",
      "Market Return : -36.99%   |   Portfolio Return : 57.78%   |   Position Changes : 18.22%   |   Max Drawdown : -29.46%\n",
      "Market Return : -24.95%   |   Portfolio Return : 29.92%   |   Position Changes : 16.79%   |   Max Drawdown : -25.34%\n",
      "Market Return : -36.78%   |   Portfolio Return : 17.60%   |   Position Changes : 16.97%   |   Max Drawdown : -44.78%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -36.96%   |   Portfolio Return : 40.07%   |   Position Changes : 11.19%   |   Max Drawdown : -43.99%\n",
      "Market Return : -36.87%   |   Portfolio Return : -11.72%   |   Position Changes : 11.40%   |   Max Drawdown : -49.01%\n",
      "Market Return : -36.78%   |   Portfolio Return : 24.84%   |   Position Changes : 11.72%   |   Max Drawdown : -48.95%\n",
      "Market Return : -36.78%   |   Portfolio Return : 29.95%   |   Position Changes : 11.77%   |   Max Drawdown : -47.84%\n",
      "Market Return : -24.86%   |   Portfolio Return : -25.99%   |   Position Changes : 10.94%   |   Max Drawdown : -48.52%\n",
      "___________________________________________ TRAINING ___________________________________________\n",
      "Market Return : 908.76%   |   Portfolio Return : 14845.15%   |   Position Changes : 11.28%   |   Max Drawdown : -46.16%\n",
      "↳ Env 0 : 023 :  1350194   |   05:39:46   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6068E+00   |   Tot. Rewards :  4005.58   |   Rewards (/1000 steps) :    68.84   |   Length :  58187\n",
      "Market Return : 1751.91%   |   Portfolio Return : 41091.52%   |   Position Changes : 11.37%   |   Max Drawdown : -60.84%\n",
      "↳ Env 2 : 023 :  1362229   |   05:45:05   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6044E+00   |   Tot. Rewards :  4816.65   |   Rewards (/1000 steps) :    82.78   |   Length :  58187\n",
      "Market Return : 908.92%   |   Portfolio Return : 14422.54%   |   Position Changes : 11.43%   |   Max Drawdown : -42.78%\n",
      "↳ Env 4 : 023 :  1362358   |   05:45:08   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6044E+00   |   Tot. Rewards :  3982.63   |   Rewards (/1000 steps) :    68.60   |   Length :  58058\n",
      "Market Return : 908.92%   |   Portfolio Return : 20367.94%   |   Position Changes : 10.84%   |   Max Drawdown : -37.12%\n",
      "↳ Env 3 : 023 :  1363379   |   05:45:36   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6041E+00   |   Tot. Rewards :  4257.16   |   Rewards (/1000 steps) :    73.33   |   Length :  58058\n",
      "Market Return : 908.92%   |   Portfolio Return : 11930.67%   |   Position Changes : 11.13%   |   Max Drawdown : -39.26%\n",
      "↳ Env 1 : 023 :  1367413   |   05:47:24   |   Epsilon :  0.10%   |   Mean Loss (last 10k) : 2.6032E+00   |   Tot. Rewards :  3832.04   |   Rewards (/1000 steps) :    66.00   |   Length :  58058\n",
      "___________________________________________ VALIDATION ___________________________________________\n",
      "Market Return : -24.83%   |   Portfolio Return : -1.48%   |   Position Changes :  5.10%   |   Max Drawdown : -37.52%\n",
      "Market Return : -24.98%   |   Portfolio Return :  2.05%   |   Position Changes :  6.08%   |   Max Drawdown : -33.61%\n",
      "Market Return : -24.98%   |   Portfolio Return :  0.05%   |   Position Changes :  6.08%   |   Max Drawdown : -34.37%\n",
      "Market Return : -36.94%   |   Portfolio Return : 120.76%   |   Position Changes :  6.16%   |   Max Drawdown : -40.91%\n",
      "Market Return : -36.93%   |   Portfolio Return : 97.62%   |   Position Changes :  6.72%   |   Max Drawdown : -41.34%\n",
      "___________________________________________ TRAINING ___________________________________________\n"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    train(steps = 30_000)\n",
    "    evaluation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dill, pickle\n",
    "#agent.model = None\n",
    "#agent.target_model = None\n",
    "#agent.replay_memory = None\n",
    "\n",
    "with open(\"test.pkl\", \"wb\") as file:\n",
    "    dill.dump(agent, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_indexes, states, actions, rewards, states_prime, dones, importance_weights = agent.replay_memory.sample(\n",
    "    256,\n",
    "    agent.prioritized_replay_beta_function(agent.episode_count, agent.steps)\n",
    ")\n",
    "results = agent.model(states)\n",
    "\n",
    "action_colors=[\"blue\", \"orange\",\"purple\",\"red\"]\n",
    "fig, axes = plt.subplots(nrows = 2, ncols = 2, figsize=(16,9), dpi=300)\n",
    "for action in range(4):\n",
    "    for i in range(256):\n",
    "        axes[action%2, action//2%2].plot(agent.zs, results[i, action, :], color = action_colors[action], alpha = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_indexes, states, actions, rewards, states_prime, dones, importance_weights = agent.replay_memory.sample(\n",
    "    256,\n",
    "    agent.prioritized_replay_beta_function(agent.episode_count, agent.steps)\n",
    ")\n",
    "results = agent.model(states)\n",
    "\n",
    "action_colors=[\"blue\", \"orange\",\"purple\",\"red\"]\n",
    "fig, axes = plt.subplots(nrows = 2, ncols = 2, figsize=(16,9), dpi=300)\n",
    "for action in range(4):\n",
    "    for i in range(1):\n",
    "        axes[action%2, action//2%2].plot(agent.zs, results[i, action, :], color = action_colors[action], alpha = 0.2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}