ray_rllib_trader_23.10.2020_15.04.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ray==0.8.4\n",
      "  Downloading ray-0.8.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 20.2 MB 7.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.16 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (1.18.5)\n",
      "Requirement already satisfied: jsonschema in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (3.2.0)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (3.12.3)\n",
      "Requirement already satisfied: grpcio in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (1.27.2)\n",
      "Requirement already satisfied: aiohttp in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (3.6.2)\n",
      "Collecting colorama\n",
      "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
      "Collecting redis>=3.3.2\n",
      "  Downloading redis-3.5.3-py2.py3-none-any.whl (72 kB)\n",
      "\u001b[K     |████████████████████████████████| 72 kB 713 kB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting py-spy>=0.2.0\n",
      "  Downloading py_spy-0.3.3-py2.py3-none-manylinux1_x86_64.whl (2.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9 MB 50.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting google\n",
      "  Downloading google-3.0.0-py2.py3-none-any.whl (45 kB)\n",
      "\u001b[K     |████████████████████████████████| 45 kB 7.5 MB/s  eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: click in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (7.1.2)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from ray==0.8.4) (5.3.1)\n",
      "Collecting filelock\n",
      "  Downloading filelock-3.0.12-py3-none-any.whl (7.6 kB)\n",
      "Requirement already satisfied: setuptools in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from jsonschema->ray==0.8.4) (47.3.1.post20200622)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from jsonschema->ray==0.8.4) (0.16.0)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from jsonschema->ray==0.8.4) (19.3.0)\n",
      "Requirement already satisfied: six>=1.11.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from jsonschema->ray==0.8.4) (1.15.0)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from jsonschema->ray==0.8.4) (1.6.1)\n",
      "Requirement already satisfied: multidict<5.0,>=4.5 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from aiohttp->ray==0.8.4) (4.7.6)\n",
      "Requirement already satisfied: async-timeout<4.0,>=3.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from aiohttp->ray==0.8.4) (3.0.1)\n",
      "Requirement already satisfied: chardet<4.0,>=2.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from aiohttp->ray==0.8.4) (3.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from aiohttp->ray==0.8.4) (1.5.1)\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from google->ray==0.8.4) (4.9.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema->ray==0.8.4) (3.1.0)\n",
      "Requirement already satisfied: idna>=2.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from yarl<2.0,>=1.0->aiohttp->ray==0.8.4) (2.9)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4; python_version < \"3.8\" in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from yarl<2.0,>=1.0->aiohttp->ray==0.8.4) (3.7.4.2)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from beautifulsoup4->google->ray==0.8.4) (2.0.1)\n",
      "Installing collected packages: colorama, redis, py-spy, google, filelock, ray\n",
      "Successfully installed colorama-0.4.4 filelock-3.0.12 google-3.0.0 py-spy-0.3.3 ray-0.8.4 redis-3.5.3\n"
     ]
    }
   ],
   "source": [
    "!pip install ray==0.8.4\n",
    "#!pip install ray==1.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting dm-tree==0.1.5\n",
      "  Downloading dm_tree-0.1.5-cp37-cp37m-manylinux1_x86_64.whl (294 kB)\n",
      "\u001b[K     |████████████████████████████████| 294 kB 9.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six>=1.12.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from dm-tree==0.1.5) (1.15.0)\n",
      "Installing collected packages: dm-tree\n",
      "Successfully installed dm-tree-0.1.5\n"
     ]
    }
   ],
   "source": [
    "!pip install dm-tree==0.1.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: failed with initial frozen solve. Retrying with flexible solve.\n",
      "Solving environment: failed with repodata from current_repodata.json, will retry with next repodata source.\n",
      "Collecting package metadata (repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "## Package Plan ##\n",
      "\n",
      "  environment location: /opt/conda/envs/Python-3.7-main\n",
      "\n",
      "  added / updated specs:\n",
      "    - opencv==3.4.2\n",
      "\n",
      "\n",
      "The following packages will be downloaded:\n",
      "\n",
      "    package                    |            build\n",
      "    ---------------------------|-----------------\n",
      "    ffmpeg-4.0                 |       hcdf2ecd_0        53.3 MB\n",
      "    freeglut-3.0.0             |       hf484d3e_5         176 KB\n",
      "    graphite2-1.3.14           |       h23475e2_0          99 KB\n",
      "    h5py-2.8.0                 |   py37h989c5e5_3         910 KB\n",
      "    hdf5-1.10.2                |       hba1933b_1         3.8 MB\n",
      "    ibm-wsrt-py37main-keep-0.0.0|             1433           2 KB\n",
      "    ibm-wsrt-py37main-main-custom|             1433           2 KB\n",
      "    jasper-2.0.14              |       h07fcdf6_1         707 KB\n",
      "    libglu-9.0.0               |       hf484d3e_1         271 KB\n",
      "    libopencv-3.4.2            |       hb342d67_1        21.8 MB\n",
      "    libopus-1.3.1              |       h7b6447c_0         491 KB\n",
      "    libvpx-1.7.0               |       h439df22_0         1.2 MB\n",
      "    opencv-3.4.2               |   py37h6fd60c2_1          11 KB\n",
      "    py-opencv-3.4.2            |   py37hb342d67_1         1.0 MB\n",
      "    pytables-3.4.4             |   py37ha205bf6_0         1.2 MB\n",
      "    ------------------------------------------------------------\n",
      "                                           Total:        85.1 MB\n",
      "\n",
      "The following NEW packages will be INSTALLED:\n",
      "\n",
      "  cairo              pkgs/main/linux-64::cairo-1.14.12-h8948797_3\n",
      "  ffmpeg             pkgs/main/linux-64::ffmpeg-4.0-hcdf2ecd_0\n",
      "  freeglut           pkgs/main/linux-64::freeglut-3.0.0-hf484d3e_5\n",
      "  graphite2          pkgs/main/linux-64::graphite2-1.3.14-h23475e2_0\n",
      "  harfbuzz           pkgs/main/linux-64::harfbuzz-1.8.8-hffaf4a1_0\n",
      "  ibm-wsrt-py37main~ opt/ibm/build/channels/wscloud/noarch::ibm-wsrt-py37main-keep-0.0.0-1433\n",
      "  jasper             pkgs/main/linux-64::jasper-2.0.14-h07fcdf6_1\n",
      "  libglu             pkgs/main/linux-64::libglu-9.0.0-hf484d3e_1\n",
      "  libopencv          pkgs/main/linux-64::libopencv-3.4.2-hb342d67_1\n",
      "  libopus            pkgs/main/linux-64::libopus-1.3.1-h7b6447c_0\n",
      "  libvpx             pkgs/main/linux-64::libvpx-1.7.0-h439df22_0\n",
      "  opencv             pkgs/main/linux-64::opencv-3.4.2-py37h6fd60c2_1\n",
      "  pixman             pkgs/main/linux-64::pixman-0.40.0-h7b6447c_0\n",
      "  py-opencv          pkgs/main/linux-64::py-opencv-3.4.2-py37hb342d67_1\n",
      "\n",
      "The following packages will be DOWNGRADED:\n",
      "\n",
      "  h5py                                2.10.0-py37h7918eee_0 --> 2.8.0-py37h989c5e5_3\n",
      "  hdf5                                    1.10.4-hb1b8bf9_0 --> 1.10.2-hba1933b_1\n",
      "  ibm-wsrt-py37main~                             0.0.0-1433 --> custom-1433\n",
      "  pytables                             3.6.1-py37h71ec239_0 --> 3.4.4-py37ha205bf6_0\n",
      "\n",
      "\n",
      "\n",
      "Downloading and Extracting Packages\n",
      "py-opencv-3.4.2      | 1.0 MB    | ##################################### | 100% \n",
      "opencv-3.4.2         | 11 KB     | ##################################### | 100% \n",
      "h5py-2.8.0           | 910 KB    | ##################################### | 100% \n",
      "libglu-9.0.0         | 271 KB    | ##################################### | 100% \n",
      "ibm-wsrt-py37main-ke | 2 KB      | ##################################### | 100% \n",
      "jasper-2.0.14        | 707 KB    | ##################################### | 100% \n",
      "libopencv-3.4.2      | 21.8 MB   | ##################################### | 100% \n",
      "ibm-wsrt-py37main-ma | 2 KB      | ##################################### | 100% \n",
      "libopus-1.3.1        | 491 KB    | ##################################### | 100% \n",
      "ffmpeg-4.0           | 53.3 MB   | ##################################### | 100% \n",
      "libvpx-1.7.0         | 1.2 MB    | ##################################### | 100% \n",
      "hdf5-1.10.2          | 3.8 MB    | ##################################### | 100% \n",
      "pytables-3.4.4       | 1.2 MB    | ##################################### | 100% \n",
      "graphite2-1.3.14     | 99 KB     | ##################################### | 100% \n",
      "freeglut-3.0.0       | 176 KB    | ##################################### | 100% \n",
      "Preparing transaction: done\n",
      "Verifying transaction: done\n",
      "Executing transaction: done\n"
     ]
    }
   ],
   "source": [
    "!conda install opencv==3.4.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.4.2'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import cv2\n",
    "cv2.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install torch==1.6.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gym==0.17.3\n",
      "  Downloading gym-0.17.3.tar.gz (1.6 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.6 MB 10.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: scipy in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from gym==0.17.3) (1.5.0)\n",
      "Requirement already satisfied: numpy>=1.10.4 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from gym==0.17.3) (1.18.5)\n",
      "Collecting pyglet<=1.5.0,>=1.4.0\n",
      "  Downloading pyglet-1.5.0-py2.py3-none-any.whl (1.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0 MB 49.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from gym==0.17.3) (1.5.0)\n",
      "Requirement already satisfied: future in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from pyglet<=1.5.0,>=1.4.0->gym==0.17.3) (0.18.2)\n",
      "Building wheels for collected packages: gym\n",
      "  Building wheel for gym (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for gym: filename=gym-0.17.3-py3-none-any.whl size=1654654 sha256=9352b5bba501971bc4967c51982fb90681fd752317e6e18a1e57884517dfab86\n",
      "  Stored in directory: /tmp/wsuser/.cache/pip/wheels/d1/81/4b/dd9c029691022cb957398d1f015e66b75e37637dda61abdf58\n",
      "Successfully built gym\n",
      "Installing collected packages: pyglet, gym\n",
      "Successfully installed gym-0.17.3 pyglet-1.5.0\n"
     ]
    }
   ],
   "source": [
    "!pip install gym==0.17.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install ibm-cos-sdk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow==1.15\n",
      "  Downloading tensorflow-1.15.0-cp37-cp37m-manylinux2010_x86_64.whl (412.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 412.3 MB 45 kB/s s eta 0:00:01     |███████████████▌                | 200.2 MB 69.2 MB/s eta 0:00:04MB/s eta 0:00:02��███     | 347.2 MB 69.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: keras-applications>=1.0.8 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.0.8)\n",
      "Requirement already satisfied: gast==0.2.2 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (0.2.2)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (3.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.15.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (0.8.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (3.12.3)\n",
      "Requirement already satisfied: wheel>=0.26 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (0.34.2)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (0.2.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (0.9.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.1.0)\n",
      "Collecting tensorflow-estimator==1.15.1\n",
      "  Downloading tensorflow_estimator-1.15.1-py2.py3-none-any.whl (503 kB)\n",
      "\u001b[K     |████████████████████████████████| 503 kB 57.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.16.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.18.5)\n",
      "Collecting tensorboard<1.16.0,>=1.15.0\n",
      "  Downloading tensorboard-1.15.0-py3-none-any.whl (3.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.8 MB 60.1 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.12.1)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorflow==1.15) (1.27.2)\n",
      "Requirement already satisfied: h5py in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from keras-applications>=1.0.8->tensorflow==1.15) (2.8.0)\n",
      "Requirement already satisfied: setuptools in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from protobuf>=3.6.1->tensorflow==1.15) (47.3.1.post20200622)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (1.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (3.1.1)\n",
      "Installing collected packages: tensorflow-estimator, tensorboard, tensorflow\n",
      "  Attempting uninstall: tensorflow-estimator\n",
      "    Found existing installation: tensorflow-estimator 2.1.0\n",
      "    Uninstalling tensorflow-estimator-2.1.0:\n",
      "      Successfully uninstalled tensorflow-estimator-2.1.0\n",
      "  Attempting uninstall: tensorboard\n",
      "    Found existing installation: tensorboard 2.1.0\n",
      "    Uninstalling tensorboard-2.1.0:\n",
      "      Successfully uninstalled tensorboard-2.1.0\n",
      "  Attempting uninstall: tensorflow\n",
      "    Found existing installation: tensorflow 2.1.0\n",
      "    Uninstalling tensorflow-2.1.0:\n",
      "      Successfully uninstalled tensorflow-2.1.0\n",
      "Successfully installed tensorboard-1.15.0 tensorflow-1.15.0 tensorflow-estimator-1.15.1\n"
     ]
    }
   ],
   "source": [
    "#!pip install tensorflow==2.3.1\n",
    "!pip install tensorflow==1.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ray\n",
    "import gym\n",
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.8.4'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ray.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.17.3'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gym.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.15.0'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import types\n",
    "import pandas as pd\n",
    "from botocore.client import Config\n",
    "import ibm_boto3\n",
    "\n",
    "def __iter__(self): return 0\n",
    "\n",
    "\n",
    "\n",
    "def is_watson_studio():\n",
    "    import os\n",
    "    return 'wsuser' in os.getcwd()\n",
    "\n",
    "\n",
    "def load_market_data (file_name):\n",
    "    \n",
    "\n",
    "\n",
    "    if is_watson_studio():\n",
    "        \n",
    "        client_244519148910498e82791851c5dd0328 = ibm_boto3.client(service_name='s3',\n",
    "        ibm_api_key_id='B4M4KvSyuL9a46802UJTEu2i5YqsKLu5JTk1bDSdiw40',\n",
    "        ibm_auth_endpoint=\"https://iam.cloud.ibm.com/oidc/token\",\n",
    "        config=Config(signature_version='oauth'),\n",
    "        endpoint_url='https://s3-api.us-geo.objectstorage.service.networklayer.com')\n",
    "        body = client_244519148910498e82791851c5dd0328.get_object(Bucket='degray12102020-donotdelete-pr-44gg35tcgd2qty',Key=file_name)['Body']\n",
    "        # add missing __iter__ method, so pandas accepts body as file-like object\n",
    "        if not hasattr(body, \"__iter__\"): body.__iter__ = types.MethodType( __iter__, body )\n",
    "        df_data_1 = pd.read_csv(body)\n",
    "\n",
    "    else:\n",
    "        df_data_1 = pd.read_csv(file_name)\n",
    "        \n",
    "    return df_data_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "assets = ['DJI', 'BTC-USD', 'IBM', 'VOW.DE']\n",
    "df = None\n",
    "for asset in assets:\n",
    "    df_asset = load_market_data(f'{asset}.csv')\n",
    "    df_asset = df_asset[['Date','Close']]\n",
    "    df_asset.columns = ['date', asset]\n",
    "    if df is None:\n",
    "        df = df_asset\n",
    "    else:\n",
    "        df = pd.merge(df,df_asset, on='date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>DJI</th>\n",
       "      <th>BTC-USD</th>\n",
       "      <th>IBM</th>\n",
       "      <th>VOW.DE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014-09-17</td>\n",
       "      <td>17156.849609</td>\n",
       "      <td>457.334015</td>\n",
       "      <td>192.800003</td>\n",
       "      <td>173.300003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014-09-18</td>\n",
       "      <td>17265.990234</td>\n",
       "      <td>424.440002</td>\n",
       "      <td>193.750000</td>\n",
       "      <td>174.300003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014-09-19</td>\n",
       "      <td>17279.740234</td>\n",
       "      <td>394.795990</td>\n",
       "      <td>194.000000</td>\n",
       "      <td>174.149994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014-09-22</td>\n",
       "      <td>17172.679688</td>\n",
       "      <td>402.152008</td>\n",
       "      <td>193.110001</td>\n",
       "      <td>172.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014-09-23</td>\n",
       "      <td>17055.869141</td>\n",
       "      <td>435.790985</td>\n",
       "      <td>191.619995</td>\n",
       "      <td>169.149994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1496</th>\n",
       "      <td>2020-10-08</td>\n",
       "      <td>28425.509766</td>\n",
       "      <td>10923.627930</td>\n",
       "      <td>131.490005</td>\n",
       "      <td>147.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1497</th>\n",
       "      <td>2020-10-09</td>\n",
       "      <td>28586.900391</td>\n",
       "      <td>11079.465820</td>\n",
       "      <td>127.790001</td>\n",
       "      <td>146.699997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1498</th>\n",
       "      <td>2020-10-12</td>\n",
       "      <td>28837.519531</td>\n",
       "      <td>11555.363281</td>\n",
       "      <td>127.209999</td>\n",
       "      <td>149.100006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1499</th>\n",
       "      <td>2020-10-13</td>\n",
       "      <td>28679.810547</td>\n",
       "      <td>11425.899414</td>\n",
       "      <td>125.099998</td>\n",
       "      <td>147.199997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500</th>\n",
       "      <td>2020-10-14</td>\n",
       "      <td>28514.000000</td>\n",
       "      <td>11429.506836</td>\n",
       "      <td>125.940002</td>\n",
       "      <td>147.800003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1498 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            date           DJI       BTC-USD         IBM      VOW.DE\n",
       "0     2014-09-17  17156.849609    457.334015  192.800003  173.300003\n",
       "1     2014-09-18  17265.990234    424.440002  193.750000  174.300003\n",
       "2     2014-09-19  17279.740234    394.795990  194.000000  174.149994\n",
       "3     2014-09-22  17172.679688    402.152008  193.110001  172.750000\n",
       "4     2014-09-23  17055.869141    435.790985  191.619995  169.149994\n",
       "...          ...           ...           ...         ...         ...\n",
       "1496  2020-10-08  28425.509766  10923.627930  131.490005  147.000000\n",
       "1497  2020-10-09  28586.900391  11079.465820  127.790001  146.699997\n",
       "1498  2020-10-12  28837.519531  11555.363281  127.209999  149.100006\n",
       "1499  2020-10-13  28679.810547  11425.899414  125.099998  147.199997\n",
       "1500  2020-10-14  28514.000000  11429.506836  125.940002  147.800003\n",
       "\n",
       "[1498 rows x 5 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.dropna()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym.spaces import Discrete, Box\n",
    "from gym.utils import seeding\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MarketBandit (gym.Env):\n",
    "    \n",
    "    def __init__ (self, config={}):\n",
    "        self.max_inflation = np.array([[100.]])\n",
    "        self.assets = assets\n",
    "\n",
    "        #0: do nothing, 1-4 buy, 5-9 sell for amount x\n",
    "        self.action_space = Discrete(9)\n",
    "        self.observation_space = Box(-self.max_inflation,self.max_inflation)\n",
    "        self.df = df\n",
    "        self.portfolio =  dict.fromkeys(self.assets, 0.0)\n",
    "        self.trade_amount_per_step = 100.\n",
    "        self.money_invested = 0\n",
    "\n",
    "\n",
    "    def reset (self):\n",
    "        self.index = 0\n",
    "        self.done = False\n",
    "        self.info = {}\n",
    "\n",
    "        return np.array([[0.]])\n",
    "\n",
    "    def get_portfolio_value(self,row):\n",
    "        portfolio_sum = 0.\n",
    "        for asset in self.assets:\n",
    "            portfolio_sum += self.portfolio[asset]*row[asset]\n",
    "        return portfolio_sum\n",
    "\n",
    "    def step (self, action):\n",
    "        if self.done:\n",
    "            reward = 0.\n",
    "            regret = 0.\n",
    "        else:\n",
    "            row = self.df.iloc[self.index]\n",
    "\n",
    "            portfolio_value_before = self.get_portfolio_value(row)\n",
    "            \n",
    "            if action == 0:\n",
    "                reward = 0.\n",
    "                regret = 0\n",
    "            else:\n",
    "                buy = False\n",
    "                \n",
    "                if action < 5: # buy\n",
    "                    buy = True\n",
    "                    asset_to_trade = self.assets[action-1]\n",
    "                else: # sell\n",
    "                    asset_to_trade = self.assets[action-5]\n",
    "                    \n",
    "                price = row[asset_to_trade]\n",
    "                asset_to_trade_fraction = self.trade_amount_per_step / price\n",
    "                \n",
    "                if buy:\n",
    "                    self.portfolio[asset_to_trade]+=asset_to_trade_fraction\n",
    "                    self.money_invested += self.trade_amount_per_step\n",
    "                else:\n",
    "                    self.portfolio[asset_to_trade]+=asset_to_trade_fraction\n",
    "\n",
    "            portfolio_value_after = self.get_portfolio_value(row)\n",
    "\n",
    "            reward = float(portfolio_value_after-portfolio_value_before) / 20000\n",
    "\n",
    "            self.index += 1\n",
    "\n",
    "            if self.index >= len(self.df):\n",
    "                self.done = True\n",
    "\n",
    "        #context = [self.cur_context]\n",
    "        #context = self.observation_space.sample()\n",
    "\n",
    "        self.info = {\n",
    "            \"regret\": 0.,\n",
    "            \"index\": self.index\n",
    "        }\n",
    "         \n",
    "        return [np.array([[0.]]), reward, self.done, self.info]\n",
    "\n",
    "\n",
    "    def seed (self, seed=None):\n",
    "        self.np_random, seed = seeding.np_random(seed)\n",
    "        return [seed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1}]\n",
      "8 [array([[0.]]), 0.005000000000000001, False, {'regret': 0.0, 'index': 2}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 3}]\n",
      "8 [array([[0.]]), 0.005000000000000003, False, {'regret': 0.0, 'index': 4}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 5}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 6}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 7}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 8}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 9}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 10}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 11}]\n",
      "1 [array([[0.]]), 0.005000000000000011, False, {'regret': 0.0, 'index': 12}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 13}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 14}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 15}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 16}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 17}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 18}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 19}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 20}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 21}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 22}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 23}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 24}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 25}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 26}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 27}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 28}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 29}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 30}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 31}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 32}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 33}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 34}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 35}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 36}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 37}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 38}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 39}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 40}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 41}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 42}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 43}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 44}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 45}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 46}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 47}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 48}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 49}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 50}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 51}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 52}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 53}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 54}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 55}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 56}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 57}]\n",
      "3 [array([[0.]]), 0.005000000000000045, False, {'regret': 0.0, 'index': 58}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 59}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 60}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 61}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 62}]\n",
      "6 [array([[0.]]), 0.004999999999999954, False, {'regret': 0.0, 'index': 63}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 64}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 65}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 66}]\n",
      "6 [array([[0.]]), 0.005000000000000045, False, {'regret': 0.0, 'index': 67}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 68}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 69}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 70}]\n",
      "1 [array([[0.]]), 0.005000000000000045, False, {'regret': 0.0, 'index': 71}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 72}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 73}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 74}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 75}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 76}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 77}]\n",
      "4 [array([[0.]]), 0.004999999999999954, False, {'regret': 0.0, 'index': 78}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 79}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 80}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 81}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 82}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 83}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 84}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 85}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 86}]\n",
      "8 [array([[0.]]), 0.005000000000000045, False, {'regret': 0.0, 'index': 87}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 88}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 89}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 90}]\n",
      "7 [array([[0.]]), 0.004999999999999954, False, {'regret': 0.0, 'index': 91}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 92}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 93}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 94}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 95}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 96}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 97}]\n",
      "1 [array([[0.]]), 0.004999999999999909, False, {'regret': 0.0, 'index': 98}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 99}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 100}]\n",
      "2 [array([[0.]]), 0.005000000000000091, False, {'regret': 0.0, 'index': 101}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 102}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 103}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 104}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 105}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 106}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 107}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 108}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 109}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 110}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 111}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 112}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 113}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 114}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 115}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 116}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 117}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 118}]\n",
      "1 [array([[0.]]), 0.004999999999999909, False, {'regret': 0.0, 'index': 119}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 120}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 121}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 122}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 123}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 124}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 125}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 126}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 127}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 128}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 129}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 130}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 131}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 132}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 133}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 134}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 135}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 136}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 137}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 138}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 139}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 140}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 141}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 142}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 143}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 144}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 145}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 146}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 147}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 148}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 149}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 150}]\n",
      "5 [array([[0.]]), 0.005000000000000182, False, {'regret': 0.0, 'index': 151}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 152}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 153}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 154}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 155}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 156}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 157}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 158}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 159}]\n",
      "3 [array([[0.]]), 0.004999999999999909, False, {'regret': 0.0, 'index': 160}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 161}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 162}]\n",
      "3 [array([[0.]]), 0.004999999999999909, False, {'regret': 0.0, 'index': 163}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 164}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 165}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 166}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 167}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 168}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 169}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 170}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 171}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 172}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 173}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 174}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 175}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 176}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 177}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 178}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 179}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 180}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 181}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 182}]\n",
      "8 [array([[0.]]), 0.005000000000000091, False, {'regret': 0.0, 'index': 183}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 184}]\n",
      "1 [array([[0.]]), 0.004999999999999818, False, {'regret': 0.0, 'index': 185}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 186}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 187}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 188}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 189}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 190}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 191}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 192}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 193}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 194}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 195}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 196}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 197}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 198}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 199}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 200}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 201}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 202}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 203}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 204}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 205}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 206}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 207}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 208}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 209}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 210}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 211}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 212}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 213}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 214}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 215}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 216}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 217}]\n",
      "2 [array([[0.]]), 0.005000000000000182, False, {'regret': 0.0, 'index': 218}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 219}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 220}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 221}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 222}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 223}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 224}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 225}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 226}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 227}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 228}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 229}]\n",
      "6 [array([[0.]]), 0.004999999999999818, False, {'regret': 0.0, 'index': 230}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 231}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 232}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 233}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 234}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 235}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 236}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 237}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 238}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 239}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 240}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 241}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 242}]\n",
      "6 [array([[0.]]), 0.004999999999999818, False, {'regret': 0.0, 'index': 243}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 244}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 245}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 246}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 247}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 248}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 249}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 250}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 251}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 252}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 253}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 254}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 255}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 256}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 257}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 258}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 259}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 260}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 261}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 262}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 263}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 264}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 265}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 266}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 267}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 268}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 269}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 270}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 271}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 272}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 273}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 274}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 275}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 276}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 277}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 278}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 279}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 280}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 281}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 282}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 283}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 284}]\n",
      "2 [array([[0.]]), 0.004999999999999818, False, {'regret': 0.0, 'index': 285}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 286}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 287}]\n",
      "2 [array([[0.]]), 0.004999999999999818, False, {'regret': 0.0, 'index': 288}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 289}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 290}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 291}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 292}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 293}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 294}]\n",
      "2 [array([[0.]]), 0.005000000000000182, False, {'regret': 0.0, 'index': 295}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 296}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 297}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 298}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 299}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 300}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 301}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 302}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 303}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 304}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 305}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 306}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 307}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 308}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 309}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 310}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 311}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 312}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 313}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 314}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 315}]\n",
      "4 [array([[0.]]), 0.005000000000000182, False, {'regret': 0.0, 'index': 316}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 317}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 318}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 319}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 320}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 321}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 322}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 323}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 324}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 325}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 326}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 327}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 328}]\n",
      "6 [array([[0.]]), 0.005000000000000182, False, {'regret': 0.0, 'index': 329}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 330}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 331}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 332}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 333}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 334}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 335}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 336}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 337}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 338}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 339}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 340}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 341}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 342}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 343}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 344}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 345}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 346}]\n",
      "2 [array([[0.]]), 0.004999999999999636, False, {'regret': 0.0, 'index': 347}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 348}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 349}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 350}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 351}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 352}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 353}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 354}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 355}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 356}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 357}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 358}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 359}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 360}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 361}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 362}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 363}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 364}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 365}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 366}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 367}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 368}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 369}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 370}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 371}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 372}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 373}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 374}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 375}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 376}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 377}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 378}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 379}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 380}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 381}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 382}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 383}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 384}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 385}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 386}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 387}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 388}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 389}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 390}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 391}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 392}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 393}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 394}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 395}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 396}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 397}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 398}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 399}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 400}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 401}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 402}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 403}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 404}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 405}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 406}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 407}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 408}]\n",
      "4 [array([[0.]]), 0.0050000000000003635, False, {'regret': 0.0, 'index': 409}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 410}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 411}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 412}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 413}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 414}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 415}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 416}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 417}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 418}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 419}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 420}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 421}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 422}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 423}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 424}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 425}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 426}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 427}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 428}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 429}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 430}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 431}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 432}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 433}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 434}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 435}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 436}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 437}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 438}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 439}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 440}]\n",
      "6 [array([[0.]]), 0.004999999999999636, False, {'regret': 0.0, 'index': 441}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 442}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 443}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 444}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 445}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 446}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 447}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 448}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 449}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 450}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 451}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 452}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 453}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 454}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 455}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 456}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 457}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 458}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 459}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 460}]\n",
      "6 [array([[0.]]), 0.0050000000000003635, False, {'regret': 0.0, 'index': 461}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 462}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 463}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 464}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 465}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 466}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 467}]\n",
      "4 [array([[0.]]), 0.004999999999999636, False, {'regret': 0.0, 'index': 468}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 469}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 470}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 471}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 472}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 473}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 474}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 475}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 476}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 477}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 478}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 479}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 480}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 481}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 482}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 483}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 484}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 485}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 486}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 487}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 488}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 489}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 490}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 491}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 492}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 493}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 494}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 495}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 496}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 497}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 498}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 499}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 500}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 501}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 502}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 503}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 504}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 505}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 506}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 507}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 508}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 509}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 510}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 511}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 512}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 513}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 514}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 515}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 516}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 517}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 518}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 519}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 520}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 521}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 522}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 523}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 524}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 525}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 526}]\n",
      "2 [array([[0.]]), 0.004999999999999272, False, {'regret': 0.0, 'index': 527}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 528}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 529}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 530}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 531}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 532}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 533}]\n",
      "6 [array([[0.]]), 0.0050000000000003635, False, {'regret': 0.0, 'index': 534}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 535}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 536}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 537}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 538}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 539}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 540}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 541}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 542}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 543}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 544}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 545}]\n",
      "3 [array([[0.]]), 0.004999999999999636, False, {'regret': 0.0, 'index': 546}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 547}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 548}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 549}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 550}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 551}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 552}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 553}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 554}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 555}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 556}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 557}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 558}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 559}]\n",
      "6 [array([[0.]]), 0.005000000000000728, False, {'regret': 0.0, 'index': 560}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 561}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 562}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 563}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 564}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 565}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 566}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 567}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 568}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 569}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 570}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 571}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 572}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 573}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 574}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 575}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 576}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 577}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 578}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 579}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 580}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 581}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 582}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 583}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 584}]\n",
      "1 [array([[0.]]), 0.005000000000000728, False, {'regret': 0.0, 'index': 585}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 586}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 587}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 588}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 589}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 590}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 591}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 592}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 593}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 594}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 595}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 596}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 597}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 598}]\n",
      "6 [array([[0.]]), 0.004999999999999272, False, {'regret': 0.0, 'index': 599}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 600}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 601}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 602}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 603}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 604}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 605}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 606}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 607}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 608}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 609}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 610}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 611}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 612}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 613}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 614}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 615}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 616}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 617}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 618}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 619}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 620}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 621}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 622}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 623}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 624}]\n",
      "3 [array([[0.]]), 0.004999999999999272, False, {'regret': 0.0, 'index': 625}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 626}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 627}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 628}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 629}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 630}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 631}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 632}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 633}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 634}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 635}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 636}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 637}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 638}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 639}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 640}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 641}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 642}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 643}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 644}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 645}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 646}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 647}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 648}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 649}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 650}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 651}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 652}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 653}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 654}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 655}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 656}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 657}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 658}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 659}]\n",
      "2 [array([[0.]]), 0.005000000000000728, False, {'regret': 0.0, 'index': 660}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 661}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 662}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 663}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 664}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 665}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 666}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 667}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 668}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 669}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 670}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 671}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 672}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 673}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 674}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 675}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 676}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 677}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 678}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 679}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 680}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 681}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 682}]\n",
      "8 [array([[0.]]), 0.0050000000000014555, False, {'regret': 0.0, 'index': 683}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 684}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 685}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 686}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 687}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 688}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 689}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 690}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 691}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 692}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 693}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 694}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 695}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 696}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 697}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 698}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 699}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 700}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 701}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 702}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 703}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 704}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 705}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 706}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 707}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 708}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 709}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 710}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 711}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 712}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 713}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 714}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 715}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 716}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 717}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 718}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 719}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 720}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 721}]\n",
      "2 [array([[0.]]), 0.0050000000000014555, False, {'regret': 0.0, 'index': 722}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 723}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 724}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 725}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 726}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 727}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 728}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 729}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 730}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 731}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 732}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 733}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 734}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 735}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 736}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 737}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 738}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 739}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 740}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 741}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 742}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 743}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 744}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 745}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 746}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 747}]\n",
      "6 [array([[0.]]), 0.0050000000000014555, False, {'regret': 0.0, 'index': 748}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 749}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 750}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 751}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 752}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 753}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 754}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 755}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 756}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 757}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 758}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 759}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 760}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 761}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 762}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 763}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 764}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 765}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 766}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 767}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 768}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 769}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 770}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 771}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 772}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 773}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 774}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 775}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 776}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 777}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 778}]\n",
      "7 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 779}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 780}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 781}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 782}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 783}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 784}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 785}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 786}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 787}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 788}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 789}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 790}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 791}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 792}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 793}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 794}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 795}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 796}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 797}]\n",
      "6 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 798}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 799}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 800}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 801}]\n",
      "6 [array([[0.]]), 0.004999999999994179, False, {'regret': 0.0, 'index': 802}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 803}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 804}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 805}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 806}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 807}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 808}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 809}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 810}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 811}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 812}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 813}]\n",
      "6 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 814}]\n",
      "2 [array([[0.]]), 0.004999999999994179, False, {'regret': 0.0, 'index': 815}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 816}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 817}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 818}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 819}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 820}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 821}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 822}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 823}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 824}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 825}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 826}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 827}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 828}]\n",
      "2 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 829}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 830}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 831}]\n",
      "8 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 832}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 833}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 834}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 835}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 836}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 837}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 838}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 839}]\n",
      "6 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 840}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 841}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 842}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 843}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 844}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 845}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 846}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 847}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 848}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 849}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 850}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 851}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 852}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 853}]\n",
      "2 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 854}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 855}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 856}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 857}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 858}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 859}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 860}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 861}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 862}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 863}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 864}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 865}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 866}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 867}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 868}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 869}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 870}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 871}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 872}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 873}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 874}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 875}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 876}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 877}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 878}]\n",
      "2 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 879}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 880}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 881}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 882}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 883}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 884}]\n",
      "2 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 885}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 886}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 887}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 888}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 889}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 890}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 891}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 892}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 893}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 894}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 895}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 896}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 897}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 898}]\n",
      "7 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 899}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 900}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 901}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 902}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 903}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 904}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 905}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 906}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 907}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 908}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 909}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 910}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 911}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 912}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 913}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 914}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 915}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 916}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 917}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 918}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 919}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 920}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 921}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 922}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 923}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 924}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 925}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 926}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 927}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 928}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 929}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 930}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 931}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 932}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 933}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 934}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 935}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 936}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 937}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 938}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 939}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 940}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 941}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 942}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 943}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 944}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 945}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 946}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 947}]\n",
      "6 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 948}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 949}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 950}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 951}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 952}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 953}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 954}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 955}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 956}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 957}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 958}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 959}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 960}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 961}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 962}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 963}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 964}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 965}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 966}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 967}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 968}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 969}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 970}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 971}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 972}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 973}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 974}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 975}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 976}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 977}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 978}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 979}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 980}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 981}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 982}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 983}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 984}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 985}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 986}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 987}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 988}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 989}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 990}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 991}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 992}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 993}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 994}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 995}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 996}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 997}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 998}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 999}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1000}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1001}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1002}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1003}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1004}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1005}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1006}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1007}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1008}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1009}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1010}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1011}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1012}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1013}]\n",
      "2 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1014}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1015}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1016}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1017}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1018}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1019}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1020}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1021}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1022}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1023}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1024}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1025}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1026}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1027}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1028}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1029}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1030}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1031}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1032}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1033}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1034}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1035}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1036}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1037}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1038}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1039}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1040}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1041}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1042}]\n",
      "6 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1043}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1044}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1045}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1046}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1047}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1048}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1049}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1050}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1051}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1052}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1053}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1054}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1055}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1056}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1057}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1058}]\n",
      "2 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1059}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1060}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1061}]\n",
      "8 [array([[0.]]), 0.0050000000000014555, False, {'regret': 0.0, 'index': 1062}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1063}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1064}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1065}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1066}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1067}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1068}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1069}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1070}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1071}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1072}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1073}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1074}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1075}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1076}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1077}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1078}]\n",
      "2 [array([[0.]]), 0.0050000000000014555, False, {'regret': 0.0, 'index': 1079}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1080}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1081}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1082}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1083}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1084}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1085}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1086}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1087}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1088}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1089}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1090}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1091}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1092}]\n",
      "2 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1093}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1094}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1095}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1096}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1097}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1098}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1099}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1100}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1101}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1102}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1103}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1104}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1105}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1106}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1107}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1108}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1109}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1110}]\n",
      "2 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1111}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1112}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1113}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1114}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1115}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1116}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1117}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1118}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1119}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1120}]\n",
      "6 [array([[0.]]), 0.004999999999998545, False, {'regret': 0.0, 'index': 1121}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1122}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1123}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1124}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1125}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1126}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1127}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1128}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1129}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1130}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1131}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1132}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1133}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1134}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1135}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1136}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1137}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1138}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1139}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1140}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1141}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1142}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1143}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1144}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1145}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1146}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1147}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1148}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1149}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1150}]\n",
      "6 [array([[0.]]), 0.004999999999994179, False, {'regret': 0.0, 'index': 1151}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1152}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1153}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1154}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1155}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1156}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1157}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1158}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1159}]\n",
      "5 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1160}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1161}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1162}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1163}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1164}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1165}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1166}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1167}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1168}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1169}]\n",
      "7 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1170}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1171}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1172}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1173}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1174}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1175}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1176}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1177}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1178}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1179}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1180}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1181}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1182}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1183}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1184}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1185}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1186}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1187}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1188}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1189}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1190}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1191}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1192}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1193}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1194}]\n",
      "2 [array([[0.]]), 0.004999999999994179, False, {'regret': 0.0, 'index': 1195}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1196}]\n",
      "6 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1197}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1198}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1199}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1200}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1201}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1202}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1203}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1204}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1205}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1206}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1207}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1208}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1209}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1210}]\n",
      "6 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 1211}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1212}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1213}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1214}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1215}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1216}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1217}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1218}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1219}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1220}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1221}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1222}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1223}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1224}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1225}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1226}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1227}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1228}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1229}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1230}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1231}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1232}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1233}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1234}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1235}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1236}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1237}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1238}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1239}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1240}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1241}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1242}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1243}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1244}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1245}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1246}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1247}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1248}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1249}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1250}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1251}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1252}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1253}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1254}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1255}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1256}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1257}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1258}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1259}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1260}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1261}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1262}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1263}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1264}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1265}]\n",
      "6 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 1266}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1267}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1268}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1269}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1270}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1271}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1272}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1273}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1274}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1275}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1276}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1277}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1278}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1279}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1280}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1281}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1282}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1283}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1284}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1285}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1286}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1287}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1288}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1289}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1290}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1291}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1292}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1293}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1294}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1295}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1296}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1297}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1298}]\n",
      "5 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1299}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1300}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1301}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1302}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1303}]\n",
      "2 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1304}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1305}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1306}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1307}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1308}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1309}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1310}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1311}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1312}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1313}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1314}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1315}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1316}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1317}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1318}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1319}]\n",
      "6 [array([[0.]]), 0.004999999999997089, False, {'regret': 0.0, 'index': 1320}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1321}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1322}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1323}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1324}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1325}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1326}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1327}]\n",
      "6 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 1328}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1329}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1330}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1331}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1332}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1333}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1334}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1335}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1336}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1337}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1338}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1339}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1340}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1341}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1342}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1343}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1344}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1345}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1346}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1347}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1348}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1349}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1350}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1351}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1352}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1353}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1354}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1355}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1356}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1357}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1358}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1359}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1360}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1361}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1362}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1363}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1364}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1365}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1366}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1367}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1368}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1369}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1370}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1371}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1372}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1373}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1374}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1375}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1376}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1377}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1378}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1379}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1380}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1381}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1382}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1383}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1384}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1385}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1386}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1387}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1388}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1389}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1390}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1391}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1392}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1393}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1394}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1395}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1396}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1397}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1398}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1399}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1400}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1401}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1402}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1403}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1404}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1405}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1406}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1407}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1408}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1409}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1410}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1411}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1412}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1413}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1414}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1415}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1416}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1417}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1418}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1419}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1420}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1421}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1422}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1423}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1424}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1425}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1426}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1427}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1428}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1429}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1430}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1431}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1432}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1433}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1434}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1435}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1436}]\n",
      "6 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1437}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1438}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1439}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1440}]\n",
      "6 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 1441}]\n",
      "2 [array([[0.]]), 0.00500000000000291, False, {'regret': 0.0, 'index': 1442}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1443}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1444}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1445}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1446}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1447}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1448}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1449}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1450}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1451}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1452}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1453}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1454}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1455}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1456}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1457}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1458}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1459}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1460}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1461}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1462}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1463}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1464}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1465}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1466}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1467}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1468}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1469}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1470}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1471}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1472}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1473}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1474}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1475}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1476}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1477}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1478}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1479}]\n",
      "6 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1480}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1481}]\n",
      "7 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1482}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1483}]\n",
      "2 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1484}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1485}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1486}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1487}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1488}]\n",
      "8 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1489}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1490}]\n",
      "0 [array([[0.]]), 0.0, False, {'regret': 0.0, 'index': 1491}]\n",
      "3 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1492}]\n",
      "4 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1493}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1494}]\n",
      "1 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1495}]\n",
      "5 [array([[0.]]), 0.005, False, {'regret': 0.0, 'index': 1496}]\n",
      "2 [array([[0.]]), 0.005000000000005821, False, {'regret': 0.0, 'index': 1497}]\n",
      "6 [array([[0.]]), 0.005, True, {'regret': 0.0, 'index': 1498}]\n"
     ]
    }
   ],
   "source": [
    "bandit = MarketBandit()\n",
    "bandit.reset()\n",
    "\n",
    "for i in range(len(df)):\n",
    "    action = bandit.action_space.sample()\n",
    "    obs = bandit.step(action)\n",
    "    print(action, obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 1}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 2}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 3}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 4}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 5}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 6}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 7}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 8}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 9}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 10}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 11}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 12}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 13}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 14}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 15}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 16}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 17}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 18}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 19}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 20}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 21}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 22}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 23}\n",
      "3 [[0.]] 0.005 False {'regret': 0.0, 'index': 24}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 25}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 26}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 27}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 28}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 29}\n",
      "3 [[0.]] 0.0050000000000014555 False {'regret': 0.0, 'index': 30}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 31}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 32}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 33}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 34}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 35}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 36}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 37}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 38}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 39}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 40}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 41}\n",
      "3 [[0.]] 0.005 False {'regret': 0.0, 'index': 42}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 43}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 44}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 45}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 46}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 47}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 48}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 49}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 50}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 51}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 52}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 53}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 54}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 55}\n",
      "4 [[0.]] 0.004999999999999272 False {'regret': 0.0, 'index': 56}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 57}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 58}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 59}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 60}\n",
      "3 [[0.]] 0.005 False {'regret': 0.0, 'index': 61}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 62}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 63}\n",
      "3 [[0.]] 0.004999999999998545 False {'regret': 0.0, 'index': 64}\n",
      "7 [[0.]] 0.004999999999999272 False {'regret': 0.0, 'index': 65}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 66}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 67}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 68}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 69}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 70}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 71}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 72}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 73}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 74}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 75}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 76}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 77}\n",
      "3 [[0.]] 0.005 False {'regret': 0.0, 'index': 78}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 79}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 80}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 81}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 82}\n",
      "2 [[0.]] 0.005 False {'regret': 0.0, 'index': 83}\n",
      "5 [[0.]] 0.005 False {'regret': 0.0, 'index': 84}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 85}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 86}\n",
      "4 [[0.]] 0.005 False {'regret': 0.0, 'index': 87}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 88}\n",
      "8 [[0.]] 0.005 False {'regret': 0.0, 'index': 89}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 90}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 91}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 92}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 93}\n",
      "3 [[0.]] 0.005 False {'regret': 0.0, 'index': 94}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 95}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 96}\n",
      "6 [[0.]] 0.005 False {'regret': 0.0, 'index': 97}\n",
      "7 [[0.]] 0.005 False {'regret': 0.0, 'index': 98}\n",
      "0 [[0.]] 0.0 False {'regret': 0.0, 'index': 99}\n",
      "1 [[0.]] 0.005 False {'regret': 0.0, 'index': 100}\n"
     ]
    }
   ],
   "source": [
    "done = 1\n",
    "reward_list = []\n",
    "iterations = 100\n",
    "\n",
    "for i in range(iterations):\n",
    "    if done == 1:\n",
    "        bandit.reset()\n",
    "\n",
    "    action = bandit.action_space.sample()\n",
    "    obs = bandit.step(action)\n",
    "    context, reward, done, info = obs\n",
    "    reward_list.append(reward)\n",
    "    print(action, context, reward, done, info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lz4 not available, disabling sample compression. This will significantly impact RLlib performance. To install lz4, run `pip install lz4`.\n"
     ]
    }
   ],
   "source": [
    "import ray\n",
    "from ray.rllib.agents.trainer import with_base_config, with_common_config\n",
    "from ray.rllib.contrib.bandits.agents.lin_ts import TS_CONFIG\n",
    "from ray.rllib.contrib.bandits.agents.lin_ts import LinTSTrainer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ray.init(temp_dir='.', memory=52428800, object_store_memory=78643200, num_cpus=1, num_gpus=0, local_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ray.rllib.agents.ppo import PPOTrainer, DEFAULT_CONFIG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer_config = DEFAULT_CONFIG.copy()\n",
    "trainer_config['num_workers'] = 1\n",
    "trainer_config[\"train_batch_size\"] = 400\n",
    "trainer_config[\"sgd_minibatch_size\"] = 64\n",
    "trainer_config[\"num_sgd_iter\"] = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-10-23 12:30:17,929\tINFO trainer.py:428 -- Tip: set 'eager': true or the --eager flag to enable TensorFlow eager execution\n",
      "2020-10-23 12:30:17,950\tERROR logger.py:193 -- pip install 'ray[tune]' to see TensorBoard files.\n",
      "2020-10-23 12:30:17,951\tWARNING logger.py:308 -- Could not instantiate TBXLogger: No module named 'tensorboardX'.\n",
      "2020-10-23 12:30:17,952\tERROR syncer.py:39 -- Log sync requires rsync to be installed.\n",
      "2020-10-23 12:30:17,955\tINFO trainer.py:585 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
      "/opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n",
      "/opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n",
      "2020-10-23 12:30:22,484\tINFO trainable.py:217 -- Getting current IP.\n",
      "2020-10-23 12:30:22,486\tWARNING util.py:37 -- Install gputil for GPU system monitoring.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training iterations: ....................done\n"
     ]
    }
   ],
   "source": [
    "trainer = PPOTrainer(trainer_config, MarketBandit)\n",
    "print(f'Training iterations: ', end='')\n",
    "for i in range(20):\n",
    "    print('.', end='')\n",
    "    trainer.train()\n",
    "print('done')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cumulative reward you received is: 147699.99999999674. Congratulations!\n",
      "Portfolio value: 542608.3900087596\n",
      "Money invested 91500.0\n"
     ]
    }
   ],
   "source": [
    "env = MarketBandit()\n",
    "state = env.reset()\n",
    "\n",
    "done = False\n",
    "max_state = -1\n",
    "cumulative_reward = 0\n",
    "\n",
    "while not done:\n",
    "    action = trainer.compute_action(state)\n",
    "    state, reward, done, results = env.step(action)\n",
    "    max_state = max(max_state, state)\n",
    "    cumulative_reward += reward\n",
    "\n",
    "print(f'Cumulative reward you received is: {cumulative_reward*20000}. Congratulations!')\n",
    "print(f'Portfolio value: {env.get_portfolio_value(df.iloc[len(df)-1])}')\n",
    "print(f'Money invested {env.money_invested}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.export_policy_model('./model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_model = tf.keras.models.load_model('./model')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ls -lahr model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!tar -zcvf model.tar.gz -C model ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-10-23 12:31:15,891 - ibm_watson_machine_learning.wml_client_error - WARNING - Deployment creation failed. Error: 400. {\"trace\":\"4e7a2048dd1e5ff2c773baa77179b8d5\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}\n",
      "2020-10-23 12:43:25,803 - ibm_watson_machine_learning.wml_client_error - WARNING - Deployment creation failed. Error: 400. {\"trace\":\"bbe681ee99e92511f9914b76b3942d8a\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}\n",
      "2020-10-23 12:43:54,650 - ibm_watson_machine_learning.wml_client_error - WARNING - Deployment creation failed. Error: 400. {\"trace\":\"4089e2b887b542dcd08120b39818950f\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}\n",
      "2020-10-23 12:44:20,420 - ibm_watson_machine_learning.wml_client_error - WARNING - Deployment creation failed. Error: 400. {\"trace\":\"53e7f53e5aeec16b8d29bf2a84ed11dd\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}\n"
     ]
    }
   ],
   "source": [
    "from ibm_watson_machine_learning import APIClient\n",
    "\n",
    "wml_credentials = {\n",
    "                   \"url\": \"https://us-south.ml.cloud.ibm.com\",\n",
    "                   \"apikey\":\"qIGYA_DUnBucYCuqpeszKRqxEEjM-Npt2vacqabOuybX\"\n",
    "                  }\n",
    "\n",
    "client = APIClient(wml_credentials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2b73a275-7cbf-420b-a912-eae7f436e0bc'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sw_spec_id = client.software_specifications.get_id_by_name('tensorflow_1.15-py3.6')\n",
    "sw_spec_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CUSTOM',\n",
       " 'DESCRIPTION',\n",
       " 'DOMAIN',\n",
       " 'HYPER_PARAMETERS',\n",
       " 'IMPORT',\n",
       " 'INPUT_DATA_SCHEMA',\n",
       " 'LABEL_FIELD',\n",
       " 'METRICS',\n",
       " 'MODEL_DEFINITION_UID',\n",
       " 'NAME',\n",
       " 'OUTPUT_DATA_SCHEMA',\n",
       " 'PIPELINE_UID',\n",
       " 'RUNTIME_UID',\n",
       " 'SIZE',\n",
       " 'SOFTWARE_SPEC_UID',\n",
       " 'SPACE_UID',\n",
       " 'TAGS',\n",
       " 'TF_MODEL_PARAMS',\n",
       " 'TRAINING_DATA_REFERENCES',\n",
       " 'TRAINING_LIB_UID',\n",
       " 'TRANSFORMED_LABEL_FIELD',\n",
       " 'TYPE']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "model_props = {\n",
    "    client.repository.ModelMetaNames.NAME: '{}'.format('Trading Agent Model'),\n",
    "    client.repository.ModelMetaNames.TYPE: \"tensorflow_1.15\",\n",
    "    client.repository.ModelMetaNames.SOFTWARE_SPEC_UID: sw_spec_id\n",
    "    #client.repository.ModelMetaNames.FRAMEWORK_LIBRARIES: [{\"name\": \"tensorflow\", \"version\": \"1.15\"}]\n",
    "}\n",
    "\n",
    "# get a list of all available props\n",
    "client.repository.ModelMetaNames.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'SUCCESS'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.set.default_space('a323e591-4254-4571-a8f9-96aeeaac649e')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'entity': {'software_spec': {'id': '2b73a275-7cbf-420b-a912-eae7f436e0bc',\n",
       "   'name': 'tensorflow_1.15-py3.6'},\n",
       "  'type': 'tensorflow_1.15'},\n",
       " 'metadata': {'created_at': '2020-10-23T12:31:13.092Z',\n",
       "  'id': '9901dde5-3b72-4de3-8704-bdea91bf5522',\n",
       "  'modified_at': '2020-10-23T12:31:15.184Z',\n",
       "  'name': 'Trading Agent Model',\n",
       "  'owner': 'IBMid-270002573Y',\n",
       "  'space_id': 'a323e591-4254-4571-a8f9-96aeeaac649e'}}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.repository.store_model('./model', meta_props=model_props)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_props = {\n",
    "    client.deployments.ConfigurationMetaNames.NAME: \"Trading Agent Deployed Model\",\n",
    "    client.deployments.ConfigurationMetaNames.BATCH: {'yes'},\n",
    "    client.deployments.ConfigurationMetaNames.HARDWARE_SPEC : { \"id\":  \"e7ed1d6c-2e89-42d7-aed5-8sb972c1d2b\"}\n",
    "}\n",
    "#client.deployments.ConfigurationMetaNames.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"trace\":\"53e7f53e5aeec16b8d29bf2a84ed11dd\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}\n",
      "\n",
      "\n",
      "--------------------------\n",
      "Deployment creation failed\n",
      "--------------------------\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "WMLClientError",
     "evalue": "Deployment creation failed. Error: 400. {\"trace\":\"53e7f53e5aeec16b8d29bf2a84ed11dd\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mWMLClientError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-c52511d43c0e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeployments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martifact_uid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'570eda1d-894a-4d26-9b71-faf3a0d5ebd2'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Trading Agent Deployed Model'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta_props\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel_props\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages/ibm_watson_machine_learning/deployments.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(self, artifact_uid, meta_props, rev_id, **kwargs)\u001b[0m\n\u001b[1;32m    262\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreason\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m                 \u001b[0mprint_text_header_h2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mWMLClientError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_msg\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'. Error: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'. '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mreason\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m             \u001b[0;31m#return self._handle_response(202, u'created deployment', response)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mWMLClientError\u001b[0m: Deployment creation failed. Error: 400. {\"trace\":\"53e7f53e5aeec16b8d29bf2a84ed11dd\",\"errors\":[{\"code\":\"deployment_type_undefined\",\"message\":\"Invalid content. Deployment needs to have at least one type defined.\",\"target\":{\"type\":\"Json\",\"name\":\"payload\"}}]}"
     ]
    }
   ],
   "source": [
    "client.deployments.create(artifact_uid='570eda1d-894a-4d26-9b71-faf3a0d5ebd2',name='Trading Agent Deployed Model', meta_props=model_props)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}