cartpole.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Importing libraries\n",
    "\n",
    "Check that the libraries are installed correctly by running the following import statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "print(gym.__version__)\n",
    "import keras\n",
    "print(keras.__version__)\n",
    "import random\n",
    "import math\n",
    "import numpy as np\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Importing Gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "env = gym.make('CartPole-v0')\n",
    "import gym\n",
    "env = gym.make('CartPole-v0')\n",
    "print(env.observation_space.high)\n",
    "print(env.observation_space.low)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Define Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training Parameters\n",
    "n_episodes=1000\n",
    "n_win_ticks=195\n",
    "max_env_steps=None\n",
    "gamma=1.0 # Discount factor. Consideration of future rewards - same policies (the ones that balance the pole) are optimal \n",
    "          # at every step.\n",
    "epsilon=1.0 # Exploration. Agent chooses action that it believes has the best long-term effect with probability 1- epsilon,\n",
    "            # and it chooses an action uniformly at random, otherwise.  \n",
    "epsilon_min=0.01 # we want to put a little randomness in decision eventually\n",
    "epsilon_decay=0.995 #we want to decay epsilon as agent stops taking actions randomly\n",
    "alpha=0.01 # the learning rate, determines to what extent the newly acquired information will override the old information. \n",
    "alpha_decay=0.01 # we want to decay alpha so that agent will not change the actions/learning everytime even after achieveing enough learning\n",
    "batch_size=64\n",
    "monitor=False\n",
    "quiet=False\n",
    "\n",
    "# Environment Parameters\n",
    "memory = deque(maxlen=100000)\n",
    "env = gym.make('CartPole-v0')\n",
    "if max_env_steps is not None: env.max_episode_steps = max_env_steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Building the Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "# Model Definition\n",
    "model = Sequential()\n",
    "model.add(Dense(64, input_dim=4, activation='relu'))\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Dense(16, activation='relu'))\n",
    "model.add(Dense(4, activation='relu'))\n",
    "model.add(Dense(2, activation='relu'))\n",
    "model.compile(loss='mse', optimizer=Adam(lr=alpha, decay=alpha_decay))\n",
    "\n",
    "def remember(state, action, reward, next_state, done):\n",
    "    memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "def choose_action(state, epsilon):\n",
    "    return env.action_space.sample() if (np.random.random() <= epsilon) else np.argmax(\n",
    "        model.predict(state))\n",
    "\n",
    "def get_epsilon(t):\n",
    "    return max(epsilon_min, min(epsilon, 1.0 - math.log10((t + 1) * epsilon_decay)))\n",
    "\n",
    "def preprocess_state(state):\n",
    "    return np.reshape(state, [1, 4])\n",
    "\n",
    "def replay(batch_size, epsilon):\n",
    "    x_batch, y_batch = [], []\n",
    "    minibatch = random.sample(\n",
    "        memory, min(len(memory), batch_size))\n",
    "    for state, action, reward, next_state, done in minibatch:\n",
    "        y_target = model.predict(state)\n",
    "        y_target[0][action] = reward if done else reward + gamma * np.max(model.predict(next_state)[0])\n",
    "        x_batch.append(state[0])\n",
    "        y_batch.append(y_target[0])\n",
    "\n",
    "    model.fit(np.array(x_batch), np.array(y_batch), batch_size=len(x_batch), verbose=0)\n",
    "    \n",
    "    if epsilon > epsilon_min:\n",
    "        epsilon *= epsilon_decay\n",
    "\n",
    "def run():\n",
    "    scores = deque(maxlen=100)\n",
    "\n",
    "    for e in range(n_episodes):\n",
    "        state = preprocess_state(env.reset())\n",
    "        done = False\n",
    "        i = 0\n",
    "        while not done:\n",
    "            action = choose_action(state, get_epsilon(e))\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            env.render()\n",
    "            next_state = preprocess_state(next_state)\n",
    "            remember(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            i += 1\n",
    "\n",
    "        scores.append(i)\n",
    "        mean_score = np.mean(scores)\n",
    "        if mean_score >= n_win_ticks and e >= 100:\n",
    "            if not quiet: print('Ran {} episodes. Solved after {} trials'.format(e, e - 100))\n",
    "            return e - 100\n",
    "        if e % 100 == 0 and not quiet:\n",
    "            print('[Episode {}] - Mean survival time over last 100 episodes was {} ticks.'.format(e, mean_score))\n",
    "   \n",
    "        replay(batch_size, get_epsilon(epsilon))\n",
    "\n",
    "    if not quiet: print('Did not solve after {} episodes'.format(e))\n",
    "    return e\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}