Taxi-v2-OpenAI-9.47.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import random\n",
    "class Agent:\n",
    "\n",
    "    def __init__(self, epsilon , alpha , gamma, nA = 6):\n",
    "        \"\"\" Initialize agent.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        - nA: number of actions available to the agent\n",
    "        \"\"\"\n",
    "        #self.env = env\n",
    "        self.nA = nA\n",
    "        self.Q = defaultdict(lambda: np.zeros(self.nA))\n",
    "        self.epsilon = epsilon\n",
    "        self.alpha = alpha\n",
    "        self.gamma = gamma\n",
    "\n",
    "    def select_action(self, state, i_episode, eps_decay = 0.999, eps_min = 0.0005):\n",
    "        \"\"\" Given the state, select an action.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        - state: the current state of the environment\n",
    "\n",
    "        Returns\n",
    "        =======\n",
    "        - action: an integer, compatible with the task's action space\n",
    "        \"\"\"\n",
    "        #self.epsilon -= 0.0005 \n",
    "        #self.alpha -= 0.0002\n",
    "        #self.gamma -= -.0002\n",
    "        self.epsilon = max( self.epsilon*eps_decay , eps_min)\n",
    "        \n",
    "        \n",
    "        if random.random() > self.epsilon:\n",
    "            return (np.argmax(self.Q[state]))\n",
    "        else:\n",
    "            return np.random.choice(np.array(self.nA))\n",
    "            #return self.env.action_space.sample()\n",
    "       # return np.random.choice(self.nA)\n",
    "\n",
    "    def step(self, state, action, reward, done , next_state = None, next_action = None ):\n",
    "        \"\"\" Update the agent's knowledge, using the most recently sampled tuple.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        - state: the previous state of the environment\n",
    "        - action: the agent's previous choice of action\n",
    "        - reward: last reward received\n",
    "        - next_state: the current state of the environment\n",
    "        - done: whether the episode is complete (True or False)\n",
    "        \"\"\"\n",
    "        current = self.Q[state][action]\n",
    "        Q_s = self.Q[next_state][next_action] if next_state is not None else 0\n",
    "        target = reward + self.gamma * Q_s\n",
    "        new_value = current + self.alpha * (target - current)\n",
    "        self.Q[state][action] = new_value\n",
    "        \n",
    "        \n",
    "        \n",
    "        #self.Q[state][action] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "import sys\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "def interact(env, agent, num_episodes=20000, window=100):\n",
    "    \"\"\" Monitor agent's performance.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "    - env: instance of OpenAI Gym's Taxi-v1 environment\n",
    "    - agent: instance of class Agent (see Agent.py for details)\n",
    "    - num_episodes: number of episodes of agent-environment interaction\n",
    "    - window: number of episodes to consider when calculating average rewards\n",
    "\n",
    "    Returns\n",
    "    =======\n",
    "    - avg_rewards: deque containing average rewards\n",
    "    - best_avg_reward: largest value in the avg_rewards deque\n",
    "    \"\"\"\n",
    "    # initialize average rewards\n",
    "    avg_rewards = deque(maxlen=num_episodes)\n",
    "    # initialize best average reward\n",
    "    best_avg_reward = -math.inf\n",
    "    # initialize monitor for most recent rewards\n",
    "    samp_rewards = deque(maxlen=window)\n",
    "    # for each episode\n",
    "    for i_episode in range(1, num_episodes+1):\n",
    "        # begin the episode\n",
    "        state = env.reset()\n",
    "        # initialize the sampled reward\n",
    "        samp_reward = 0\n",
    "        action = agent.select_action(state, i_episode)\n",
    "        \n",
    "        while True:\n",
    "              \n",
    "            # agent performs the selected action\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            samp_reward += reward\n",
    "            if not done:\n",
    "                \n",
    "                next_action = agent.select_action(next_state, i_episode)\n",
    "            \n",
    "            # agent performs internal updates based on sampled experience\n",
    "                agent.step(state, action, reward, done , next_state, next_action )\n",
    "            \n",
    "                state = next_state\n",
    "                action = next_action\n",
    "            # update the sampled reward\n",
    "            #samp_reward += reward\n",
    "            # update the state (s <- s') to next time step\n",
    "            #state = next_state\n",
    "            if done:\n",
    "                agent.step(state, action, reward, done, next_state = None, next_action = None )\n",
    "                \n",
    "                # save final sampled reward\n",
    "                samp_rewards.append(samp_reward)\n",
    "                break\n",
    "        if (i_episode >= 100):\n",
    "            # get average reward from last 100 episodes\n",
    "            avg_reward = np.mean(samp_rewards)\n",
    "            # append to deque\n",
    "            avg_rewards.append(avg_reward)\n",
    "            # update best average reward\n",
    "            if avg_reward > best_avg_reward:\n",
    "                best_avg_reward = avg_reward\n",
    "        # monitor progress\n",
    "        print(\"\\rEpisode {}/{} || Best average reward {}\".format(i_episode, num_episodes, best_avg_reward), end=\"\")\n",
    "        sys.stdout.flush()\n",
    "        # check if task is solved (according to OpenAI Gym)\n",
    "        if best_avg_reward >= 9.7:\n",
    "            print('\\nEnvironment solved in {} episodes.'.format(i_episode), end=\"\")\n",
    "            break\n",
    "        if i_episode == num_episodes: print('\\n')\n",
    "    return avg_rewards, best_avg_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 97249/100000 || Best average reward 9.4718"
     ]
    }
   ],
   "source": [
    "#from agent import Agent\n",
    "#from monitor import interact\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "env = gym.make('Taxi-v2')\n",
    "agent = Agent(epsilon = 0.9, alpha = 0.02, gamma = 0.9)\n",
    "avg_rewards, best_avg_reward= interact(env, agent, num_episodes=100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}