openai-gym-taxi-v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "from collections import defaultdict\n",
    "import gym\n",
    "import sys\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interact(env, agent, num_episodes=20000, window=100):\n",
    "    \"\"\" Monitor agent's performance.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "    - env: instance of OpenAI Gym's Taxi-v1 environment\n",
    "    - agent: instance of class Agent (see Agent.py for details)\n",
    "    - num_episodes: number of episodes of agent-environment interaction\n",
    "    - window: number of episodes to consider when calculating average rewards\n",
    "\n",
    "    Returns\n",
    "    =======\n",
    "    - avg_rewards: deque containing average rewards\n",
    "    - best_avg_reward: largest value in the avg_rewards deque\n",
    "    \"\"\"\n",
    "    # initialize average rewards\n",
    "    avg_rewards = deque(maxlen=num_episodes)\n",
    "    # initialize best average reward\n",
    "    best_avg_reward = -math.inf\n",
    "    # initialize monitor for most recent rewards\n",
    "    samp_rewards = deque(maxlen=window)\n",
    "    # for each episode\n",
    "    for i_episode in range(1, num_episodes+1):\n",
    "        # begin the episode\n",
    "        state = env.reset()\n",
    "        # initialize the sampled reward\n",
    "        samp_reward = 0\n",
    "        while True:\n",
    "            # agent selects an action\n",
    "            action = agent.select_action(state)\n",
    "            # agent performs the selected action\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            # agent performs internal updates based on sampled experience\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            # update the sampled reward\n",
    "            samp_reward += reward\n",
    "            # update the state (s <- s') to next time step\n",
    "            state = next_state\n",
    "            if done:\n",
    "                # save final sampled reward\n",
    "                samp_rewards.append(samp_reward)\n",
    "                break\n",
    "        if (i_episode >= 100):\n",
    "            # get average reward from last 100 episodes\n",
    "            avg_reward = np.mean(samp_rewards)\n",
    "            # append to deque\n",
    "            avg_rewards.append(avg_reward)\n",
    "            # update best average reward\n",
    "            if avg_reward > best_avg_reward:\n",
    "                best_avg_reward = avg_reward\n",
    "        # monitor progress\n",
    "        print(\"\\rEpisode {}/{} || Best average reward {}\".format(i_episode, num_episodes, best_avg_reward), end=\"\")\n",
    "        sys.stdout.flush()\n",
    "        # check if task is solved (according to OpenAI Gym)\n",
    "        if best_avg_reward >= 9.7:\n",
    "            print('\\nEnvironment solved in {} episodes.'.format(i_episode), end=\"\")\n",
    "            break\n",
    "        if i_episode == num_episodes: print('\\n')\n",
    "    return avg_rewards, best_avg_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "\n",
    "    def __init__(self, nA=6):\n",
    "        self.nA = nA\n",
    "        self.Q = defaultdict(lambda: np.zeros(self.nA))\n",
    "        self.epsilon = 0\n",
    "        self.decay = 0\n",
    "        self.gamma = 0\n",
    "        self.alpha = 0\n",
    "        self.epsilon_function = None;\n",
    "        self.next_value_function = None;\n",
    "        #best score for self.init_for_sarsa() is 9.261 | slower to learn that q-learning\n",
    "        #self.init_for_sarsa()\n",
    "        #best score for self.init_for_q_learning is 9.26\n",
    "        self.init_for_q_learning()\n",
    "        #fixed epsilon=0.005 with alpha = 1 and gamma = 1 was very bad with a score of 8.655\n",
    "        #current settings succeeded with 9.253\n",
    "        #self.init_for_expected_sarsa()\n",
    "    def init_for_sarsa(self):\n",
    "        self.epsilon = 1\n",
    "        self.decay = 0.5\n",
    "        #discounted return rate\n",
    "        self.gamma = 0.9\n",
    "        #tendency to use recent event\n",
    "        self.alpha = 0.5\n",
    "        self.epsilon_function = self.decayed_epsilon\n",
    "        def calculate(next_state):\n",
    "            return self.Q[next_state][self.select_action(next_state)]\n",
    "        self.next_value_function = calculate \n",
    "    def init_for_q_learning(self):\n",
    "        self.epsilon = 1\n",
    "        self.decay = 0.9\n",
    "        #discounted return rate\n",
    "        self.gamma = 0.9\n",
    "        #tendency to use recent event\n",
    "        self.alpha = 0.3\n",
    "        self.epsilon_function = self.decayed_min_epsilon\n",
    "        def calculate(next_state):\n",
    "            return np.max(self.Q[next_state])\n",
    "        self.next_value_function = calculate\n",
    "    def init_for_expected_sarsa(self):\n",
    "        self.epsilon = 0.05\n",
    "        self.decay = 0.5\n",
    "        #discounted return rate\n",
    "        self.gamma = 0.8\n",
    "        #tendency to use recent event\n",
    "        self.alpha = 0.7\n",
    "        self.epsilon_function = self.fixed_epsilon\n",
    "        def calculate(next_state):\n",
    "            policy_s = self.get_action_probability(next_state)\n",
    "            next_action_value = np.dot(self.Q[next_state], policy_s)\n",
    "            return np.max(self.Q[next_state])\n",
    "        self.next_value_function = calculate\n",
    "    def decayed_min_epsilon(self):\n",
    "        self.epsilon = max(self.epsilon*self.decay,0.01)\n",
    "    def decayed_epsilon(self):\n",
    "        self.epsilon *= self.decay\n",
    "    def fixed_epsilon(self):\n",
    "        pass\n",
    "    def get_action_probability(self, state):\n",
    "        policy_s = np.ones(self.nA) * self.epsilon / self.nA\n",
    "        #find the best action and assign it the other side of the probabiliy epsilon\n",
    "        best_action = np.argmax(self.Q[state])\n",
    "        policy_s[best_action] += 1 - self.epsilon\n",
    "        return policy_s\n",
    "    def select_action(self, state):\n",
    "        policy_s = self.get_action_probability(state)\n",
    "        return np.random.choice(np.arange(self.nA), p=policy_s)\n",
    "    \n",
    "    def step(self, state, action, reward, next_state, done):\n",
    "        self.epsilon_function()\n",
    "        next_action_value = 0\n",
    "        if not done:\n",
    "            next_action_value = self.next_value_function(next_state)\n",
    "        self.Q[state][action] += self.alpha * (reward  + self.gamma * next_action_value - self.Q[state][action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Taxi-v2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sarsa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20000/20000 || Best average reward 9.446\n",
      "\n"
     ]
    }
   ],
   "source": [
    "agent = Agent()\n",
    "agent.init_for_sarsa()\n",
    "avg_rewards, best_avg_reward = interact(env, agent)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sarsa-max / q-learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20000/20000 || Best average reward 9.33\n",
      "\n"
     ]
    }
   ],
   "source": [
    "agent_qlearning = Agent()\n",
    "agent.init_for_q_learning()\n",
    "avg_rewards, best_avg_reward = interact(env, agent)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expected Sarsa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20000/20000 || Best average reward 9.37\n",
      "\n"
     ]
    }
   ],
   "source": [
    "agent_expected_sarsa = Agent()\n",
    "agent.init_for_expected_sarsa()\n",
    "avg_rewards, best_avg_reward = interact(env, agent)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openai",
   "language": "python",
   "name": "openai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}