sarsa0-implementation-scoring-9.426.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque, defaultdict\n",
    "import sys, math, random\n",
    "import numpy as np\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interact(env, agent, num_episodes=20000, window=100):\n",
    "    avg_rewards = deque(maxlen=num_episodes)\n",
    "    best_avg_reward = -math.inf\n",
    "    samp_rewards = deque(maxlen=window)\n",
    "    for i_episode in range(1, num_episodes+1):\n",
    "        state = env.reset()\n",
    "        samp_reward = 0\n",
    "        epsilon = 1.0 / i_episode\n",
    "        while True:\n",
    "            action = agent.select_action(state, epsilon)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            agent.step(state, action, reward, next_state, done, epsilon)\n",
    "            samp_reward += reward\n",
    "            state = next_state\n",
    "            if done:\n",
    "                samp_rewards.append(samp_reward)\n",
    "                break\n",
    "        if (i_episode >= 100):\n",
    "            avg_reward = np.mean(samp_rewards)\n",
    "            avg_rewards.append(avg_reward)\n",
    "            if avg_reward > best_avg_reward:\n",
    "                best_avg_reward = avg_reward\n",
    "        print(\"\\rEpisode {}/{} || Best average reward {}\".format(i_episode, num_episodes, best_avg_reward), end=\"\")\n",
    "        sys.stdout.flush()\n",
    "        if best_avg_reward >= 9.7:\n",
    "            print('\\nEnvironment solved in {} episodes.'.format(i_episode), end=\"\")\n",
    "            break\n",
    "        if i_episode == num_episodes: print('\\n')\n",
    "    return avg_rewards, best_avg_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "\n",
    "    def __init__(self, nA=6, gamma=1.0, alpha=0.1):\n",
    "        self.nA = nA\n",
    "        self.gamma = gamma\n",
    "        self.alpha = alpha\n",
    "        self.Q = defaultdict(lambda: np.zeros(self.nA))\n",
    "\n",
    "    def select_action(self, state, epsilon):\n",
    "        if random.random() > epsilon: # select greedy action with probability epsilon\n",
    "            return np.argmax(self.Q[state])\n",
    "        else:\n",
    "            return np.random.choice(self.nA)\n",
    "\n",
    "    def step(self, state, action, reward, next_state, done, epsilon):\n",
    "        one_step_reward = reward * self.gamma\n",
    "        old_q = self.Q[state][action]\n",
    "        if done:\n",
    "            self.Q[state][action] += self.alpha * (one_step_reward - old_q)\n",
    "        else:\n",
    "            next_action = self.select_action(next_state, epsilon)\n",
    "            next_q = self.Q[next_state][next_action]\n",
    "            self.Q[state][action] += self.alpha * (one_step_reward + next_q - old_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 20000/20000 || Best average reward 9.426\n",
      "\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Taxi-v2')\n",
    "agent = Agent()\n",
    "avg_rewards, best_avg_reward = interact(env, agent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.figure(figsize=(20, 10))\n",
    "plt.plot(np.exp(avg_rewards))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}