ars_practice.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ARS Practice\n",
    "This notebook is a recreation of the ARS (Augmented Random Search) reinforcement learning algorithm example workflow tutorial by Colin Skow in [this repository](https://github.com/colinskow/move37/blob/master/ars/ars.py). I want to get a keen feel of the mechanics of ARS so that I might be able to apply it to Carla. I will be using the BipedalWalker-v3 environment from the gym package to test this algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gym\n",
    "import numpy as np\n",
    "from gym import wrappers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_values([EnvSpec(Copy-v0), EnvSpec(RepeatCopy-v0), EnvSpec(ReversedAddition-v0), EnvSpec(ReversedAddition3-v0), EnvSpec(DuplicatedInput-v0), EnvSpec(Reverse-v0), EnvSpec(CartPole-v0), EnvSpec(CartPole-v1), EnvSpec(MountainCar-v0), EnvSpec(MountainCarContinuous-v0), EnvSpec(Pendulum-v0), EnvSpec(Acrobot-v1), EnvSpec(LunarLander-v2), EnvSpec(LunarLanderContinuous-v2), EnvSpec(BipedalWalker-v3), EnvSpec(BipedalWalkerHardcore-v3), EnvSpec(CarRacing-v0), EnvSpec(Blackjack-v0), EnvSpec(KellyCoinflip-v0), EnvSpec(KellyCoinflipGeneralized-v0), EnvSpec(FrozenLake-v0), EnvSpec(FrozenLake8x8-v0), EnvSpec(CliffWalking-v0), EnvSpec(NChain-v0), EnvSpec(Roulette-v0), EnvSpec(Taxi-v3), EnvSpec(GuessingGame-v0), EnvSpec(HotterColder-v0), EnvSpec(Reacher-v2), EnvSpec(Pusher-v2), EnvSpec(Thrower-v2), EnvSpec(Striker-v2), EnvSpec(InvertedPendulum-v2), EnvSpec(InvertedDoublePendulum-v2), EnvSpec(HalfCheetah-v2), EnvSpec(HalfCheetah-v3), EnvSpec(Hopper-v2), EnvSpec(Hopper-v3), EnvSpec(Swimmer-v2), EnvSpec(Swimmer-v3), EnvSpec(Walker2d-v2), EnvSpec(Walker2d-v3), EnvSpec(Ant-v2), EnvSpec(Ant-v3), EnvSpec(Humanoid-v2), EnvSpec(Humanoid-v3), EnvSpec(HumanoidStandup-v2), EnvSpec(FetchSlide-v1), EnvSpec(FetchPickAndPlace-v1), EnvSpec(FetchReach-v1), EnvSpec(FetchPush-v1), EnvSpec(HandReach-v0), EnvSpec(HandManipulateBlockRotateZ-v0), EnvSpec(HandManipulateBlockRotateZTouchSensors-v0), EnvSpec(HandManipulateBlockRotateZTouchSensors-v1), EnvSpec(HandManipulateBlockRotateParallel-v0), EnvSpec(HandManipulateBlockRotateParallelTouchSensors-v0), EnvSpec(HandManipulateBlockRotateParallelTouchSensors-v1), EnvSpec(HandManipulateBlockRotateXYZ-v0), EnvSpec(HandManipulateBlockRotateXYZTouchSensors-v0), EnvSpec(HandManipulateBlockRotateXYZTouchSensors-v1), EnvSpec(HandManipulateBlockFull-v0), EnvSpec(HandManipulateBlock-v0), EnvSpec(HandManipulateBlockTouchSensors-v0), EnvSpec(HandManipulateBlockTouchSensors-v1), EnvSpec(HandManipulateEggRotate-v0), EnvSpec(HandManipulateEggRotateTouchSensors-v0), EnvSpec(HandManipulateEggRotateTouchSensors-v1), EnvSpec(HandManipulateEggFull-v0), EnvSpec(HandManipulateEgg-v0), EnvSpec(HandManipulateEggTouchSensors-v0), EnvSpec(HandManipulateEggTouchSensors-v1), EnvSpec(HandManipulatePenRotate-v0), EnvSpec(HandManipulatePenRotateTouchSensors-v0), EnvSpec(HandManipulatePenRotateTouchSensors-v1), EnvSpec(HandManipulatePenFull-v0), EnvSpec(HandManipulatePen-v0), EnvSpec(HandManipulatePenTouchSensors-v0), EnvSpec(HandManipulatePenTouchSensors-v1), EnvSpec(FetchSlideDense-v1), EnvSpec(FetchPickAndPlaceDense-v1), EnvSpec(FetchReachDense-v1), EnvSpec(FetchPushDense-v1), EnvSpec(HandReachDense-v0), EnvSpec(HandManipulateBlockRotateZDense-v0), EnvSpec(HandManipulateBlockRotateZTouchSensorsDense-v0), EnvSpec(HandManipulateBlockRotateZTouchSensorsDense-v1), EnvSpec(HandManipulateBlockRotateParallelDense-v0), EnvSpec(HandManipulateBlockRotateParallelTouchSensorsDense-v0), EnvSpec(HandManipulateBlockRotateParallelTouchSensorsDense-v1), EnvSpec(HandManipulateBlockRotateXYZDense-v0), EnvSpec(HandManipulateBlockRotateXYZTouchSensorsDense-v0), EnvSpec(HandManipulateBlockRotateXYZTouchSensorsDense-v1), EnvSpec(HandManipulateBlockFullDense-v0), EnvSpec(HandManipulateBlockDense-v0), EnvSpec(HandManipulateBlockTouchSensorsDense-v0), EnvSpec(HandManipulateBlockTouchSensorsDense-v1), EnvSpec(HandManipulateEggRotateDense-v0), EnvSpec(HandManipulateEggRotateTouchSensorsDense-v0), EnvSpec(HandManipulateEggRotateTouchSensorsDense-v1), EnvSpec(HandManipulateEggFullDense-v0), EnvSpec(HandManipulateEggDense-v0), EnvSpec(HandManipulateEggTouchSensorsDense-v0), EnvSpec(HandManipulateEggTouchSensorsDense-v1), EnvSpec(HandManipulatePenRotateDense-v0), EnvSpec(HandManipulatePenRotateTouchSensorsDense-v0), EnvSpec(HandManipulatePenRotateTouchSensorsDense-v1), EnvSpec(HandManipulatePenFullDense-v0), EnvSpec(HandManipulatePenDense-v0), EnvSpec(HandManipulatePenTouchSensorsDense-v0), EnvSpec(HandManipulatePenTouchSensorsDense-v1), EnvSpec(Adventure-v0), EnvSpec(Adventure-v4), EnvSpec(AdventureDeterministic-v0), EnvSpec(AdventureDeterministic-v4), EnvSpec(AdventureNoFrameskip-v0), EnvSpec(AdventureNoFrameskip-v4), EnvSpec(Adventure-ram-v0), EnvSpec(Adventure-ram-v4), EnvSpec(Adventure-ramDeterministic-v0), EnvSpec(Adventure-ramDeterministic-v4), EnvSpec(Adventure-ramNoFrameskip-v0), EnvSpec(Adventure-ramNoFrameskip-v4), EnvSpec(AirRaid-v0), EnvSpec(AirRaid-v4), EnvSpec(AirRaidDeterministic-v0), EnvSpec(AirRaidDeterministic-v4), EnvSpec(AirRaidNoFrameskip-v0), EnvSpec(AirRaidNoFrameskip-v4), EnvSpec(AirRaid-ram-v0), EnvSpec(AirRaid-ram-v4), EnvSpec(AirRaid-ramDeterministic-v0), EnvSpec(AirRaid-ramDeterministic-v4), EnvSpec(AirRaid-ramNoFrameskip-v0), EnvSpec(AirRaid-ramNoFrameskip-v4), EnvSpec(Alien-v0), EnvSpec(Alien-v4), EnvSpec(AlienDeterministic-v0), EnvSpec(AlienDeterministic-v4), EnvSpec(AlienNoFrameskip-v0), EnvSpec(AlienNoFrameskip-v4), EnvSpec(Alien-ram-v0), EnvSpec(Alien-ram-v4), EnvSpec(Alien-ramDeterministic-v0), EnvSpec(Alien-ramDeterministic-v4), EnvSpec(Alien-ramNoFrameskip-v0), EnvSpec(Alien-ramNoFrameskip-v4), EnvSpec(Amidar-v0), EnvSpec(Amidar-v4), EnvSpec(AmidarDeterministic-v0), EnvSpec(AmidarDeterministic-v4), EnvSpec(AmidarNoFrameskip-v0), EnvSpec(AmidarNoFrameskip-v4), EnvSpec(Amidar-ram-v0), EnvSpec(Amidar-ram-v4), EnvSpec(Amidar-ramDeterministic-v0), EnvSpec(Amidar-ramDeterministic-v4), EnvSpec(Amidar-ramNoFrameskip-v0), EnvSpec(Amidar-ramNoFrameskip-v4), EnvSpec(Assault-v0), EnvSpec(Assault-v4), EnvSpec(AssaultDeterministic-v0), EnvSpec(AssaultDeterministic-v4), EnvSpec(AssaultNoFrameskip-v0), EnvSpec(AssaultNoFrameskip-v4), EnvSpec(Assault-ram-v0), EnvSpec(Assault-ram-v4), EnvSpec(Assault-ramDeterministic-v0), EnvSpec(Assault-ramDeterministic-v4), EnvSpec(Assault-ramNoFrameskip-v0), EnvSpec(Assault-ramNoFrameskip-v4), EnvSpec(Asterix-v0), EnvSpec(Asterix-v4), EnvSpec(AsterixDeterministic-v0), EnvSpec(AsterixDeterministic-v4), EnvSpec(AsterixNoFrameskip-v0), EnvSpec(AsterixNoFrameskip-v4), EnvSpec(Asterix-ram-v0), EnvSpec(Asterix-ram-v4), EnvSpec(Asterix-ramDeterministic-v0), EnvSpec(Asterix-ramDeterministic-v4), EnvSpec(Asterix-ramNoFrameskip-v0), EnvSpec(Asterix-ramNoFrameskip-v4), EnvSpec(Asteroids-v0), EnvSpec(Asteroids-v4), EnvSpec(AsteroidsDeterministic-v0), EnvSpec(AsteroidsDeterministic-v4), EnvSpec(AsteroidsNoFrameskip-v0), EnvSpec(AsteroidsNoFrameskip-v4), EnvSpec(Asteroids-ram-v0), EnvSpec(Asteroids-ram-v4), EnvSpec(Asteroids-ramDeterministic-v0), EnvSpec(Asteroids-ramDeterministic-v4), EnvSpec(Asteroids-ramNoFrameskip-v0), EnvSpec(Asteroids-ramNoFrameskip-v4), EnvSpec(Atlantis-v0), EnvSpec(Atlantis-v4), EnvSpec(AtlantisDeterministic-v0), EnvSpec(AtlantisDeterministic-v4), EnvSpec(AtlantisNoFrameskip-v0), EnvSpec(AtlantisNoFrameskip-v4), EnvSpec(Atlantis-ram-v0), EnvSpec(Atlantis-ram-v4), EnvSpec(Atlantis-ramDeterministic-v0), EnvSpec(Atlantis-ramDeterministic-v4), EnvSpec(Atlantis-ramNoFrameskip-v0), EnvSpec(Atlantis-ramNoFrameskip-v4), EnvSpec(BankHeist-v0), EnvSpec(BankHeist-v4), EnvSpec(BankHeistDeterministic-v0), EnvSpec(BankHeistDeterministic-v4), EnvSpec(BankHeistNoFrameskip-v0), EnvSpec(BankHeistNoFrameskip-v4), EnvSpec(BankHeist-ram-v0), EnvSpec(BankHeist-ram-v4), EnvSpec(BankHeist-ramDeterministic-v0), EnvSpec(BankHeist-ramDeterministic-v4), EnvSpec(BankHeist-ramNoFrameskip-v0), EnvSpec(BankHeist-ramNoFrameskip-v4), EnvSpec(BattleZone-v0), EnvSpec(BattleZone-v4), EnvSpec(BattleZoneDeterministic-v0), EnvSpec(BattleZoneDeterministic-v4), EnvSpec(BattleZoneNoFrameskip-v0), EnvSpec(BattleZoneNoFrameskip-v4), EnvSpec(BattleZone-ram-v0), EnvSpec(BattleZone-ram-v4), EnvSpec(BattleZone-ramDeterministic-v0), EnvSpec(BattleZone-ramDeterministic-v4), EnvSpec(BattleZone-ramNoFrameskip-v0), EnvSpec(BattleZone-ramNoFrameskip-v4), EnvSpec(BeamRider-v0), EnvSpec(BeamRider-v4), EnvSpec(BeamRiderDeterministic-v0), EnvSpec(BeamRiderDeterministic-v4), EnvSpec(BeamRiderNoFrameskip-v0), EnvSpec(BeamRiderNoFrameskip-v4), EnvSpec(BeamRider-ram-v0), EnvSpec(BeamRider-ram-v4), EnvSpec(BeamRider-ramDeterministic-v0), EnvSpec(BeamRider-ramDeterministic-v4), EnvSpec(BeamRider-ramNoFrameskip-v0), EnvSpec(BeamRider-ramNoFrameskip-v4), EnvSpec(Berzerk-v0), EnvSpec(Berzerk-v4), EnvSpec(BerzerkDeterministic-v0), EnvSpec(BerzerkDeterministic-v4), EnvSpec(BerzerkNoFrameskip-v0), EnvSpec(BerzerkNoFrameskip-v4), EnvSpec(Berzerk-ram-v0), EnvSpec(Berzerk-ram-v4), EnvSpec(Berzerk-ramDeterministic-v0), EnvSpec(Berzerk-ramDeterministic-v4), EnvSpec(Berzerk-ramNoFrameskip-v0), EnvSpec(Berzerk-ramNoFrameskip-v4), EnvSpec(Bowling-v0), EnvSpec(Bowling-v4), EnvSpec(BowlingDeterministic-v0), EnvSpec(BowlingDeterministic-v4), EnvSpec(BowlingNoFrameskip-v0), EnvSpec(BowlingNoFrameskip-v4), EnvSpec(Bowling-ram-v0), EnvSpec(Bowling-ram-v4), EnvSpec(Bowling-ramDeterministic-v0), EnvSpec(Bowling-ramDeterministic-v4), EnvSpec(Bowling-ramNoFrameskip-v0), EnvSpec(Bowling-ramNoFrameskip-v4), EnvSpec(Boxing-v0), EnvSpec(Boxing-v4), EnvSpec(BoxingDeterministic-v0), EnvSpec(BoxingDeterministic-v4), EnvSpec(BoxingNoFrameskip-v0), EnvSpec(BoxingNoFrameskip-v4), EnvSpec(Boxing-ram-v0), EnvSpec(Boxing-ram-v4), EnvSpec(Boxing-ramDeterministic-v0), EnvSpec(Boxing-ramDeterministic-v4), EnvSpec(Boxing-ramNoFrameskip-v0), EnvSpec(Boxing-ramNoFrameskip-v4), EnvSpec(Breakout-v0), EnvSpec(Breakout-v4), EnvSpec(BreakoutDeterministic-v0), EnvSpec(BreakoutDeterministic-v4), EnvSpec(BreakoutNoFrameskip-v0), EnvSpec(BreakoutNoFrameskip-v4), EnvSpec(Breakout-ram-v0), EnvSpec(Breakout-ram-v4), EnvSpec(Breakout-ramDeterministic-v0), EnvSpec(Breakout-ramDeterministic-v4), EnvSpec(Breakout-ramNoFrameskip-v0), EnvSpec(Breakout-ramNoFrameskip-v4), EnvSpec(Carnival-v0), EnvSpec(Carnival-v4), EnvSpec(CarnivalDeterministic-v0), EnvSpec(CarnivalDeterministic-v4), EnvSpec(CarnivalNoFrameskip-v0), EnvSpec(CarnivalNoFrameskip-v4), EnvSpec(Carnival-ram-v0), EnvSpec(Carnival-ram-v4), EnvSpec(Carnival-ramDeterministic-v0), EnvSpec(Carnival-ramDeterministic-v4), EnvSpec(Carnival-ramNoFrameskip-v0), EnvSpec(Carnival-ramNoFrameskip-v4), EnvSpec(Centipede-v0), EnvSpec(Centipede-v4), EnvSpec(CentipedeDeterministic-v0), EnvSpec(CentipedeDeterministic-v4), EnvSpec(CentipedeNoFrameskip-v0), EnvSpec(CentipedeNoFrameskip-v4), EnvSpec(Centipede-ram-v0), EnvSpec(Centipede-ram-v4), EnvSpec(Centipede-ramDeterministic-v0), EnvSpec(Centipede-ramDeterministic-v4), EnvSpec(Centipede-ramNoFrameskip-v0), EnvSpec(Centipede-ramNoFrameskip-v4), EnvSpec(ChopperCommand-v0), EnvSpec(ChopperCommand-v4), EnvSpec(ChopperCommandDeterministic-v0), EnvSpec(ChopperCommandDeterministic-v4), EnvSpec(ChopperCommandNoFrameskip-v0), EnvSpec(ChopperCommandNoFrameskip-v4), EnvSpec(ChopperCommand-ram-v0), EnvSpec(ChopperCommand-ram-v4), EnvSpec(ChopperCommand-ramDeterministic-v0), EnvSpec(ChopperCommand-ramDeterministic-v4), EnvSpec(ChopperCommand-ramNoFrameskip-v0), EnvSpec(ChopperCommand-ramNoFrameskip-v4), EnvSpec(CrazyClimber-v0), EnvSpec(CrazyClimber-v4), EnvSpec(CrazyClimberDeterministic-v0), EnvSpec(CrazyClimberDeterministic-v4), EnvSpec(CrazyClimberNoFrameskip-v0), EnvSpec(CrazyClimberNoFrameskip-v4), EnvSpec(CrazyClimber-ram-v0), EnvSpec(CrazyClimber-ram-v4), EnvSpec(CrazyClimber-ramDeterministic-v0), EnvSpec(CrazyClimber-ramDeterministic-v4), EnvSpec(CrazyClimber-ramNoFrameskip-v0), EnvSpec(CrazyClimber-ramNoFrameskip-v4), EnvSpec(Defender-v0), EnvSpec(Defender-v4), EnvSpec(DefenderDeterministic-v0), EnvSpec(DefenderDeterministic-v4), EnvSpec(DefenderNoFrameskip-v0), EnvSpec(DefenderNoFrameskip-v4), EnvSpec(Defender-ram-v0), EnvSpec(Defender-ram-v4), EnvSpec(Defender-ramDeterministic-v0), EnvSpec(Defender-ramDeterministic-v4), EnvSpec(Defender-ramNoFrameskip-v0), EnvSpec(Defender-ramNoFrameskip-v4), EnvSpec(DemonAttack-v0), EnvSpec(DemonAttack-v4), EnvSpec(DemonAttackDeterministic-v0), EnvSpec(DemonAttackDeterministic-v4), EnvSpec(DemonAttackNoFrameskip-v0), EnvSpec(DemonAttackNoFrameskip-v4), EnvSpec(DemonAttack-ram-v0), EnvSpec(DemonAttack-ram-v4), EnvSpec(DemonAttack-ramDeterministic-v0), EnvSpec(DemonAttack-ramDeterministic-v4), EnvSpec(DemonAttack-ramNoFrameskip-v0), EnvSpec(DemonAttack-ramNoFrameskip-v4), EnvSpec(DoubleDunk-v0), EnvSpec(DoubleDunk-v4), EnvSpec(DoubleDunkDeterministic-v0), EnvSpec(DoubleDunkDeterministic-v4), EnvSpec(DoubleDunkNoFrameskip-v0), EnvSpec(DoubleDunkNoFrameskip-v4), EnvSpec(DoubleDunk-ram-v0), EnvSpec(DoubleDunk-ram-v4), EnvSpec(DoubleDunk-ramDeterministic-v0), EnvSpec(DoubleDunk-ramDeterministic-v4), EnvSpec(DoubleDunk-ramNoFrameskip-v0), EnvSpec(DoubleDunk-ramNoFrameskip-v4), EnvSpec(ElevatorAction-v0), EnvSpec(ElevatorAction-v4), EnvSpec(ElevatorActionDeterministic-v0), EnvSpec(ElevatorActionDeterministic-v4), EnvSpec(ElevatorActionNoFrameskip-v0), EnvSpec(ElevatorActionNoFrameskip-v4), EnvSpec(ElevatorAction-ram-v0), EnvSpec(ElevatorAction-ram-v4), EnvSpec(ElevatorAction-ramDeterministic-v0), EnvSpec(ElevatorAction-ramDeterministic-v4), EnvSpec(ElevatorAction-ramNoFrameskip-v0), EnvSpec(ElevatorAction-ramNoFrameskip-v4), EnvSpec(Enduro-v0), EnvSpec(Enduro-v4), EnvSpec(EnduroDeterministic-v0), EnvSpec(EnduroDeterministic-v4), EnvSpec(EnduroNoFrameskip-v0), EnvSpec(EnduroNoFrameskip-v4), EnvSpec(Enduro-ram-v0), EnvSpec(Enduro-ram-v4), EnvSpec(Enduro-ramDeterministic-v0), EnvSpec(Enduro-ramDeterministic-v4), EnvSpec(Enduro-ramNoFrameskip-v0), EnvSpec(Enduro-ramNoFrameskip-v4), EnvSpec(FishingDerby-v0), EnvSpec(FishingDerby-v4), EnvSpec(FishingDerbyDeterministic-v0), EnvSpec(FishingDerbyDeterministic-v4), EnvSpec(FishingDerbyNoFrameskip-v0), EnvSpec(FishingDerbyNoFrameskip-v4), EnvSpec(FishingDerby-ram-v0), EnvSpec(FishingDerby-ram-v4), EnvSpec(FishingDerby-ramDeterministic-v0), EnvSpec(FishingDerby-ramDeterministic-v4), EnvSpec(FishingDerby-ramNoFrameskip-v0), EnvSpec(FishingDerby-ramNoFrameskip-v4), EnvSpec(Freeway-v0), EnvSpec(Freeway-v4), EnvSpec(FreewayDeterministic-v0), EnvSpec(FreewayDeterministic-v4), EnvSpec(FreewayNoFrameskip-v0), EnvSpec(FreewayNoFrameskip-v4), EnvSpec(Freeway-ram-v0), EnvSpec(Freeway-ram-v4), EnvSpec(Freeway-ramDeterministic-v0), EnvSpec(Freeway-ramDeterministic-v4), EnvSpec(Freeway-ramNoFrameskip-v0), EnvSpec(Freeway-ramNoFrameskip-v4), EnvSpec(Frostbite-v0), EnvSpec(Frostbite-v4), EnvSpec(FrostbiteDeterministic-v0), EnvSpec(FrostbiteDeterministic-v4), EnvSpec(FrostbiteNoFrameskip-v0), EnvSpec(FrostbiteNoFrameskip-v4), EnvSpec(Frostbite-ram-v0), EnvSpec(Frostbite-ram-v4), EnvSpec(Frostbite-ramDeterministic-v0), EnvSpec(Frostbite-ramDeterministic-v4), EnvSpec(Frostbite-ramNoFrameskip-v0), EnvSpec(Frostbite-ramNoFrameskip-v4), EnvSpec(Gopher-v0), EnvSpec(Gopher-v4), EnvSpec(GopherDeterministic-v0), EnvSpec(GopherDeterministic-v4), EnvSpec(GopherNoFrameskip-v0), EnvSpec(GopherNoFrameskip-v4), EnvSpec(Gopher-ram-v0), EnvSpec(Gopher-ram-v4), EnvSpec(Gopher-ramDeterministic-v0), EnvSpec(Gopher-ramDeterministic-v4), EnvSpec(Gopher-ramNoFrameskip-v0), EnvSpec(Gopher-ramNoFrameskip-v4), EnvSpec(Gravitar-v0), EnvSpec(Gravitar-v4), EnvSpec(GravitarDeterministic-v0), EnvSpec(GravitarDeterministic-v4), EnvSpec(GravitarNoFrameskip-v0), EnvSpec(GravitarNoFrameskip-v4), EnvSpec(Gravitar-ram-v0), EnvSpec(Gravitar-ram-v4), EnvSpec(Gravitar-ramDeterministic-v0), EnvSpec(Gravitar-ramDeterministic-v4), EnvSpec(Gravitar-ramNoFrameskip-v0), EnvSpec(Gravitar-ramNoFrameskip-v4), EnvSpec(Hero-v0), EnvSpec(Hero-v4), EnvSpec(HeroDeterministic-v0), EnvSpec(HeroDeterministic-v4), EnvSpec(HeroNoFrameskip-v0), EnvSpec(HeroNoFrameskip-v4), EnvSpec(Hero-ram-v0), EnvSpec(Hero-ram-v4), EnvSpec(Hero-ramDeterministic-v0), EnvSpec(Hero-ramDeterministic-v4), EnvSpec(Hero-ramNoFrameskip-v0), EnvSpec(Hero-ramNoFrameskip-v4), EnvSpec(IceHockey-v0), EnvSpec(IceHockey-v4), EnvSpec(IceHockeyDeterministic-v0), EnvSpec(IceHockeyDeterministic-v4), EnvSpec(IceHockeyNoFrameskip-v0), EnvSpec(IceHockeyNoFrameskip-v4), EnvSpec(IceHockey-ram-v0), EnvSpec(IceHockey-ram-v4), EnvSpec(IceHockey-ramDeterministic-v0), EnvSpec(IceHockey-ramDeterministic-v4), EnvSpec(IceHockey-ramNoFrameskip-v0), EnvSpec(IceHockey-ramNoFrameskip-v4), EnvSpec(Jamesbond-v0), EnvSpec(Jamesbond-v4), EnvSpec(JamesbondDeterministic-v0), EnvSpec(JamesbondDeterministic-v4), EnvSpec(JamesbondNoFrameskip-v0), EnvSpec(JamesbondNoFrameskip-v4), EnvSpec(Jamesbond-ram-v0), EnvSpec(Jamesbond-ram-v4), EnvSpec(Jamesbond-ramDeterministic-v0), EnvSpec(Jamesbond-ramDeterministic-v4), EnvSpec(Jamesbond-ramNoFrameskip-v0), EnvSpec(Jamesbond-ramNoFrameskip-v4), EnvSpec(JourneyEscape-v0), EnvSpec(JourneyEscape-v4), EnvSpec(JourneyEscapeDeterministic-v0), EnvSpec(JourneyEscapeDeterministic-v4), EnvSpec(JourneyEscapeNoFrameskip-v0), EnvSpec(JourneyEscapeNoFrameskip-v4), EnvSpec(JourneyEscape-ram-v0), EnvSpec(JourneyEscape-ram-v4), EnvSpec(JourneyEscape-ramDeterministic-v0), EnvSpec(JourneyEscape-ramDeterministic-v4), EnvSpec(JourneyEscape-ramNoFrameskip-v0), EnvSpec(JourneyEscape-ramNoFrameskip-v4), EnvSpec(Kangaroo-v0), EnvSpec(Kangaroo-v4), EnvSpec(KangarooDeterministic-v0), EnvSpec(KangarooDeterministic-v4), EnvSpec(KangarooNoFrameskip-v0), EnvSpec(KangarooNoFrameskip-v4), EnvSpec(Kangaroo-ram-v0), EnvSpec(Kangaroo-ram-v4), EnvSpec(Kangaroo-ramDeterministic-v0), EnvSpec(Kangaroo-ramDeterministic-v4), EnvSpec(Kangaroo-ramNoFrameskip-v0), EnvSpec(Kangaroo-ramNoFrameskip-v4), EnvSpec(Krull-v0), EnvSpec(Krull-v4), EnvSpec(KrullDeterministic-v0), EnvSpec(KrullDeterministic-v4), EnvSpec(KrullNoFrameskip-v0), EnvSpec(KrullNoFrameskip-v4), EnvSpec(Krull-ram-v0), EnvSpec(Krull-ram-v4), EnvSpec(Krull-ramDeterministic-v0), EnvSpec(Krull-ramDeterministic-v4), EnvSpec(Krull-ramNoFrameskip-v0), EnvSpec(Krull-ramNoFrameskip-v4), EnvSpec(KungFuMaster-v0), EnvSpec(KungFuMaster-v4), EnvSpec(KungFuMasterDeterministic-v0), EnvSpec(KungFuMasterDeterministic-v4), EnvSpec(KungFuMasterNoFrameskip-v0), EnvSpec(KungFuMasterNoFrameskip-v4), EnvSpec(KungFuMaster-ram-v0), EnvSpec(KungFuMaster-ram-v4), EnvSpec(KungFuMaster-ramDeterministic-v0), EnvSpec(KungFuMaster-ramDeterministic-v4), EnvSpec(KungFuMaster-ramNoFrameskip-v0), EnvSpec(KungFuMaster-ramNoFrameskip-v4), EnvSpec(MontezumaRevenge-v0), EnvSpec(MontezumaRevenge-v4), EnvSpec(MontezumaRevengeDeterministic-v0), EnvSpec(MontezumaRevengeDeterministic-v4), EnvSpec(MontezumaRevengeNoFrameskip-v0), EnvSpec(MontezumaRevengeNoFrameskip-v4), EnvSpec(MontezumaRevenge-ram-v0), EnvSpec(MontezumaRevenge-ram-v4), EnvSpec(MontezumaRevenge-ramDeterministic-v0), EnvSpec(MontezumaRevenge-ramDeterministic-v4), EnvSpec(MontezumaRevenge-ramNoFrameskip-v0), EnvSpec(MontezumaRevenge-ramNoFrameskip-v4), EnvSpec(MsPacman-v0), EnvSpec(MsPacman-v4), EnvSpec(MsPacmanDeterministic-v0), EnvSpec(MsPacmanDeterministic-v4), EnvSpec(MsPacmanNoFrameskip-v0), EnvSpec(MsPacmanNoFrameskip-v4), EnvSpec(MsPacman-ram-v0), EnvSpec(MsPacman-ram-v4), EnvSpec(MsPacman-ramDeterministic-v0), EnvSpec(MsPacman-ramDeterministic-v4), EnvSpec(MsPacman-ramNoFrameskip-v0), EnvSpec(MsPacman-ramNoFrameskip-v4), EnvSpec(NameThisGame-v0), EnvSpec(NameThisGame-v4), EnvSpec(NameThisGameDeterministic-v0), EnvSpec(NameThisGameDeterministic-v4), EnvSpec(NameThisGameNoFrameskip-v0), EnvSpec(NameThisGameNoFrameskip-v4), EnvSpec(NameThisGame-ram-v0), EnvSpec(NameThisGame-ram-v4), EnvSpec(NameThisGame-ramDeterministic-v0), EnvSpec(NameThisGame-ramDeterministic-v4), EnvSpec(NameThisGame-ramNoFrameskip-v0), EnvSpec(NameThisGame-ramNoFrameskip-v4), EnvSpec(Phoenix-v0), EnvSpec(Phoenix-v4), EnvSpec(PhoenixDeterministic-v0), EnvSpec(PhoenixDeterministic-v4), EnvSpec(PhoenixNoFrameskip-v0), EnvSpec(PhoenixNoFrameskip-v4), EnvSpec(Phoenix-ram-v0), EnvSpec(Phoenix-ram-v4), EnvSpec(Phoenix-ramDeterministic-v0), EnvSpec(Phoenix-ramDeterministic-v4), EnvSpec(Phoenix-ramNoFrameskip-v0), EnvSpec(Phoenix-ramNoFrameskip-v4), EnvSpec(Pitfall-v0), EnvSpec(Pitfall-v4), EnvSpec(PitfallDeterministic-v0), EnvSpec(PitfallDeterministic-v4), EnvSpec(PitfallNoFrameskip-v0), EnvSpec(PitfallNoFrameskip-v4), EnvSpec(Pitfall-ram-v0), EnvSpec(Pitfall-ram-v4), EnvSpec(Pitfall-ramDeterministic-v0), EnvSpec(Pitfall-ramDeterministic-v4), EnvSpec(Pitfall-ramNoFrameskip-v0), EnvSpec(Pitfall-ramNoFrameskip-v4), EnvSpec(Pong-v0), EnvSpec(Pong-v4), EnvSpec(PongDeterministic-v0), EnvSpec(PongDeterministic-v4), EnvSpec(PongNoFrameskip-v0), EnvSpec(PongNoFrameskip-v4), EnvSpec(Pong-ram-v0), EnvSpec(Pong-ram-v4), EnvSpec(Pong-ramDeterministic-v0), EnvSpec(Pong-ramDeterministic-v4), EnvSpec(Pong-ramNoFrameskip-v0), EnvSpec(Pong-ramNoFrameskip-v4), EnvSpec(Pooyan-v0), EnvSpec(Pooyan-v4), EnvSpec(PooyanDeterministic-v0), EnvSpec(PooyanDeterministic-v4), EnvSpec(PooyanNoFrameskip-v0), EnvSpec(PooyanNoFrameskip-v4), EnvSpec(Pooyan-ram-v0), EnvSpec(Pooyan-ram-v4), EnvSpec(Pooyan-ramDeterministic-v0), EnvSpec(Pooyan-ramDeterministic-v4), EnvSpec(Pooyan-ramNoFrameskip-v0), EnvSpec(Pooyan-ramNoFrameskip-v4), EnvSpec(PrivateEye-v0), EnvSpec(PrivateEye-v4), EnvSpec(PrivateEyeDeterministic-v0), EnvSpec(PrivateEyeDeterministic-v4), EnvSpec(PrivateEyeNoFrameskip-v0), EnvSpec(PrivateEyeNoFrameskip-v4), EnvSpec(PrivateEye-ram-v0), EnvSpec(PrivateEye-ram-v4), EnvSpec(PrivateEye-ramDeterministic-v0), EnvSpec(PrivateEye-ramDeterministic-v4), EnvSpec(PrivateEye-ramNoFrameskip-v0), EnvSpec(PrivateEye-ramNoFrameskip-v4), EnvSpec(Qbert-v0), EnvSpec(Qbert-v4), EnvSpec(QbertDeterministic-v0), EnvSpec(QbertDeterministic-v4), EnvSpec(QbertNoFrameskip-v0), EnvSpec(QbertNoFrameskip-v4), EnvSpec(Qbert-ram-v0), EnvSpec(Qbert-ram-v4), EnvSpec(Qbert-ramDeterministic-v0), EnvSpec(Qbert-ramDeterministic-v4), EnvSpec(Qbert-ramNoFrameskip-v0), EnvSpec(Qbert-ramNoFrameskip-v4), EnvSpec(Riverraid-v0), EnvSpec(Riverraid-v4), EnvSpec(RiverraidDeterministic-v0), EnvSpec(RiverraidDeterministic-v4), EnvSpec(RiverraidNoFrameskip-v0), EnvSpec(RiverraidNoFrameskip-v4), EnvSpec(Riverraid-ram-v0), EnvSpec(Riverraid-ram-v4), EnvSpec(Riverraid-ramDeterministic-v0), EnvSpec(Riverraid-ramDeterministic-v4), EnvSpec(Riverraid-ramNoFrameskip-v0), EnvSpec(Riverraid-ramNoFrameskip-v4), EnvSpec(RoadRunner-v0), EnvSpec(RoadRunner-v4), EnvSpec(RoadRunnerDeterministic-v0), EnvSpec(RoadRunnerDeterministic-v4), EnvSpec(RoadRunnerNoFrameskip-v0), EnvSpec(RoadRunnerNoFrameskip-v4), EnvSpec(RoadRunner-ram-v0), EnvSpec(RoadRunner-ram-v4), EnvSpec(RoadRunner-ramDeterministic-v0), EnvSpec(RoadRunner-ramDeterministic-v4), EnvSpec(RoadRunner-ramNoFrameskip-v0), EnvSpec(RoadRunner-ramNoFrameskip-v4), EnvSpec(Robotank-v0), EnvSpec(Robotank-v4), EnvSpec(RobotankDeterministic-v0), EnvSpec(RobotankDeterministic-v4), EnvSpec(RobotankNoFrameskip-v0), EnvSpec(RobotankNoFrameskip-v4), EnvSpec(Robotank-ram-v0), EnvSpec(Robotank-ram-v4), EnvSpec(Robotank-ramDeterministic-v0), EnvSpec(Robotank-ramDeterministic-v4), EnvSpec(Robotank-ramNoFrameskip-v0), EnvSpec(Robotank-ramNoFrameskip-v4), EnvSpec(Seaquest-v0), EnvSpec(Seaquest-v4), EnvSpec(SeaquestDeterministic-v0), EnvSpec(SeaquestDeterministic-v4), EnvSpec(SeaquestNoFrameskip-v0), EnvSpec(SeaquestNoFrameskip-v4), EnvSpec(Seaquest-ram-v0), EnvSpec(Seaquest-ram-v4), EnvSpec(Seaquest-ramDeterministic-v0), EnvSpec(Seaquest-ramDeterministic-v4), EnvSpec(Seaquest-ramNoFrameskip-v0), EnvSpec(Seaquest-ramNoFrameskip-v4), EnvSpec(Skiing-v0), EnvSpec(Skiing-v4), EnvSpec(SkiingDeterministic-v0), EnvSpec(SkiingDeterministic-v4), EnvSpec(SkiingNoFrameskip-v0), EnvSpec(SkiingNoFrameskip-v4), EnvSpec(Skiing-ram-v0), EnvSpec(Skiing-ram-v4), EnvSpec(Skiing-ramDeterministic-v0), EnvSpec(Skiing-ramDeterministic-v4), EnvSpec(Skiing-ramNoFrameskip-v0), EnvSpec(Skiing-ramNoFrameskip-v4), EnvSpec(Solaris-v0), EnvSpec(Solaris-v4), EnvSpec(SolarisDeterministic-v0), EnvSpec(SolarisDeterministic-v4), EnvSpec(SolarisNoFrameskip-v0), EnvSpec(SolarisNoFrameskip-v4), EnvSpec(Solaris-ram-v0), EnvSpec(Solaris-ram-v4), EnvSpec(Solaris-ramDeterministic-v0), EnvSpec(Solaris-ramDeterministic-v4), EnvSpec(Solaris-ramNoFrameskip-v0), EnvSpec(Solaris-ramNoFrameskip-v4), EnvSpec(SpaceInvaders-v0), EnvSpec(SpaceInvaders-v4), EnvSpec(SpaceInvadersDeterministic-v0), EnvSpec(SpaceInvadersDeterministic-v4), EnvSpec(SpaceInvadersNoFrameskip-v0), EnvSpec(SpaceInvadersNoFrameskip-v4), EnvSpec(SpaceInvaders-ram-v0), EnvSpec(SpaceInvaders-ram-v4), EnvSpec(SpaceInvaders-ramDeterministic-v0), EnvSpec(SpaceInvaders-ramDeterministic-v4), EnvSpec(SpaceInvaders-ramNoFrameskip-v0), EnvSpec(SpaceInvaders-ramNoFrameskip-v4), EnvSpec(StarGunner-v0), EnvSpec(StarGunner-v4), EnvSpec(StarGunnerDeterministic-v0), EnvSpec(StarGunnerDeterministic-v4), EnvSpec(StarGunnerNoFrameskip-v0), EnvSpec(StarGunnerNoFrameskip-v4), EnvSpec(StarGunner-ram-v0), EnvSpec(StarGunner-ram-v4), EnvSpec(StarGunner-ramDeterministic-v0), EnvSpec(StarGunner-ramDeterministic-v4), EnvSpec(StarGunner-ramNoFrameskip-v0), EnvSpec(StarGunner-ramNoFrameskip-v4), EnvSpec(Tennis-v0), EnvSpec(Tennis-v4), EnvSpec(TennisDeterministic-v0), EnvSpec(TennisDeterministic-v4), EnvSpec(TennisNoFrameskip-v0), EnvSpec(TennisNoFrameskip-v4), EnvSpec(Tennis-ram-v0), EnvSpec(Tennis-ram-v4), EnvSpec(Tennis-ramDeterministic-v0), EnvSpec(Tennis-ramDeterministic-v4), EnvSpec(Tennis-ramNoFrameskip-v0), EnvSpec(Tennis-ramNoFrameskip-v4), EnvSpec(TimePilot-v0), EnvSpec(TimePilot-v4), EnvSpec(TimePilotDeterministic-v0), EnvSpec(TimePilotDeterministic-v4), EnvSpec(TimePilotNoFrameskip-v0), EnvSpec(TimePilotNoFrameskip-v4), EnvSpec(TimePilot-ram-v0), EnvSpec(TimePilot-ram-v4), EnvSpec(TimePilot-ramDeterministic-v0), EnvSpec(TimePilot-ramDeterministic-v4), EnvSpec(TimePilot-ramNoFrameskip-v0), EnvSpec(TimePilot-ramNoFrameskip-v4), EnvSpec(Tutankham-v0), EnvSpec(Tutankham-v4), EnvSpec(TutankhamDeterministic-v0), EnvSpec(TutankhamDeterministic-v4), EnvSpec(TutankhamNoFrameskip-v0), EnvSpec(TutankhamNoFrameskip-v4), EnvSpec(Tutankham-ram-v0), EnvSpec(Tutankham-ram-v4), EnvSpec(Tutankham-ramDeterministic-v0), EnvSpec(Tutankham-ramDeterministic-v4), EnvSpec(Tutankham-ramNoFrameskip-v0), EnvSpec(Tutankham-ramNoFrameskip-v4), EnvSpec(UpNDown-v0), EnvSpec(UpNDown-v4), EnvSpec(UpNDownDeterministic-v0), EnvSpec(UpNDownDeterministic-v4), EnvSpec(UpNDownNoFrameskip-v0), EnvSpec(UpNDownNoFrameskip-v4), EnvSpec(UpNDown-ram-v0), EnvSpec(UpNDown-ram-v4), EnvSpec(UpNDown-ramDeterministic-v0), EnvSpec(UpNDown-ramDeterministic-v4), EnvSpec(UpNDown-ramNoFrameskip-v0), EnvSpec(UpNDown-ramNoFrameskip-v4), EnvSpec(Venture-v0), EnvSpec(Venture-v4), EnvSpec(VentureDeterministic-v0), EnvSpec(VentureDeterministic-v4), EnvSpec(VentureNoFrameskip-v0), EnvSpec(VentureNoFrameskip-v4), EnvSpec(Venture-ram-v0), EnvSpec(Venture-ram-v4), EnvSpec(Venture-ramDeterministic-v0), EnvSpec(Venture-ramDeterministic-v4), EnvSpec(Venture-ramNoFrameskip-v0), EnvSpec(Venture-ramNoFrameskip-v4), EnvSpec(VideoPinball-v0), EnvSpec(VideoPinball-v4), EnvSpec(VideoPinballDeterministic-v0), EnvSpec(VideoPinballDeterministic-v4), EnvSpec(VideoPinballNoFrameskip-v0), EnvSpec(VideoPinballNoFrameskip-v4), EnvSpec(VideoPinball-ram-v0), EnvSpec(VideoPinball-ram-v4), EnvSpec(VideoPinball-ramDeterministic-v0), EnvSpec(VideoPinball-ramDeterministic-v4), EnvSpec(VideoPinball-ramNoFrameskip-v0), EnvSpec(VideoPinball-ramNoFrameskip-v4), EnvSpec(WizardOfWor-v0), EnvSpec(WizardOfWor-v4), EnvSpec(WizardOfWorDeterministic-v0), EnvSpec(WizardOfWorDeterministic-v4), EnvSpec(WizardOfWorNoFrameskip-v0), EnvSpec(WizardOfWorNoFrameskip-v4), EnvSpec(WizardOfWor-ram-v0), EnvSpec(WizardOfWor-ram-v4), EnvSpec(WizardOfWor-ramDeterministic-v0), EnvSpec(WizardOfWor-ramDeterministic-v4), EnvSpec(WizardOfWor-ramNoFrameskip-v0), EnvSpec(WizardOfWor-ramNoFrameskip-v4), EnvSpec(YarsRevenge-v0), EnvSpec(YarsRevenge-v4), EnvSpec(YarsRevengeDeterministic-v0), EnvSpec(YarsRevengeDeterministic-v4), EnvSpec(YarsRevengeNoFrameskip-v0), EnvSpec(YarsRevengeNoFrameskip-v4), EnvSpec(YarsRevenge-ram-v0), EnvSpec(YarsRevenge-ram-v4), EnvSpec(YarsRevenge-ramDeterministic-v0), EnvSpec(YarsRevenge-ramDeterministic-v4), EnvSpec(YarsRevenge-ramNoFrameskip-v0), EnvSpec(YarsRevenge-ramNoFrameskip-v4), EnvSpec(Zaxxon-v0), EnvSpec(Zaxxon-v4), EnvSpec(ZaxxonDeterministic-v0), EnvSpec(ZaxxonDeterministic-v4), EnvSpec(ZaxxonNoFrameskip-v0), EnvSpec(ZaxxonNoFrameskip-v4), EnvSpec(Zaxxon-ram-v0), EnvSpec(Zaxxon-ram-v4), EnvSpec(Zaxxon-ramDeterministic-v0), EnvSpec(Zaxxon-ramDeterministic-v4), EnvSpec(Zaxxon-ramNoFrameskip-v0), EnvSpec(Zaxxon-ramNoFrameskip-v4), EnvSpec(CubeCrash-v0), EnvSpec(CubeCrashSparse-v0), EnvSpec(CubeCrashScreenBecomesBlack-v0), EnvSpec(MemorizeDigits-v0)])\n"
     ]
    }
   ],
   "source": [
    "# A look at different options for gym environments\n",
    "print(gym.envs.registry.all())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameter class. Makes it easy to keep training initialization organized\n",
    "class HP():\n",
    "    # Hyperparameters\n",
    "    def __init__(self,\n",
    "                 nb_steps=1000,\n",
    "                 episode_length=2000,\n",
    "                 learning_rate=0.02,\n",
    "                 num_deltas=32,\n",
    "                 num_best_deltas=16,\n",
    "                 noise=0.03,\n",
    "                 seed=1,\n",
    "                 env_name='BipedalWalker-v2',\n",
    "                 record_every=50):\n",
    "\n",
    "        self.nb_steps = nb_steps\n",
    "        self.episode_length = episode_length\n",
    "        self.learning_rate = learning_rate\n",
    "        self.num_deltas = num_deltas\n",
    "        self.num_best_deltas = num_best_deltas\n",
    "        assert self.num_best_deltas <= self.num_deltas\n",
    "        self.noise = noise\n",
    "        self.seed = seed\n",
    "        self.env_name = env_name\n",
    "        self.record_every = record_every"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creates running statistical distributions for observations\n",
    "class Normalizer():\n",
    "    # Normalizes the inputs\n",
    "    def __init__(self, nb_inputs):\n",
    "        self.n = np.zeros(nb_inputs)\n",
    "        self.mean = np.zeros(nb_inputs)\n",
    "        self.mean_diff = np.zeros(nb_inputs)\n",
    "        self.var = np.zeros(nb_inputs)\n",
    "\n",
    "    def observe(self, x):\n",
    "        self.n += 1.0\n",
    "        last_mean = self.mean.copy()\n",
    "        self.mean += (x - self.mean) / self.n\n",
    "        self.mean_diff += (x - last_mean) * (x - self.mean)\n",
    "        self.var = (self.mean_diff / self.n).clip(min = 1e-2)\n",
    "\n",
    "    def normalize(self, inputs):\n",
    "        obs_mean = self.mean\n",
    "        obs_std = np.sqrt(self.var)\n",
    "        return (inputs - obs_mean) / obs_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Policy class. Keeps track of weights and contains methods related to updating them.\n",
    "class Policy():\n",
    "    def __init__(self, input_size, output_size, hp):\n",
    "      #this creates a zero matrix of rows,column\n",
    "        self.theta = np.zeros((output_size, input_size))\n",
    "        self.hp = hp\n",
    "\n",
    "    def evaluate(self, input, delta = None, direction = None):\n",
    "        if direction is None:\n",
    "          #.dot is numpy function for dot product\n",
    "            return self.theta.dot(input)\n",
    "        elif direction == \"+\":\n",
    "            return (self.theta + self.hp.noise * delta).dot(input)\n",
    "        elif direction == \"-\":\n",
    "            return (self.theta - self.hp.noise * delta).dot(input)\n",
    "\n",
    "    def sample_deltas(self):\n",
    "        return [np.random.randn(*self.theta.shape) for _ in range(self.hp.num_deltas)]\n",
    "#This code above here is super important \n",
    "#This is how the weights are updated according to which configuration of weights led to the biggest reward\n",
    "    def update(self, rollouts, sigma_rewards):\n",
    "        # sigma_rewards is the standard deviation of the rewards\n",
    "        step = np.zeros(self.theta.shape)\n",
    "        for r_pos, r_neg, delta in rollouts:\n",
    "            step += (r_pos - r_neg) * delta\n",
    "        self.theta += self.hp.learning_rate / (self.hp.num_best_deltas * sigma_rewards) * step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Skow's ARS agent which makes use of the above classes\n",
    "class ARSTrainer():\n",
    "    def __init__(self,\n",
    "                 hp=None,\n",
    "                 input_size=None,\n",
    "                 output_size=None,\n",
    "                 normalizer=None,\n",
    "                 policy=None,\n",
    "                 monitor_dir=None):\n",
    "\n",
    "        self.hp = hp or HP()\n",
    "        np.random.seed(self.hp.seed)\n",
    "        self.env = gym.make(self.hp.env_name)\n",
    "        if monitor_dir is not None:\n",
    "            should_record = lambda i: self.record_video\n",
    "            self.env = wrappers.Monitor(self.env, monitor_dir, video_callable=should_record, force=True)\n",
    "        self.hp.episode_length = self.hp.episode_length\n",
    "        self.input_size = input_size or self.env.observation_space.shape[0]\n",
    "        self.output_size = output_size or self.env.action_space.shape[0]\n",
    "        self.normalizer = normalizer or Normalizer(self.input_size)\n",
    "        self.policy = policy or Policy(self.input_size, self.output_size, self.hp)\n",
    "        self.record_video = False\n",
    "        self.history = {}\n",
    "        self.history['step'] = []\n",
    "        self.history['reward'] = []\n",
    "\n",
    "    # Explore the policy on one specific direction and over one episode\n",
    "    def explore(self, direction=None, delta=None):\n",
    "        state = self.env.reset()\n",
    "        done = False\n",
    "        num_plays = 0.0\n",
    "        sum_rewards = 0.0\n",
    "        while not done and num_plays < self.hp.episode_length:\n",
    "            self.normalizer.observe(state)\n",
    "            state = self.normalizer.normalize(state)\n",
    "            action = self.policy.evaluate(state, delta, direction)\n",
    "            state, reward, done, _ = self.env.step(action)\n",
    "            reward = max(min(reward, 1), -1)\n",
    "            sum_rewards += reward\n",
    "            num_plays += 1\n",
    "        return sum_rewards\n",
    "\n",
    "    def train(self):\n",
    "        for step in range(self.hp.nb_steps):\n",
    "            # initialize the random noise deltas and the positive/negative rewards\n",
    "            deltas = self.policy.sample_deltas()\n",
    "            positive_rewards = [0] * self.hp.num_deltas\n",
    "            negative_rewards = [0] * self.hp.num_deltas\n",
    "\n",
    "            # play an episode each with positive deltas and negative deltas, collect rewards\n",
    "            for k in range(self.hp.num_deltas):\n",
    "                positive_rewards[k] = self.explore(direction=\"+\", delta=deltas[k])\n",
    "                negative_rewards[k] = self.explore(direction=\"-\", delta=deltas[k])\n",
    "                \n",
    "            # Compute the standard deviation of all rewards\n",
    "            sigma_rewards = np.array(positive_rewards + negative_rewards).std()\n",
    "\n",
    "            # Sort the rollouts by the max(r_pos, r_neg) and select the deltas with best rewards\n",
    "            scores = {k:max(r_pos, r_neg) for k,(r_pos,r_neg) in enumerate(zip(positive_rewards, negative_rewards))}\n",
    "            order = sorted(scores.keys(), key = lambda x:scores[x], reverse = True)[:self.hp.num_best_deltas]\n",
    "            rollouts = [(positive_rewards[k], negative_rewards[k], deltas[k]) for k in order]\n",
    "\n",
    "            # Update the policy\n",
    "            self.policy.update(rollouts, sigma_rewards)\n",
    "\n",
    "            # Only record video during evaluation, every n steps\n",
    "            if step % self.hp.record_every == 0:\n",
    "                self.record_video = True\n",
    "            # Play an episode with the new weights and print the score\n",
    "            reward_evaluation = self.explore()\n",
    "            print('Step: ', step, 'Reward: ', reward_evaluation)\n",
    "            self.history['step'].append(step)\n",
    "            self.history['reward'].append(reward_evaluation)\n",
    "            # Record score to history every n steps\n",
    "            #if step % self.hp.record_every == 0:\n",
    "            #    self.history['step'].append(step)\n",
    "            #    self.history['reward'].append(reward_evaluation)\n",
    "            self.record_video = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mkdir(base, name):\n",
    "    path = os.path.join(base, name)\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    return path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  0 Reward:  6.998605306755646\n",
      "Step:  1 Reward:  6.329438480241898\n",
      "Step:  2 Reward:  6.128041341355143\n",
      "Step:  3 Reward:  5.373827039827025\n",
      "Step:  4 Reward:  4.813151372186638\n",
      "Step:  5 Reward:  6.064876862792623\n",
      "Step:  6 Reward:  6.461119420408277\n",
      "Step:  7 Reward:  6.998481927326692\n",
      "Step:  8 Reward:  -15.075066064962813\n",
      "Step:  9 Reward:  -17.019307712121684\n",
      "Step:  10 Reward:  -16.93668466660058\n",
      "Step:  11 Reward:  6.351556079405391\n",
      "Step:  12 Reward:  -13.626644421729708\n",
      "Step:  13 Reward:  5.628572899435834\n",
      "Step:  14 Reward:  7.330288377048349\n",
      "Step:  15 Reward:  6.2765273648154345\n",
      "Step:  16 Reward:  -13.406336567355053\n",
      "Step:  17 Reward:  -10.553758186327606\n",
      "Step:  18 Reward:  6.815041637589285\n",
      "Step:  19 Reward:  6.016768832067139\n",
      "Step:  20 Reward:  5.390543608236722\n",
      "Step:  21 Reward:  5.978307020506692\n",
      "Step:  22 Reward:  5.7577679331297835\n",
      "Step:  23 Reward:  5.695070883720288\n",
      "Step:  24 Reward:  6.480034452275325\n",
      "Step:  25 Reward:  7.236423773016281\n",
      "Step:  26 Reward:  5.842905506861084\n",
      "Step:  27 Reward:  6.278259185461926\n",
      "Step:  28 Reward:  -7.491007545986496\n",
      "Step:  29 Reward:  6.706907577472396\n",
      "Step:  30 Reward:  7.2560861889903805\n",
      "Step:  31 Reward:  7.064390511697409\n",
      "Step:  32 Reward:  6.14063250951657\n",
      "Step:  33 Reward:  5.585378647139906\n",
      "Step:  34 Reward:  5.476412849701868\n",
      "Step:  35 Reward:  6.561066801908242\n",
      "Step:  36 Reward:  6.355695042724805\n",
      "Step:  37 Reward:  5.306077819130007\n",
      "Step:  38 Reward:  6.531901077131075\n",
      "Step:  39 Reward:  6.072742091385577\n",
      "Step:  40 Reward:  6.4864765613121715\n",
      "Step:  41 Reward:  6.618609916612115\n",
      "Step:  42 Reward:  6.050272906432314\n",
      "Step:  43 Reward:  6.1894074945126345\n",
      "Step:  44 Reward:  6.468651767551487\n",
      "Step:  45 Reward:  6.4208769289955985\n",
      "Step:  46 Reward:  6.817712326499944\n",
      "Step:  47 Reward:  6.125269648169768\n",
      "Step:  48 Reward:  6.664581718491441\n",
      "Step:  49 Reward:  6.388609582435674\n",
      "Step:  50 Reward:  6.275305328520828\n",
      "Step:  51 Reward:  6.362035279720562\n",
      "Step:  52 Reward:  6.5673433475465846\n",
      "Step:  53 Reward:  6.533302342740072\n",
      "Step:  54 Reward:  6.398497581628795\n",
      "Step:  55 Reward:  6.323655575846477\n",
      "Step:  56 Reward:  6.454579902399518\n",
      "Step:  57 Reward:  6.611592644411844\n",
      "Step:  58 Reward:  6.361039374580156\n",
      "Step:  59 Reward:  6.837140710113394\n",
      "Step:  60 Reward:  6.758268033373906\n",
      "Step:  61 Reward:  6.742832257803554\n",
      "Step:  62 Reward:  6.648135400279271\n",
      "Step:  63 Reward:  6.721014073132246\n",
      "Step:  64 Reward:  6.709663333619757\n",
      "Step:  65 Reward:  6.274811203231948\n",
      "Step:  66 Reward:  6.918192518709658\n",
      "Step:  67 Reward:  6.400130220170334\n",
      "Step:  68 Reward:  6.605737669750254\n",
      "Step:  69 Reward:  6.592008207076283\n",
      "Step:  70 Reward:  6.382329223640315\n",
      "Step:  71 Reward:  6.476061204567108\n",
      "Step:  72 Reward:  6.39363230561454\n",
      "Step:  73 Reward:  6.448096032475214\n",
      "Step:  74 Reward:  6.328316680311828\n",
      "Step:  75 Reward:  6.539489428534641\n",
      "Step:  76 Reward:  6.641271207896652\n",
      "Step:  77 Reward:  6.577460865652587\n",
      "Step:  78 Reward:  6.492083369491515\n",
      "Step:  79 Reward:  6.600474190583436\n",
      "Step:  80 Reward:  6.326209322302834\n",
      "Step:  81 Reward:  6.720300699762355\n",
      "Step:  82 Reward:  6.463202650985034\n",
      "Step:  83 Reward:  6.149120119835603\n",
      "Step:  84 Reward:  6.297271055171887\n",
      "Step:  85 Reward:  6.283980742003427\n",
      "Step:  86 Reward:  6.435976276545116\n",
      "Step:  87 Reward:  6.605470962506047\n",
      "Step:  88 Reward:  6.316056246442439\n",
      "Step:  89 Reward:  6.379608141227598\n",
      "Step:  90 Reward:  6.1919692643715125\n",
      "Step:  91 Reward:  6.452263669041672\n",
      "Step:  92 Reward:  6.272255854041068\n",
      "Step:  93 Reward:  6.264952407060708\n",
      "Step:  94 Reward:  6.276663945153123\n",
      "Step:  95 Reward:  6.632447417334007\n",
      "Step:  96 Reward:  6.489536609815823\n",
      "Step:  97 Reward:  6.225515455211769\n",
      "Step:  98 Reward:  6.277006050770242\n",
      "Step:  99 Reward:  6.293487979926497\n",
      "Step:  100 Reward:  6.444380104325838\n",
      "Step:  101 Reward:  6.3601888898266\n",
      "Step:  102 Reward:  6.301948686644642\n",
      "Step:  103 Reward:  6.43789223611902\n",
      "Step:  104 Reward:  6.337633671359766\n",
      "Step:  105 Reward:  6.38728352683899\n",
      "Step:  106 Reward:  6.366688180959967\n",
      "Step:  107 Reward:  6.2504847265712815\n",
      "Step:  108 Reward:  5.97773634365583\n",
      "Step:  109 Reward:  6.118793378370489\n",
      "Step:  110 Reward:  6.389959516191266\n",
      "Step:  111 Reward:  6.0908219129199255\n",
      "Step:  112 Reward:  6.054790705923996\n",
      "Step:  113 Reward:  6.191668821420619\n",
      "Step:  114 Reward:  6.355932462401533\n",
      "Step:  115 Reward:  6.172304924136627\n",
      "Step:  116 Reward:  6.116418242983039\n",
      "Step:  117 Reward:  6.448202769851077\n",
      "Step:  118 Reward:  6.728610457564959\n",
      "Step:  119 Reward:  7.0925589025291895\n",
      "Step:  120 Reward:  6.886014213024224\n",
      "Step:  121 Reward:  6.97121555892226\n",
      "Step:  122 Reward:  7.048672234761893\n",
      "Step:  123 Reward:  6.877129766105846\n",
      "Step:  124 Reward:  6.722302127138844\n",
      "Step:  125 Reward:  6.686822265431094\n",
      "Step:  126 Reward:  6.876973556776685\n",
      "Step:  127 Reward:  6.785488463175927\n",
      "Step:  128 Reward:  6.145952629325283\n",
      "Step:  129 Reward:  6.085025317385891\n",
      "Step:  130 Reward:  6.001890626973102\n",
      "Step:  131 Reward:  5.607728932365928\n",
      "Step:  132 Reward:  6.2085720858823645\n",
      "Step:  133 Reward:  6.296295743338519\n",
      "Step:  134 Reward:  6.471770042679562\n",
      "Step:  135 Reward:  6.3697303893271\n",
      "Step:  136 Reward:  6.352288992930378\n",
      "Step:  137 Reward:  6.47161737988219\n",
      "Step:  138 Reward:  6.314539254583879\n",
      "Step:  139 Reward:  6.512431357013222\n",
      "Step:  140 Reward:  6.154934447135566\n",
      "Step:  141 Reward:  6.520118481842305\n",
      "Step:  142 Reward:  6.526611701436555\n",
      "Step:  143 Reward:  6.558933725158524\n",
      "Step:  144 Reward:  6.4683089799582785\n",
      "Step:  145 Reward:  6.776280360405815\n",
      "Step:  146 Reward:  6.561078728220927\n",
      "Step:  147 Reward:  6.5674298744458675\n",
      "Step:  148 Reward:  6.618799398815983\n",
      "Step:  149 Reward:  6.711145477789679\n",
      "Step:  150 Reward:  6.8207585841995595\n",
      "Step:  151 Reward:  6.864734767504847\n",
      "Step:  152 Reward:  6.793105112239087\n",
      "Step:  153 Reward:  6.630528218037385\n",
      "Step:  154 Reward:  6.687999610935041\n",
      "Step:  155 Reward:  6.736958090558662\n",
      "Step:  156 Reward:  6.918089923663338\n",
      "Step:  157 Reward:  6.928426060255068\n",
      "Step:  158 Reward:  6.4051551411834975\n",
      "Step:  159 Reward:  6.578866679038579\n",
      "Step:  160 Reward:  6.710938540029205\n",
      "Step:  161 Reward:  6.345930069881851\n",
      "Step:  162 Reward:  6.30554343166225\n",
      "Step:  163 Reward:  6.658436383250896\n",
      "Step:  164 Reward:  6.954173608861858\n",
      "Step:  165 Reward:  6.787167312520458\n",
      "Step:  166 Reward:  6.954354126494875\n",
      "Step:  167 Reward:  6.741747212359597\n",
      "Step:  168 Reward:  6.825262290617309\n",
      "Step:  169 Reward:  6.489249943302987\n",
      "Step:  170 Reward:  6.597832888340833\n",
      "Step:  171 Reward:  7.057602568569493\n",
      "Step:  172 Reward:  6.5882170896258145\n",
      "Step:  173 Reward:  7.151037667268847\n",
      "Step:  174 Reward:  6.91594500104158\n",
      "Step:  175 Reward:  7.190627679668658\n",
      "Step:  176 Reward:  7.279277891009778\n",
      "Step:  177 Reward:  7.062500649209561\n",
      "Step:  178 Reward:  6.88043124382791\n",
      "Step:  179 Reward:  6.5594019024131995\n",
      "Step:  180 Reward:  6.557108738673549\n",
      "Step:  181 Reward:  6.699917630613378\n",
      "Step:  182 Reward:  7.202691889329554\n",
      "Step:  183 Reward:  7.175532237360562\n",
      "Step:  184 Reward:  6.328294467256375\n",
      "Step:  185 Reward:  6.338639833383809\n",
      "Step:  186 Reward:  6.195141675951316\n",
      "Step:  187 Reward:  6.550018788608231\n",
      "Step:  188 Reward:  6.560063423706774\n",
      "Step:  189 Reward:  6.39274249984375\n",
      "Step:  190 Reward:  6.444023285247044\n",
      "Step:  191 Reward:  6.819025327368531\n",
      "Step:  192 Reward:  6.607648481656393\n",
      "Step:  193 Reward:  6.502977958263622\n",
      "Step:  194 Reward:  6.418592862647219\n",
      "Step:  195 Reward:  6.259656236601141\n",
      "Step:  196 Reward:  6.933710275681265\n",
      "Step:  197 Reward:  6.7936217099357386\n",
      "Step:  198 Reward:  6.679313354809647\n",
      "Step:  199 Reward:  6.895229794170349\n",
      "Step:  200 Reward:  6.684410639735438\n",
      "Step:  201 Reward:  6.9143158607373305\n",
      "Step:  202 Reward:  6.583242998047533\n",
      "Step:  203 Reward:  6.569996221854639\n",
      "Step:  204 Reward:  6.689619501172292\n",
      "Step:  205 Reward:  6.533453704684214\n",
      "Step:  206 Reward:  6.406509896619708\n",
      "Step:  207 Reward:  6.543951687233216\n",
      "Step:  208 Reward:  6.327649580164163\n",
      "Step:  209 Reward:  6.427146313320296\n",
      "Step:  210 Reward:  6.4852124018442225\n",
      "Step:  211 Reward:  6.29299649780577\n",
      "Step:  212 Reward:  6.5480404444397\n",
      "Step:  213 Reward:  7.012935827993811\n",
      "Step:  214 Reward:  6.833147153553708\n",
      "Step:  215 Reward:  6.937977516908105\n",
      "Step:  216 Reward:  6.710548926174495\n",
      "Step:  217 Reward:  6.54792483037854\n",
      "Step:  218 Reward:  6.684088135397828\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  219 Reward:  7.042125277506303\n",
      "Step:  220 Reward:  6.733986408002886\n",
      "Step:  221 Reward:  6.55309301403958\n",
      "Step:  222 Reward:  6.825554474703357\n",
      "Step:  223 Reward:  6.4800626002299655\n",
      "Step:  224 Reward:  6.539258904157239\n",
      "Step:  225 Reward:  6.507534745116608\n",
      "Step:  226 Reward:  6.540433778016744\n",
      "Step:  227 Reward:  6.509938633734018\n",
      "Step:  228 Reward:  6.690441036215109\n",
      "Step:  229 Reward:  6.580294559978918\n",
      "Step:  230 Reward:  6.433953213134096\n",
      "Step:  231 Reward:  6.543349537359719\n",
      "Step:  232 Reward:  6.6627470696196545\n",
      "Step:  233 Reward:  6.710622121925188\n",
      "Step:  234 Reward:  6.441768966770211\n",
      "Step:  235 Reward:  6.6161827758163305\n",
      "Step:  236 Reward:  6.81124988921295\n",
      "Step:  237 Reward:  6.726968949546192\n",
      "Step:  238 Reward:  6.532836408280604\n",
      "Step:  239 Reward:  6.759712276363724\n",
      "Step:  240 Reward:  6.842729700224997\n",
      "Step:  241 Reward:  6.5083111425030715\n",
      "Step:  242 Reward:  6.392003477414071\n",
      "Step:  243 Reward:  6.305585181194758\n",
      "Step:  244 Reward:  6.649379138833567\n",
      "Step:  245 Reward:  6.730234007196245\n",
      "Step:  246 Reward:  6.201980001338522\n",
      "Step:  247 Reward:  6.566066587622738\n",
      "Step:  248 Reward:  6.944087291089251\n",
      "Step:  249 Reward:  6.897377641236909\n",
      "Step:  250 Reward:  6.994774839299391\n",
      "Step:  251 Reward:  6.87393132203598\n",
      "Step:  252 Reward:  6.825956958685519\n",
      "Step:  253 Reward:  6.899940299508057\n",
      "Step:  254 Reward:  6.789474988427076\n",
      "Step:  255 Reward:  7.124660032043698\n",
      "Step:  256 Reward:  6.758015755511336\n",
      "Step:  257 Reward:  6.977211433086464\n",
      "Step:  258 Reward:  6.4881951379990905\n",
      "Step:  259 Reward:  6.755181311372112\n",
      "Step:  260 Reward:  7.305245342099202\n",
      "Step:  261 Reward:  7.118599430208816\n",
      "Step:  262 Reward:  7.134915649062027\n",
      "Step:  263 Reward:  7.09467843379101\n",
      "Step:  264 Reward:  6.552883638142124\n",
      "Step:  265 Reward:  6.639784626498501\n",
      "Step:  266 Reward:  7.0074606316225445\n",
      "Step:  267 Reward:  6.876643088741626\n",
      "Step:  268 Reward:  7.031765893221634\n",
      "Step:  269 Reward:  6.8577853001221625\n",
      "Step:  270 Reward:  6.925935590099495\n",
      "Step:  271 Reward:  6.719023871737473\n",
      "Step:  272 Reward:  6.907908911402224\n",
      "Step:  273 Reward:  6.938925601677311\n",
      "Step:  274 Reward:  6.781449481108558\n",
      "Step:  275 Reward:  6.605832663391806\n",
      "Step:  276 Reward:  6.632443342535419\n",
      "Step:  277 Reward:  6.674080186062613\n",
      "Step:  278 Reward:  6.658549506888013\n",
      "Step:  279 Reward:  6.606761047532833\n",
      "Step:  280 Reward:  6.687132318670397\n",
      "Step:  281 Reward:  6.985786822445145\n",
      "Step:  282 Reward:  6.605308908577703\n",
      "Step:  283 Reward:  6.621820763141288\n",
      "Step:  284 Reward:  6.793233517865265\n",
      "Step:  285 Reward:  6.908027619037806\n",
      "Step:  286 Reward:  6.696147030033297\n",
      "Step:  287 Reward:  6.862728782619792\n",
      "Step:  288 Reward:  6.897273340980606\n",
      "Step:  289 Reward:  6.833041990657186\n",
      "Step:  290 Reward:  6.6667156275055515\n",
      "Step:  291 Reward:  6.822036780846029\n",
      "Step:  292 Reward:  6.909974928005162\n",
      "Step:  293 Reward:  6.428696732490974\n",
      "Step:  294 Reward:  6.57862747672079\n",
      "Step:  295 Reward:  6.514164512275449\n",
      "Step:  296 Reward:  7.090132383834424\n",
      "Step:  297 Reward:  6.953105233009566\n",
      "Step:  298 Reward:  6.5217100564629815\n",
      "Step:  299 Reward:  6.574015447953776\n",
      "Step:  300 Reward:  6.639146007907991\n",
      "Step:  301 Reward:  6.432454978652311\n",
      "Step:  302 Reward:  6.5408384753115945\n",
      "Step:  303 Reward:  6.370542483192502\n",
      "Step:  304 Reward:  6.441929277051475\n",
      "Step:  305 Reward:  6.654107327801865\n",
      "Step:  306 Reward:  6.538476313411703\n",
      "Step:  307 Reward:  6.5713441505827035\n",
      "Step:  308 Reward:  6.73851581856487\n",
      "Step:  309 Reward:  6.404535562837773\n",
      "Step:  310 Reward:  6.874918722763365\n",
      "Step:  311 Reward:  6.51535791776588\n",
      "Step:  312 Reward:  6.652743334320535\n",
      "Step:  313 Reward:  6.628652894115374\n",
      "Step:  314 Reward:  6.975009628295417\n",
      "Step:  315 Reward:  6.780165073506403\n",
      "Step:  316 Reward:  6.957752545893851\n",
      "Step:  317 Reward:  6.509561304545182\n",
      "Step:  318 Reward:  6.609833347628535\n",
      "Step:  319 Reward:  6.320042624012381\n",
      "Step:  320 Reward:  6.762708276524664\n",
      "Step:  321 Reward:  6.705635165485053\n",
      "Step:  322 Reward:  6.684802436244318\n",
      "Step:  323 Reward:  6.852644548588573\n",
      "Step:  324 Reward:  6.872660997309103\n",
      "Step:  325 Reward:  6.552887409749725\n",
      "Step:  326 Reward:  6.445823616637678\n",
      "Step:  327 Reward:  6.298504109775541\n",
      "Step:  328 Reward:  6.769594461789262\n",
      "Step:  329 Reward:  6.463297425646917\n",
      "Step:  330 Reward:  6.723901413570928\n",
      "Step:  331 Reward:  6.476570601554928\n",
      "Step:  332 Reward:  6.670176696240766\n",
      "Step:  333 Reward:  6.719461167695236\n",
      "Step:  334 Reward:  6.769725030464048\n",
      "Step:  335 Reward:  6.729504084674624\n",
      "Step:  336 Reward:  6.2845457060497365\n",
      "Step:  337 Reward:  6.419714648481181\n",
      "Step:  338 Reward:  6.212312493423558\n",
      "Step:  339 Reward:  6.5020695034023746\n",
      "Step:  340 Reward:  6.278530247550797\n",
      "Step:  341 Reward:  6.25055514894964\n",
      "Step:  342 Reward:  6.284800690747053\n",
      "Step:  343 Reward:  6.299858166630026\n",
      "Step:  344 Reward:  6.492492969460967\n",
      "Step:  345 Reward:  6.395632491293251\n",
      "Step:  346 Reward:  6.447246017369681\n",
      "Step:  347 Reward:  6.677308056730987\n",
      "Step:  348 Reward:  6.40519254224547\n",
      "Step:  349 Reward:  6.632704708684133\n",
      "Step:  350 Reward:  6.57635569586473\n",
      "Step:  351 Reward:  6.755787484552637\n",
      "Step:  352 Reward:  6.533000617096326\n",
      "Step:  353 Reward:  6.501609348290552\n",
      "Step:  354 Reward:  6.5504664838949935\n",
      "Step:  355 Reward:  6.478312088753725\n",
      "Step:  356 Reward:  6.631703815176786\n",
      "Step:  357 Reward:  6.64315561673086\n",
      "Step:  358 Reward:  6.391775404521858\n",
      "Step:  359 Reward:  6.502609585755944\n",
      "Step:  360 Reward:  6.245575551694303\n",
      "Step:  361 Reward:  6.548674735233839\n",
      "Step:  362 Reward:  6.461653620282424\n",
      "Step:  363 Reward:  6.389867474460342\n",
      "Step:  364 Reward:  6.570608194559592\n",
      "Step:  365 Reward:  6.687487879346364\n",
      "Step:  366 Reward:  6.736427354732655\n",
      "Step:  367 Reward:  6.783608738171818\n",
      "Step:  368 Reward:  7.159739986993298\n",
      "Step:  369 Reward:  7.1763197152532285\n",
      "Step:  370 Reward:  6.764817545362378\n",
      "Step:  371 Reward:  6.7695938287069115\n",
      "Step:  372 Reward:  6.8648938539775655\n",
      "Step:  373 Reward:  6.764472976360651\n",
      "Step:  374 Reward:  6.9572080086274575\n",
      "Step:  375 Reward:  6.88473146885941\n",
      "Step:  376 Reward:  7.354022505083925\n",
      "Step:  377 Reward:  6.842101441214833\n",
      "Step:  378 Reward:  7.045231027566853\n",
      "Step:  379 Reward:  6.8110145997729035\n",
      "Step:  380 Reward:  6.860805378483992\n",
      "Step:  381 Reward:  6.932716142352249\n",
      "Step:  382 Reward:  7.030996467180495\n",
      "Step:  383 Reward:  6.979854899485583\n",
      "Step:  384 Reward:  6.935278485626901\n",
      "Step:  385 Reward:  6.9625071431955705\n",
      "Step:  386 Reward:  6.828466873677082\n",
      "Step:  387 Reward:  6.950774227769327\n",
      "Step:  388 Reward:  7.319367095992066\n",
      "Step:  389 Reward:  7.2745562691760846\n",
      "Step:  390 Reward:  7.170769982503334\n",
      "Step:  391 Reward:  6.999814295226065\n",
      "Step:  392 Reward:  7.0035634791334775\n",
      "Step:  393 Reward:  6.8072245398620215\n",
      "Step:  394 Reward:  7.208720336512485\n",
      "Step:  395 Reward:  7.295868204808951\n",
      "Step:  396 Reward:  7.118046582013891\n",
      "Step:  397 Reward:  7.141516999383656\n",
      "Step:  398 Reward:  7.083069502195679\n",
      "Step:  399 Reward:  7.205021420264266\n",
      "Step:  400 Reward:  7.28273439076141\n",
      "Step:  401 Reward:  7.253553968820517\n",
      "Step:  402 Reward:  7.209444815230421\n",
      "Step:  403 Reward:  6.962901739028385\n",
      "Step:  404 Reward:  7.257385896035945\n",
      "Step:  405 Reward:  6.895414114238089\n",
      "Step:  406 Reward:  7.050629235880489\n",
      "Step:  407 Reward:  6.782820768400374\n",
      "Step:  408 Reward:  6.824560013188146\n",
      "Step:  409 Reward:  7.068345240462563\n",
      "Step:  410 Reward:  7.127341882797415\n",
      "Step:  411 Reward:  6.865917134927243\n",
      "Step:  412 Reward:  7.1043264719576715\n",
      "Step:  413 Reward:  7.172249997716751\n",
      "Step:  414 Reward:  7.24035645956039\n",
      "Step:  415 Reward:  7.215952812854949\n",
      "Step:  416 Reward:  6.922935683199174\n",
      "Step:  417 Reward:  7.253188488420898\n",
      "Step:  418 Reward:  7.675252343548944\n",
      "Step:  419 Reward:  7.016006172177699\n",
      "Step:  420 Reward:  7.107910300626651\n",
      "Step:  421 Reward:  7.249340053874784\n",
      "Step:  422 Reward:  7.27516810213676\n",
      "Step:  423 Reward:  7.364798425955925\n",
      "Step:  424 Reward:  7.27179432261652\n",
      "Step:  425 Reward:  7.320853221974859\n",
      "Step:  426 Reward:  7.390660289140831\n",
      "Step:  427 Reward:  7.171678670812513\n",
      "Step:  428 Reward:  6.736366457339413\n",
      "Step:  429 Reward:  6.840570758162609\n",
      "Step:  430 Reward:  7.2379848837882115\n",
      "Step:  431 Reward:  6.810670881442473\n",
      "Step:  432 Reward:  7.265528659332272\n",
      "Step:  433 Reward:  7.289108158327563\n",
      "Step:  434 Reward:  6.8946011843463975\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  435 Reward:  7.617378820975597\n",
      "Step:  436 Reward:  7.452059910239312\n",
      "Step:  437 Reward:  6.854936395979463\n",
      "Step:  438 Reward:  7.338130292276466\n",
      "Step:  439 Reward:  7.024104248512694\n",
      "Step:  440 Reward:  7.203664147530366\n",
      "Step:  441 Reward:  7.21630236625788\n",
      "Step:  442 Reward:  6.96913944046346\n",
      "Step:  443 Reward:  7.300694082072562\n",
      "Step:  444 Reward:  7.218967933954824\n",
      "Step:  445 Reward:  7.3394267248876055\n",
      "Step:  446 Reward:  7.394177878090725\n",
      "Step:  447 Reward:  7.405043853272913\n",
      "Step:  448 Reward:  7.339610611596134\n",
      "Step:  449 Reward:  7.415155944434709\n",
      "Step:  450 Reward:  7.176659446403372\n",
      "Step:  451 Reward:  7.193607826063875\n",
      "Step:  452 Reward:  7.129801135228298\n",
      "Step:  453 Reward:  7.064625530604493\n",
      "Step:  454 Reward:  7.19377283526239\n",
      "Step:  455 Reward:  7.22733220002387\n",
      "Step:  456 Reward:  7.239289140448005\n",
      "Step:  457 Reward:  7.247306242188637\n",
      "Step:  458 Reward:  6.975974085259283\n",
      "Step:  459 Reward:  7.043192593733089\n",
      "Step:  460 Reward:  7.241029944035034\n",
      "Step:  461 Reward:  7.340977806276923\n",
      "Step:  462 Reward:  7.418184776858002\n",
      "Step:  463 Reward:  7.181570422059552\n",
      "Step:  464 Reward:  6.902416642443033\n",
      "Step:  465 Reward:  7.096598774380391\n",
      "Step:  466 Reward:  7.201933599765203\n",
      "Step:  467 Reward:  7.283355701253452\n",
      "Step:  468 Reward:  7.342408521088409\n",
      "Step:  469 Reward:  6.80472416397451\n",
      "Step:  470 Reward:  6.78429757882402\n",
      "Step:  471 Reward:  7.215796461931262\n",
      "Step:  472 Reward:  7.022836913178775\n",
      "Step:  473 Reward:  7.150754970840168\n",
      "Step:  474 Reward:  7.3373436006846084\n",
      "Step:  475 Reward:  7.13787210922937\n",
      "Step:  476 Reward:  7.12682074836065\n",
      "Step:  477 Reward:  6.811171641511366\n",
      "Step:  478 Reward:  6.941019050666381\n",
      "Step:  479 Reward:  6.835905158073839\n",
      "Step:  480 Reward:  6.91856287339758\n",
      "Step:  481 Reward:  6.975460743835088\n",
      "Step:  482 Reward:  7.063532201991725\n",
      "Step:  483 Reward:  6.679518954076474\n",
      "Step:  484 Reward:  6.969367443621266\n",
      "Step:  485 Reward:  6.547324211000935\n",
      "Step:  486 Reward:  6.829232249466458\n",
      "Step:  487 Reward:  7.203375980215263\n",
      "Step:  488 Reward:  6.934389075345192\n",
      "Step:  489 Reward:  7.088967516997842\n",
      "Step:  490 Reward:  6.93387206382066\n",
      "Step:  491 Reward:  7.040483729448816\n",
      "Step:  492 Reward:  7.123919212686612\n",
      "Step:  493 Reward:  6.8195223764761135\n",
      "Step:  494 Reward:  7.133203469173193\n",
      "Step:  495 Reward:  6.83936782141783\n",
      "Step:  496 Reward:  6.759711574340002\n",
      "Step:  497 Reward:  6.878788679298839\n",
      "Step:  498 Reward:  6.9702270487277085\n",
      "Step:  499 Reward:  7.082128773829904\n",
      "Step:  500 Reward:  7.057963027869313\n",
      "Step:  501 Reward:  7.042814062062735\n",
      "Step:  502 Reward:  7.00262404446981\n",
      "Step:  503 Reward:  7.360338673024122\n",
      "Step:  504 Reward:  6.928013379213691\n",
      "Step:  505 Reward:  6.763883782024839\n",
      "Step:  506 Reward:  6.952170524325463\n",
      "Step:  507 Reward:  6.94245622454542\n",
      "Step:  508 Reward:  7.050033165007221\n",
      "Step:  509 Reward:  6.9513013238508\n",
      "Step:  510 Reward:  6.860405228346824\n",
      "Step:  511 Reward:  6.859385393549744\n",
      "Step:  512 Reward:  6.908817755467569\n",
      "Step:  513 Reward:  7.005145898655382\n",
      "Step:  514 Reward:  6.962277439947134\n",
      "Step:  515 Reward:  7.11344181915489\n",
      "Step:  516 Reward:  7.082916208372602\n",
      "Step:  517 Reward:  6.984357421789238\n",
      "Step:  518 Reward:  6.858868288827342\n",
      "Step:  519 Reward:  7.030774838992999\n",
      "Step:  520 Reward:  7.226356123003399\n",
      "Step:  521 Reward:  6.90902690644475\n",
      "Step:  522 Reward:  7.275366299246363\n",
      "Step:  523 Reward:  7.379860299332728\n",
      "Step:  524 Reward:  7.24562707400978\n",
      "Step:  525 Reward:  7.188036720447938\n",
      "Step:  526 Reward:  6.9059212914740655\n",
      "Step:  527 Reward:  7.207730294226495\n",
      "Step:  528 Reward:  7.094675114654523\n",
      "Step:  529 Reward:  7.124559040742767\n",
      "Step:  530 Reward:  7.061180051326755\n",
      "Step:  531 Reward:  6.938196042556806\n",
      "Step:  532 Reward:  6.9463925837086595\n",
      "Step:  533 Reward:  6.88425879603246\n",
      "Step:  534 Reward:  6.944111221725989\n",
      "Step:  535 Reward:  6.901492860990683\n",
      "Step:  536 Reward:  6.627117969240414\n",
      "Step:  537 Reward:  6.666099752428695\n",
      "Step:  538 Reward:  6.902548812136712\n",
      "Step:  539 Reward:  7.00407582301332\n",
      "Step:  540 Reward:  6.741917845617608\n",
      "Step:  541 Reward:  6.773639943794154\n",
      "Step:  542 Reward:  7.222078933769341\n",
      "Step:  543 Reward:  6.970989545165087\n",
      "Step:  544 Reward:  6.919265236515226\n",
      "Step:  545 Reward:  7.146171563250752\n",
      "Step:  546 Reward:  7.339603009086577\n",
      "Step:  547 Reward:  7.085528473125024\n",
      "Step:  548 Reward:  7.013741824145054\n",
      "Step:  549 Reward:  6.930185485302436\n",
      "Step:  550 Reward:  7.046626960973864\n",
      "Step:  551 Reward:  7.127229554112958\n",
      "Step:  552 Reward:  7.297995685853527\n",
      "Step:  553 Reward:  7.137797357746978\n",
      "Step:  554 Reward:  7.233508666102857\n",
      "Step:  555 Reward:  7.389561747735071\n",
      "Step:  556 Reward:  7.413416286393\n",
      "Step:  557 Reward:  7.9496201828435655\n",
      "Step:  558 Reward:  7.334381164161741\n",
      "Step:  559 Reward:  7.439304552690228\n",
      "Step:  560 Reward:  7.198846851925627\n",
      "Step:  561 Reward:  7.625292744982952\n",
      "Step:  562 Reward:  7.7509353213979395\n",
      "Step:  563 Reward:  7.520516507567004\n",
      "Step:  564 Reward:  7.794480954125412\n",
      "Step:  565 Reward:  7.828554166260691\n",
      "Step:  566 Reward:  7.163578990582135\n",
      "Step:  567 Reward:  7.731662851819371\n",
      "Step:  568 Reward:  8.024165867864756\n",
      "Step:  569 Reward:  8.483053187709915\n",
      "Step:  570 Reward:  7.859059797400457\n",
      "Step:  571 Reward:  8.513234622722695\n",
      "Step:  572 Reward:  9.049802825223034\n",
      "Step:  573 Reward:  8.082046206124366\n",
      "Step:  574 Reward:  8.514190990325604\n",
      "Step:  575 Reward:  8.537860975576475\n",
      "Step:  576 Reward:  9.748204749939283\n",
      "Step:  577 Reward:  8.193113397855207\n",
      "Step:  578 Reward:  8.32962101545835\n",
      "Step:  579 Reward:  9.190347002809274\n",
      "Step:  580 Reward:  9.554314712659908\n",
      "Step:  581 Reward:  8.410392891239109\n",
      "Step:  582 Reward:  9.07628827256157\n",
      "Step:  583 Reward:  8.155985403430048\n",
      "Step:  584 Reward:  8.474239151433899\n",
      "Step:  585 Reward:  7.591704431165059\n",
      "Step:  586 Reward:  9.713437054903915\n",
      "Step:  587 Reward:  7.989016450108995\n",
      "Step:  588 Reward:  7.859621658234646\n",
      "Step:  589 Reward:  8.992479848636956\n",
      "Step:  590 Reward:  9.695061120237108\n",
      "Step:  591 Reward:  9.019387553151677\n",
      "Step:  592 Reward:  8.78499664307854\n",
      "Step:  593 Reward:  8.902797964392958\n",
      "Step:  594 Reward:  9.144624831022275\n",
      "Step:  595 Reward:  8.736813175435083\n",
      "Step:  596 Reward:  7.790243548429151\n",
      "Step:  597 Reward:  8.702263574528807\n",
      "Step:  598 Reward:  9.462088975769042\n",
      "Step:  599 Reward:  10.469765877789692\n",
      "Step:  600 Reward:  9.535260687713127\n",
      "Step:  601 Reward:  10.738490094888704\n",
      "Step:  602 Reward:  9.748988786547352\n",
      "Step:  603 Reward:  11.433833536603077\n",
      "Step:  604 Reward:  10.336943212962023\n",
      "Step:  605 Reward:  12.263952771660092\n",
      "Step:  606 Reward:  11.440317381447894\n",
      "Step:  607 Reward:  11.497148982321981\n",
      "Step:  608 Reward:  12.533396641199541\n",
      "Step:  609 Reward:  12.110768606126129\n",
      "Step:  610 Reward:  13.3828334330606\n",
      "Step:  611 Reward:  13.827718380801713\n",
      "Step:  612 Reward:  12.501596759679984\n",
      "Step:  613 Reward:  12.605438065808698\n",
      "Step:  614 Reward:  12.893121516785499\n",
      "Step:  615 Reward:  13.359405463926679\n",
      "Step:  616 Reward:  12.9697004913509\n",
      "Step:  617 Reward:  13.496055073861529\n",
      "Step:  618 Reward:  12.801239908905993\n",
      "Step:  619 Reward:  12.595234745294809\n",
      "Step:  620 Reward:  12.918267269505755\n",
      "Step:  621 Reward:  13.740644929146796\n",
      "Step:  622 Reward:  12.677339791680767\n",
      "Step:  623 Reward:  13.199563324170445\n",
      "Step:  624 Reward:  12.820662486380975\n",
      "Step:  625 Reward:  13.820401780708984\n",
      "Step:  626 Reward:  14.214159026490288\n",
      "Step:  627 Reward:  13.491173584920046\n",
      "Step:  628 Reward:  13.181200298756078\n",
      "Step:  629 Reward:  13.599494940837227\n",
      "Step:  630 Reward:  14.727538295957249\n",
      "Step:  631 Reward:  13.804042947184698\n",
      "Step:  632 Reward:  12.78993969801894\n",
      "Step:  633 Reward:  16.823036233631182\n",
      "Step:  634 Reward:  15.041813999013428\n",
      "Step:  635 Reward:  14.943793813510977\n",
      "Step:  636 Reward:  17.32947943807815\n",
      "Step:  637 Reward:  18.229896994190476\n",
      "Step:  638 Reward:  27.716420178199233\n",
      "Step:  639 Reward:  25.145028653527174\n",
      "Step:  640 Reward:  40.42192354638346\n",
      "Step:  641 Reward:  56.9432325857583\n",
      "Step:  642 Reward:  59.3383179083888\n",
      "Step:  643 Reward:  24.310641461191693\n",
      "Step:  644 Reward:  60.41297892289664\n",
      "Step:  645 Reward:  29.605352103752455\n",
      "Step:  646 Reward:  205.09479152624635\n",
      "Step:  647 Reward:  199.63405632671055\n",
      "Step:  648 Reward:  66.62759630065119\n",
      "Step:  649 Reward:  125.88021683380485\n",
      "Step:  650 Reward:  33.472688347284404\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  651 Reward:  189.22885130231208\n",
      "Step:  652 Reward:  172.18773382125255\n",
      "Step:  653 Reward:  181.0445233885804\n",
      "Step:  654 Reward:  177.35085494953609\n",
      "Step:  655 Reward:  192.6627135199184\n",
      "Step:  656 Reward:  154.43477900807346\n",
      "Step:  657 Reward:  199.17292042522578\n",
      "Step:  658 Reward:  188.9317631720433\n",
      "Step:  659 Reward:  194.9414625504449\n",
      "Step:  660 Reward:  205.99107498189528\n",
      "Step:  661 Reward:  205.4438193191707\n",
      "Step:  662 Reward:  224.0756692290723\n",
      "Step:  663 Reward:  234.4732212745785\n",
      "Step:  664 Reward:  219.27280923578562\n",
      "Step:  665 Reward:  229.46511749586452\n",
      "Step:  666 Reward:  233.7690962850002\n",
      "Step:  667 Reward:  243.5227999109979\n",
      "Step:  668 Reward:  254.94216242025408\n",
      "Step:  669 Reward:  247.75596730036997\n",
      "Step:  670 Reward:  249.52004058129387\n",
      "Step:  671 Reward:  246.39014363525072\n",
      "Step:  672 Reward:  259.4183879483405\n",
      "Step:  673 Reward:  263.78305733487895\n",
      "Step:  674 Reward:  262.8495295732538\n",
      "Step:  675 Reward:  272.31986963585325\n",
      "Step:  676 Reward:  265.8500156667384\n",
      "Step:  677 Reward:  244.74038941636917\n",
      "Step:  678 Reward:  244.1506085883999\n",
      "Step:  679 Reward:  55.4117578502105\n",
      "Step:  680 Reward:  279.93261534258477\n",
      "Step:  681 Reward:  271.6288700213936\n",
      "Step:  682 Reward:  283.33484383802204\n",
      "Step:  683 Reward:  286.6631746033079\n",
      "Step:  684 Reward:  284.3123660706437\n",
      "Step:  685 Reward:  282.64220414251116\n",
      "Step:  686 Reward:  289.8833079347083\n",
      "Step:  687 Reward:  289.13126828291735\n",
      "Step:  688 Reward:  290.6747308706247\n",
      "Step:  689 Reward:  288.0931973527783\n",
      "Step:  690 Reward:  288.8893556080136\n",
      "Step:  691 Reward:  291.69353801395164\n",
      "Step:  692 Reward:  291.3690148019358\n",
      "Step:  693 Reward:  291.1861397882984\n",
      "Step:  694 Reward:  290.3467811249694\n",
      "Step:  695 Reward:  292.51489238863377\n",
      "Step:  696 Reward:  291.5580869725756\n",
      "Step:  697 Reward:  163.86161714345604\n",
      "Step:  698 Reward:  294.764862621303\n",
      "Step:  699 Reward:  177.7568261124128\n",
      "Step:  700 Reward:  295.5091596072252\n",
      "Step:  701 Reward:  297.29869437816086\n",
      "Step:  702 Reward:  298.33163851425826\n",
      "Step:  703 Reward:  301.3642168231695\n",
      "Step:  704 Reward:  241.7530881853163\n",
      "Step:  705 Reward:  297.30791622267634\n",
      "Step:  706 Reward:  252.05082813867202\n",
      "Step:  707 Reward:  297.64854526366264\n",
      "Step:  708 Reward:  82.55542576746319\n",
      "Step:  709 Reward:  299.27940481242644\n",
      "Step:  710 Reward:  303.4929978145113\n",
      "Step:  711 Reward:  302.8605319715494\n",
      "Step:  712 Reward:  301.8712346968758\n",
      "Step:  713 Reward:  112.81012519505255\n",
      "Step:  714 Reward:  132.7810232249013\n",
      "Step:  715 Reward:  304.69264001067984\n",
      "Step:  716 Reward:  302.672808422668\n",
      "Step:  717 Reward:  300.5428910582085\n",
      "Step:  718 Reward:  301.62258890615254\n",
      "Step:  719 Reward:  303.8507088482292\n",
      "Step:  720 Reward:  304.73623917146335\n",
      "Step:  721 Reward:  304.7691339032591\n",
      "Step:  722 Reward:  306.48149035944675\n",
      "Step:  723 Reward:  306.91323077076515\n",
      "Step:  724 Reward:  309.5156607756111\n",
      "Step:  725 Reward:  308.59312665524493\n",
      "Step:  726 Reward:  303.891191750274\n",
      "Step:  727 Reward:  308.8404910131692\n",
      "Step:  728 Reward:  306.995554490362\n",
      "Step:  729 Reward:  308.07399468371256\n",
      "Step:  730 Reward:  310.5319083228841\n",
      "Step:  731 Reward:  309.36403748003085\n",
      "Step:  732 Reward:  310.1311824002711\n",
      "Step:  733 Reward:  309.56409698146797\n",
      "Step:  734 Reward:  310.2312160288098\n",
      "Step:  735 Reward:  309.14183807787646\n",
      "Step:  736 Reward:  311.5369408597688\n",
      "Step:  737 Reward:  313.26578016517726\n",
      "Step:  738 Reward:  311.82618034688323\n",
      "Step:  739 Reward:  311.39623841333173\n",
      "Step:  740 Reward:  311.9992836789837\n",
      "Step:  741 Reward:  309.1159582734317\n",
      "Step:  742 Reward:  309.56142535098326\n",
      "Step:  743 Reward:  312.7525157380931\n",
      "Step:  744 Reward:  310.0358114759551\n",
      "Step:  745 Reward:  230.79230422826922\n",
      "Step:  746 Reward:  311.5463881687091\n",
      "Step:  747 Reward:  308.3853732793939\n",
      "Step:  748 Reward:  307.12008035949145\n",
      "Step:  749 Reward:  309.9731518706875\n",
      "Step:  750 Reward:  310.52075085438963\n",
      "Step:  751 Reward:  309.6170770342221\n",
      "Step:  752 Reward:  309.00594974749185\n",
      "Step:  753 Reward:  311.1131129681686\n",
      "Step:  754 Reward:  310.13309834219393\n",
      "Step:  755 Reward:  310.760273493147\n",
      "Step:  756 Reward:  307.66674565595866\n",
      "Step:  757 Reward:  311.94745680595287\n",
      "Step:  758 Reward:  312.26544857789344\n",
      "Step:  759 Reward:  311.2680401287606\n",
      "Step:  760 Reward:  312.59386734126576\n",
      "Step:  761 Reward:  312.3180604624863\n",
      "Step:  762 Reward:  311.2039325864726\n",
      "Step:  763 Reward:  312.4708184624004\n",
      "Step:  764 Reward:  312.6934643970576\n",
      "Step:  765 Reward:  314.50519075192256\n",
      "Step:  766 Reward:  313.9199970416134\n",
      "Step:  767 Reward:  315.55588441206413\n",
      "Step:  768 Reward:  316.1609903820398\n",
      "Step:  769 Reward:  313.56940134661426\n",
      "Step:  770 Reward:  316.5708962463802\n",
      "Step:  771 Reward:  314.9117694590472\n",
      "Step:  772 Reward:  313.45219089082275\n",
      "Step:  773 Reward:  316.191277953773\n",
      "Step:  774 Reward:  316.5819984611074\n",
      "Step:  775 Reward:  316.4594755269241\n",
      "Step:  776 Reward:  316.68796553573725\n",
      "Step:  777 Reward:  313.7830027452852\n",
      "Step:  778 Reward:  315.78919520693705\n",
      "Step:  779 Reward:  313.37084159074635\n",
      "Step:  780 Reward:  316.13376900758993\n",
      "Step:  781 Reward:  316.49686580721533\n",
      "Step:  782 Reward:  315.21063164922674\n",
      "Step:  783 Reward:  317.9601499395322\n",
      "Step:  784 Reward:  315.1679581157865\n",
      "Step:  785 Reward:  316.59057585747934\n",
      "Step:  786 Reward:  317.768009702771\n",
      "Step:  787 Reward:  315.62722568639674\n",
      "Step:  788 Reward:  315.36090551778625\n",
      "Step:  789 Reward:  172.53617593949593\n",
      "Step:  790 Reward:  315.62111041047774\n",
      "Step:  791 Reward:  316.7904488845704\n",
      "Step:  792 Reward:  314.1087528160062\n",
      "Step:  793 Reward:  315.57324396401833\n",
      "Step:  794 Reward:  316.498887711593\n",
      "Step:  795 Reward:  124.26571185596012\n",
      "Step:  796 Reward:  318.10396403756295\n",
      "Step:  797 Reward:  317.89963450625265\n",
      "Step:  798 Reward:  317.4000671865334\n",
      "Step:  799 Reward:  316.8122134871624\n",
      "Step:  800 Reward:  315.6517986761941\n",
      "Step:  801 Reward:  317.8679392351833\n",
      "Step:  802 Reward:  319.2356614893759\n",
      "Step:  803 Reward:  317.4590891204637\n",
      "Step:  804 Reward:  317.9795679961159\n",
      "Step:  805 Reward:  316.2718808378282\n",
      "Step:  806 Reward:  317.5110318345473\n",
      "Step:  807 Reward:  319.70145033976115\n",
      "Step:  808 Reward:  318.87061726306064\n",
      "Step:  809 Reward:  320.34972669514855\n",
      "Step:  810 Reward:  320.16505858375615\n",
      "Step:  811 Reward:  319.8015827793187\n",
      "Step:  812 Reward:  317.9567921127956\n",
      "Step:  813 Reward:  317.713467799916\n",
      "Step:  814 Reward:  318.39121034945856\n",
      "Step:  815 Reward:  319.8719748311579\n",
      "Step:  816 Reward:  319.92396705397164\n",
      "Step:  817 Reward:  317.4913293759428\n",
      "Step:  818 Reward:  317.72611500502984\n",
      "Step:  819 Reward:  319.4396719025425\n",
      "Step:  820 Reward:  319.2972293092051\n",
      "Step:  821 Reward:  317.8677366883844\n",
      "Step:  822 Reward:  317.20968189031413\n",
      "Step:  823 Reward:  316.32800009976074\n",
      "Step:  824 Reward:  317.9948084137061\n",
      "Step:  825 Reward:  317.068700296518\n",
      "Step:  826 Reward:  317.92128505398114\n",
      "Step:  827 Reward:  319.9467789389749\n",
      "Step:  828 Reward:  315.2339015829294\n",
      "Step:  829 Reward:  316.05702542162044\n",
      "Step:  830 Reward:  319.388006539721\n",
      "Step:  831 Reward:  319.22637849945994\n",
      "Step:  832 Reward:  318.0590226504435\n",
      "Step:  833 Reward:  316.5311893049124\n",
      "Step:  834 Reward:  320.5381427576658\n",
      "Step:  835 Reward:  317.5462948457149\n",
      "Step:  836 Reward:  316.9149690992684\n",
      "Step:  837 Reward:  319.6015813881776\n",
      "Step:  838 Reward:  316.6109513651811\n",
      "Step:  839 Reward:  317.2734738763591\n",
      "Step:  840 Reward:  318.9178634648851\n",
      "Step:  841 Reward:  320.06214401481844\n",
      "Step:  842 Reward:  318.76607223513577\n",
      "Step:  843 Reward:  321.5123991179235\n",
      "Step:  844 Reward:  319.99050075514253\n",
      "Step:  845 Reward:  318.7832904390619\n",
      "Step:  846 Reward:  321.4964649239581\n",
      "Step:  847 Reward:  318.5939984348388\n",
      "Step:  848 Reward:  321.9786973013237\n",
      "Step:  849 Reward:  318.0881911355077\n",
      "Step:  850 Reward:  319.09652849549036\n",
      "Step:  851 Reward:  320.29815687801164\n",
      "Step:  852 Reward:  319.5204418757005\n",
      "Step:  853 Reward:  321.67656854153216\n",
      "Step:  854 Reward:  322.8527834716203\n",
      "Step:  855 Reward:  321.7295598452567\n",
      "Step:  856 Reward:  322.07114506064056\n",
      "Step:  857 Reward:  321.6850833494067\n",
      "Step:  858 Reward:  320.1769084661001\n",
      "Step:  859 Reward:  321.2181913851104\n",
      "Step:  860 Reward:  321.52326954888775\n",
      "Step:  861 Reward:  319.33804551877887\n",
      "Step:  862 Reward:  320.7195802648499\n",
      "Step:  863 Reward:  320.28560952549753\n",
      "Step:  864 Reward:  319.55276949418834\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  865 Reward:  321.9671904634939\n",
      "Step:  866 Reward:  319.6975563667266\n",
      "Step:  867 Reward:  319.407801322776\n",
      "Step:  868 Reward:  320.7387384639884\n",
      "Step:  869 Reward:  321.4871634207794\n",
      "Step:  870 Reward:  322.4623776777662\n",
      "Step:  871 Reward:  320.09565183386593\n",
      "Step:  872 Reward:  322.9604226322185\n",
      "Step:  873 Reward:  320.32285440631733\n",
      "Step:  874 Reward:  321.09660356997756\n",
      "Step:  875 Reward:  319.8704258861878\n",
      "Step:  876 Reward:  322.7838334933718\n",
      "Step:  877 Reward:  322.2843878609591\n",
      "Step:  878 Reward:  322.3250573427822\n",
      "Step:  879 Reward:  321.30525741518966\n",
      "Step:  880 Reward:  324.0098535112502\n",
      "Step:  881 Reward:  320.33524567722594\n",
      "Step:  882 Reward:  323.52146212741906\n",
      "Step:  883 Reward:  322.6065443460234\n",
      "Step:  884 Reward:  320.6321971159979\n",
      "Step:  885 Reward:  323.19037593640263\n",
      "Step:  886 Reward:  322.3098352721045\n",
      "Step:  887 Reward:  320.28900221825637\n",
      "Step:  888 Reward:  320.94358868736776\n",
      "Step:  889 Reward:  322.20308980464154\n",
      "Step:  890 Reward:  322.27379497275126\n",
      "Step:  891 Reward:  323.3503035057233\n",
      "Step:  892 Reward:  322.1641781737589\n",
      "Step:  893 Reward:  322.7339743627475\n",
      "Step:  894 Reward:  318.51816985515194\n",
      "Step:  895 Reward:  322.89549309395943\n",
      "Step:  896 Reward:  323.5543058450304\n",
      "Step:  897 Reward:  320.81538863033825\n",
      "Step:  898 Reward:  320.48585060827526\n",
      "Step:  899 Reward:  322.8740379761294\n",
      "Step:  900 Reward:  319.6789546043593\n",
      "Step:  901 Reward:  319.7262430222595\n",
      "Step:  902 Reward:  321.43010468180415\n",
      "Step:  903 Reward:  321.09535990853635\n",
      "Step:  904 Reward:  320.0081712254409\n",
      "Step:  905 Reward:  324.36928167293763\n",
      "Step:  906 Reward:  324.5746828418315\n",
      "Step:  907 Reward:  322.3009524239779\n",
      "Step:  908 Reward:  321.0724615565031\n",
      "Step:  909 Reward:  324.16633000197106\n",
      "Step:  910 Reward:  322.4929537503743\n",
      "Step:  911 Reward:  323.5832362230889\n",
      "Step:  912 Reward:  325.56922703909083\n",
      "Step:  913 Reward:  323.30233003756643\n",
      "Step:  914 Reward:  321.96717183526266\n",
      "Step:  915 Reward:  322.4123828060959\n",
      "Step:  916 Reward:  322.3314085986711\n",
      "Step:  917 Reward:  326.112014420026\n",
      "Step:  918 Reward:  325.3356342777462\n",
      "Step:  919 Reward:  323.3154041031013\n",
      "Step:  920 Reward:  324.39070911211223\n",
      "Step:  921 Reward:  324.6511379146241\n",
      "Step:  922 Reward:  326.18453966057785\n",
      "Step:  923 Reward:  324.3733230217801\n",
      "Step:  924 Reward:  326.12598462532776\n",
      "Step:  925 Reward:  325.8816642776601\n",
      "Step:  926 Reward:  323.77223046072453\n",
      "Step:  927 Reward:  323.2646227148346\n",
      "Step:  928 Reward:  326.5866867307181\n",
      "Step:  929 Reward:  322.93563440134477\n",
      "Step:  930 Reward:  324.33123615484516\n",
      "Step:  931 Reward:  324.7037315774168\n",
      "Step:  932 Reward:  322.67583740594955\n",
      "Step:  933 Reward:  324.2264211352726\n",
      "Step:  934 Reward:  323.81880100095657\n",
      "Step:  935 Reward:  325.8209884767945\n",
      "Step:  936 Reward:  323.47888529281954\n",
      "Step:  937 Reward:  324.16247636403034\n",
      "Step:  938 Reward:  323.59171989241855\n",
      "Step:  939 Reward:  325.61534718257434\n",
      "Step:  940 Reward:  323.05232286251436\n",
      "Step:  941 Reward:  325.6929063780069\n",
      "Step:  942 Reward:  326.52150219354036\n",
      "Step:  943 Reward:  324.72291543434136\n",
      "Step:  944 Reward:  323.3716648414978\n",
      "Step:  945 Reward:  323.7524015215313\n",
      "Step:  946 Reward:  324.856507653496\n",
      "Step:  947 Reward:  323.77125794206904\n",
      "Step:  948 Reward:  326.3312938885146\n",
      "Step:  949 Reward:  324.719251703322\n",
      "Step:  950 Reward:  325.61276048792854\n",
      "Step:  951 Reward:  324.7895944448567\n",
      "Step:  952 Reward:  325.13126769062535\n",
      "Step:  953 Reward:  325.589902938485\n",
      "Step:  954 Reward:  324.46005724925703\n",
      "Step:  955 Reward:  324.0459899081894\n",
      "Step:  956 Reward:  325.7103570587748\n",
      "Step:  957 Reward:  324.72992877799993\n",
      "Step:  958 Reward:  323.8007638262176\n",
      "Step:  959 Reward:  326.9430433636467\n",
      "Step:  960 Reward:  326.2010898999974\n",
      "Step:  961 Reward:  326.3491479325159\n",
      "Step:  962 Reward:  325.99715472907815\n",
      "Step:  963 Reward:  326.8362090746563\n",
      "Step:  964 Reward:  324.3905522256298\n",
      "Step:  965 Reward:  323.1450369131125\n",
      "Step:  966 Reward:  324.09761185546466\n",
      "Step:  967 Reward:  325.26176284704974\n",
      "Step:  968 Reward:  323.1997203117613\n",
      "Step:  969 Reward:  326.7685839841724\n",
      "Step:  970 Reward:  325.93912912454505\n",
      "Step:  971 Reward:  327.10630815869916\n",
      "Step:  972 Reward:  325.64845933605653\n",
      "Step:  973 Reward:  325.2113737252794\n",
      "Step:  974 Reward:  327.0883378983572\n",
      "Step:  975 Reward:  326.47260508984397\n",
      "Step:  976 Reward:  324.1060974164732\n",
      "Step:  977 Reward:  324.94458137949357\n",
      "Step:  978 Reward:  326.41355482359626\n",
      "Step:  979 Reward:  324.10441609662865\n",
      "Step:  980 Reward:  327.3220331983402\n",
      "Step:  981 Reward:  327.5716167818407\n",
      "Step:  982 Reward:  328.06671550432884\n",
      "Step:  983 Reward:  325.51378196163733\n",
      "Step:  984 Reward:  325.09064474640184\n",
      "Step:  985 Reward:  326.1172672161966\n",
      "Step:  986 Reward:  324.8953226638927\n",
      "Step:  987 Reward:  326.1482602351649\n",
      "Step:  988 Reward:  325.5671499899039\n",
      "Step:  989 Reward:  325.63975948778807\n",
      "Step:  990 Reward:  325.98594853311545\n",
      "Step:  991 Reward:  326.73424001095907\n",
      "Step:  992 Reward:  323.96086316453705\n",
      "Step:  993 Reward:  327.04615353278854\n",
      "Step:  994 Reward:  326.1431386219342\n",
      "Step:  995 Reward:  325.67109241201194\n",
      "Step:  996 Reward:  325.2840031137363\n",
      "Step:  997 Reward:  326.5312005861776\n",
      "Step:  998 Reward:  326.0743145351608\n",
      "Step:  999 Reward:  324.7250702858516\n",
      "Time elapsed: 3:38:01.635916\n"
     ]
    }
   ],
   "source": [
    "# Run the training process, and time it\n",
    "start = datetime.datetime.now()\n",
    "ENV_NAME = 'BipedalWalker-v3'\n",
    "\n",
    "videos_dir = mkdir('.', 'videos')\n",
    "monitor_dir = mkdir(videos_dir, ENV_NAME)\n",
    "\n",
    "hp = HP(env_name=ENV_NAME)\n",
    "trainer = ARSTrainer(hp=hp, monitor_dir=monitor_dir)\n",
    "trainer.train()\n",
    "end = datetime.datetime.now()\n",
    "print(\"Time elapsed:\", end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtcAAAH0CAYAAAD2XgFYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3xT5f4H8M9J2iTddFBKS4EWZLQgslpAVksBZTlQEVDLdaGIXkVUFBUU9eIA7gVFvMhPxQnKEkW5lLJlD9m77JbSFuhu0uT5/VFySNqkTUuapOnn/XrxIufk7NPxydPveR5JCCFARERERES3TOHsAyAiIiIichcM10REREREdsJwTURERERkJwzXRERERER2wnBNRERERGQnDNdERERERHbCcE3kgtavXw9JknDhwoVb2k5ubi7uu+8++Pv7Q5IknDlzxunHZIszZ85AkiRs3rzZ5nW+/vpreHh41OJR3VT+Wjjy2tQFlu6fJEn47rvvnHhU7qdbt24YP358tdaZNGkS2rVrV0tH5Hr+/PNPSJKErKwsZx8K1SMM1+RWxowZA0mSIEkSlEolmjRpgsceewwXL1509qE5xeeff46tW7diy5YtSE9PR2RkZIVljEHI+M/DwwORkZEYN24c8vLy5OV69OiB9PR0hIeHO/IUaqSoqAgajQaff/652fyZM2dCkiSL8zUaDYqKihx5mLXq/PnzGDBgAMLDw6FWqxEeHo7HHnusyg8AU6dOlb8WFAoFIiIiMHLkSJw9e/aWjic9PR0PPPDALW2jLjH9WWTt3/r1629pH6tWrcK//vWvaq3z5ptvYsOGDbe0X1uVlpZi+vTpiImJgbe3NwIDA9GxY0e8++67Dtk/ACQmJiI9PR3BwcEO2ycRwzW5nV69eiE9PR3nzp3DDz/8gL179+LBBx909mFVIISATqer1X2cOHECsbGxaN++PcLCwqBUKq0uu2LFCqSnp+PMmTOYP38+VqxYgRdffFF+X6VSISwsDAqF6//Y8PLyQvfu3bF27Vqz+ampqWjatKnF+T169ICXl5cjD7NKWq22xut6eHhg+PDhWLlyJU6cOIHFixfj+PHjGDp0aJXrNm/eHOnp6bhw4QIWLlyIXbt2YejQodDr9TU+nrCwMGg0mhqv78os3af//Oc/SE9Pl/81b94cL7/8stm8Hj16VFhPr9fbfJ2DgoLg5+dXrWP19fV1WNB888038cknn+Ctt97CwYMHsWnTJrz66qtmH9prm/HnliRJDtsnkev/liSqJuMP04iICPTu3RtPP/00tm7ditzcXLPl5syZgzZt2kCj0eC2227D+++/j9LSUgDAl19+iSZNmsjLGlt3H3nkEXneV199hUaNGsE4yOnkyZPRtm1beHt7IzIyEs888wyuX78uL28sW1i3bh06duwItVqN1atXy8fSpEkTeHt7Y+DAgTh37lyV56nT6TBp0iRERERApVIhJiYGP/zwg/x+8+bNsWDBAqSmpkKSJPTt27fS7QUFBSEsLAxNmjTBXXfdhYcffhi7du2S37dWCrFy5UrExcVBo9EgNjYWa9asMdvuyZMnMXz4cDRo0ACBgYEYMGAADhw4YLbM4sWL0bJlS2g0GvTo0QP79+83e18IgaeeegotWrSAl5cXoqOj8cYbb6CkpMTq+fTr1w/r1q2DwWAAUBZaNm3ahLfeesvi/KSkJHndqu5lVQwGA8aPH48mTZrg4MGDAMpa8aZOnYqoqCj5Wn3xxRdm60mShNmzZ2PUqFEICAjA6NGjLW5/8uTJaN26dYX5zz77LLp16wYAaNy4McaOHYvOnTujadOm6NmzJ15//XXs27evynNRKpUICwtDeHg4+vXrh6lTp+LAgQM4efIkgLJW6IcffhgNGjSAl5cX+vbta/a1Ykn5spD8/Hy8+OKLiIyMhFqtRvPmzfHBBx8AAPr06YOnn37abH0hBFq0aIGpU6da3Udlx2UwGNC0aVN5H0YlJSUIDAzEvHnz5HmV/WwAyr633nzzTYwbNw7BwcG48847KxxLQEAAwsLC5H9KpRK+vr5m81QqlVym8d1336FVq1ZQq9VIS0vD9u3bMWDAADRs2BB+fn6Ij4+v8KGwfFlIt27d8Nxzz+Htt99GaGgogoOD8dRTT5n9RaZ8WYhx+ueff0arVq3g6+uLpKSkCn+pWLhwofy126tXLyxfvhySJFV635cvX45nnnkGI0eORHR0NNq1a4eRI0fi448/Nltu1apV6NatG7y8vNCkSRM89dRTuHr1qvz+33//jaSkJDRo0AA+Pj6IiYnBokWL5Pfnzp2L1q1bQ6PRIDg4GAkJCbh8+TIAy2UhmzdvRs+ePaHRaBAUFITHHnsM2dnZ1b4mRFYJIjeSnJws+vXrJ09fvHhR9O7dWyiVSpGfny/PnzJlimjatKlYunSpOH36tPj9999FZGSkePPNN4UQQpw+fVoAEEePHhVCCPHll1+Khg0bisaNG8vbeOSRR8SIESPk6WnTpomNGzeKtLQ0kZKSIlq3bi0ee+wx+f2vvvpKSJIkunTpItauXStOnTolMjMzxfLly4VSqRQzZswQx44dE19++aUIDQ0VAMT58+etnuvEiRNFUFCQWLx4sTh27Jh4//33hSRJIiUlRQghRGZmpnjooYdEr169RHp6usjOzra4nbS0NAFAbNq0SZ536tQpERMTI5555hl53rp168yOyTjdsmVLsXLlSnH48GHx+OOPC41GIy5cuCCEECIjI0M0atRIPPPMM2L//v3i6NGjYvz48SIoKEhkZmYKIYTYs2ePkCRJTJo0SRw9elQsWbJENG/e3OyY9Hq9mDx5sti2bZtIS0sTK1asEGFhYeLtt982u75KpVKe/uuvvwQAsXv3biGEEFu3bhVBQUFCp9OJwMBAs/kAxLZt22y+l9auxfnz50VRUZG4//77Rdu2bcXZs2fldZKTk0X79u3F6tWrxenTp8VPP/0kAgICxJdffikvA0AEBQWJ2bNni5MnT4pjx45ZvGfHjh0TAMRff/0lzyspKRFBQUFi7ty5Fte5cuWKGD58uOjUqZPF942mTJkiWrRoYTZvyZIlAoA4cOCAMBgMIi4uTnTo0EFs2rRJ7N+/Xzz00EOiQYMG4sqVK0IIy19TAMS3334rhBDCYDCIPn36iKioKLFs2TJx6tQpsWHDBvHf//5XCCHEDz/8IHx9fUVeXp68fkpKilAoFGbX1JQtxzVp0iTRunVrs/V+/vlnoVarRU5Ojnz+lf1sEEKIZs2aCT8/PzFlyhRx7NgxcejQoUqvqRBCtGjRQkyZMqXC/Ndee014e3uLxMREsX37dnHkyBGRn58v1qxZIxYuXCgOHz4sjh49Kl555RWhVqvF6dOn5XXj4+PFc889ZzYdEBAgXn31VXH06FHx22+/CV9fX/HBBx+Y7S82NtZs2tfXVwwaNEjs2bNH7N69W8TGxooBAwbIy2zZskVIkiTeeecdcezYMfHLL7+IqKgoAUDs3LnT6jn37dtX9OjRQ1y6dMnqMqtWrRLe3t5i7ty54sSJE2Lbtm3izjvvFP3795eXue2220RycrI4fPiwOHXqlPjtt9/EqlWrhBBCbN68WXh6eooffvhBnDlzRvz9999i3rx5IiMjQwghxB9//CEAyF8D586dE97e3iI5OVkcOHBArF+/XrRp00YkJSVV65oQVYbhmtxKcnKyUCqVwsfHR3h5eQkAAoB4+eWX5WUKCgqEl5eX+OOPP8zW/eabb0RAQIA83bx5c/HZZ58JIYQYNWqUePvtt4Wfn5/8izQiIkJ88cUXVo9l6dKlQqVSCb1eL4QoC38AxMaNG82Wu/POO8WoUaPM5r388suVhuuCggKhUqnk4zO69957RUJCgtn1MP2wYYkxCHl5eQkfHx+hVqsFAJGQkGAWbqwFStNwqNPpRNOmTcXkyZOFEGVBJT4+3mx/BoNBREdHi1mzZgkhhBg9erTo3r272TJz5sypEM7KmzlzpmjZsqU8XT5c63Q64e/vLz766CMhhBDvvfeeuO+++4QQQtxzzz1m8wMCAkRpaanVfZW/l9auxYEDB0SvXr1Ejx49zD7MnD59WkiSJI4cOWK23XfeeUd06NBBngYgHn/8cavHYSo+Pt7sw8+SJUuESqWq8CHq4Ycflr8XunfvLn+osaZ8uD579qyIi4sTkZGRQqvVipSUFAHALFAWFxeLsLAw8c477wghqg7Xxm1YC2YlJSUiJCREzJ8/3+w8Bg0aZPW4bTmuI0eOVPggNXToUPHAAw8IIWz/2dCsWTORmJho9VgsqSxcK5XKSgOoUatWrcQnn3wiT1sK1127djVbJzk5WfTt29dsf+XDtUqlkj9cCFH2veTh4SF/T9x///1m4VMIIWbNmlVluN6/f79o06aNkCRJtGnTRowZM0b8+OOPZt9r8fHxFa6L8cPjkSNHhMFgEGq1Wvz4448W9/HDDz+I4OBgs8YTU+XD9cSJE0VUVJTQ6XTyMtu2bRMAxPbt222+JkSVYVkIuZ34+Hjs27cPO3bswFtvvYVu3bph2rRp8vuHDh1CUVERhg8fDl9fX/nf2LFjcf36dVy5cgUAkJCQgNTUVADAunXrMHDgQPTq1Qupqak4duwYLl68iMTERHm7S5cuRe/evREeHg5fX1+MHj0aWq0WGRkZZsfXtWtXs+nDhw9XqL3s2bNnped48uRJaLVa9O7d22x+nz59cOjQIRuvlLmvvvoK+/btw/79+7F69Wrk5+dj2LBhcvmENd27d5dfe3h4IC4uDocPHwYA7Ny5E7t37za7zn5+fjhz5gxOnDgBoOz8y/9Z3dL5z58/H/Hx8WjUqBF8fX3x+uuvV/pnWg8PD/Tu3RspKSkAyuqqjfcrMTHRbH5CQoJZPbqt97K8QYMGAQBSUlIQFBQkz9+1axeEEOjSpYvZtfjggw/k62AUFxdnNv3MM8+YrWMsGXrsscewaNEiud7322+/xdChQ832CwCzZs3C3r178ccffwAAHn744Sprek+fPg1fX194e3ujWbNmEEJg2bJl8PT0xKFDhxAcHIyYmBh5ebVajfj4eJu/9nbv3o3AwEB06dLF4vsqlQpjxozB/PnzAQDZ2dlYtmwZnnrqKavbtOW42rRpg65du2LhwoUAgKysLPz5559ITk6Wt2HLzwag4n26FZGRkWjcuLHZvIyMDIwdOxatW7dGQEAAfH19cfLkySpLE+644w6z6YiICLlEwppmzZohMDDQbJ3S0lK5VOLw4cNyuZGR6fe9Ne3bt8ehQ4ewY8cOPPvssygsLERycjJ69eoFrVYLIQR2796N6dOnm13vTp06ASh7ZkSSJEycOBGPPvooEhMT8e677+Lvv/+W9zFo0CCEhYWhefPmGDVqFL788kvk5ORYPaZDhw6hR48eZj0LGcvaTL9+q7omRJVxTL9VRA7k5eWFli1bAgDatWuH48eP47nnnsP//d//AYAcFo31dOUZw0liYiL++c9/4tChQ8jLy0NcXBwSExOxdu1aKJVKREZGyvvZvn07HnzwQbz++uv4+OOPERgYiG3btiE5OdnsYSelUmnxoa6aPmxTfj0hRI23FRERIZ9Pq1at4Ofnhx49emD9+vVmHyKqIm7UoANl17pfv3749NNPKywXEBBg8zH//PPPeO655zB9+nT06dMH/v7++PnnnzF58uRK10tKSsIbb7yB3Nxc/PXXX5gzZw6Asg9Or7/+ujzftAbU1ntpybBhw/DVV19h69atZtfM+DX3119/wdvb22yd8ufu4+NjNv3uu+9i4sSJ8rSxt5aHH34YL730ElauXImEhASsWrUKP//8c4VjMtb3tm7dGh06dEB4eDjWrFmDu+66y+p5REZGYu3atVAoFAgLC6vymIHqf+1VtezYsWMxY8YM7N+/H6mpqQgKCsKQIUOqvc3yx5WcnIwpU6Zg1qxZ+PHHHxEYGChfC1t/NgAV79OtsLSt0aNHIycnBzNmzECzZs3g5eWF++67r8qvQZVKZTYtSVKVH5AtrQPAbL2a/lxRKBTo0qULunTpghdeeAEpKSno378/li1bhgcffBAGgwHvvPMOHnrooQrrGj9wvPfeexgzZgz+/PNPrF27Fu+99x7eeustvPXWWwgICMC+ffuwadMmrF27FnPmzMGrr76KDRs2oH379haPydq5mM635ZoQWcNwTW5v6tSpiI2Nxbhx49ClSxfExsZCo9Hg9OnTckujJf369UNOTg5mzZqF3r17w8PDA4mJiXj//fehUCjMwtPmzZsREhKC9957T573yy+/2HR8MTEx2LJlC8aNGyfP27JlS6XrtGzZEmq1Ghs2bEBsbKw8f+PGjWbTt8LYslNYWFjpctu2bZNbC0tLS7Fz5075wc8uXbrg66+/RkREhNWeOGJjYyucb/npjRs3omPHjpgwYYI8z5Y+u/v164fCwkLMnDkTDRo0kI+zXbt28PHxwcyZM1FcXGz2MOOt3MtJkyahefPmGDJkCJYvX44BAwYAADp37gwAOHfuXJUBsbzQ0FCEhoZWmG8MmwsXLsTly5cREBCAu+++u9JtGYNBZQ+CAoCnp6f8Qau82NhYZGVl4fDhw/L1LCkpwY4dO8y+hivTuXNn5OTkYNeuXVZbr1u2bInExETMnz8f69atwz/+8Y9K+zG39bhGjhyJCRMm4Pfff8e3336LUaNGydu19WdDbRNCYNOmTfjiiy/kr5fr16/j7NmzFh+erG0xMTHYunWr2bxt27bVaFtt27YFAGRmZkKhUKBTp044fPiw1a83o5YtW2L8+PEYP348pk6dinnz5uGtt94CUPazKiEhAQkJCXj33Xdx22234aeffrIYrmNjY7FkyRKUlpbK933Hjh0oLi62289OIoZrcntt2rTBkCFD8Prrr2PNmjXw9fXFG2+8gTfeeAMA0L9/f5SWluLAgQPYu3cvPvzwQwBlrSatW7fGN998g+nTpwMo+5OrQqHAr7/+igULFsj7aN26Na5cuYIFCxYgISEBmzdvxty5c206vpdffhkPPvgg4uLiMGjQIGzevBnffvttpet4e3vjhRdewFtvvYWGDRvijjvuwM8//4wVK1ZU6K3DVjk5OcjIyIDBYMDZs2fx6quvIjQ01GJ3YaamT5+OsLAwREVFYebMmbh8+TKeffZZAMD48eOxYMEC3HvvvXjzzTcRGRmJCxcu4I8//sDgwYPRo0cPvPTSS+jatSsmT56M5ORkHDp0CDNmzDDbR+vWrbFgwQKsWLEC7dq1w2+//YalS5dWeU7t2rVDo0aNMHPmTLNQa+w9ZebMmYiIiECbNm3M9lXTewkAEydOhEqlwj333INffvkFgwcPRsuWLfH444/jqaeewkcffYTu3bujoKAAu3fvxpUrV/Daa6/ZvH1TycnJeOCBB3Dq1CmMHDkSnp6e8ntLlixBQUEBOnXqBD8/P5w4cQJTpkxBkyZNqvWXiPISExMRFxeHUaNG4bPPPkNAQACmTZuG4uJi+b7bso1evXphxIgRmDlzJm6//XZcunQJR44cwZNPPikvN3bsWDzyyCPQ6XRYuXKlXY4rKCgIgwcPxrvvvot9+/aZ9dhi68+G2iZJElq1aoVvv/0WcXFxKCkpqfKvNLXp5ZdfRs+ePfHee+9hxIgROHDgAGbPni0fqzXDhg1DQkICunfvjsaNG+P8+fN45513oNFo5A+C7733HgYPHozw8HCMHj0aPj4+OHHiBBYtWoQFCxbg+vXrmDp1Ku677z5ERUUhOzsba9askT9A/fLLL7h06RJ69uyJkJAQbN++HZcuXTIrDzL1z3/+E3PnzsWTTz6JV155BVlZWXj22WeRlJRUoWSPqMacVu1NVAusPcC3efNmAUDuSUOIsh5AOnToINRqtWjQoIGIi4ur0NPCuHHjBACxZ88eed79999v8WHDN998U4SGhgpvb29x9913ix9++EEAEGlpaUKIig/cmfr3v/8twsPDhUajEf369RNff/11lb2FaLVa8dprr4nw8HDh6ekp2rZtK77//nubrocp48Nnxn+SJIlGjRqJe++9Vxw4cEBeztpDfCtWrBCdOnUSKpVKtG3bVvz5559m2z9z5owYNWqUCAkJESqVSjRt2lSMHj3arNeDH3/8UURHRwuVSiXi4uLE8uXLzR6I02q14umnnxaBgYHCz89PjBw5Un7o0cja9R05cqQAYPZwnBBCzJ07VwAw6wXEqKp7WVlvIabbV6vVYvny5UIIIUpLS8WHH34oWrduLTw9PUVwcLDo3bu3WLx4sbwOTB76s4VWqxUNGzYUAMSuXbvM3lu5cqXce4RarRbR0dHimWeeEefOnat0m5Z6Cynv0qVLYsSIESIgIEBoNBrRu3dvswfbqnqgUQghcnNzxfjx40VYWJjw9PQUzZs3F//6178snp+tvTRUdVxGxq+vdu3aWdxOVT8bmjVrJqZNm2bTMRlV9kCj6QOGRnv27BFxcXFCo9GIqKgoMX/+fHHnnXeKsWPHystYeqDRdFoIISZPnmzWQ4qlBxrL73/NmjUCgEhPT5fnff3116J58+ZCrVaLnj17iu+//14AEAcPHrR6znPnzhWJiYmiUaNGQqVSiYiICHHfffeJHTt2mC23du1a0bdvX+Hj4yN8fHxETEyMeOmll4TBYBB5eXlixIgRolmzZkKlUonQ0FAxatQo+QHQlJQU0adPHxEUFCTUarVo1aqV+Pjjj+Vtl3+gUQghNm3aJHr06CHUarUIDAwUjz76qMjKyqr2NSGyRhLCpECSiMhG69evR0JCAs6fP2/WJziRveTk5CAiIgLfffcdhg8f7uzDIRP//e9/8dxzz+H69esVavKJ6juWhRARkUvR6XS4fPkypk2bhvDwcNx7773OPqR676OPPpIHctm+fTsmT56M0aNHM1gTWcBwTURELmXLli1ISEhAVFQUFi5caNZNIjnHnj17MHPmTFy9ehVNmzbFk08+iSlTpjj7sIhcEstCiIiIiIjshIPIEBERERHZCcM1EREREZGdMFwTEREREdmJWz3QeOnSJafsNyQkBFlZWU7ZNzkO77P74z2uH3if6wfe5/rBWfc5PDzc6ntsuSYiIiIishOGayIiIiIiO2G4JiIiIiKyE7equS5PCIHi4mIYDAZIklRr+7l8+TJKSkpqbfu1RQgBhUIBjUZTq9eHiIiIqL5w63BdXFwMT09PeHjU7ml6eHjU2RHESktLUVxcDC8vL2cfChEREVGd59ZlIQaDodaDdV3n4eEBg8Hg7MMgIiIicgtuHa5Z6mAbXiciIiIi+3DrcO0KIiMj0b9/fyQmJiI5ORnXr193ynGcP38eiYmJTtk3ERERUX3BcF3LNBoN1qxZg9TUVDRo0ABff/21Q/ar1+sdsh8iIiIiuonh2oE6d+6MjIwMefrzzz/HoEGDkJSUhE8++QQAMHfuXCxYsAAAMGXKFDz44IMAgE2bNuH5558HAEyaNAl33303EhIS5PUAID4+HrNmzcK9996L3377Dfv370dSUhKGDh3qsFBPREREVJ/Vm6f9tr69FdmHs+26zeCYYHR/t7tNy+r1emzevBkjR44EAGzYsAFpaWn4/fffIYTAmDFjsG3bNsTHx+OLL77AE088gf3790Or1UKn02Hnzp2Ii4sDALz22msIDAyEXq/HiBEjcPjwYcTExAAA1Go1li9fDgBISkrCtGnT0L17d0ybNs2u505EREREFbHlupYVFxejf//+aNeuHa5du4bevXsDKAvXGzZswIABAzBw4ECcOnUKaWlpuP3223HgwAHk5+dDpVKhc+fO+Pvvv7F9+3bEx8cDAFauXImBAwdi4MCBOHbsGE6cOCHvb9iwYQCA3NxcXL9+Hd27l4X/4cOHO/jMiYiIiOqfetNybWsLs70Za65zc3ORnJyMr7/+Gk888QSEEBg/fjweffTRCus0adIEixYtQpcuXdC2bVv89ddfOHv2LG677TacO3cOX3zxBX7//Xc0aNAAL774IoqLi+V1vb29AZQNEMNeQIiIiIgciy3XDuLv749p06Zh3rx50Ol06Nu3LxYtWoSCggIAQHp6OrKysgAA3bp1w7x58xAfH4/4+Hh8++23iI2NhSRJyMvLg5eXF/z9/XHlyhWsW7fO4v4CAgLg7++PHTt2AACWLVvmmBMlIiIiqsfqTcu1K2jXrh1iYmKwYsUKPPDAAzhx4oRcxuHt7Y05c+YgJCQEcXFxmD17Nrp06QJvb2+o1Wq53jo2Nhbt2rVDQkICmjZtiq5du1rd38yZMzFhwgR4eXmhb9++jjhFIiIionpNEkIIZx+EvVy6dMlsurCwUC6TqE0eHh4oLS2t9f3UFkddp7ouJCRE/usCuSfe4/qB97l+4H2uH5x1n8PDw62+x7IQIiIiInIZdb3dl+GaiIiIiJzu2slrWJK0BOv/ud7Zh3JLGK6JiIiICAUZBbh28lqly2Qfzoah1CBPW2tlPrjgIPZ9ug955/Ow7d1tKC0uK5+9vPMyUsam4Jd+v2DfZ/vwxyN/QJuvRVF2EX4b/htyjuTg5JKTEIab29Xma7F/3n78es+vSPsjDdpcLXLP5NrhjGuHWz/QWNf/rOAovE5ERETuSwiBoitF8A71xtXjV7F92nb0ntEbxdnFSH0uFUq1ElGDo7Bn5h7oS/R46uJTAABtnhZn/3cWLe9viYL0AlzeeRmp41IR/3Y8bh97Ow5/cxhb3tiC4anD0aBlAxRdKYLCQ4HCjEJsfXsrAGDnv3YCACJ6RiAyMRJ/Jv8J7XVt2XsflL2X+mwqzqeeBwAExwYj+1A2Vt63ErpCHQJvC8TV41eRcyQHAHD5ycsI7RyKzN2ZuOvbuxDyUIhDr6Ut3DpcKxQKlJaWwsPDrU/zlpSWlkKh4B8wiIiI3InpeBf75uzDrg93YeSOkUhblYbzqeexYcIGSJKEq8euAgCy9t98KLDwSiE8fTyxedJmnFp+Cv7N/PHrPb/K759LOYfmdzfHlje2AACWJC4pe0MCgtoG4fqp6/J0426Nkb41HZl7M5G2Kk0O1qaMwRoAYsbEYNMrm3B512UAQM7hnArLZ+7OBADs/Zy4YS8AACAASURBVM9edHyoY00vUa1x69Sp0WhQXFyMkpKSWh1QRa1Wo6SkpNa2X1uEEFAoFNBoNM4+FCIiIjJxZd8VeDfyhk9jn2qve+ynY9j06ib0+rgXCtILcOC/BwAAGTszUJRZBAC4sO6C1fW/v+N7eId5ozCjEADMgjUAZOzIwKLuiyquKMzDcNIXSYgaHIUlSUvw99y/oS/WAwA6T+yM6KHROPjlQeSeycXFTRfldaIGR6Hkegn2zdkHba4WCbMT0PCOhljcazEAYMRfI3B80XEIg8C+OfuQdSQLaFjtS1Sr3DpcS5IELy+vWt8Pu/shIiIiUwa9ARCAwqPqvw5rc7WAAlD5qgAAwiCwfPByAEBMcgyihkRBE6RBUJsgaPO12P3xbnQY3wG6fB3WjV+HntN7IrBVIJRqJQDgxC8nIPQCGydsNNvP5R2Xcf3MdbN5vT7uBUkpVVjWGKzLa/dEOxxccFCebnFPC3Sb2g0KTwXO/HkGO97bgdBOoTifeh7BscEAgKb9mmLfp/vkdTTBGjRo2QA9p/fEhY0XkHc+DxG9I5C1PwvqADU6PNsBHZ7tAIPeAIWy7Po17t4YGTsz4N/MH11e7YLinGKE9whHcOtgZOdkV3mNHcmt+7l2FIbr+oH32f3xHtcPvM/1gzPus2kpxqpRq6DL16H/l/3hHepdYTkjY8mGJliDuxbehTVPrEFw+2CcW3OuwvaTjyZj44SNSFuVZja/QasGuHb8Gh7c8CAatGyAn7r/hLxzeWbLePp5QpenAwBEJkZCm69F/OR4NOrSCABw9n9nsem1TXLLNgBE9I5A84HNce30NeSm5eJ86nkMTx2Oo98dhbqBGk16N0Fwu2B4eN1sqxVCQOgFsg5kIbRjKAAg/2I+fkn8BS3uaYGj3x/FQ5sfQkBUQLWurUFngEFvgIfGvF3YFfu5Zri2A/6grh94n90f73H9wPtcPzjiPhekF+Bcyjk06tIIu2fsxpk/zuCR/Y+gIL0AywYuk5fr+WFPaPO0uO2B26DyVeGrll+h29Ru8G3ii5QnU+TlOv6zI/b+Z6/V/fk396+0l4xuU7rBu5E3Up9LhTpAjZJrZSWrbR9riyZ9m2DN42vKjmd6T7R9tK3FbWQfzoZvhC8O/PcA2j/dHuoAtfxeUXYRvIJrVhFgKDXY1IpfXQzXtYzhmmoT77P74z2uH3if6wdL9zn3XC58wnygVCmrvb2MHRnwi/Qzq4FePWZ1hRbmIUuG4FzKOez/fD+ih0XjzB9nYNCVdV3nFeqFFkNbyGUVpnXNRj7hPgiOtdxyDQDhd4bjru/uwtL+SyvtNi/uzTjseG8H1A3UeOzQYxBC4Pza8wjrFiaXn7gDVwzX7CaCiIiI3Jo2V4tt72zDou6L8H3H73Hl7yvI3JOJ//3jfzi75iwKrxTK/TAbl1/3/DoUZBRAV6iDEAIr71uJpQOW4vLOy9CX6LHv030WA/D6F9Zj/+f74dXQC/0+74eOL97szaIos8isXrkwoxDeYeYlI7p8Hfov6I/4t+IBAG1Gt0FwbLBcvtHt7W5QqpTQBJd1RhA9LNriOQdEB2DYr8Mw7NdhAMqeQ2ua1NStgrWrcusHGomIiKh+OrnqJFLfTMXQZUPx9+d/yz1mlFwrwd7Ze2HQGnA+9Tzyzufhetp1qHxVeGjLQ1D5qnDoq0M4ufQkrh6/iuyD2WjavykAoDinGL/e+yuih0Xj9K+nLe43/2I+gJuh9/axt8M33BenfzuN82vPmy3bsGND3LXwLpxcehKNujTCwf87iKjBUVAoFWg/tj0iekcgOKbsocDCy4UovlqMoDZBZSvf6ATttuG3IWF2Aja/sRnHfjiGPjP7IPtQNiJ6RcDT29Ou15Rsw7IQO+CfGOsH3mf3x3tcP/A+111n/3cWDTs0hDZPi4AWAWbd7AohsGfWHgREB6DlvS0xP2I+AKBp/6YoLSrFpc2XEJMcA2EQOPLtEYvb7z2zNyRJwtHvj8r9LFel1UOtcHzxcXla8pAwbNkwBMUEmT18V5xTjCt/X0Hj7o2hL9Yj/0I+1EFq+Ib71uRSYPu07dg/bz/uX30/gtsFyw8S1kZdsytzxbIQtlwTERGRyyu5VoL//eN/8nS/ef0QPTQa2jwtPH09UZxTjD0z9gAoG2LbyFi60XpUa9z5wZ3I2JGBI98egSZYg/ZPtcfO6TuhDlTDK8SrQnd0gHkvG6Ya3NYAw1OGQ+GhQO8ZvZF7NhdHFh6Byl+F0E6hFZbXBGkQmRAJAPDQeEDdQF1hmero8loXNO3fFMHtylq2JUmC5FF7Y3qQ7RiuiYiIyKmEQeDMn2dw4pcTaD6wOfIu5OH4ouPQFehwz2/3oPByIf585E+zdS5uuoiA6AAsHbAUnSZ0wvl1N0suDn99GABwxwt3YN/ssv6Vo4eWlWmExYVhxNYRUCgV0ARpICklRA2KgjZPi2V33ejhQwJU/ipor2sRfmc4zv55tsIxt320rdxKLCkkBEQFoNuUbna/NtYoVUo07tbYYfsj2zFcExERkVPteH8H9s/bD6CsV46SqzdHPV7cc7HFdU6tOCX3lrFn5h55fvd3u2Pr21sBAJ0ndIZXsBfOrzuPiF4R8jL+Tf3l1x3GdZBfx70Zh2vHr6H3zN5YdOciaK9rEdw2GB1f6Ij0beloM7oNrp++jit7rqBtsuWu7IgYromIiMhpTq88LQdrAHKwvm/1fWZ9RRtJHhLufP9ObH5tMzK2Z1R4X+GpwF3f3QVcL3vd7sl2aPdkO5uOpcOzN4N29JBo/P3Z34h9PBaaIA0adigbY7vh7Q3R8HYXG2+bXArDNREREdWIvkQPhUph9mBhdRRmFmLtM2srzFeoFAhqHWQ2b+A3AxEWHwZdvg4+jX2wbeo2lBaVVlgXACITIm/5Qbeuk7qi44sd2eMGVVv9eqSUiIiI7EKbp8X/Rf8f9s0pq2nW5mpx7KdjsKUTsqyDWSjKKsLRH44CAKIGReGO5++AUq1E0/5N0ahzIyg8Fejz7z7yOqGdQqHyU90cxOVGnm/QsgHaj20vL1fToF+epJAYrKlG2HJNRERE1XZh/QUAwK4Pd6Eoqwh5Z/NwLuUcAloEIKR9iFk3dEZHfzyKza9uhjDcDOCBrQPR77/9IEkSuk7qitKiUvn9Vg+2Qst7WyLvQh40QRqzbRlDtFdDL4T3CMeBLw7ceKM2zpbIdgzXREREVG3n1t4cnfDQgkPy698e+A2iVODJC09CkiSUXCvBvk/34bbht2HHezvMgjUADPlliFlrs4eXeTRReCoQEBVQ8QAkK6+JnIzhmoiIiKqUfTgbq5NXo+V9LaHUKHHi5xNofndztLinhVndtCgtC8/n1pzDjg92wCvEC+lb07H/87KHFht2bIgWw1rAUGqAOlBdoUXaZtLN/03Dub3KQohqiuGaiIiILNo4cSMiEyIRNTgKJ5eeRMGlAvz92d/y+8HtghE9NBq7Pt6F66eum61rHPDl2olrZvNjkmPQ6sFWt3xsxhBdIUwzW5OTMVwTERFRBYVXCnHsx2M49uMxs/mNujSCQW/Alb1XEJlYNuJg1KAo7JuzD0qNEvpivbxsaMdQZO7NRIt7WuDUilMAAL9IP/scoEnLNQM1uRKGayIiIqoga3/Fbuxue/A29P13XwBAaVGpXB/deWJndJrQCatGrkLGtpt9T7ca0QqJnydCHaDGudRz0OXp4NvE1y7HZ9ZyzfprciEM10RERGRGCCG3NJtqM7qN/Nr0wUN5GPByJRpBbYLkluq7v78bh78+fLMrPXthzTW5GIZrIiIiMnN512WcXHISUUOikPZbGgCg8yudEdY1rPIVb+RaTz9P9JnZB6FdQuW3GnVuhEadG9nvIKVy/5efT+QkHESGiIiontOX6HH9dNkDiQadAbs+3AWFSoHen/SWl+n4z45VbkdSlCVbD40HogZF1WorstWyECInY7gmIiKq5/bM3IPFvRYj91wutr27Delb0xGZGAmVn0pexpagbAzXxv9rlWlXfCbpWmLSJidjuCYiIqqHzv7vLFYMXYHCK4XIu5gHAEj7LQ2HvzmMqCFR6Pd5PwCAX1M/eIV62bRNOVQ7IluzKz5yUQ6pudZqtZgyZQpKS0uh1+vRrVs3PPTQQ8jMzMS///1v5OfnIyoqCs8//zw8PDyg0+nw6aef4vTp0/Dz88OLL76I0NDQqndERERENjm1/BQy92Ri82ub0ahrI5xadgr75uyD0Au0Hd0WSpUSAPDQxods36gxWzvioUL2EEIuyiEt156enpgyZQo+/vhjfPTRR9i3bx+OHz+O7777DoMHD8bs2bPh4+OD1NRUAEBqaip8fHwwZ84cDB48GN9//70jDpOIiMhtZR/MRvbBbHlam68FAJxdfVYeAEabq4WklBDa+WaDlsJTAYWnbXFBbk12cFkIgza5EoeEa0mSoNGUDW+q1+uh1+shSRIOHTqEbt26AQD69u2LnTt3AgB27dqFvn37AgC6deuGgwcPQgjhiEMlIiJyS0sHLsXSgUvl6bzzefD08QQAFOcUy/MDWgTI86vLWWUh7IqPXInDuuIzGAx47bXXkJGRgYEDB6JRo0bw9vaGUln2Z6egoCDk5OQAAHJychAcHAwAUCqV8Pb2Rl5eHvz9/R11uERERG7DUGowmxZCIP98Pvya+eHq0avAjfYrrxAv9JnZp+Y7MmZrR7Rcl9snkatwWLhWKBT4+OOPUVBQgE8++QQXL160uqylVmpLn0RTUlKQkpICAJg+fTpCQkLsd8DV4OHh4bR9k+PwPrs/3uP6oT7e5+yjN8tBQkJCUJhdiNKiUgRHB+Pq0avw9ChrqR53fBxUPiprm6mSWqMGACg9lLV+jRXKsj++q9VqBDQIkOf7+fshJCSkXt7n+sgV77PDB5Hx8fFBTEwMTpw4gcLCQuj1eiiVSuTk5CAoKAgAEBwcjOzsbAQHB0Ov16OwsBC+vhWHS01KSkJSUpI8nZVVcahWRwgJCXHavslxeJ/dH+9x/VAf7/Px1OMAAIVKgStXriD7UFnYNijLWrRLiksAlP3l2KOo5tFAV6or267BUOvXWBjKGuK0Wi1yc3Pl+fn5+cjKyqqX97k+ctZ9Dg8Pt/qeQ2quc3NzUVBQAKDsm+DAgQOIiIhAbGwstm3bBgBYv349unTpAgDo3Lkz1q9fDwDYtm0bYmNjWUNFRERUA0II7J+3HwBg0BrwZZMvsWzgsrI37fyr1WkPNFqaT+QkDmm5vnr1Kj777DMYDAYIIdC9e3d07twZTZo0wb///W/89NNPiIqKQmJiIgAgMTERn376KZ5//nn4+vrixRdfdMRhEhERuZ2rx67i6rGrFt8zhmC7dRrgyJpra72FEDmZQ8J1s2bN8NFHH1WY36hRI/zrX/+qMF+lUmHChAmOODQiIiK3dm7tOQBASPsQZB0o9+fzG6HUWGJxqyHVaYPIsCs+ciEcoZGIiMhNpY5Lxc4PdkIdqIZPuE+F9+WSS0O56Rpy2vDn7IqPXAjDNRERkZs6teIUgLKwW1nglctCbjWXOqEsRGJTNbkYh/cWQkRERLWrKKsIJddK5OnyA63I840h2F4l104oC2HNNbkahmsiIiI3kbEjA/5R/ljz+Bpk7sm8+YYESErr4dpYc33LJRXGvOuI0gzTshDTdM2gTU7GcE1ERFTHFWUXQZenw8r7VlpeQLJSqmFsuLZTWYgja67NHmi0MJ/IWRiuiYiI6rDMPZlYMXRFpctIkmT5KStjDhXlpmtIDraOfKKLZSHkYvhAIxERUR1VmFlYIVg/vP1htHuinfmCVlqu7V4WciNVOLIshF3xkathuCYiIqqjCjMLzaZ9wnzg18QP3aZ2M5svSZZ7C5FDsJ0faHREuDZ9oJFd8ZErYbgmIiKqo0pyynoEueP5O8r+f6Hsf0tDglfaFZ+9BpFxZFmIteHPiZyMNddERER10JFvjyD3XC4AoMV9LdB6ZGv4N/MHYPkhv8oeaDS2XNel3kI4QiO5KoZrIiKiOkav1WPzpM3ytCZQA+9Qb+srVFVzLexTFyLvw5Et1ygX5hmuyclYFkJERFTHXE+7bjatbqCuuFC51tzKaq6FEHYJpc4a/pzIlTBcExER1THXTlwzm1aqlBWWKf+Qny1lIbfMBcpC+EAjORvDNRERUR1TPlxbZBo4FVKVIzTaI5Q6cvhzs5Zr1lyTC2G4JiIiqmOKsoqgClBVvlC5kGkxPBt74jPYqSxEclxZCEdoJFfFBxqJiIjqmNLCUnj6eGLI4iEwlBosLiMpJAiYdLFnoTlNktM17Ftz7ciAKzFQk2thuCYiIqpjdAU6eHp7IrhdsNVlbKq5vhG4hbBPWYgc0Dn8OdVjLAshIiKqQzZO3Ii039Pg4VNF+1j53kIqqbm2+wiNjuwtBCYt8OXmEzkDwzUREVEdcuzHYwAAT29Pm9epavhzu9VcO2H4c5aEkKthuCYiIqqDPLwrb7k2C52Kyh9ohLBTSHVk39NWegth2CZnY7gmIiKqgzx9qmi5Lhc4LZWFuE1vIeyKj1wIwzUREVEdVK2Wa1tGaLQDZ9VcE7kShmsiIqI6SKGs4ld4+ZbrSsK1vR9odHRZSPmeUYicieGaiIioDrLWv7WsfMa0lDmN2drOXfE5c/hztmiTszFcExER1UEGXeXhunxZiKWWbtPhz+3ZWwiHP6f6jOGaiIjIxWnztSgtKjWbV52Wa0mSLP/Gr63eQuxUZlL5rtgVH7kmhmsiIiIX903rb7C452KzeVWFa7M6ZIWDaq4dPOy58X/WXJMrYbgmIiKqAwoyCgAA6kA1AKDFsBa2r2yltxB5+HN7l4U4AstCyEVVMXYqERERuRK/pn4I7RiK6KHRlS9Yfvjzqrris0codWCTHUdoJFfFlmsiIiIXVqEPahtLOEzDtLWu+ORNGuzTW4izWq7NykLYdE1OxnBNRETkwvTFevm13EOIDfnRpkFkjPOEbdus1j5rm9wxiWRxPpGzMFwTERG5MG2+Vn5dnFNs+8OHZtnaSsu1vXv3cGS2luR0zUBNLoXhmoiIyIVpc2+G66LsItsHfDFdRFFFzXUdLwuxOJ/ISRiuiYiIXJguXye/Ls4uLntRzbIQqzXXpr2F2IEjy0JMH2hkV3zkShiuiYiIXJhpy7WuUFejshBIgKSspJ/r8svXlDNSBbviIxfDcE1EROTC0lalya/1JXqby0Ik86Jri+uYDn9eV8tC2FJNrobhmoiIyEVp87Q4svCIPF2d3kJsGv78hgrd/dWQM8pCOEIjuRqGayIiIheVdz7PbFqv1de8LKSq3kLskUmdkWtZFkIuhuGaiIjIReVfyAcAJMxJAAAYtDXr51qSJMs11ywLIbI7hmsiIiIXlXehrOU65PYQADdarm1VvuXaUs21SVd8dhlExoHh2urDmMza5GQM10RERC4q/0I+lBolfBr7ACirua5JP9dWu+Kzcyh1ieHP2ZJNTsZwTURE5KIKMwvh3cgbSrUSgEnLdW0Mf24Pjsy1psOfM0+TC2G4JiIiclGlBaXw9PEsq5eWbtRc12T4cys118Zl7FZz7aTeQszfcNghEFnEcE1EROSiSotK4eHlAUmSoFQrodeVtVxLNiRIm1quUXdrrq2WhTBdk5MxXBMREbkoXaEOnt6eAACFpwKGEoPtfVKXr6e2lDlNuuKzSyh1ZLY2Gf7cWcdAZAnDNRERkYu5duoaDDqD3HINAErPGy3XNvZJbdaaq5CgUFb8lS93xWevQWQc2XIt79TxuySqjIezD4CIiIhuKsoqws+9f0bbx9qitLAUHt43wrVKWa0HGiuM0Gip5Nr0gcY6WhZS4dwYtsnJGK6JiIhciDZXCwA4v+48DDqD3HKtUCmq9UCjTeUS9g6lxqxup5bwSnfF4c/JRTkkXGdlZeGzzz7DtWvXIEkSkpKSMGjQICxevBhr166Fv78/AGDkyJHo1KkTAGDZsmVITU2FQqHAP/7xD9xxxx2OOFQiIiKnKi0qBVDWp7W1luvq9nNtNTiXb92+RU4JttbqyYmcxCHhWqlU4tFHH0V0dDSKioowadIk3H777QCAwYMHY9iwYWbLX7hwAX/99RdmzpyJq1evYtq0afjPf/4DhYIl4kRE5N50BToAgL5Eb1ZzrfBUyIPI2MSG4GxWxmHHshCndMknz3DYroksckhaDQwMRHR0NADAy8sLERERyMnJsbr8zp070aNHD3h6eiI0NBRhYWE4efKkIw6ViIjIqYzhurSo1KwsRKlWwqAz1OiBRkiwWE5i927rHFgWYrZbloWQC3F4U3BmZibS0tLQsmVLAMDq1asxceJEzJ07F/n5+QCAnJwcBAcHy+sEBQVVGsaJiIjchS7/Rst1cVkJiGnLtb6kGmUhJkxbqL1CvUzegOXXdUn5spC6eh7kNhz6QGNxcTFmzJiBMWPGwNvbGwMGDMADDzwAAFi0aBEWLlyIcePG2fyJNyUlBSkpKQCA6dOnIyQkpNaOvTIeHh5O2zc5Du+z++M9rh9c/T5flC6aTQeGBiIkJAQaHw30Oj0UCgVUalWV56BUKuXXarVafr7JtMTSz99Pfu2hvPXrku2fLe+vtq+xSqUCAPj4+iAoKEie36BBA4SEhLj8fSb7cMX77LBwXVpaihkzZqBXr16Ij48HUPYNYNSvXz98+OGHAIDg4GBkZ2fL7+Xk5Jh94xglJSUhKSlJns7Kyqqtw69USEiI0/ZNjsP77P54j+sHV7/PVzOumk0X64uRlZUFvaRHSUEJ9Ho9tFptleeg1+vl11qdFrm5uQDMSzbyC/JvLi/0t3xdjPuw5fhulVZb1qtKYWEhcq7e/Ov2tevX4Jnl6fL3mezDWfc5PDzc6nsOKQsRQmDevHmIiIjAkCFD5PlXr978AbJjxw5ERkYCALp06YK//voLOp0OmZmZSE9Pl8tIiIiI3Jk2X2s2rdSUtUDLvYXUpJzZStlEbdUqO7LmWpIk1lyTS3FIy/WxY8ewceNGNG3aFK+88gqAsm73tmzZgjNnzkCSJDRs2BBPP/00ACAyMhLdu3fHhAkToFAo8MQTT7CnECIiqheMDzT6NPZBQXoBirOKAdzoLURrKFuomvnR9MFFs4cY3SGHsis+cjEOCddt2rTB4sWLK8w39mltyf3334/777+/Ng+LiIjI5ejyddAEadD///pj+d3L0fjOxgButFzr9BVaam1ireW6lkZUdGjrcfldMWiTk3GERiIiIheiy9fBw8cDDW9viKcuPiXPV6gU0Bfr5TKR6pAU0s1yEhsGlLlVjigLMe6jQlkI0zU5GWstiIiIXIiuUAeVr6rC/ODYYBReLkTe2bzqb9RBIzQ6RR09bHJfDNdEREQuRJevg6ePZ4X5sWNia1zGYe2BP3uP0OgMHKGRXA3DNRERkQvRFejg6VsxXEsK6WZwtCVAWhtYxQE11w7HQWTIhTBcExERuRBrLdcA5OBYkwcavcO8AQANOzSssL0abdPKfhzFamt8XS1vIbfBBxqJiIhciC7fcss1UBYcRQ06upYkCcExwbh/9f2QPCWk/Z4mz7+5UI0O15zjure+iVmaXAxbromIiFyIrqCSlmujGgbK4HbBUChNfvW7QTmFJElucR7kPhiuiYiIXIQQwqaykOqy9uCi3cspnBFspVpogSe6BQzXRERELkJfrIcwiErLQkz/t5kD+rZ2lgot10ROxnBNRETkIoxDn1sL17LqZmtbHv6rqwG1fE98fKCRnIzhmoiIyEXo8m+EazuXhVhjWi5SZ0OpBNZck0thuCYiInIRcri2c8u11XXdIIiWH/6cyNkYromIiJystKgUvz3wGzJ2ZQAAVD4Vhz8HbqHm2sI2yiZqvBnLXKArPgZtcjb2c01ERORkl3dfRvrWdKRvTQcAePg45tezWwx/Dj7QSK6FLddEREROJvTmTb4+YT6WF6zO8Oemm7Q2/DnsXHPNrviIGK6JiIiczVBqkF9LHpI8VHl5dg/A7lB/XVePm9wWwzUREZGTGXQ3w7VvuK/5KIqmbgTJaodsK7XQ7tBbSPnjrqvnQe6D4ZqIiMjJjP1bA4BvE99a3VdtllCE9wiHX1M/dHqpk303XBl2xUcuhg80EhEROZkuzyRcR1QSrqsTHG0YldHeQVsdoMbDWx++9Q1VA7viI1fDlmsiIiIn0xZo5deaQI3V5eQQaa8saZoC6mo+ZVd85GIYromIiJzMOHgMYMMAMjVhpbXa6lDodQnLQsjFMFwTERE5mWnNtdWhz4GaP9BobXNu0IUdy0LI1TBcExEROZlpy7XKz/LojMAtlIVYGzmxDmdSIW6cFFuuycUwXBMRETlZrZeFmLJhQJm6pPxxsxWbnI3hmoiIyMnMykJsCNd2Kwtxg+HPAbDlmlwKwzUREZGTlRaXyq8rDdf2GKDR2kOMdSyUmpbIsLWaXAnDNRERkZMZSm6O0KjyqYWaa7ONWH5dZwMqu+IjF8NwTURE5GSlJSYt13421FxXNz9aWd6sLKSOkiSpzrW6k3tjuCYiInIyg/Zmy3WtlIWY9BZitRSkrgZUloWQi2G4JiIicjK9Vi+/VvlWXRZyS2HSWg8hdTSfVrgWdfQ8yH0wXBMRETmZvuRmuFaqldYXtHdwZM01kd0xXBMRETmZvkSPmH/E4KmLT9m2Qi080FhnW3zLH3ddPQ9yGwzXRERETqbX6qFUVdJibSR3FlILw5/XUe5wDuReGK6JiIicTF9iW7iuTpC09uBihfmShfl1mLucB9VdDNdEREROZCg1QOhF5bXWRlK5/21l0luI25VNsCyEXAzDNRERkRMZewqxKVzbm2k3dnU0lLKlmlwNwzUREZETGXsKsaksBLcehCsMf17Xy0LYck0uhuGaiIjIiYwDyFSnLKTOBuFaUP5a8NqQszFcExEROZFx6HObeguxP4+iSgAAIABJREFUN5MHGutsi29dPW5yWwzXRERETmRsuVaobfiVXI0gLISw/Ea5nkPsMuqjM7EshFwMwzUREZETyTXXNpSF1EoQruMt1ywLIVfDcE1ERORE1Xmg0R4B2O3Cp5udDtV9DNdEREROJHfFV52aa3sNfw52xUdkbwzXRERETlStfq7tnSPr8AiNttSUEzkDwzUREZET5V/MB1C9mmu79nMtT9R8m66krn1IIPfDcE1ERFRLDHoDtLnaSpfZNHETgOqVhdxSgLSyKkMpkX0wXBMREdWSbe9swzdtv0FpcanF94XhZmmDb4Rv1RushbKQuhqqrR533TwdciMM10RERLXkzB9nAABFV4osvl9aVBa649+Kh8pfVeX2qlMWYkv4NB3+3F3U1Q8L5D4YromIiGqJd6g3AKDwcqHF93X5OgCAh7eHw46JLb5Etcsh381ZWVn47LPPcO3aNUiShKSkJAwaNAj5+fmYNWsWrly5goYNG+Kll16Cr68vhBD46quvsHfvXqjVaowbNw7R0dGOOFQiIiK7kcP1FSvhuqAsXKt8q261NiXZKwmblIW4TYuvm5wG1V0OablWKpV49NFHMWvWLLz//vtYvXo1Lly4gOXLl6N9+/aYPXs22rdvj+XLlwMA9u7di4yMDMyePRtPP/00vvzyS0ccJhERkV15NfQCABRlWi4L0RXeaLn2sbGtyx6jKVobLtxdQqm7nAfVWQ4J14GBgXLLs5eXFyIiIpCTk4OdO3eiT58+AIA+ffpg586dAIBdu3ahd+/ekCQJrVq1QkFBAa5eveqIQyUiIrIbTbAGgPWaa2NZiKe3p03bs3vrshsGUbdpgac6y3FFXjdkZmYiLS0NLVu2xPXr1xEYGAigLIDn5uYCAHJychASEiKvExwcjJycHHlZo5SUFKSkpAAApk+fbraOI3l4eDht3+Q4vM/uj/e4fnDkffb1L+sBxHDdUGGf2cezUXKuBADQsElDm45J6VHWXZ+3j3eVyyuVN7v2U2vU8vLG1nIACAoKgkJR1s6mVqvr1Ne/SlVWSuPv718hM6h8Vfx+ridc8T47NFwXFxdjxowZGDNmDLy9va0uZ2nUJUufRJOSkpCUlCRPZ2Vl2edAqykkJMRp+ybH4X12f7zH9YMj73NhQVmtdf71/Ar7nN9+vvy6QFdg0zHp9WWjORYWFla5vHFZACgpLpGXN/ZQAgBXr16FQNnvXK1OW6e+/rXasv7Dc3NzzY47OycbnsWe/H6uJ5x1n8PDw62+57DeQkpLSzFjxgz06tUL8fHxAICAgAC53OPq1avw9/cHUPap0+wbJTu7Qqs1ERGRq5Mbi6yM1G3k6VO9shC7lT6YDH/uLiUiLAshZ3NIuBZCYN68eYiIiMCQIUPk+V26dMGGDRsAABs2bEDXrl3l+Rs3boQQAsePH4e3tzfDNRER1T1VhGojm7vis/MDjVaHQieiGnNIWcixY8ewceNGNG3aFK+88goAYOTIkbj33nsxa9YspKamIiQkBBMmTAAAdOzYEXv27MELL7wAlUqFcePGOeIwiYiI7MrYcm2p3NGUrS3X9lA+RDNUE9mXQ8J1mzZtsHjxYovvvf322xXmSZKEJ598srYPi4iIyDEqydYKlQJKldL6AqaMAzTWRiB2l4ztLudBdRZHaCQiIqototz/FlRnABm7hGor/VzbbWAaonqO4ZqIiKi2GJ9nrKQsxNOvBiUh9srBkpXXdRjLXMjZHN7PNRERUX0hULG3ECEEfuzyozztoanBr+JbyI/lH2KUp90lk7rLeVCdxZZrIiKiWmbacm3QGVCQUSBPKzU21lsD9gmO1oY/dxNsuSZnY7gmIiKqLRZqrvXFerNFPLyq33Jt136u7b1NonqO4ZqIiKiWWBpEprS41GyZGpWF3IpygZplIUT2xXBNRERUW+RsLXB80XHMj5iP4qvFZotUqyzEDiq0UNdm937O4CanQXUXwzUREVFtMSkL2Tt7LwAg90yu2SLxk+Orv1179hbCMEpkVwzXREREtUwIIbdQa69p5fntnmiHgOgAxx6MtTDtJiHbbVrgqc5iuCYiIqolpjXXxlEYS3JL5PclZc2C4K0ESLN1JbDmmsjOGK6JiIhqi0lZiDFca3NvtlxXN1zbu1VWkiS3q7l2l/OguovhmoiIqJYYW66FEFCoyn7laq/XPFzfXPGWD42IagnDNRERUW0TgFJ9o+U6zyRcKxxfFmK+IUBiUieyK4ZrIiKi2mIsuRbCYs21QuncX8OmZSHM2ET2wXBNRERUWyzVXLtaWYib1VwTORvDNRERUS0RJkMzGmuu07emy/NqWhZSK63MzNZEdsFwTUREVFtMykIsDXNe45ZrezHpio8t10T2wXBNRERU28TNlmtTzujnuuLG7LcpImK4JiIiqjWmXfEZa65N1bgsxE74QCOR/TFcExER1RYLDzSaqnFvIdUMwsah1+25TSKyjOGaiOj/27v36Kjqe+/jn7nmNrlMMkkgyEUICJH7pSK1ooLWSu1BT9WlRzzWngcpbS3Spy4fn9XaHmurbTnYFKztkQPa03OK9VGOuqhayhK7oGgot2oEpQIKJIRkhmQmyWQyM/v5Y8yQGxLInsxk8n75hzM7e/b8Jr/ZzGd++e7fD0iUzuE6o5eRa3viy0JGXzdan3noM2c5EDXXgNkI1wAAJFhHeUiH/HH5kgamLGTW/54lZ66z159RFgKYj3ANAECCxEN112wdD9XJXkQGgPk4qwEASJROU/F1H72WmIoPSEeEawAAEuRsI9fxVRGTvYgMeRowHeEaAIBEMc5y+xMpNc/1IAvaDpdDkmR1EGWQWnouF9XJxo0b+3SQ2267zZTGAACQjgzDiIfrixddrNOHTktKfllI55A+2MpCPvvoZ+W+xK2L5l+U7KYAXXxquG5oaIjfDoVCeuutt1ReXi6Px6P6+nodOnRIl112WcIbCQDAoGacKRFZ+OuFen7B85JSoCxEgy9Ud8goyNCM+2YkuxlAD58arpcvXx6//cQTT+hb3/qW5s6dG9/21ltv6S9/+UviWgcAwCDWeYVGGeoRis97thCzc7BFg7YsBEhVfT6r9+zZo898pusk9HPmzNGePXtMbxQAAGmh0yIyhmH0mJkj2TXXg7ksBEhVfQ7Xw4YN06uvvtpl22uvvaZhw4aZ3igAANKJEe195DqlykLI1oApPrUspLNly5bpZz/7mV566SUVFhbK6/XKZrPp29/+diLbBwDAoNVlKr7ewnWy57mWCNWAyfocrkePHq2f//zn+uCDD+Tz+VRQUKAJEybIbu/zIQAAGFqMrv/vXnpxweHazHmuqbkGTNWnZByNRrVkyRJt2LBBkyZNSnSbAABID/GB69h/cZ8EWas1deZopuYaMEefzmqr1aqysjL5/f5EtwcAgLRxtrKQVLqgkZprwFx9rum44oor9Pjjj+sLX/iCioqKupzYkydPTkjjAABIB4ZhdJktpEMiy0KyirOk9ySr8xzjaPFsTboGzNDncP36669Lkn7/+9932W6xWLRmzRpzWwUAQDroXHPd+YLGjkB7nuH6fEasr1l7jY68ekTu8e5POeB5PT2APuhzuF67dm0i2wEAQNrpURbSzYVOxdeXUebMwkxNvGPipx+HshDAdKlzJQUAAOmmI1t3Kwvp+P95r9CYSIRrwBR9HrluaWnR73//e1VXV8vv95/5Ni7pl7/8ZUIaBwBAWjjLPNcXHGgTMBUfs4UA5ujzV+ann35ahw8f1pe//GUFAgHdc8898ng8WrRoUSLbBwDAoNcxINWjBKOXUpE+MTNcm31MYIjrc7jev3+/vv3tb2vOnDmyWq2aM2eO7r//fv35z39OZPsAABi0Otdcd/6LbyphxBowV5/DtWEYys7OliRlZmaqublZBQUFqq2tTVjjAAAY1Ixut7vl2AsN3GbOc01ZCGCu81r+vLq6WlOmTNHEiRO1bt06ZWZmavjw4YlsHwAAg1enCxp7m4rvgpmYg5ktBDBXn0eu7733XhUXF0uS7rnnHjmdTjU3N+sb3/hGwhoHAEBaMGJLoKfc6DA114Dp+jxyXVpaGr+dl5enZcuWJaRBAACki46yj+4j1/0N2ZSFAKmrzyPXDzzwgDZs2KC3335bgUAgkW0CACA9dFqhsfM8156pHklSRkHG+R0vEfmXTA2Yqs8j10uWLNF7772nzZs3q7KyUsOGDVNFRYUqKio0d+7cRLYRAIBB6WwrNF7+g8s14dYJyr84/8IOTMgGUlafw/WUKVM0ZcoUSZLf79crr7yiV199Va+99po2btz4qY998skntXv3buXn52vVqlWSpOeee05/+tOflJeXJ0m6/fbbNXPmTEnSiy++qK1bt8pqteorX/mKpk+ffkEvDgCAVBAP2Z8EWJvTppIZJRd+wARc0NiXJdUBnFufw/XevXtVXV2t6upqNTQ0aPz48brjjjtUUVFxzsdeddVVuv7667V27dou2xctWqQvfelLXbYdO3ZMO3bs0L/927/J5/PpkUce0c9//nNZrSm0RCwAAH3RqSzkgheMSTSzZjABIOk8wvWPf/xjlZaWavHixZo/f75sNlufn6SiokJ1dXV92reqqkrz5s2Tw+FQSUmJhg0bpkOHDmnChAl9fj4AAFJCp6n4Otdc95eZFx/GR6wJ14Ap+hyuf/CDH+i9997Tzp07tXHjRo0cOVIVFRWaNGmSJk2adEFP/tprr+nNN9/U2LFjddddd8nlcsnr9Wr8+PHxfQoLC+X1ei/o+AAAJJOhbjXXBFgg7fU5XE+cOFETJ07UTTfdpMbGRm3evFn/8z//o40bN56z5ro31113nb785S9LkjZu3Khnn31Wy5cvP6/VqrZs2aItW7ZIkh577DF5PJ7zbocZ7HZ70p4bA4d+Tn/08dAwkP2c4YzNBmK1WpWZkSmrzdqv57bbYx/bubm5prwGj8cjuzN2zOyc7LR4/3e8Bs7noSEV+7nP4frtt9/Wu+++q+rqatXU1Gjs2LG6/vrr+1Rz3ZuCgoL47QULFujxxx+XJBUVFamhoSH+M6/Xq8LCwl6PsXDhQi1cuDB+v76+/oLa0l8ejydpz42BQz+nP/p4aBjIfm5ra5MkRSIRtba2yjCMfj13OByWJAUCAVNeQ319ffyYra2tafH+73gNnM9DQ7L6uays7Kw/63O43rx5syoqKvTP//zPmjBhgpxOZ78a5fP55Ha7JcWC+8iRIyVJs2fPVmVlpb74xS/K5/OppqZG5eXl/XouAACSoctUfFK/y0ISudALi8gA5uhzuP7+979/wU/yxBNPqLq6Wn6/X8uWLdOtt96qd999V0eOHJHFYlFxcbGWLl0qSRo5cqQuv/xyrVy5UlarVV/96leZKQQAMDgl6ILGREzFB8AcfQ7X7e3tev7557V9+3b5/X4988wz2rdvn2pqanT99dd/6mNXrFjRY9s111xz1v1vvvlm3XzzzX1tGgAAqcno9v9UzLFMxQeYqs9Dwhs2bNDHH3+s++67L/4td+TIkXr99dcT1jgAANKC2fNcmxmEUzn4A4NQn0euq6qqVFlZqczMzHi4Zpo8AADOrqPmOl4WYlKCTUQpB+UhgDn6PHJtt9sVjUa7bGtqalJubq7pjQIAIC10X6GR/AqkvT6H67lz52rNmjXxlRZ9Pp/WrVunefPmJaxxAAAMZl1mCzEzXBPSgZTV53B9xx13qKSkRN/+9rfV0tKi++67T263W7fccksi2wcAwKBnyKTZQhIZqgnsgCn6XHNtt9t199136+67746Xgxw9elSVlZVauXJlItsIAMDgZHZZyCfHo+YaSF3nDNdtbW168cUXdeTIEQ0fPly33HKLWltb9etf/1r79+/XlVdeORDtBABg0Oq4oDGly0LI1oApzhmu161bp8OHD2vatGnau3evPvroI504cULz58/X0qVLlZeXNxDtBABg0Om+QmMql4Uwcg2Y45zhet++ffrJT36i/Px8feELX9Dy5cv18MMPq6KiYiDaBwDA4NW9LMQs5GAgZZ3zgsZgMKj8/HxJUlFRkTIzMwnWAACch/gIdiqH4lRuGzCInHPkOhKJ6J133umyrfv9yZMnm9sqAADSQOeyEFNmC/lEQko4CNeAKc4ZrvPz8/XLX/4yft/lcnW5b7FYtGbNmsS0DgCAwawjWxuGuWUhJur4AkDNNWCOc4brtWvXDkQ7AABIP4NphcZUbhswiPR5ERkAAHDhzCwLMTMIM2INmItwDQBAgnSUXMTLQvqbYxOQg40oZSGAmQjXAAAkSkfNtckB1swgHG2PSpKsTiIBYAbOJAAAEqT7bCGmMXGQORqOhWub02beQYEhjHANAECiDIILGiOhiCTCNWAWwjUAAAlmWs31J8wsC4mH6wzCNWAGwjUAAIlmSIZSc7YQaq4Bc3EmAQCQIKbPFpIA0dAnNdcORq4BMxCuAQBIlE9qriPBiOrfqU/JketIOzXXgJkI1wAADIDAx4F+HyMRc1HHy0IyiASAGTiTAABIkB7T7/UzG3ccz9R5rkNMxQeYyZ7sBgAAkLZMnNq6CxMHsDsWuBns4XryVycrcKL/fx0A+otwDQBAonQfuO7niHMilygf7FPxXf6vlye7CYAkykIAAEgYs8tC4odJQMi2OogEgBk4kwAAGCCJHHnur8FeFgKkCsI1AACJ0r3mOnWz9aAvCwFSBeEaAIBESVS4TkBIZ4VGwBycSQAAJIjRLV1TFgKkP8I1AAADhQsagbTHmQQAQKKYPc91Age+U3lUHRhMCNcAACRI96n4LKYNXZtzGADmI1wDAJAoCbqgkVFmIHURrgEASJRBNBUfAHMQrgEAGCCmjTgT0oGURbgGACBBzF7+3LSa7U5GXz9aGe4M048LDFX2ZDcAAIC0ZXJZSHzebBMz9nXrrjPvYAAYuQYAIFF6zBZiUlkIFzQCqYtwDQDAIJGIshAA5iJcAwCQKImaLYSMDaQswjUAAAOEcg4g/RGuAQBIkB6zhZiE8hAgdRGuAQBIFMpCgCGHcA0AwAChLARIf4RrAAASxOxFZEw/DgDTEa4BAEiU7tmaea6BtDcgKzQ++eST2r17t/Lz87Vq1SpJUiAQ0OrVq3Xq1CkVFxfr/vvvl8vlkmEYWr9+vfbs2aOMjAwtX75cY8eOHYhmAgBgrsRczwgghQ3IyPVVV12lhx56qMu2TZs2acqUKaqsrNSUKVO0adMmSdKePXtUW1uryspKLV26VE8//fRANBEAgMSjLARIewMSrisqKuRyubpsq6qq0vz58yVJ8+fPV1VVlSRp165duvLKK2WxWDRhwgQ1NzfL5/MNRDMBADBVj5rr/o5kE6qBlJe0muvGxka53W5JktvtVlNTkyTJ6/XK4/HE9ysqKpLX601KGwEA6JcElYVQcw2krgGpuT4fvU24f7Z/RLZs2aItW7ZIkh577LEuoXwg2e32pD03Bg79nP7o46FhIPvZau06huWwO/r13HZH7GM7vyDflNeQzu93zuehIRX7OWnhOj8/Xz6fT263Wz6fT3l5eZJiI9X19fXx/RoaGuIj3N0tXLhQCxcujN/v/LiB5PF4kvbcGDj0c/qjj4eGgeznSDjS5X6oPdSv5w63hyXF/vprxmtI5/c75/PQkKx+LisrO+vPklYWMnv2bG3btk2StG3bNs2ZMye+/c0335RhGHr//feVnZ191nANAMCgYlaZCFUhQMoakJHrJ554QtXV1fL7/Vq2bJluvfVWLV68WKtXr9bWrVvl8Xi0cuVKSdKMGTO0e/du3XfffXI6nVq+fPlANBEAgITrrfTxvBCqgZQ3IOF6xYoVvW7/3ve+12ObxWLRv/zLvyS6SQAAJJzps4V88nguaARSFys0AgCQKIlaRIZsDaQswjUAAIMFoRpIeYRrAAASpN811mdhIWUDKYtwDQBAonQvuU5Q2AaQOgjXAAAkClkaGHII1wAADDIGqR1IWYRrAAASxPSp+ACkPMI1AACJQs01MOQQrgEASJTuWbq/CzSyeAyQ8gjXAAAkiNm10Yx8A6mPcA0AwEAhGwNpj3ANAECimBymKQsBUh/hGgCABOlexmFaWQcj4EDKIlwDAJAohGBgyCFcAwAwUMwK21SHACnLnuwGAACQtkyeis+s41z7H9eq3d9uTlsAdEG4BgAgQXrUXPd7ouv+PbzDmM+PMedAAHqgLAQAgESh5hoYcgjXAAAkSI/ZQQjbQNojXAMAAAAmIVwDAJAo3QeuWb4cSHuEawAABgrZGkh7hGsAABKEkWpg6CFcAwAwUMxa/ZzQDqQswjUAAIlCBgaGHMI1AAAJ0mMRGUacgbRHuAYAIFHMztImrdAIIHEI1wAAJEr3cN3fsM3AN5DyCNcAAACASQjXAAAkiOk115SFACmPcA0AQKKYXRZi9nEAmI5wDQBAgjA7CDD0EK4BABhsKA8BUhbhGgCABBq5YGT8tmkj2QyIAymLcA0AQKIYkvsSt27YeIMph7NYGLIGUh3hGgCABDEMQxaLRRbrJ6GYEWcg7RGuAQBIFEOShRFnYCghXAMAkEAWWeKftsweAqQ/wjUAAAnSEabjI9dkayDtEa4BAEiAxsONMsJG17IQwjWQ9gjXAACYrK2xTc9d8VzsjkXmz0tNSAdSFuEaAAATGYahZyuejd+3WCzmhWuuiwRSHuEaAAATNf69sce2jqn4+n1BIyPWQMojXAMAYKK6v9Z13cBUfMCQQrgGAMBEvkO+LvctljNT8fV75JmMDqQ8wjUAACaKBCNdN3QauWaeayD9Ea4BADBROBjuusHEqfiu+PEVGrlgpEpmlvTvQAASxp7sBgAAkE56jFxLppVzuCe4df2z15tzMAAJwcg1AAAm6h6uLRZLfLYQAOkv6SPXX//615WZmSmr1SqbzabHHntMgUBAq1ev1qlTp1RcXKz7779fLpcr2U0FAOCcepSFSFyICAwhSQ/XkvTwww8rLy8vfn/Tpk2aMmWKFi9erE2bNmnTpk268847k9hCAAD6JtLWywWNZs1zDSDlpWRZSFVVlebPny9Jmj9/vqqqqpLcIgAA+oaRa2BoS4mR60cffVSSdO2112rhwoVqbGyU2+2WJLndbjU1NfX6uC1btmjLli2SpMcee0wej2dgGtyN3W5P2nNj4NDP6Y8+HhoS3c+WSNcknZ2drcLCQkmSzWLjPTZAOJ+HhlTs56SH60ceeUSFhYVqbGzUD3/4Q5WVlfX5sQsXLtTChQvj9+vr6xPRxHPyeDxJe24MHPo5/dHHQ0Oi+7kt0NblfnNTs3ynYwvLhCNh3mMDhPN5aEhWP39aXk16WUjHt/n8/HzNmTNHhw4dUn5+vny+2D9EPp+vSz02AACpLNzWtSwkGomemS2Ekmsg7SU1XAeDQbW2tsZv79+/X6NGjdLs2bO1bds2SdK2bds0Z86cZDYTAIA+6z4VnxExziwiAyDtJbUspLGxUT/72c8kSZFIRFdccYWmT5+ucePGafXq1dq6das8Ho9WrlyZzGYCANBn3WcLiYajZ4ayGLkG0l5Sw3Vpaal++tOf9tiem5ur733ve0loEQAA/dN9thBGroGhJek11wAApItoJKpoKNpjW8dUfMxzDaQ/wjUAACbpHqwlyQgzcg0MJYRrAABMEm7tuYBMNNIzcANIX4RrAABM0mPpc8Vqrs/cGcDGAEgKwjUAACaJX8zYqQrEiBhn7hOugbRHuAYAwCQdI9f2zDOTcXUuCzFI10DaI1wDAGCSjgVkbBm2+DYuaASGFsI1AAAm6SgLOdvINQPXQPojXAMAYJKOshBbZqeR68411wDSHuEaAACTdIxc9wjXHbdZRAZIe4RrAABM0lvNdTQSpeYaGEII1wAAmKQjXNuzztRcG2Gm4gOGEsI1AAAmCbf1ckFjOEq4BoYQwjUAACbpdSq+qKHMgkxlFGTo8n+9PFlNAzBA7OfeBQAA9EWvU/GFo7I6rLrr3buS1SwAA4iRawAATBJpi0gWyeo48/HaebYQAOmPcA0AgEkiwUisJKTT5CCEa2BoIVwDAGCScFu4S0mIJBVeWpik1gBIBmquAQAwSSQY6bKAzKQlkzT3+3OT2CIAA41wDQCACWqranXwvw92meO6ZEZJj5FsAOmNshAAAExw4LcHJEnh1nB8m8HE1sCQQ7gGAMAEuaNy47dZ7hwYugjXAACYwAgzSg2AcA0AgClC/pAk6abXboqPYmcVZSWzSQCSgKssAAAwQcgfkmuES57JHhVOLFTRpUUauWBkspsFYIARrgEAMEF7oF3OPKckyWq3avR1o5PcIgDJQFkIAAAmCPlDcrgcyW4GgCQjXAMAYIKQPyRnrjPZzQCQZIRrAABM0O5vJ1wDIFwDAGCGUICyEACEawAA+i3oDarN16aMgoxkNwVAkjFbCAAAF6jtdJuObTsmb7VXRtRQ+c3lyW4SgCQjXAMAcIHe/tHbOvDbA5KkYZcNU+HEwiS3CECyURYCAMAFiIQi8VUZJalkVkkSWwMgVTByDQCAJMMwJEkWi+Ws+9RW1SrcEpYjx6FXbnmlS431iCtGJLyNAFIf4RoAMOQFjgf0hzv/oNJZpZq5cqYO/tdBNZ9s1rTl0/TO0++o5WSLPFM92vX4LkmSPcuuaCiq1rpWjf78aH32R59Vdml2kl8FgFRAuAYADEmRtogCJwIKeoN6efHLMqKGTr9/Wh/8vw8UDUUlSQf/+6AUG9DWkT8ckdVhlesil/wf+TXq2lH66I8fKW9MnnKG5STxlQBIJYRrAMCQ01LXop3/ulN/f/Hv8W3DLx+upiNNKigv0LjF4xQOhnVk8xHlj81XKBBSpC2iqfdOVfH0YgW9QWUVZ2nfmn0aff3oJL4SAKmGcA0AGDKC3qAObj+oF259Ib7NYrdoytIpmvHNGXLmdV1h8dK7L+31ONkCYpbwAAAdXElEQVQlsRKQ6d+cnrjGAhiUCNcAgEHFMAwd//NxDb9seOx+1JA9K/ZxdmrfKbknuBVpi6i1oVUF4wrijztZdVIvLX6py7EuvuFizXt0XjwsA0B/Ea4BAIPCqb2n5Hvfp6ziLL1656u6eNHFOrnrpILeoOY8OEflN5Vr0w2bNGrhKPk+8Ml/1K/LvnuZ/B/5VX5zubb/3+1djjf7gdmquLtCGfmsqgjAPBajY+6hNHDixImkPK/H41F9fX1SnhsDh35Of/Rx8jTXNCvSHlHeqDxF26OKRqIKHAvIdZFL1Ruq9dYjb5nyPFc8foXGzhsr23BbfLQb6YnzeWhIVj+XlZWd9Wf8y9JPH7/xsdxfcsfvt5xqkSPbIUeOQ5JUt6dO9X+rV8VdFTIMo8v8qUdfP6rckbkqnFSow5sPK9IW0bjF4z51jtXux+guGonKYrXE92lvaZc90y6L9eyPSaTmmmb5P/bLYrHIme9UwfgCtTe3y5Hj+NTXcTahQEj2LLusNqtaTrXIarMqszAz/vOWky2KRqLKdGfKlmlT498bFfQF1fRhkwInAsofm6+Lb7hY0XBUHzz/gQomFMT/tNxduDWscGtY9my7ZMQWjOiNETXU+GGjbBk25ZTlKOgNqj3QrvyL88/5ejr6MxqOymrvuqZTa0OrWk+1ymq3ypnvVHNNs6KhqFwjXGp4t0He97xyjXCp7LNlCgfDajvdJnuWXVnFWQo1hZTpzuxSP9pc0yxnnlO2TJvaA+2SEWt7hjtD7YF2OXNjzxE4EVDJzJKz9k+kLSJbhk2hppBCgZCcuU45c888T6jpkz5yJH6NquNvHpf/uF/li8tly7TJd9Cnv2/6u/LH5mvYZ4bJnmVX0BuUrFJWUZaOv3lcDe82qGB8gQrKC9TwboNaTrbINcKlSHtEWfYs+f1+RUNR5Y7OVcmMErX52tR0tEmySG3eNoX8IXnf86r2rVrNWDFDgeMBtdS1SBbJareq4p8rdLIqNpoaOB7Q0dePKm9UngorCpU7KldG1FB7oF2Rtoj8x/yKtkeVWZip4unFysjLUMupFoUaQ8oeli17ll01O2pkdVo1dtFYNVQ3yP+xX2M+P0YtdS1qD7Sr8cNGuSe5VTKzJP54I2woqzhLFptFJ7afUEZ+hpy5TgV9QdmcNhVPL1a0PaqG6gbJkPzH/Ao1hjThtglqqWtRuCWsvNF5CreG1drQqpbaFmUVZ8lqs+rk7pNqqW1RW1Obhs8drmh7VLYMm5y5TuWOypXNaYufC/6jftkybDq1/5QsFotC/pB8B30qu6JMFqtFJ3edlAypzdemA/91QOHWsIqnF8t30Kdwa7hHfztyHJIlthJizvAc1f21TjNWzJDvfZ/e/937ChwPxPctnVOqL/7+i2o83ChJev7q5+W+xK2J/zRRxcXFhC4ACcPIdT80VDfohWtf0OQ7J2vWd2fJW+3Vyze9rAm3TtDc78/V8T8f15+/82eFmkK6YeMN2vK/tmjWd2ap4q4K1e2u08s3v6yMggxNWTolPnfq5Y9cLiNiKPBxQBa7Rd4DXlltVpXMKpH/qF8fvvyhwq1hjf/yeLlGuHRq/ynlDM+JfejtOinfBz7ZM2IBq+10m9pOt8mR41BhRaHyxuTJkR0L/b73fTIMQ7YMm/wf+ZU7Mlejrh2lUGNIdbvrlD8uX06XU7JI9my7jr1xTEUVRbGr5v2h2MpkjaF4WCuZXqKG9xrknuBWy8kWRdoiajnZon1r93X5kHTmORVqCmnE/BGa98g82TPt+vv//F1NR5rkyHEoHAyrPdCucDAsq92qpsNNCreG5XA55P/Ir6A3KIvNokx3plrrWyVJ7oluZRVlyZ5t10d//Cj+XNnDstVa1yoj2vUtnjMsR7ZMm5qONEmSLv3qpWquaZbFaok/R9kVZWp4p0EN7zRIFkmGYuFhdK4cOQ5ll2YrqzhL4ZawGj9sVO1btZKkLE9WPJQ785xyjXApb0yewsGwIm0R2bPsioQi8n/klyPbEf8Td8vJFhVVFKlocpFq/lKj9kC72hrbFG2PXvD702K3yD3BLSNiyGK1yPe+T85cpwzDUKjxzKpy9iy7wq1h5Y/LV9PhJhlRQ/lj85VRkKHsYdnKdGeqobpBVodV9fvrFQlGlDcmLzbS2BaRxWqR+xJ3/HcY9AaVVZKlYXOGqfbtWtkybHLkOGKBMhRRyawS5Y7MlcViUe7oXIWaQvJ/7FewPhgLhlFDrXWtKpxUKEeOQ/6P/bHf+bBshVvDyijIUOOHjfId8KntdFv8dVgd1n79vvrze850Z0qW2BeLSLD3L2Hd2TJtyhudJ5vTpsYjjWr3t/e6X4Y7Q+GW2PtHkqxOa3yauGRy5DrO2mZJyizKVLAh2Kdj2bPsyizMVDQSVbglrKLJRXJPcCsajurQC4c05//M0cU3XCyb06YMd8ZZv/jV/KVGLadaVFBeoJxhOV2+eHsPxL6MOnOdjGgOEfTz0JCKI9eE636q+nGV9q7Zq9xRuQqcCMgIGyqeXqys4qwuQW/0daN19PWjXR5rsVq6BL/i6cU6tffUWZ/LlmGLf8B2cF/ySZgNRZSRFwtDJbNKFKwPqulok07tPSWL1SLPNE8sOBmGwi1huSe4Zc+xx/Y70iSLzRIPJlmerHhw7ZBRkKGQPyQjcqa9Fpuly/3ejP78aI26dpSyi7MVOBGIhVVJB357oMt+mYWZseNHDRmR2MVJOcNzYsGlMFP2TLtyynKUNzpP7c3tCjYE5RrhUs3OGrWcbIkFvVBEl9x2iVwXudR6qlV1u+sUbg0rZ3iOLr7hYo1cMFJ1e+p04D8PKBKKaMznx2j/r/fLf9QvZ54zNppusyh3ZK68B7wKNYZU8ZUKGeHYawzVh9Tqb1Xb6TY1n2hW0BuUPccuRaVxi8fJPcGtmrdqZLVZZc+xq+lIU2zU1JDsmXbZMmwKeoNy5DiUUZChoDeo0jmlajrSJNcIl2qratV4qFG5o3LVdKRJo68frbGLxioaiSrUGFJ7c+xLR81fauSe4Na0b0xT84lm7X9qv3JH5ap4arEioYiaa5qVVZQl/3G/fAdjX7ai4ajsWXa1t7QrqzhLWYVZsufYY8Huw0a1nW7T6UOnlV2SrYIJBarfV69oOKqWutgoasH4AlntVrWdblNrQ6uKKoqUXZot1wiX6v9WL+97XjlyHBo2d5hyhuXoZNVJ1b5dK8Mw5B7vls1pU355vlpPterYG8d6vE+ceU7Zs+2y2q3KGZ4jh8sRe18HI8oujb13mk80dxkVLZlRIvcl7vjvTpJcI1xyjXCp5WRs9NWWYVNmYWYssNe3yjXCpVELR+n0odPyve+TI8eh4XOHq+10W+xL4vAS1RypUXNNs1pqW9T0UZMyCjKUOzJXRsRQYUVhrC9a2pU3Ok/Ntc3KvzhfNqdNUmwhkoMbDyqnNEf2LLuKpxcr3BL7smWxWZTpyZQRNmTPtiu7NDseEiOhSPzLcJYnSxnuDNXvq5ctwyb3JW4FfUGd3HVSJTNKZHVYdeC/Dij3otzYl+bRefJWe1W3t04ypOzSbJ0+dFp1u+vkyHGo/KZyWeyxc9WIGLH3ri8oZ55TnikeWR1WGVFDIX9IDe80KH9svmwZNgWOBWKhtyhTWZ4sBb1BtZ1uU97ovNi/KVM9OrX3lCJtkdiX+cY2BY4FYu+bky06te+USmaUqL0l9peqgnEFcrgccl/ilrfaq0h7RCXTS2SxWuRwOeJ/XTOihmQ5s0pib3/V6S9C19BAPw8NhOsES1bN9cebPtarX39VkuTMd2rYnGHyH/PLd8AX36f0M6U6+fZJSbHwWnhpoaYtm6bNt2+O7WCR/umv/6Q/fvWP8n/sV87wHNX/rV7X//Z6lcwoUf3f6lU6q1RWu1X+Y37lXpSrSHtEjmyHDMOQETVktfX8AIqEIvEP/g7dS0uMqKFwa1iBEwHlDMuRM9epaHssWDnznAq3hmMfrr6goqGonPlOWawW2Zw2GYah1lOtOv7mcbknuhU4HpCrzCVbpk3R9qg8kz29/s4aqht0YvsJHd58WPMemSfPZE9s6WFDaq1vjY1QFfT9IiMjaigajvZ4recSjUQV9AbldDm71F8ahqF2f3uXsoruJ3BHCJAh08puopFovOQly5N1QaUzydDRd51/Dx2vpbtoe1RtjbG/qLQ3t8vqsPbpgrJoezReepEofBgPDfTz0EA/Dw2E6wRL5gWNb/zkDYWbwzr6+lE5ch1qrWuV9z1vfJ/CSYXyvufVdRuu06iFo+Khafv/3a7qDdWyZ9n1lUNfie8faYso5A8py5M14K8HveMf6vRHHw8N9PPQQD8PDakYrrmg0SST75ksKXaBYzQUG2HrLOSP1bg685xdRiPtmbEusGV2HXG1ZdiUlUGwBgAAGExSOlzv3btX69evVzQa1YIFC7R48eJkN+mcbA5brL6y25/zO8K1PaPrr7yjFKH7dgAAAAw+iZ8r6wJFo1GtW7dODz30kFavXq3t27fr2LGeF0KlGqvDqkh7pOfIdVMsXFudXX/ltqxPRqxTticAAADQVykb6Q4dOqRhw4aptLRUdrtd8+bNU1VVVbKbdU4d02R1Hrm2Zdgko9PtTljEAAAAIH2kbLj2er0qKiqK3y8qKpLX6/2UR6QGm9PWY7ERh8sRv91RYx2/T7gGAABIGymb7HqbxKR7HfOWLVu0ZcsWSdJjjz0mj6f3ad8SzW63x587Ozdb3ohXdseZX21m/pnFFIqHFyvHkxP/mbsktrqjzWZLWvvRN537GemJPh4a6OehgX4eGlKxn1M2XBcVFamhoSF+v6GhQW63u8s+Cxcu1MKFC+P3kzXlTudpYNqj7WoPtsseOfOrtWWfKQVpbG7sskBLazh2OxKOMGVQimNap/RHHw8N9PPQQD8PDak4FV/KloWMGzdONTU1qqurUzgc1o4dOzR79uxkN+ucOi5o7LyYhjP3zEIk1FwDAACkr5RNdjabTffcc48effRRRaNRXX311Ro5cmSym3VONkdsZUKLegnXllj47oxwDQAAkD5SOtnNnDlTM2fOTHYzzovVae1xQaM9+5OFYjJsPerGu1/gCAAAgMErZctCBiub09ZjhUabM1YK0r0kRGLkGgAAIJ0Qrk1mdVhlRI0us510LBzTW7iOL3vec3IUAAAADDKEa5N1BOloezS+rSNU9xauO1/4CAAAgMGNmgST2RyxAB1uDZ/Z9klZiD2j5687uyRbpXNKNWvlrIFpIAAAABKGcG2yjpHrcEvPcN3xsy7726360qYvDUzjAAAAkFCUhZisY+S6vaU9vq1j+j2rnV83AABAOiPtmaxjdLo9cCZcd9RaU18NAACQ3gjXJusoATEinWYL+WTk2nWRKyltAgAAwMAgXJus+wqMkhSsD0qS3OPdA90cAAAADCDCtcl6DdenPwnXEwnXAAAA6YzZQkzWURbSYdo3pmnyv0xWdnG2xlw/JjmNAgAAwIAgXJusc7ieunyqPvN/PiNJmvW/mccaAAAg3VEWYrKC8QXx2x3T8gEAAGBoIFybLMuTpcJJhZJ6r78GAABA+iL9JcC0r0/TRVddpNHXjk52UwAAADCAqLlOgPKbylV+U3mymwEAAIABxsg1AAAAYBLCNQAAAGASwjUAAABgEsI1AAAAYBLCNQAAAGASwjUAAABgEsI1AAAAYBLCNQAAAGASwjUAAABgEsI1AAAAYBLCNQAAAGASwjUAAABgEsI1AAAAYBLCNQAAAGASwjUAAABgEsI1AAAAYBLCNQAAAGASwjUAAABgEothGEayGwEAAACkA0auTfDggw8muwkYAPRz+qOPhwb6eWign4eGVOxnwjUAAABgEsI1AAAAYBLb97///e8nuxHpYOzYscluAgYA/Zz+6OOhgX4eGujnoSHV+pkLGgEAAACTUBYCAAAAmMSe7AYMZnv37tX69esVjUa1YMECLV68ONlNwgWqr6/X2rVrdfr0aVksFi1cuFA33HCDAoGAVq9erVOnTqm4uFj333+/XC6XDMPQ+vXrtWfPHmVkZGj58uUp92cp9C4ajerBBx9UYWGhHnzwQdXV1emJJ55QIBDQxRdfrG9+85uy2+1qb2/XmjVr9OGHHyo3N1crVqxQSUlJspuPPmpubtZTTz2ljz/+WBaLRV/72tdUVlbG+ZxGXnnlFW3dulUWi0UjR47U8uXLdfr0ac7nNPDkk09q9+7dys/P16pVqyTpgj6P33jjDb3wwguSpJtvvllXXXXVgLSfkesLFI1GtW7dOj300ENavXq1tm/frmPHjiW7WbhANptNS5Ys0erVq/Xoo4/qtdde07Fjx7Rp0yZNmTJFlZWVmjJlijZt2iRJ2rNnj2pra1VZWamlS5fq6aefTvIrQF9t3rxZI0aMiN//z//8Ty1atEiVlZXKycnR1q1bJUlbt25VTk6OfvGLX2jRokX67W9/m6wm4wKsX79e06dP1xNPPKGf/vSnGjFiBOdzGvF6vfrDH/6gxx57TKtWrVI0GtWOHTs4n9PEVVddpYceeqjLtvM9fwOBgJ5//nn96Ec/0o9+9CM9//zzCgQCA9J+wvUFOnTokIYNG6bS0lLZ7XbNmzdPVVVVyW4WLpDb7Y5/083KytKIESPk9XpVVVWl+fPnS5Lmz58f7+Ndu3bpyiuvlMVi0YQJE9Tc3Cyfz5e09qNvGhoatHv3bi1YsECSZBiG3n33Xc2dO1dS7B/0zn3cMcoxd+5cvfPOO+ISlcGhpaVF7733nq655hpJkt1uV05ODudzmolGowqFQopEIgqFQiooKOB8ThMVFRVyuVxdtp3v+bt3715NnTpVLpdLLpdLU6dO1d69ewek/ZSFXCCv16uioqL4/aKiIn3wwQdJbBHMUldXp8OHD6u8vFyNjY1yu92SYgG8qalJUqz/PR5P/DFFRUXyer3xfZGaNmzYoDvvvFOtra2SJL/fr+zsbNlsNklSYWGhvF6vpK7nuM1mU3Z2tvx+v/Ly8pLTePRZXV2d8vLy9OSTT+ro0aMaO3as7r77bs7nNFJYWKgbb7xRX/va1+R0OjVt2jSNHTuW8zmNne/52z2ndX4/JBoj1xeot2+8FoslCS2BmYLBoFatWqW7775b2dnZZ92P/h98/vrXvyo/P7/PtbT08eAViUR0+PBhXXfddfrJT36ijIyM+J+Qe0NfDz6BQEBVVVVau3atfvWrXykYDH7qqCR9nL7Op28Hqs8Zub5ARUVFamhoiN9vaGhglGOQC4fDWrVqlT73uc/psssukyTl5+fL5/PJ7XbL5/PFRzmKiopUX18ffyz9n/oOHjyoXbt2ac+ePQqFQmptbdWGDRvU0tKiSCQim80mr9erwsJCSWfO8aKiIkUiEbW0tPT4MyVSU1FRkYqKijR+/HhJsTKATZs2cT6nkb/97W8qKSmJ9+Fll12mgwcPcj6nsfM9fwsLC1VdXR3f7vV6VVFRMSBtZeT6Ao0bN041NTWqq6tTOBzWjh07NHv27GQ3CxfIMAw99dRTGjFihL74xS/Gt8+ePVvbtm2TJG3btk1z5syJb3/zzTdlGIbef/99ZWdn82Gc4u644w499dRTWrt2rVasWKHJkyfrvvvu06WXXqqdO3dKil1Z3nEez5o1S2+88YYkaefOnbr00ksZ6RokCgoKVFRUpBMnTkiKBbGLLrqI8zmNeDweffDBB2pra5NhGPE+5nxOX+d7/k6fPl379u1TIBBQIBDQvn37NH369AFpK4vI9MPu3bv1zDPPKBqN6uqrr9bNN9+c7CbhAh04cEDf+973NGrUqPg/uLfffrvGjx+v1atXq76+Xh6PRytXroxP/bNu3Trt27dPTqdTy5cv17hx45L8KtBX7777rl5++WU9+OCDOnnyZI+puxwOh0KhkNasWaPDhw/L5XJpxYoVKi0tTXbT0UdHjhzRU089pXA4rJKSEi1fvlyGYXA+p5HnnntOO3bskM1m05gxY7Rs2TJ5vV7O5zTwxBNPqLq6Wn6/X/n5+br11ls1Z86c8z5/t27dqhdffFFSbCq+q6++ekDaT7gGAAAATEJZCAAAAGASwjUAAABgEsI1AAAAYBLCNQAAAGASwjUAAABgEsI1AKS4aDSqJUuWdFkowYx9AQDmYyo+ADDZkiVL4rdDoZDsdrus1thYxtKlS/W5z30uWU1LiNraWt1333167rnnkt0UAEg6lj8HAJP95je/id/++te/rnvvvVdTp0496/4dyzUDAAY/wjUADLDf/e53qqmpkcVi0e7du3XPPfeorKxMzzzzjI4fPy6n06m5c+fqrrvukt1uVyQS0e233641a9aopKRElZWVcrlcqq2t1YEDBzRy5Eh961vfUklJyXntK0l79uzRhg0bdPr0ac2fP1+HDx/WggULdNVVV/Vo9/vvv69169aptrZWTqdTV155pZYsWaKHH35Y0pkR+4cffljl5eX605/+pJdfflmNjY0aP368li5dKo/HE2/jV77yFb3yyisKBoO65pprdPvtt8tqterEiRP61a9+pSNHjshut2vq1Kn61re+NWD9AwD9Qc01ACTB22+/rSuuuEIbNmzQvHnzZLVadffdd2vdunV65JFHtG/fPm3ZsuWsj9++fbtuu+02/cd//Ic8Ho9+97vfnfe+jY2NWr16te68806tW7dOJSUlOnTo0FmPs379et1444165plnVFlZqblz50qSfvCDH0iKjdj/5je/UXl5uXbu3KmXXnpJDzzwgJ5++mmVl5ersrKyy/F27dqlxx9/XD/+8Y+1c+dOvfnmm5JiXz5mzJih9evX65e//KU+//nP9+2XCgApgHANAEkwceJEzZ49W1arVU6nU+Xl5Ro/frxsNptKS0u1YMECVVdXn/Xxl112mcaNGye73a7Pfe5zOnr06Hnv+9e//lVjxozRnDlzZLfbtWjRIuXm5p71ODabTbW1tfL7/crKytL48ePPuu8f//hH3XTTTSorK5PNZtM//uM/6tChQ/J6vfF9/uEf/kEul0slJSX6whe+oO3bt8ef59SpUzp9+rScTqcmTpx41ucBgFRDWQgAJEFRUVGX+8ePH9ezzz6rDz/8UKFQSJFI5FPDa0FBQfy20+lUMBg87319Pl+Xdlgslh7t6mz58uV67rnntGLFCpWUlOiWW27RzJkze923vr5e69at04YNG7ocv6GhQfn5+ZIkj8cT/1lxcXE8eN91113auHGjHnzwQeXm5urGG2/stUwFAFIR4RoAksBisXS5/+tf/1rjx4/X/fffr8zMTL300kvavXt3Qtvgdru1f//++H3DMLqMLHdXVlamFStWKBqNaufOnVq1apXWr1/f675FRUW67bbbNG/evB4/i0QikmIBvKysLH67sLAw3q5ly5ZJkqqrq/XDH/5QFRUV8TpxAEhllIUAQAoIBoPKzs5WRkaGjh079qn11maZOXOmPvzwQ+3atUuRSESbN29WU1PTWfd/88031dTUJKvVquzsbFksFlmtVuXn58tisejkyZPxfa+99lq98MILOnbsmCSpublZO3fu7HK8l156Sc3Nzaqvr9cf/vAHXX755ZKkHTt2xEN+Tk5O/HkAYDBg5BoAUsCSJUv07//+73rxxRc1duxYzZs3TwcOHEjocxYUFOj+++/Xhg0b9Itf/ELz58/XmDFjZLf3/tGwe/duPfPMM2pvb1dxcbFWrFghu90uu92uxYsX66GHHlIkEtF3v/tdXX755Wpra9Pq1atVX1+v7OxsTZs2LX4RpCTNmjVLDzzwgFpbW3XNNdfESz8OHTqkZ555Ri0tLSooKNBXv/rVLiUkAJDKWEQGACAptrrjvffeq5UrV2rSpEkJe57u0wUCQDrh72wAMITt3btXLS0tam9v1/PPPy+r1ary8vJkNwsABi3KQgBgCDtw4IAqKysVDoc1cuRIfec735HD4Uh2swBg0KIsBAAAADAJZSEAAACASQjXAAAAgEkI1wAAAIBJCNcAAACASQjXAAAAgEkI1wAAAIBJ/j9n1gX6mSRlJAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing the results of the training\n",
    "plt.figure(figsize=(12, 8))\n",
    "plt.plot(trainer.history['step'], trainer.history['reward'], color='purple', label='Reward')\n",
    "plt.title('Reward of BipedalWalker-v3 Policy over Training Session')\n",
    "plt.xlabel('Training steps')\n",
    "plt.ylabel('Reward')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  0 Reward:  7.201754848995174\n",
      "Step:  1 Reward:  5.910035305126789\n",
      "Step:  2 Reward:  -16.325964244665336\n",
      "Step:  3 Reward:  6.125203420809023\n",
      "Step:  4 Reward:  3.3585454726996655\n",
      "Step:  5 Reward:  2.7376328079717105\n",
      "Step:  6 Reward:  2.9403991365301834\n",
      "Step:  7 Reward:  2.251316898508081\n",
      "Step:  8 Reward:  3.1733946027911557\n",
      "Step:  9 Reward:  2.8667793908242096\n",
      "Step:  10 Reward:  3.869940680240518\n",
      "Step:  11 Reward:  4.009611958498104\n",
      "Step:  12 Reward:  4.07066822153234\n",
      "Step:  13 Reward:  3.6079994315117068\n",
      "Step:  14 Reward:  3.522110822229344\n",
      "Step:  15 Reward:  6.324855491124692\n",
      "Step:  16 Reward:  5.620884462282735\n",
      "Step:  17 Reward:  4.568540984787917\n",
      "Step:  18 Reward:  4.268912809356342\n",
      "Step:  19 Reward:  4.689601616519572\n",
      "Step:  20 Reward:  5.06420993279569\n",
      "Step:  21 Reward:  4.296691491191144\n",
      "Step:  22 Reward:  3.819935711374087\n",
      "Step:  23 Reward:  3.7273745162465337\n",
      "Step:  24 Reward:  4.071615534971969\n",
      "Step:  25 Reward:  3.4852839784281517\n",
      "Step:  26 Reward:  4.532317356613178\n",
      "Step:  27 Reward:  5.332020376700524\n",
      "Step:  28 Reward:  -5.164218407322484\n",
      "Step:  29 Reward:  2.7173704920275705\n",
      "Step:  30 Reward:  3.9660698449673264\n",
      "Step:  31 Reward:  3.819903846574065\n",
      "Step:  32 Reward:  4.079570282799206\n",
      "Step:  33 Reward:  4.098904979106965\n",
      "Step:  34 Reward:  4.132609766683576\n",
      "Step:  35 Reward:  4.334228975871789\n",
      "Step:  36 Reward:  4.3041762584323555\n",
      "Step:  37 Reward:  4.569254793468168\n",
      "Step:  38 Reward:  4.793187740298545\n",
      "Step:  39 Reward:  3.991942843698725\n",
      "Step:  40 Reward:  5.156465095381589\n",
      "Step:  41 Reward:  4.303718925042791\n",
      "Step:  42 Reward:  5.1218650265087025\n",
      "Step:  43 Reward:  4.483875210259766\n",
      "Step:  44 Reward:  4.610639206063661\n",
      "Step:  45 Reward:  5.146845932084862\n",
      "Step:  46 Reward:  5.2807442400522335\n",
      "Step:  47 Reward:  4.527099472365265\n",
      "Step:  48 Reward:  5.006879644885913\n",
      "Step:  49 Reward:  4.046096997279206\n",
      "Step:  50 Reward:  3.8837464248041504\n",
      "Step:  51 Reward:  0.39941693114259613\n",
      "Step:  52 Reward:  3.4384065538535875\n",
      "Step:  53 Reward:  4.343927244336811\n",
      "Step:  54 Reward:  4.317756669441688\n",
      "Step:  55 Reward:  5.0105947734187435\n",
      "Step:  56 Reward:  4.776572766003689\n",
      "Step:  57 Reward:  3.570573935952175\n",
      "Step:  58 Reward:  3.8318396211834997\n",
      "Step:  59 Reward:  5.11268272473103\n",
      "Step:  60 Reward:  4.7190250873703095\n",
      "Step:  61 Reward:  4.670237685686447\n",
      "Step:  62 Reward:  5.294826599517827\n",
      "Step:  63 Reward:  4.639656685408121\n",
      "Step:  64 Reward:  4.300606391384523\n",
      "Step:  65 Reward:  5.545741738370243\n",
      "Step:  66 Reward:  4.967846622047138\n",
      "Step:  67 Reward:  5.484978825628214\n",
      "Step:  68 Reward:  4.762442818149275\n",
      "Step:  69 Reward:  4.683855278895476\n",
      "Step:  70 Reward:  5.317465440299817\n",
      "Step:  71 Reward:  4.893212917824129\n",
      "Step:  72 Reward:  5.016034132853312\n",
      "Step:  73 Reward:  5.1763090535738\n",
      "Step:  74 Reward:  5.228386781172389\n",
      "Step:  75 Reward:  4.944808979689977\n",
      "Step:  76 Reward:  5.7230890677609425\n",
      "Step:  77 Reward:  5.808720142458278\n",
      "Step:  78 Reward:  5.77594798592432\n",
      "Step:  79 Reward:  5.90295008147939\n",
      "Step:  80 Reward:  5.527592089210439\n",
      "Step:  81 Reward:  5.618733879261005\n",
      "Step:  82 Reward:  5.6345635644727095\n",
      "Step:  83 Reward:  5.216562901803674\n",
      "Step:  84 Reward:  4.834279042245785\n",
      "Step:  85 Reward:  4.7428445381052615\n",
      "Step:  86 Reward:  5.813897167461776\n",
      "Step:  87 Reward:  5.374069903208536\n",
      "Step:  88 Reward:  5.1721658351708975\n",
      "Step:  89 Reward:  5.887578536918604\n",
      "Step:  90 Reward:  5.763143060522876\n",
      "Step:  91 Reward:  5.821675819004346\n",
      "Step:  92 Reward:  5.712397521349863\n",
      "Step:  93 Reward:  5.814786942294835\n",
      "Step:  94 Reward:  5.720318853477761\n",
      "Step:  95 Reward:  6.095181073766357\n",
      "Step:  96 Reward:  6.209007116150057\n",
      "Step:  97 Reward:  6.068181193489911\n",
      "Step:  98 Reward:  6.918483334705117\n",
      "Step:  99 Reward:  5.682532558387096\n",
      "Step:  100 Reward:  6.585124875942434\n",
      "Step:  101 Reward:  6.02110572990565\n",
      "Step:  102 Reward:  5.55736605094426\n",
      "Step:  103 Reward:  6.488996524967062\n",
      "Step:  104 Reward:  6.854504227676463\n",
      "Step:  105 Reward:  6.270559804809958\n",
      "Step:  106 Reward:  6.269101209381587\n",
      "Step:  107 Reward:  6.3377514418639995\n",
      "Step:  108 Reward:  6.181766899024041\n",
      "Step:  109 Reward:  5.227657887576995\n",
      "Step:  110 Reward:  6.889210139027537\n",
      "Step:  111 Reward:  5.42975322826268\n",
      "Step:  112 Reward:  6.861099138566986\n",
      "Step:  113 Reward:  6.2385737594479584\n",
      "Step:  114 Reward:  6.710323093313339\n",
      "Step:  115 Reward:  6.725357685050897\n",
      "Step:  116 Reward:  6.766820822091445\n",
      "Step:  117 Reward:  6.9248053983091475\n",
      "Step:  118 Reward:  6.835202743077072\n",
      "Step:  119 Reward:  6.893143166283329\n",
      "Step:  120 Reward:  6.120080978494566\n",
      "Step:  121 Reward:  6.543483245325637\n",
      "Step:  122 Reward:  6.416493301255689\n",
      "Step:  123 Reward:  5.55488588967691\n",
      "Step:  124 Reward:  6.187258267860491\n",
      "Step:  125 Reward:  6.7678033093352585\n",
      "Step:  126 Reward:  -26.015464517659897\n",
      "Step:  127 Reward:  7.371545289032229\n",
      "Step:  128 Reward:  -34.50829639049599\n",
      "Step:  129 Reward:  8.307219063255593\n",
      "Step:  130 Reward:  7.841141988841267\n",
      "Step:  131 Reward:  7.767748221813827\n",
      "Step:  132 Reward:  6.110041689771599\n",
      "Step:  133 Reward:  6.478897261445043\n",
      "Step:  134 Reward:  6.095747135019729\n",
      "Step:  135 Reward:  6.817298865727808\n",
      "Step:  136 Reward:  7.250683995056189\n",
      "Step:  137 Reward:  7.254347051207997\n",
      "Step:  138 Reward:  6.546096197773505\n",
      "Step:  139 Reward:  7.349214317041209\n",
      "Step:  140 Reward:  7.603589349053777\n",
      "Step:  141 Reward:  6.8755579093784025\n",
      "Step:  142 Reward:  7.438243342276381\n",
      "Step:  143 Reward:  7.523839570656984\n",
      "Step:  144 Reward:  6.932021062453535\n",
      "Step:  145 Reward:  7.533937105465352\n",
      "Step:  146 Reward:  8.149427163661548\n",
      "Step:  147 Reward:  9.219544118906821\n",
      "Step:  148 Reward:  7.5947183560369815\n",
      "Step:  149 Reward:  8.902130107693772\n",
      "Step:  150 Reward:  9.522880045904193\n",
      "Step:  151 Reward:  8.00555356662781\n",
      "Step:  152 Reward:  7.848025648423466\n",
      "Step:  153 Reward:  8.328584588969928\n",
      "Step:  154 Reward:  9.327004420537891\n",
      "Step:  155 Reward:  8.993520678067773\n",
      "Step:  156 Reward:  7.565801444271537\n",
      "Step:  157 Reward:  8.083160799273122\n",
      "Step:  158 Reward:  8.421785130231443\n",
      "Step:  159 Reward:  7.43480361990202\n",
      "Step:  160 Reward:  7.293120690185116\n",
      "Step:  161 Reward:  7.102702325748076\n",
      "Step:  162 Reward:  8.327809089649787\n",
      "Step:  163 Reward:  8.005750729778553\n",
      "Step:  164 Reward:  8.785550924899255\n",
      "Step:  165 Reward:  8.827965704959437\n",
      "Step:  166 Reward:  8.250403467697492\n",
      "Step:  167 Reward:  8.056148194629193\n",
      "Step:  168 Reward:  7.492627521664737\n",
      "Step:  169 Reward:  7.788361240431186\n",
      "Step:  170 Reward:  7.82468476767208\n",
      "Step:  171 Reward:  8.25535236475746\n",
      "Step:  172 Reward:  7.410653381192224\n",
      "Step:  173 Reward:  7.166967739531161\n",
      "Step:  174 Reward:  7.937517643672278\n",
      "Step:  175 Reward:  7.8090879481034285\n",
      "Step:  176 Reward:  8.35291761902252\n",
      "Step:  177 Reward:  7.431161152518932\n",
      "Step:  178 Reward:  8.354822363157247\n",
      "Step:  179 Reward:  8.36365173329716\n",
      "Step:  180 Reward:  9.071882079710095\n",
      "Step:  181 Reward:  8.910132442627217\n",
      "Step:  182 Reward:  9.969671693604557\n",
      "Step:  183 Reward:  9.357019287715012\n",
      "Step:  184 Reward:  9.095351348505826\n",
      "Step:  185 Reward:  8.181716497597602\n",
      "Step:  186 Reward:  8.711836263194908\n",
      "Step:  187 Reward:  8.303180675862516\n",
      "Step:  188 Reward:  8.236271041486653\n",
      "Step:  189 Reward:  8.1212785325374\n",
      "Step:  190 Reward:  8.771484265448043\n",
      "Step:  191 Reward:  9.27445844092687\n",
      "Step:  192 Reward:  10.08940095103435\n",
      "Step:  193 Reward:  10.552174494893562\n",
      "Step:  194 Reward:  10.626812322405359\n",
      "Step:  195 Reward:  12.077462832001741\n",
      "Step:  196 Reward:  12.71826446960273\n",
      "Step:  197 Reward:  11.233435399735907\n",
      "Step:  198 Reward:  12.949185073552993\n",
      "Step:  199 Reward:  10.21657568283818\n",
      "Step:  200 Reward:  8.511971391966352\n",
      "Step:  201 Reward:  8.516587040615434\n",
      "Step:  202 Reward:  10.619978031931637\n",
      "Step:  203 Reward:  10.298941037332751\n",
      "Step:  204 Reward:  14.329289358474298\n",
      "Step:  205 Reward:  11.67058475735386\n",
      "Step:  206 Reward:  12.231392036077454\n",
      "Step:  207 Reward:  12.050398988486492\n",
      "Step:  208 Reward:  11.695639979875978\n",
      "Step:  209 Reward:  10.687817233992865\n",
      "Step:  210 Reward:  10.078633853757642\n",
      "Step:  211 Reward:  12.278794611010785\n",
      "Step:  212 Reward:  10.058432707354843\n",
      "Step:  213 Reward:  12.72690163201022\n",
      "Step:  214 Reward:  10.144453605966692\n",
      "Step:  215 Reward:  10.256547780445397\n",
      "Step:  216 Reward:  9.87497603507009\n",
      "Step:  217 Reward:  10.528754254889295\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  218 Reward:  10.388106388092616\n",
      "Step:  219 Reward:  11.823639200851623\n",
      "Step:  220 Reward:  10.95469222514039\n",
      "Step:  221 Reward:  11.862585806485246\n",
      "Step:  222 Reward:  10.076116092107862\n",
      "Step:  223 Reward:  8.781296957015527\n",
      "Step:  224 Reward:  11.005192637225619\n",
      "Step:  225 Reward:  9.569805977601522\n",
      "Step:  226 Reward:  10.297250523721202\n",
      "Step:  227 Reward:  9.606115799690855\n",
      "Step:  228 Reward:  9.94702147186791\n",
      "Step:  229 Reward:  9.456330663307533\n",
      "Step:  230 Reward:  9.384687600080646\n",
      "Step:  231 Reward:  10.581306421506945\n",
      "Step:  232 Reward:  8.51453925220397\n",
      "Step:  233 Reward:  11.135745582038199\n",
      "Step:  234 Reward:  11.756716155874196\n",
      "Step:  235 Reward:  13.243532529536262\n",
      "Step:  236 Reward:  14.375274265767816\n",
      "Step:  237 Reward:  13.39892490185419\n",
      "Step:  238 Reward:  12.115612531765928\n",
      "Step:  239 Reward:  10.220966168378226\n",
      "Step:  240 Reward:  9.130378672133194\n",
      "Step:  241 Reward:  11.349093594738056\n",
      "Step:  242 Reward:  8.950868095893288\n",
      "Step:  243 Reward:  10.050049527753812\n",
      "Step:  244 Reward:  11.987367371722584\n",
      "Step:  245 Reward:  12.041547926118291\n",
      "Step:  246 Reward:  12.24651635652811\n",
      "Step:  247 Reward:  10.450903058011221\n",
      "Step:  248 Reward:  11.103257560749265\n",
      "Step:  249 Reward:  12.05887976324377\n",
      "Step:  250 Reward:  10.007446697791593\n",
      "Step:  251 Reward:  10.450550930238094\n",
      "Step:  252 Reward:  10.531680849317397\n",
      "Step:  253 Reward:  10.380278248833811\n",
      "Step:  254 Reward:  9.107515728066815\n",
      "Step:  255 Reward:  9.712509242221653\n",
      "Step:  256 Reward:  10.966851276654532\n",
      "Step:  257 Reward:  11.2526328059136\n",
      "Step:  258 Reward:  9.695095706145395\n",
      "Step:  259 Reward:  11.937414011129787\n",
      "Step:  260 Reward:  13.292911683427327\n",
      "Step:  261 Reward:  11.568292764017167\n",
      "Step:  262 Reward:  11.18006540383414\n",
      "Step:  263 Reward:  10.742530737236846\n",
      "Step:  264 Reward:  11.50373112630865\n",
      "Step:  265 Reward:  8.579907117908514\n",
      "Step:  266 Reward:  11.501812629349548\n",
      "Step:  267 Reward:  9.536911355530114\n",
      "Step:  268 Reward:  8.119054019235021\n",
      "Step:  269 Reward:  10.514602345784844\n",
      "Step:  270 Reward:  10.51682222141041\n",
      "Step:  271 Reward:  11.480097563128266\n",
      "Step:  272 Reward:  10.391488981552676\n",
      "Step:  273 Reward:  12.938311223450102\n",
      "Step:  274 Reward:  11.540515824909878\n",
      "Step:  275 Reward:  9.768939498372987\n",
      "Step:  276 Reward:  5.258673128763942\n",
      "Step:  277 Reward:  10.717729223659148\n",
      "Step:  278 Reward:  11.633521635724277\n",
      "Step:  279 Reward:  10.324507170095597\n",
      "Step:  280 Reward:  9.92299264721314\n",
      "Step:  281 Reward:  11.9294799394992\n",
      "Step:  282 Reward:  11.710784647133995\n",
      "Step:  283 Reward:  12.010355247475998\n",
      "Step:  284 Reward:  12.105038066868214\n",
      "Step:  285 Reward:  9.657866846001678\n",
      "Step:  286 Reward:  11.424664878649509\n",
      "Step:  287 Reward:  10.27075931157202\n",
      "Step:  288 Reward:  11.869730246498326\n",
      "Step:  289 Reward:  11.191924653262008\n",
      "Step:  290 Reward:  11.143229865028665\n",
      "Step:  291 Reward:  11.372288257908222\n",
      "Step:  292 Reward:  10.692671536074878\n",
      "Step:  293 Reward:  8.202508807660296\n",
      "Step:  294 Reward:  11.455519166216288\n",
      "Step:  295 Reward:  9.51942909149115\n",
      "Step:  296 Reward:  10.960880787581152\n",
      "Step:  297 Reward:  10.962836507027653\n",
      "Step:  298 Reward:  13.118039159455487\n",
      "Step:  299 Reward:  10.849406924323654\n",
      "Step:  300 Reward:  28.396302161408574\n",
      "Step:  301 Reward:  13.343064215503407\n",
      "Step:  302 Reward:  29.107702593362045\n",
      "Step:  303 Reward:  30.585579455815058\n",
      "Step:  304 Reward:  25.731622084019527\n",
      "Step:  305 Reward:  62.01058325609146\n",
      "Step:  306 Reward:  51.04481374015909\n",
      "Step:  307 Reward:  54.379285063606865\n",
      "Step:  308 Reward:  52.90141309534029\n",
      "Step:  309 Reward:  31.888488635678357\n",
      "Step:  310 Reward:  64.83011706689405\n",
      "Step:  311 Reward:  15.363779416050537\n",
      "Step:  312 Reward:  56.07021845421532\n",
      "Step:  313 Reward:  10.630620138917267\n",
      "Step:  314 Reward:  30.64406425338912\n",
      "Step:  315 Reward:  15.099598874233386\n",
      "Step:  316 Reward:  109.35088623068411\n",
      "Step:  317 Reward:  197.08355441117143\n",
      "Step:  318 Reward:  92.06313952058349\n",
      "Step:  319 Reward:  115.55325891164497\n",
      "Step:  320 Reward:  34.96534594124255\n",
      "Step:  321 Reward:  20.274994634448195\n",
      "Step:  322 Reward:  -1.0147183679987086\n",
      "Step:  323 Reward:  32.49637290507533\n",
      "Step:  324 Reward:  145.60365427620764\n",
      "Step:  325 Reward:  128.89972456679186\n",
      "Step:  326 Reward:  82.01447339994407\n",
      "Step:  327 Reward:  57.6915644675214\n",
      "Step:  328 Reward:  170.74228614037946\n",
      "Step:  329 Reward:  73.36859319467666\n",
      "Step:  330 Reward:  105.78564029000488\n",
      "Step:  331 Reward:  36.97094643126235\n",
      "Step:  332 Reward:  64.3953541143189\n",
      "Step:  333 Reward:  32.19800138060309\n",
      "Step:  334 Reward:  17.939482345379172\n",
      "Step:  335 Reward:  63.884723057678684\n",
      "Step:  336 Reward:  80.82310942734634\n",
      "Step:  337 Reward:  221.55484330072338\n",
      "Step:  338 Reward:  47.859401814469415\n",
      "Step:  339 Reward:  25.35895408931258\n",
      "Step:  340 Reward:  219.08034952251214\n",
      "Step:  341 Reward:  56.636498469436404\n",
      "Step:  342 Reward:  129.47052078934604\n",
      "Step:  343 Reward:  18.320661916344914\n",
      "Step:  344 Reward:  58.73146295097014\n",
      "Step:  345 Reward:  57.772148787805236\n",
      "Step:  346 Reward:  158.57199057220333\n",
      "Step:  347 Reward:  36.12706287579873\n",
      "Step:  348 Reward:  55.73471577966802\n",
      "Step:  349 Reward:  -9.135754086767474\n",
      "Step:  350 Reward:  59.19980886732543\n",
      "Step:  351 Reward:  124.75542073918521\n",
      "Step:  352 Reward:  35.39178632108426\n",
      "Step:  353 Reward:  34.312628139250194\n",
      "Step:  354 Reward:  187.18097269116583\n",
      "Step:  355 Reward:  158.4222239384444\n",
      "Step:  356 Reward:  95.5354971091317\n",
      "Step:  357 Reward:  52.06348534234772\n",
      "Step:  358 Reward:  149.34701409392426\n",
      "Step:  359 Reward:  87.93332791417329\n",
      "Step:  360 Reward:  73.8160197617037\n",
      "Step:  361 Reward:  45.27609592973254\n",
      "Step:  362 Reward:  49.93172448754222\n",
      "Step:  363 Reward:  122.12045464636995\n",
      "Step:  364 Reward:  46.88707236776354\n",
      "Step:  365 Reward:  20.601978441271147\n",
      "Step:  366 Reward:  30.437090457433904\n",
      "Step:  367 Reward:  239.99352796059742\n",
      "Step:  368 Reward:  36.61376700501052\n",
      "Step:  369 Reward:  266.9830897733789\n",
      "Step:  370 Reward:  69.05346691769853\n",
      "Step:  371 Reward:  31.142452586666423\n",
      "Step:  372 Reward:  57.36743952628853\n",
      "Step:  373 Reward:  121.60818371482091\n",
      "Step:  374 Reward:  49.390364944940586\n",
      "Step:  375 Reward:  175.63601335230334\n",
      "Step:  376 Reward:  100.00355592245535\n",
      "Step:  377 Reward:  182.46983231213642\n",
      "Step:  378 Reward:  65.42524162214815\n",
      "Step:  379 Reward:  39.20719886762038\n",
      "Step:  380 Reward:  39.61276695828622\n",
      "Step:  381 Reward:  113.47437519158908\n",
      "Step:  382 Reward:  189.4676720539638\n",
      "Step:  383 Reward:  78.4651035470581\n",
      "Step:  384 Reward:  41.758933684439526\n",
      "Step:  385 Reward:  127.70778370650194\n",
      "Step:  386 Reward:  156.5702156461143\n",
      "Step:  387 Reward:  117.29145843054772\n",
      "Step:  388 Reward:  189.65530819355138\n",
      "Step:  389 Reward:  190.4488710462441\n",
      "Step:  390 Reward:  45.824792616639506\n",
      "Step:  391 Reward:  175.56275677749207\n",
      "Step:  392 Reward:  172.22747630333308\n",
      "Step:  393 Reward:  192.4881117212942\n",
      "Step:  394 Reward:  193.6344478765219\n",
      "Step:  395 Reward:  76.89764119281715\n",
      "Step:  396 Reward:  204.72961484911173\n",
      "Step:  397 Reward:  191.57993962951392\n",
      "Step:  398 Reward:  200.10663992980747\n",
      "Step:  399 Reward:  177.91590874637058\n",
      "Step:  400 Reward:  215.68895500773942\n",
      "Step:  401 Reward:  212.21633087244638\n",
      "Step:  402 Reward:  219.6205883370357\n",
      "Step:  403 Reward:  218.14254516123557\n",
      "Step:  404 Reward:  214.89691692190638\n",
      "Step:  405 Reward:  224.74424771500347\n",
      "Step:  406 Reward:  180.0517850873213\n",
      "Step:  407 Reward:  192.8305346203926\n",
      "Step:  408 Reward:  82.1138565761716\n",
      "Step:  409 Reward:  224.52217617728684\n",
      "Step:  410 Reward:  230.959043992228\n",
      "Step:  411 Reward:  181.9719284010895\n",
      "Step:  412 Reward:  226.1413097733424\n",
      "Step:  413 Reward:  231.1254749661274\n",
      "Step:  414 Reward:  240.50888421780846\n",
      "Step:  415 Reward:  228.83529563494315\n",
      "Step:  416 Reward:  244.43977191835705\n",
      "Step:  417 Reward:  230.99544531763138\n",
      "Step:  418 Reward:  235.9538707858265\n",
      "Step:  419 Reward:  235.95923430958464\n",
      "Step:  420 Reward:  225.85393002166757\n",
      "Step:  421 Reward:  226.3414014803564\n",
      "Step:  422 Reward:  135.92250169962043\n",
      "Step:  423 Reward:  234.99531292262157\n",
      "Step:  424 Reward:  224.45696651925795\n",
      "Step:  425 Reward:  249.70864298022568\n",
      "Step:  426 Reward:  241.103505516112\n",
      "Step:  427 Reward:  249.8461132582954\n",
      "Step:  428 Reward:  217.43859654036913\n",
      "Step:  429 Reward:  236.2391527232024\n",
      "Step:  430 Reward:  53.89852080025844\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  431 Reward:  60.21360385019215\n",
      "Step:  432 Reward:  228.5769552544754\n",
      "Step:  433 Reward:  237.0667971333898\n",
      "Step:  434 Reward:  231.69974338292147\n",
      "Step:  435 Reward:  230.54481316636264\n",
      "Step:  436 Reward:  188.0104504693506\n",
      "Step:  437 Reward:  131.4233419998278\n",
      "Step:  438 Reward:  232.54766627190745\n",
      "Step:  439 Reward:  245.32803749486615\n",
      "Step:  440 Reward:  246.92270085973354\n",
      "Step:  441 Reward:  102.08504680956176\n",
      "Step:  442 Reward:  209.68956569649373\n",
      "Step:  443 Reward:  249.38164980023592\n",
      "Step:  444 Reward:  249.1000471436739\n",
      "Step:  445 Reward:  238.85877516154142\n",
      "Step:  446 Reward:  240.30674329386568\n",
      "Step:  447 Reward:  244.18594709796807\n",
      "Step:  448 Reward:  225.7375956610299\n",
      "Step:  449 Reward:  216.62790299425902\n",
      "Step:  450 Reward:  235.45126288649504\n",
      "Step:  451 Reward:  114.3491331228251\n",
      "Step:  452 Reward:  265.2244817757962\n",
      "Step:  453 Reward:  277.29411381384745\n",
      "Step:  454 Reward:  174.33626152803936\n",
      "Step:  455 Reward:  259.9154197749472\n",
      "Step:  456 Reward:  252.5303252650397\n",
      "Step:  457 Reward:  249.64619648563428\n",
      "Step:  458 Reward:  262.71209862727125\n",
      "Step:  459 Reward:  61.191018055317144\n",
      "Step:  460 Reward:  267.1187379078906\n",
      "Step:  461 Reward:  276.61478835410327\n",
      "Step:  462 Reward:  245.73868961215595\n",
      "Step:  463 Reward:  253.18197689120495\n",
      "Step:  464 Reward:  253.5325283074611\n",
      "Step:  465 Reward:  147.6690552502524\n",
      "Step:  466 Reward:  245.14986547597448\n",
      "Step:  467 Reward:  254.24081233273367\n",
      "Step:  468 Reward:  235.4622838760092\n",
      "Step:  469 Reward:  245.4176396564796\n",
      "Step:  470 Reward:  254.12519465454395\n",
      "Step:  471 Reward:  269.0875187477675\n",
      "Step:  472 Reward:  275.7614095911859\n",
      "Step:  473 Reward:  279.0546202125807\n",
      "Step:  474 Reward:  268.68014634116696\n",
      "Step:  475 Reward:  276.27081388163856\n",
      "Step:  476 Reward:  251.14876285172284\n",
      "Step:  477 Reward:  259.8795855132405\n",
      "Step:  478 Reward:  270.9969018978585\n",
      "Step:  479 Reward:  192.0456681861912\n",
      "Step:  480 Reward:  276.23286728154324\n",
      "Step:  481 Reward:  268.2725248927064\n",
      "Step:  482 Reward:  280.1795971295487\n",
      "Step:  483 Reward:  271.7933412391228\n",
      "Step:  484 Reward:  282.7616681880869\n",
      "Step:  485 Reward:  280.53019353267524\n",
      "Step:  486 Reward:  266.52580731461813\n",
      "Step:  487 Reward:  286.89715753868654\n",
      "Step:  488 Reward:  265.6157309836133\n",
      "Step:  489 Reward:  270.23973265092803\n",
      "Step:  490 Reward:  271.157802274903\n",
      "Step:  491 Reward:  251.3400795710307\n",
      "Step:  492 Reward:  251.33240802245973\n",
      "Step:  493 Reward:  283.1969540956635\n",
      "Step:  494 Reward:  268.77051456261927\n",
      "Step:  495 Reward:  273.90153972987866\n",
      "Step:  496 Reward:  273.9370488513527\n",
      "Step:  497 Reward:  288.4910757151481\n",
      "Step:  498 Reward:  272.85202016137896\n",
      "Step:  499 Reward:  265.18394331293695\n",
      "Step:  500 Reward:  261.91050637636647\n",
      "Step:  501 Reward:  273.59530340513345\n",
      "Step:  502 Reward:  271.1256255243499\n",
      "Step:  503 Reward:  268.89680707991926\n",
      "Step:  504 Reward:  259.4880590274701\n",
      "Step:  505 Reward:  265.07616473285805\n",
      "Step:  506 Reward:  286.5386720077671\n",
      "Step:  507 Reward:  283.2984967934144\n",
      "Step:  508 Reward:  278.3273852818123\n",
      "Step:  509 Reward:  284.8162720938785\n",
      "Step:  510 Reward:  290.60371479828035\n",
      "Step:  511 Reward:  286.0799869495865\n",
      "Step:  512 Reward:  28.829127974591128\n",
      "Step:  513 Reward:  260.9525498400119\n",
      "Step:  514 Reward:  266.50818517005894\n",
      "Step:  515 Reward:  277.8105802750847\n",
      "Step:  516 Reward:  251.35227231845093\n",
      "Step:  517 Reward:  266.2991821099375\n",
      "Step:  518 Reward:  272.18360999458093\n",
      "Step:  519 Reward:  286.5235055010269\n",
      "Step:  520 Reward:  270.3421459831605\n",
      "Step:  521 Reward:  263.0133543621573\n",
      "Step:  522 Reward:  258.5715285781689\n",
      "Step:  523 Reward:  277.6950607228577\n",
      "Step:  524 Reward:  285.90393469699984\n",
      "Step:  525 Reward:  273.0331917978804\n",
      "Step:  526 Reward:  268.92390838785184\n",
      "Step:  527 Reward:  281.0422799425454\n",
      "Step:  528 Reward:  289.0438365197519\n",
      "Step:  529 Reward:  273.7144200567165\n",
      "Step:  530 Reward:  279.38613204048426\n",
      "Step:  531 Reward:  287.27375900015477\n",
      "Step:  532 Reward:  291.4074403156675\n",
      "Step:  533 Reward:  233.15369487898732\n",
      "Step:  534 Reward:  290.3169509050299\n",
      "Step:  535 Reward:  291.5519797604215\n",
      "Step:  536 Reward:  289.20252720451606\n",
      "Step:  537 Reward:  287.6849661861138\n",
      "Step:  538 Reward:  287.8225016894635\n",
      "Step:  539 Reward:  285.51932251757285\n",
      "Step:  540 Reward:  291.24857075321387\n",
      "Step:  541 Reward:  291.6584841594861\n",
      "Step:  542 Reward:  291.1367102590985\n",
      "Step:  543 Reward:  290.96399374649934\n",
      "Step:  544 Reward:  294.43406463159886\n",
      "Step:  545 Reward:  289.44802452358005\n",
      "Step:  546 Reward:  290.33152286244336\n",
      "Step:  547 Reward:  276.34282146746074\n",
      "Step:  548 Reward:  290.9252516812736\n",
      "Step:  549 Reward:  274.813528581549\n",
      "Step:  550 Reward:  288.2270846861079\n",
      "Step:  551 Reward:  290.22546469542704\n",
      "Step:  552 Reward:  272.9995395604952\n",
      "Step:  553 Reward:  293.7336277749212\n",
      "Step:  554 Reward:  290.82305903458763\n",
      "Step:  555 Reward:  290.0236261153938\n",
      "Step:  556 Reward:  290.93997462877655\n",
      "Step:  557 Reward:  288.6814160159217\n",
      "Step:  558 Reward:  291.06782372536463\n",
      "Step:  559 Reward:  283.7911810464464\n",
      "Step:  560 Reward:  287.4342405432827\n",
      "Step:  561 Reward:  292.3685829744494\n",
      "Step:  562 Reward:  291.282847529652\n",
      "Step:  563 Reward:  285.20292200145184\n",
      "Step:  564 Reward:  283.94974851559596\n",
      "Step:  565 Reward:  272.6189264722521\n",
      "Step:  566 Reward:  291.7345856671627\n",
      "Step:  567 Reward:  286.7448095298272\n",
      "Step:  568 Reward:  290.4886468806498\n",
      "Step:  569 Reward:  290.11264902391997\n",
      "Step:  570 Reward:  290.20947098842163\n",
      "Step:  571 Reward:  292.52937483765777\n",
      "Step:  572 Reward:  289.91343978429126\n",
      "Step:  573 Reward:  291.5676795240167\n",
      "Step:  574 Reward:  294.0313153856475\n",
      "Step:  575 Reward:  289.47536830154917\n",
      "Step:  576 Reward:  288.83132076801047\n",
      "Step:  577 Reward:  294.0383221242354\n",
      "Step:  578 Reward:  289.29139398549637\n",
      "Step:  579 Reward:  92.67536162751165\n",
      "Step:  580 Reward:  278.27399451276386\n",
      "Step:  581 Reward:  288.51729314529314\n",
      "Step:  582 Reward:  292.0861023577896\n",
      "Step:  583 Reward:  288.2530200387069\n",
      "Step:  584 Reward:  274.08493584778887\n",
      "Step:  585 Reward:  269.3039571685824\n",
      "Step:  586 Reward:  289.97411059867034\n",
      "Step:  587 Reward:  290.56536475549143\n",
      "Step:  588 Reward:  281.8882799198017\n",
      "Step:  589 Reward:  288.7006147595578\n",
      "Step:  590 Reward:  287.02518553684547\n",
      "Step:  591 Reward:  287.99153041706177\n",
      "Step:  592 Reward:  288.2549319820998\n",
      "Step:  593 Reward:  292.6699163523026\n",
      "Step:  594 Reward:  291.7083818392473\n",
      "Step:  595 Reward:  290.79248136620424\n",
      "Step:  596 Reward:  283.9307372655655\n",
      "Step:  597 Reward:  286.81709593609173\n",
      "Step:  598 Reward:  290.49099306058406\n",
      "Step:  599 Reward:  289.935093924273\n",
      "Step:  600 Reward:  277.2337965178755\n",
      "Step:  601 Reward:  290.000947384393\n",
      "Step:  602 Reward:  287.87354598150154\n",
      "Step:  603 Reward:  287.4935207523374\n",
      "Step:  604 Reward:  287.36683387080075\n",
      "Step:  605 Reward:  289.3441850595288\n",
      "Step:  606 Reward:  292.47549481054256\n",
      "Step:  607 Reward:  289.7644535011341\n",
      "Step:  608 Reward:  287.13534621171243\n",
      "Step:  609 Reward:  287.0537702917076\n",
      "Step:  610 Reward:  284.5880120776075\n",
      "Step:  611 Reward:  289.11559861479805\n",
      "Step:  612 Reward:  287.7458756139181\n",
      "Step:  613 Reward:  290.92259995363645\n",
      "Step:  614 Reward:  289.2245895123082\n",
      "Step:  615 Reward:  290.59174378034254\n",
      "Step:  616 Reward:  289.371317593979\n",
      "Step:  617 Reward:  288.37924673936476\n",
      "Step:  618 Reward:  290.1316767378872\n",
      "Step:  619 Reward:  290.0586549235764\n",
      "Step:  620 Reward:  287.91953564850314\n",
      "Step:  621 Reward:  288.5142852164375\n",
      "Step:  622 Reward:  286.78896803435384\n",
      "Step:  623 Reward:  288.8772942226725\n",
      "Step:  624 Reward:  292.80720200491896\n",
      "Step:  625 Reward:  286.40757699691954\n",
      "Step:  626 Reward:  286.5814249796394\n",
      "Step:  627 Reward:  286.77953834370925\n",
      "Step:  628 Reward:  281.8412596602519\n",
      "Step:  629 Reward:  287.4596535336734\n",
      "Step:  630 Reward:  288.65831442535665\n",
      "Step:  631 Reward:  285.9342777902502\n",
      "Step:  632 Reward:  277.4558013297067\n",
      "Step:  633 Reward:  287.2633621234744\n",
      "Step:  634 Reward:  291.23154389073454\n",
      "Step:  635 Reward:  289.0988638464216\n",
      "Step:  636 Reward:  290.56414651753715\n",
      "Step:  637 Reward:  290.2100902812056\n",
      "Step:  638 Reward:  292.1845866087923\n",
      "Step:  639 Reward:  289.29356309653696\n",
      "Step:  640 Reward:  289.0222137366735\n",
      "Step:  641 Reward:  293.0798475187182\n",
      "Step:  642 Reward:  291.62730534527225\n",
      "Step:  643 Reward:  285.7235349245384\n",
      "Step:  644 Reward:  288.3279669870781\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  645 Reward:  289.6701627961298\n",
      "Step:  646 Reward:  292.6090675243363\n",
      "Step:  647 Reward:  291.4154155182841\n",
      "Step:  648 Reward:  283.4990474101051\n",
      "Step:  649 Reward:  289.3130716200317\n",
      "Step:  650 Reward:  292.60474113672643\n",
      "Step:  651 Reward:  291.3077820780503\n",
      "Step:  652 Reward:  291.3916715427709\n",
      "Step:  653 Reward:  292.6167764912496\n",
      "Step:  654 Reward:  292.072852386178\n",
      "Step:  655 Reward:  293.80153479089216\n",
      "Step:  656 Reward:  292.26907170739315\n",
      "Step:  657 Reward:  294.44542860978356\n",
      "Step:  658 Reward:  120.78659668954846\n",
      "Step:  659 Reward:  289.5363347208893\n",
      "Step:  660 Reward:  292.1509220797319\n",
      "Step:  661 Reward:  289.8674394335326\n",
      "Step:  662 Reward:  286.0142107372029\n",
      "Step:  663 Reward:  291.7686056037756\n",
      "Step:  664 Reward:  290.8898126652285\n",
      "Step:  665 Reward:  289.06337454009594\n",
      "Step:  666 Reward:  289.2951369707579\n",
      "Step:  667 Reward:  292.8812687437367\n",
      "Step:  668 Reward:  291.130918644188\n",
      "Step:  669 Reward:  287.2646953154002\n",
      "Step:  670 Reward:  281.4308666700308\n",
      "Step:  671 Reward:  288.52855468461314\n",
      "Step:  672 Reward:  292.5324665675301\n",
      "Step:  673 Reward:  289.83159838638545\n",
      "Step:  674 Reward:  210.44954504648257\n",
      "Step:  675 Reward:  288.3905310307758\n",
      "Step:  676 Reward:  138.9963081004046\n",
      "Step:  677 Reward:  285.4835302130321\n",
      "Step:  678 Reward:  288.24714310913845\n",
      "Step:  679 Reward:  284.75513084095087\n",
      "Step:  680 Reward:  286.6463609667137\n",
      "Step:  681 Reward:  289.4927096970081\n",
      "Step:  682 Reward:  285.21723008897305\n",
      "Step:  683 Reward:  287.47553610400934\n",
      "Step:  684 Reward:  290.4272131836989\n",
      "Step:  685 Reward:  290.8177402106276\n",
      "Step:  686 Reward:  288.4569715455592\n",
      "Step:  687 Reward:  289.3768399650813\n",
      "Step:  688 Reward:  291.68805044332623\n",
      "Step:  689 Reward:  288.31367797376504\n",
      "Step:  690 Reward:  282.0026953492504\n",
      "Step:  691 Reward:  288.7312644644169\n",
      "Step:  692 Reward:  288.4253989914259\n",
      "Step:  693 Reward:  289.4104342606841\n",
      "Step:  694 Reward:  290.00140376625586\n",
      "Step:  695 Reward:  288.77991574745744\n",
      "Step:  696 Reward:  288.3018387141098\n",
      "Step:  697 Reward:  287.08763943959104\n",
      "Step:  698 Reward:  285.68147636015766\n",
      "Step:  699 Reward:  289.38354756343733\n",
      "Step:  700 Reward:  290.69268055953063\n",
      "Step:  701 Reward:  288.3652535776503\n",
      "Step:  702 Reward:  289.38053323263466\n",
      "Step:  703 Reward:  291.436673236565\n",
      "Step:  704 Reward:  291.87258134535267\n",
      "Step:  705 Reward:  285.9847789471292\n",
      "Step:  706 Reward:  292.3844332878802\n",
      "Step:  707 Reward:  285.66218173931463\n",
      "Step:  708 Reward:  291.5113433506288\n",
      "Step:  709 Reward:  291.15512361247056\n",
      "Step:  710 Reward:  288.3116565152463\n",
      "Step:  711 Reward:  292.04074187699035\n",
      "Step:  712 Reward:  293.7844431083283\n",
      "Step:  713 Reward:  291.54488091818945\n",
      "Step:  714 Reward:  291.7959117449778\n",
      "Step:  715 Reward:  289.9285607081487\n",
      "Step:  716 Reward:  293.7717951648531\n",
      "Step:  717 Reward:  295.3491432036583\n",
      "Step:  718 Reward:  291.7968855835273\n",
      "Step:  719 Reward:  291.3696474010601\n",
      "Step:  720 Reward:  291.254397082653\n",
      "Step:  721 Reward:  293.8346747791902\n",
      "Step:  722 Reward:  291.9023653271611\n",
      "Step:  723 Reward:  291.9022302933212\n",
      "Step:  724 Reward:  291.43663699380124\n",
      "Step:  725 Reward:  292.78417514716085\n",
      "Step:  726 Reward:  291.38879622980363\n",
      "Step:  727 Reward:  292.2416468066909\n",
      "Step:  728 Reward:  288.21143844516604\n",
      "Step:  729 Reward:  290.39381708594703\n",
      "Step:  730 Reward:  292.068960832943\n",
      "Step:  731 Reward:  292.2086454644506\n",
      "Step:  732 Reward:  292.84846389533095\n",
      "Step:  733 Reward:  294.61772344700876\n",
      "Step:  734 Reward:  295.66928646108295\n",
      "Step:  735 Reward:  292.05751944438623\n",
      "Step:  736 Reward:  294.4779530786709\n",
      "Step:  737 Reward:  297.6448167450681\n",
      "Step:  738 Reward:  292.2536302381095\n",
      "Step:  739 Reward:  290.7824917912533\n",
      "Step:  740 Reward:  294.1529799584594\n",
      "Step:  741 Reward:  293.8181983987856\n",
      "Step:  742 Reward:  295.43824974728943\n",
      "Step:  743 Reward:  290.2255133574302\n",
      "Step:  744 Reward:  289.98981800256496\n",
      "Step:  745 Reward:  292.719864666681\n",
      "Step:  746 Reward:  293.3268798384176\n",
      "Step:  747 Reward:  292.36793994252827\n",
      "Step:  748 Reward:  279.17921573383353\n",
      "Step:  749 Reward:  295.73391813437286\n",
      "Step:  750 Reward:  296.6788292077017\n",
      "Step:  751 Reward:  294.12427386887146\n",
      "Step:  752 Reward:  293.53600695564455\n",
      "Step:  753 Reward:  295.07721098756537\n",
      "Step:  754 Reward:  297.36225611799074\n",
      "Step:  755 Reward:  298.4837943340908\n",
      "Step:  756 Reward:  297.16146832357873\n",
      "Step:  757 Reward:  295.87764178287387\n",
      "Step:  758 Reward:  297.264069712604\n",
      "Step:  759 Reward:  296.63674485893455\n",
      "Step:  760 Reward:  296.483874371809\n",
      "Step:  761 Reward:  295.1603856973811\n",
      "Step:  762 Reward:  299.39874921274736\n",
      "Step:  763 Reward:  294.5064823259032\n",
      "Step:  764 Reward:  299.28271538778347\n",
      "Step:  765 Reward:  298.4620729332127\n",
      "Step:  766 Reward:  298.58138710754395\n",
      "Step:  767 Reward:  296.4451135973637\n",
      "Step:  768 Reward:  296.87846743702227\n",
      "Step:  769 Reward:  294.90280949268197\n",
      "Step:  770 Reward:  296.76119298211523\n",
      "Step:  771 Reward:  298.07136343107516\n",
      "Step:  772 Reward:  298.8634947423795\n",
      "Step:  773 Reward:  299.08190351053565\n",
      "Step:  774 Reward:  295.8148766194905\n",
      "Step:  775 Reward:  294.9651754436093\n",
      "Step:  776 Reward:  295.9470994031998\n",
      "Step:  777 Reward:  298.3216069093805\n",
      "Step:  778 Reward:  295.3218761099829\n",
      "Step:  779 Reward:  295.5878293478512\n",
      "Step:  780 Reward:  296.5163238846047\n",
      "Step:  781 Reward:  297.5152341861506\n",
      "Step:  782 Reward:  303.78185743504946\n",
      "Step:  783 Reward:  299.63847187110775\n",
      "Step:  784 Reward:  299.0358621957802\n",
      "Step:  785 Reward:  301.6662653363693\n",
      "Step:  786 Reward:  301.43541885180855\n",
      "Step:  787 Reward:  300.41226793371186\n",
      "Step:  788 Reward:  301.01785079297866\n",
      "Step:  789 Reward:  302.400009692485\n",
      "Step:  790 Reward:  301.5043526856201\n",
      "Step:  791 Reward:  299.45293585457404\n",
      "Step:  792 Reward:  295.8650614852419\n",
      "Step:  793 Reward:  305.1762675735686\n",
      "Step:  794 Reward:  304.1604055949912\n",
      "Step:  795 Reward:  302.45702743384857\n",
      "Step:  796 Reward:  302.581969969118\n",
      "Step:  797 Reward:  298.38396421224826\n",
      "Step:  798 Reward:  306.0335843224596\n",
      "Step:  799 Reward:  303.79663106076214\n",
      "Step:  800 Reward:  299.88383574228675\n",
      "Step:  801 Reward:  299.7743778361577\n",
      "Step:  802 Reward:  297.7817226802542\n",
      "Step:  803 Reward:  296.48197265392355\n",
      "Step:  804 Reward:  302.60181761673186\n",
      "Step:  805 Reward:  303.7958080859694\n",
      "Step:  806 Reward:  306.42736080302893\n",
      "Step:  807 Reward:  304.3334025007283\n",
      "Step:  808 Reward:  303.8719105905004\n",
      "Step:  809 Reward:  304.1281354684503\n",
      "Step:  810 Reward:  305.7293238831347\n",
      "Step:  811 Reward:  303.9105169926735\n",
      "Step:  812 Reward:  304.5390499826191\n",
      "Step:  813 Reward:  305.1150827489823\n",
      "Step:  814 Reward:  305.5114581630062\n",
      "Step:  815 Reward:  305.5243704667879\n",
      "Step:  816 Reward:  304.5842020444107\n",
      "Step:  817 Reward:  306.23566266606014\n",
      "Step:  818 Reward:  311.1429409255707\n",
      "Step:  819 Reward:  309.77046042726084\n",
      "Step:  820 Reward:  305.6467069184299\n",
      "Step:  821 Reward:  305.8354627321296\n",
      "Step:  822 Reward:  303.61953243644615\n",
      "Step:  823 Reward:  304.3379276349154\n",
      "Step:  824 Reward:  307.96295062176625\n",
      "Step:  825 Reward:  305.7158121291235\n",
      "Step:  826 Reward:  306.05576516284043\n",
      "Step:  827 Reward:  308.7883642974571\n",
      "Step:  828 Reward:  309.3244369822748\n",
      "Step:  829 Reward:  107.96928751654158\n",
      "Step:  830 Reward:  305.21038178499555\n",
      "Step:  831 Reward:  305.81742068137333\n",
      "Step:  832 Reward:  306.3483309711838\n",
      "Step:  833 Reward:  306.6776747120966\n",
      "Step:  834 Reward:  304.6669075849627\n",
      "Step:  835 Reward:  304.82292370147366\n",
      "Step:  836 Reward:  308.177431251616\n",
      "Step:  837 Reward:  304.8422069272635\n",
      "Step:  838 Reward:  302.54040184007056\n",
      "Step:  839 Reward:  307.0286688569674\n",
      "Step:  840 Reward:  304.9618910639588\n",
      "Step:  841 Reward:  306.96820299994647\n",
      "Step:  842 Reward:  308.6441249670736\n",
      "Step:  843 Reward:  310.18578274145784\n",
      "Step:  844 Reward:  308.37571843986325\n",
      "Step:  845 Reward:  306.54409751757527\n",
      "Step:  846 Reward:  310.6324590284364\n",
      "Step:  847 Reward:  313.8316354138764\n",
      "Step:  848 Reward:  310.23880764058447\n",
      "Step:  849 Reward:  311.23449246370035\n",
      "Step:  850 Reward:  311.44068716995275\n",
      "Step:  851 Reward:  308.3732578610361\n",
      "Step:  852 Reward:  305.37229591435977\n",
      "Step:  853 Reward:  308.20036388300673\n",
      "Step:  854 Reward:  309.9769706034841\n",
      "Step:  855 Reward:  313.27085133452977\n",
      "Step:  856 Reward:  309.5429441049491\n",
      "Step:  857 Reward:  310.0114009923796\n",
      "Step:  858 Reward:  310.32589670768976\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step:  859 Reward:  305.9644709593253\n",
      "Step:  860 Reward:  307.1572094123591\n",
      "Step:  861 Reward:  308.4353975151262\n",
      "Step:  862 Reward:  306.34233544654063\n",
      "Step:  863 Reward:  306.62627333446255\n",
      "Step:  864 Reward:  307.07067561544494\n",
      "Step:  865 Reward:  310.70949918243826\n",
      "Step:  866 Reward:  308.803012470193\n",
      "Step:  867 Reward:  311.0815136504547\n",
      "Step:  868 Reward:  310.898458531386\n",
      "Step:  869 Reward:  310.52112449558416\n",
      "Step:  870 Reward:  309.94624905624977\n",
      "Step:  871 Reward:  310.809152129623\n",
      "Step:  872 Reward:  311.1880188448004\n",
      "Step:  873 Reward:  310.446882924464\n",
      "Step:  874 Reward:  309.98147212616135\n",
      "Step:  875 Reward:  309.97139449276574\n",
      "Step:  876 Reward:  309.40632454469164\n",
      "Step:  877 Reward:  309.6277773431968\n",
      "Step:  878 Reward:  308.7200068724167\n",
      "Step:  879 Reward:  307.94695180683436\n",
      "Step:  880 Reward:  307.2604099081361\n",
      "Step:  881 Reward:  308.7492317009486\n",
      "Step:  882 Reward:  310.3033046409393\n",
      "Step:  883 Reward:  308.5706301090862\n",
      "Step:  884 Reward:  307.9040156976966\n",
      "Step:  885 Reward:  308.39018205855\n",
      "Step:  886 Reward:  310.2769801594735\n",
      "Step:  887 Reward:  309.78233457193846\n",
      "Step:  888 Reward:  310.53659648114916\n",
      "Step:  889 Reward:  308.4859567614907\n",
      "Step:  890 Reward:  308.74509920825517\n",
      "Step:  891 Reward:  305.12464415004166\n",
      "Step:  892 Reward:  305.27865933871874\n",
      "Step:  893 Reward:  308.70914228538317\n",
      "Step:  894 Reward:  309.3524044192536\n",
      "Step:  895 Reward:  312.80425658483927\n",
      "Step:  896 Reward:  307.20951660379995\n",
      "Step:  897 Reward:  306.0981994399851\n",
      "Step:  898 Reward:  307.8188782183256\n",
      "Step:  899 Reward:  308.48564766339183\n",
      "Step:  900 Reward:  106.02757764146547\n",
      "Step:  901 Reward:  309.8353945920666\n",
      "Step:  902 Reward:  305.9629595298659\n",
      "Step:  903 Reward:  308.4287339326448\n",
      "Step:  904 Reward:  308.9253494685183\n",
      "Step:  905 Reward:  306.36004971796183\n",
      "Step:  906 Reward:  305.40626901528293\n",
      "Step:  907 Reward:  304.74841914147123\n",
      "Step:  908 Reward:  304.2337610316091\n",
      "Step:  909 Reward:  305.58309068867385\n",
      "Step:  910 Reward:  306.4379604010369\n",
      "Step:  911 Reward:  306.50050566662435\n",
      "Step:  912 Reward:  306.69458815433575\n",
      "Step:  913 Reward:  309.40283127503096\n",
      "Step:  914 Reward:  307.7773195410507\n",
      "Step:  915 Reward:  307.62984698444035\n",
      "Step:  916 Reward:  307.5247910618936\n",
      "Step:  917 Reward:  306.93294363134527\n",
      "Step:  918 Reward:  305.6895899360715\n",
      "Step:  919 Reward:  303.5330150047527\n",
      "Step:  920 Reward:  308.35041531894524\n",
      "Step:  921 Reward:  306.73405223239007\n",
      "Step:  922 Reward:  311.21050866921547\n",
      "Step:  923 Reward:  307.14354070840204\n",
      "Step:  924 Reward:  307.11745501113757\n",
      "Step:  925 Reward:  307.74121087805986\n",
      "Step:  926 Reward:  310.3192705724312\n",
      "Step:  927 Reward:  306.04036629181417\n",
      "Step:  928 Reward:  307.9125778915642\n",
      "Step:  929 Reward:  310.5171603979348\n",
      "Step:  930 Reward:  310.72509202633887\n",
      "Step:  931 Reward:  308.1715939138486\n",
      "Step:  932 Reward:  309.4936697335668\n",
      "Step:  933 Reward:  310.6959331322349\n",
      "Step:  934 Reward:  306.60857000459237\n",
      "Step:  935 Reward:  306.7617633218826\n",
      "Step:  936 Reward:  308.80482556892605\n",
      "Step:  937 Reward:  313.1510470488909\n",
      "Step:  938 Reward:  286.5657433368361\n",
      "Step:  939 Reward:  308.6716344616901\n",
      "Step:  940 Reward:  313.1882462836174\n",
      "Step:  941 Reward:  311.70919924986083\n",
      "Step:  942 Reward:  306.49175152319174\n",
      "Step:  943 Reward:  306.7597407770618\n",
      "Step:  944 Reward:  310.8049366131394\n",
      "Step:  945 Reward:  311.6542212692348\n",
      "Step:  946 Reward:  311.24340522570077\n",
      "Step:  947 Reward:  308.6687642107351\n",
      "Step:  948 Reward:  311.9786888211093\n",
      "Step:  949 Reward:  312.24966766382255\n",
      "Step:  950 Reward:  310.3816118131741\n",
      "Step:  951 Reward:  311.5139406021899\n",
      "Step:  952 Reward:  310.92343003325936\n",
      "Step:  953 Reward:  308.2130273048746\n",
      "Step:  954 Reward:  309.1659945782532\n",
      "Step:  955 Reward:  312.58360065901707\n",
      "Step:  956 Reward:  311.3316268149511\n",
      "Step:  957 Reward:  310.12650900980805\n",
      "Step:  958 Reward:  308.8719660903324\n",
      "Step:  959 Reward:  311.86574756499846\n",
      "Step:  960 Reward:  309.77884549987544\n",
      "Step:  961 Reward:  312.65730226062925\n",
      "Step:  962 Reward:  310.62725154494\n",
      "Step:  963 Reward:  308.01159417756014\n",
      "Step:  964 Reward:  309.1373587355173\n",
      "Step:  965 Reward:  309.68372930237757\n",
      "Step:  966 Reward:  308.33495264210063\n",
      "Step:  967 Reward:  312.3659511186085\n",
      "Step:  968 Reward:  312.14488222140375\n",
      "Step:  969 Reward:  311.0996793934262\n",
      "Step:  970 Reward:  310.93241645087\n",
      "Step:  971 Reward:  309.52142488446356\n",
      "Step:  972 Reward:  310.4839280595881\n",
      "Step:  973 Reward:  308.12982402109026\n",
      "Step:  974 Reward:  308.2542130304972\n",
      "Step:  975 Reward:  308.7368628130614\n",
      "Step:  976 Reward:  306.8833032168013\n",
      "Step:  977 Reward:  311.8614534976471\n",
      "Step:  978 Reward:  311.2516059979804\n",
      "Step:  979 Reward:  305.68415610893675\n",
      "Step:  980 Reward:  306.28826949318744\n",
      "Step:  981 Reward:  311.6964135471695\n",
      "Step:  982 Reward:  311.12354711463314\n",
      "Step:  983 Reward:  309.17846133355783\n",
      "Step:  984 Reward:  308.76330689588633\n",
      "Step:  985 Reward:  313.01051550397744\n",
      "Step:  986 Reward:  89.06415185178847\n",
      "Step:  987 Reward:  312.4620392770981\n",
      "Step:  988 Reward:  310.56766221533064\n",
      "Step:  989 Reward:  311.95549683829034\n",
      "Step:  990 Reward:  308.56143298435535\n",
      "Step:  991 Reward:  313.54263007552527\n",
      "Step:  992 Reward:  313.2788214546157\n",
      "Step:  993 Reward:  312.54428480567793\n",
      "Step:  994 Reward:  313.90945659029404\n",
      "Step:  995 Reward:  312.2662719469618\n",
      "Step:  996 Reward:  311.45090358684496\n",
      "Step:  997 Reward:  310.62135824896296\n",
      "Step:  998 Reward:  307.15981462185533\n",
      "Step:  999 Reward:  311.44621325607704\n"
     ]
    }
   ],
   "source": [
    "# Old training session for backup\n",
    "start = datetime.datetime.now()\n",
    "ENV_NAME = 'BipedalWalker-v3'\n",
    "\n",
    "videos_dir = mkdir('.', 'videos')\n",
    "monitor_dir = mkdir(videos_dir, ENV_NAME)\n",
    "\n",
    "hp = HP(env_name=ENV_NAME)\n",
    "trainer = ARSTrainer(hp=hp, monitor_dir=monitor_dir)\n",
    "trainer.train()\n",
    "end = datetime.datetime.now()\n",
    "print(\"Time elapsed\": end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.19773867, -0.43398089,  0.0753329 , -0.40455625, -0.78022785,\n",
       "        -0.47065727, -0.28752706,  0.02663993, -0.04235575, -0.00839982,\n",
       "        -0.04940817, -0.14592225,  0.11720849,  0.04220249,  0.09243848,\n",
       "         0.05408006,  0.217542  ,  0.13483803, -0.10166978, -0.32866835,\n",
       "        -0.00126649,  0.09081812,  0.06181933,  0.05200593],\n",
       "       [-0.06774746,  0.07975838,  0.11180235, -0.083668  ,  0.21467017,\n",
       "        -0.159638  , -0.11771604, -0.28212586, -0.10031455,  0.4626594 ,\n",
       "         0.47707995, -0.02464654,  0.14722706, -0.03351459, -0.09869358,\n",
       "        -0.04679419,  0.26850789, -0.22494934,  0.48149819, -0.41699818,\n",
       "        -0.27997625,  0.25551447, -0.261145  ,  0.06094136],\n",
       "       [-0.28416599,  0.01871607, -0.13579297,  0.40673224, -0.11451224,\n",
       "         0.01893984, -0.56593534, -0.11441946,  0.1603778 , -0.63110977,\n",
       "        -0.09323498,  0.01092114,  0.06647962, -0.11395431,  0.20635529,\n",
       "        -0.48134119, -0.12073207, -0.02296331, -0.23105415,  0.29713571,\n",
       "        -0.10748665,  0.02233215, -0.06541907,  0.05559063],\n",
       "       [ 0.24716027,  0.24578585, -0.15897411, -0.25435104,  0.24297   ,\n",
       "         0.18871955, -0.09838014,  0.12231298, -0.03587745, -0.28419632,\n",
       "        -0.37318322, -0.45613835, -0.33224005, -0.04004181, -0.13613522,\n",
       "        -0.54438399, -0.05580985, -0.25016605, -0.14118687, -0.0516655 ,\n",
       "        -0.19283703,  0.09819555, -0.00102822, -0.03006884]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.policy.theta"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}