[RLlib] Bump Gymnasium to 0.28.1

.buildkite/pipeline.ml.yml

@@ -551,15 +551,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 
-    # Install mujoco necessary for the testing environments
-    - sudo apt install libosmesa6-dev libgl1-mesa-glx libglfw3 patchelf -y
-    - wget https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz
-    - mkdir /root/.mujoco
-    - mv mujoco210-linux-x86_64.tar.gz /root/.mujoco/.
-    - (cd /root/.mujoco && tar -xf /root/.mujoco/mujoco210-linux-x86_64.tar.gz)
-    - echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/root/.mujoco/mujoco210/bin' >> /root/.bashrc
     - source /root/.bashrc
-
     - (cd rllib_contrib/maml && pip install -r requirements.txt && pip install -e .)
     - ./ci/env/env_info.sh
     - pytest rllib_contrib/maml/tests/test_maml.py

ci/env/install-dependencies.sh

@@ -354,6 +354,14 @@ install_pip_packages() {
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt")
     #TODO(amogkam): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
     pip install --no-dependencies mlagents==0.28.0
+
+    # Install MuJoCo.
+    sudo apt install libosmesa6-dev libgl1-mesa-glx libglfw3 patchelf -y
+    wget https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz
+    mkdir -p /root/.mujoco
+    mv mujoco210-linux-x86_64.tar.gz /root/.mujoco/.
+    (cd /root/.mujoco && tar -xf /root/.mujoco/mujoco210-linux-x86_64.tar.gz)
+    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/root/.mujoco/mujoco210/bin
   fi
 
   # Additional Train test dependencies.

ci/run/bazel_export_options

@@ -6,5 +6,6 @@ logs_archive_flag=--test_env=RAY_TEST_FAILURE_LOGS_ARCHIVE_DIR=/artifact-mount/.
 summary_directory_flag=--test_env=RAY_TEST_SUMMARY_DIR=/artifact-mount/test-summaries
 summary_directory_host_flag=--test_env=RAY_TEST_SUMMARY_DIR_HOST=/tmp/artifacts/test-summaries
 buildkite_env_vars="--test_env=BUILDKITE_COMMIT --test_env=BUILDKITE_LABEL --test_env=BUILDKITE_JOB_ID"
+mujoco_env_var="--test_env=LD_LIBRARY_PATH=/root/.mujoco/mujoco210/bin"
 
-echo "${event_json_flag} ${logs_archive_flag} ${summary_directory_flag} ${summary_directory_host_flag} ${buildkite_env_vars}"
+echo "${event_json_flag} ${logs_archive_flag} ${summary_directory_flag} ${summary_directory_host_flag} ${buildkite_env_vars} ${mujoco_env_var}"

doc/source/ray-core/examples/plot_pong_example.ipynb

@@ -31,7 +31,7 @@
     "To run the application, first install some dependencies.\n",
     "\n",
     "```bash\n",
-    "pip install gymnasium[atari] gym==0.26.2\n",
+    "pip install gymnasium[atari]==0.28.1\n",
     "```\n",
     "\n",
     "At the moment, on a large machine with 64 physical cores, computing an update\n",
@@ -292,7 +292,7 @@
     "@ray.remote\n",
     "class RolloutWorker(object):\n",
     "    def __init__(self):\n",
-    "        self.env = gym.make(\"GymV26Environment-v0\", env_id=\"ALE/Pong-v5\")\n",
+    "        self.env = gym.make(\"ALE/Pong-v5\")\n",
     "\n",
     "    def compute_gradient(self, model):\n",
     "        # Compute a simulation episode.\n",

doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt

@@ -108,6 +108,7 @@ everett==3.2.0
 exceptiongroup==1.1.1
 executing==1.2.0
 fairscale==0.4.6
+farama-notifications==0.0.1
 fastapi==0.97.0
 fasteners==0.18
 fastjsonschema==2.17.1
@@ -143,9 +144,7 @@ graphviz==0.8.4
 greenlet==2.0.2
 grpcio==1.50.0
 gunicorn==20.1.0
-gym==0.26.2
-gym-notices==0.0.8
-Gymnasium==0.26.3
+gymnasium==0.28.1
 gymnasium-notices==0.0.1
 h11==0.14.0
 h5py==3.7.0

doc/source/rllib/doc_code/training.py

@@ -4,7 +4,7 @@
 try:
     import gymnasium as gym
 
-    env = gym.make("GymV26Environment-v0", env_id="ALE/Pong-v5")
+    env = gym.make("ALE/Pong-v5")
     obs, infos = env.reset()
 except Exception:
     import gym

doc/source/rllib/rllib-models.rst

@@ -162,7 +162,7 @@ For example, for manipulating your env's observations or rewards, do:
 
 .. code-block:: python
 
-    import gym
+    import gymnasium as gym
     from ray.rllib.utils.numpy import one_hot
 
     class OneHotEnv(gym.core.ObservationWrapper):

doc/source/serve/tutorials/rllib.md

@@ -24,7 +24,7 @@ In particular, we show:
 Check out the [Key Concepts](serve-key-concepts) page to learn more general information about Ray Serve.
 ```
 
-We will train and checkpoint a simple PPO model with the `CartPole-v0` environment from `gym`.
+We will train and checkpoint a simple PPO model with the `CartPole-v1` environment from `gymnasium`.
 In this tutorial we simply write to local disk, but in production you might want to consider using a cloud
 storage solution like S3 or a shared file system.
 
@@ -83,7 +83,7 @@ class ServePPOModel:
             .framework("torch")\
             .rollouts(num_rollout_workers=0)
         # Build the Algorithm instance using the config.
-        self.algorithm = config.build(env="CartPole-v0")
+        self.algorithm = config.build(env="CartPole-v1")
         # Restore the algo's state from the checkpoint.
         self.algorithm.restore(checkpoint_path)
 

docker/ray-ml/install-ml-docker-requirements.sh

@@ -9,10 +9,15 @@ sudo apt-get update \
     && sudo apt-get install -y gcc \
         cmake \
         libgtk2.0-dev \
-        zlib1g-dev \
         libgl1-mesa-dev \
+        libgl1-mesa-glx \
+        libosmesa6 \
+        libosmesa6-dev \
+        libglfw3 \
+        patchelf \
         unzip \
-        unrar
+        unrar \
+        zlib1g-dev
 
 pip --no-cache-dir install -U pip pip-tools
 
@@ -48,3 +53,12 @@ pip --no-cache-dir install \
 sudo apt-get clean
 
 sudo rm ./*requirements*.txt
+
+# MuJoCo Installation.
+export MUJOCO_GL=osmesa
+wget https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz
+mkdir -p ~/.mujoco
+mv mujoco210-linux-x86_64.tar.gz ~/.mujoco/.
+cd ~/.mujoco || exit
+tar -xf ~/.mujoco/mujoco210-linux-x86_64.tar.gz
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/root/.mujoco/mujoco210/bin

python/ray/tune/tests/test_api.py

@@ -10,7 +10,7 @@
 from functools import partial
 from unittest.mock import patch
 
-import gym
+import gymnasium as gym
 import numpy as np
 import pytest
 import ray

python/requirements.txt

@@ -42,6 +42,7 @@ rich
 gpustat>=1.0.0
 opentelemetry-sdk
 fastapi
+gymnasium==0.28.1
 virtualenv<20.21.1,>=20.0.24
 opentelemetry-api
 opencensus
@@ -53,7 +54,6 @@ prometheus_client>=0.7.1
 requests
 pandas
 tensorboardX<=2.6.0,>=1.9  # >=2.6.1 uses protobuf>=4, and conflicts with other packages.
-gymnasium==0.26.3
 aiohttp>=3.7
 starlette
 typer

python/requirements/ml/rllib-test-requirements.txt

@@ -3,24 +3,30 @@
 # Environment adapters.
 # ---------------------
 # Atari
-# TODO(sven): Still needed for Atari (need to be wrapped by gymnasium as it does NOT support Atari yet)
-gym==0.26.2
-gymnasium[atari,mujoco]==0.26.3
+gymnasium==0.28.1
+imageio==2.31.1
+ale_py==0.8.1
 # For testing MuJoCo envs with gymnasium.
 mujoco-py<2.2,>=2.1
+dm_control==1.0.12
+
+# For tests on PettingZoo's multi-agent envs.
+pettingzoo==1.23.1; python_version >= '3.7'
+# When installing pettingzoo, chess is missing, even though its a dependancy
+# TODO: remove if a future pettingzoo and/or ray version fixes this dependancy issue.
+chess==1.7.0
+pymunk==6.2.1
+supersuit==3.8.0; python_version >= '3.7'
+tinyscaler==1.2.6
+shimmy
+
 # Kaggle envs.
 kaggle_environments==1.7.11
 # Unity3D testing
 # TODO(sven): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
 #mlagents==0.28.0
 mlagents_envs==0.28.0
-# For tests on PettingZoo's multi-agent envs.
-pettingzoo==1.22.1; python_version >= '3.7'
-# When installing pettingzoo, chess is missing, even though its a dependancy
-# TODO: remove if a future pettingzoo and/or ray version fixes this dependancy issue.
-chess==1.7.0
-pymunk==6.2.1
-supersuit==3.7.0; python_version >= '3.7'
+
 # For tests on minigrid.
 minigrid==2.1.1
 # For tests on RecSim and Kaggle envs.

python/requirements_compiled.txt

@@ -173,9 +173,8 @@ greenlet==2.0.2
 grpcio==1.54.2
 gsutil==5.25
 gunicorn==20.1.0
-gym==0.26.2
 gym-notices==0.0.8
-gymnasium==0.26.3
+gymnasium==0.28.1
 gymnasium-notices==0.0.1
 h11==0.12.0
 h5py==3.7.0
@@ -277,7 +276,7 @@ msgpack==1.0.5
 msgpack-numpy==0.4.8
 msrest==0.7.1
 msrestazure==0.6.4
-mujoco==2.2.0
+mujoco==2.3.7
 mujoco-py==2.1.2.14
 multidict==6.0.4
 multipledispatch==1.0.0
@@ -340,7 +339,7 @@ pathspec==0.11.1
 pathtools==0.1.2
 patsy==0.5.3
 pbr==5.11.1
-pettingzoo==1.22.1 ; python_version >= "3.7"
+pettingzoo==1.23.1 ; python_version >= "3.7"
 pexpect==4.8.0
 pickleshare==0.7.5
 pillow==9.2.0 ; platform_system != "Windows"
@@ -473,7 +472,7 @@ sshpubkeys==3.3.1
 stack-data==0.6.2
 starlette==0.27.0
 statsmodels==0.14.0
-supersuit==3.7.0 ; python_version >= "3.7"
+supersuit==3.8.0 ; python_version >= "3.7"
 sympy==1.12
 tabulate==0.9.0
 tblib==2.0.0

python/setup.py

@@ -288,7 +288,7 @@ def get_packages(self):
 
     setup_spec.extras["rllib"] = setup_spec.extras["tune"] + [
         "dm_tree",
-        "gymnasium==0.26.3",
+        "gymnasium==0.28.1",
         "lz4",
         "scikit-image",
         "pyyaml",

release/rllib_tests/app_config.yaml

@@ -13,12 +13,6 @@ debian_packages:
 
 python:
   pip_packages:
-  ## These dependencies should be handled by rllib-requirements.txt
-  ## and removed here
-  - gymnasium[atari,mujoco]==0.26.3
-  - ale-py==0.8.0
-  - gym==0.26.2
-  - mujoco-py<2.2,>=2.1
   # AutoROM downloads ROMs via torrent when they are built. The torrent is unreliable,
   # so we built it for py3 and use that instead. This wheel was tested for python 3.7, 3.8,
   # and 3.9.
@@ -32,7 +26,7 @@ post_build_cmds:
   # Clone the rl-experiments repo for offline-RL files.
   - git clone https://github.com/ray-project/rl-experiments.git
   - unzip rl-experiments/halfcheetah-sac/2022-12-17/halfcheetah_1500_mean_reward_sac.zip -d ~/.
-  
+
   - pip3 install torch==2.0.0+cu118 torchvision==0.15.1+cu118 --index-url https://download.pytorch.org/whl/cu118
 
   # TODO(sven): remove once nightly image gets gymnasium and the other new dependencies.

release/rllib_tests/debug_app_config.yaml

@@ -16,8 +16,7 @@ python:
   ## These dependencies should be handled by rllib-requirements.txt
   ## and removed here
   - gymnasium[atari,mujoco]==0.26.3
-  - ale-py==0.8.0
-  - gym==0.26.2
+  - ale-py==0.8.1
   - mujoco-py<2.2,>=2.1
   # AutoROM downloads ROMs via torrent when they are built. The torrent is unreliable,
   # so we built it for py3 and use that instead. This wheel was tested for python 3.7, 3.8,

rllib/BUILD

@@ -2937,7 +2937,7 @@ py_test(
 )
 
 py_test(
-     name = "tests/test_supported_spaces_ppo_no_preproceesor_gpu",
+     name = "tests/test_supported_spaces_ppo_no_preprocessor_gpu",
      main = "tests/test_supported_spaces.py",
      tags = ["team:rllib", "gpu", "no_cpu"],
      size = "large",

rllib/algorithms/algorithm_config.py

@@ -306,11 +306,11 @@ def __init__(self, algo_class=None):
         self.normalize_actions = True
         self.clip_actions = False
         self.disable_env_checking = False
+        self.auto_wrap_old_gym_envs = True
+        self.action_mask_key = "action_mask"
         # Whether this env is an atari env (for atari-specific preprocessing).
         # If not specified, we will try to auto-detect this.
         self._is_atari = None
-        self.auto_wrap_old_gym_envs = True
-        self.action_mask_key = "action_mask"
 
         # `self.rollouts()`
         self.env_runner_cls = None
@@ -2651,10 +2651,7 @@ def is_atari(self) -> bool:
             if not type(self.env) == str:
                 return False
             try:
-                if self.env.startswith("ALE/"):
-                    env = gym.make("GymV26Environment-v0", env_id=self.env)
-                else:
-                    env = gym.make(self.env)
+                env = gym.make(self.env)
             # Any gymnasium error -> Cannot be an Atari env.
             except gym.error.Error:
                 return False

rllib/algorithms/ppo/tests/test_ppo_rl_module.py

@@ -151,10 +151,7 @@ def test_rollouts(self):
                 # LSTM not implemented in TF2 yet
                 continue
             print(f"[FW={fw} | [ENV={env_name}] | [FWD={fwd_fn}] | LSTM" f"={lstm}")
-            if env_name.startswith("ALE/"):
-                env = gym.make("GymV26Environment-v0", env_id=env_name)
-            else:
-                env = gym.make(env_name)
+            env = gym.make(env_name)
 
             preprocessor_cls = get_preprocessor(env.observation_space)
             preprocessor = preprocessor_cls(env.observation_space)
@@ -191,11 +188,7 @@ def test_forward_train(self):
         for config in itertools.product(*config_combinations):
             fw, env_name, lstm = config
             print(f"[FW={fw} | [ENV={env_name}] | LSTM={lstm}")
-            # TODO(Artur): Figure out why this is needed and fix it.
-            if env_name.startswith("ALE/"):
-                env = gym.make("GymV26Environment-v0", env_id=env_name)
-            else:
-                env = gym.make(env_name)
+            env = gym.make(env_name)
 
             preprocessor_cls = get_preprocessor(env.observation_space)
             preprocessor = preprocessor_cls(env.observation_space)

rllib/algorithms/tests/test_algorithm_config.py

@@ -1,4 +1,4 @@
-import gym
+import gymnasium as gym
 from typing import Type
 import unittest
 
@@ -158,9 +158,8 @@ def test_detect_atari_env(self):
 
         config = AlgorithmConfig().environment(
             env=lambda ctx: gym.make(
-                "GymV26Environment-v0",
-                env_id="ALE/Breakout-v5",
-                make_kwargs={"frameskip": 1},
+                "ALE/Breakout-v5",
+                frameskip=1,
             )
         )
         # We do not auto-detect callable env makers for Atari envs.

rllib/connectors/tests/test_connector.py

@@ -1,6 +1,6 @@
 import unittest
 
-import gym
+import gymnasium as gym
 
 from ray.rllib.connectors.connector import Connector, ConnectorPipeline
 from ray.rllib.connectors.connector import ConnectorContext

rllib/core/learner/tests/test_learner_group_config.py

@@ -1,4 +1,4 @@
-import gym
+import gymnasium as gym
 import unittest
 
 import ray

rllib/env/utils.py

@@ -131,15 +131,6 @@ def _gym_env_creator(
         # If class provided, call constructor directly.
         if isinstance(env_descriptor, type):
             env = env_descriptor(env_context)
-        # Special case: Atari not supported by gymnasium yet -> Need to use their
-        # GymV26 compatibility wrapper class.
-        # TODO(sven): Remove this if-block once gymnasium fully supports Atari envs.
-        elif env_descriptor.startswith("ALE/"):
-            env = gym.make(
-                "GymV26Environment-v0",
-                env_id=env_descriptor,
-                make_kwargs=env_context,
-            )
         else:
             env = gym.make(env_descriptor, **env_context)
         # If we are dealing with an old gym-env API, use the provided compatibility