Dm control humanoid ppo learnable stand, walk, run

docs/source/tasks/control/index.md

@@ -87,4 +87,65 @@ Hopper robot stands upright
 
 **Success Conditions:**
 - No specific success conditions. We can threshold the episode accumulated reward to determine success.
-:::
+:::
+
+## MS-HumanoidStand-v1
+![dense-reward][reward-badge]
+
+:::{dropdown} Task Card
+:icon: note
+:color: primary
+
+**Task Description:**
+Humanoid robot stands upright
+
+
+**Supported Robots: humanoid**
+
+**Randomizations:**
+- Humanoid robot is randomly rotated [-pi, pi] radians about z axis.
+- Humanoid qpos and qvel have added noise from uniform distribution [-1e-2, 1e-2]
+
+**Fail Conditions:**
+- Humanoid robot torso link leaves z range [0.7, 1.0]
+:::
+
+## MS-HumanoidWalk-v1
+![dense-reward][reward-badge]
+
+:::{dropdown} Task Card
+:icon: note
+:color: primary
+
+**Task Description:**
+Humanoid moves in x direction at walking pace
+
+
+**Supported Robots: humanoid**
+
+**Randomizations:**
+- Humanoid qpos and qvel have added noise from uniform distribution [-1e-2, 1e-2]
+
+**Fail Conditions:**
+- Humanoid robot torso link leaves z range [0.7, 1.0]
+:::
+
+## MS-HumanoidRun-v1
+![dense-reward][reward-badge]
+
+:::{dropdown} Task Card
+:icon: note
+:color: primary
+
+**Task Description:**
+Humanoid moves in x direction at running pace
+
+
+**Supported Robots: humanoid**
+
+**Randomizations:**
+- Humanoid qpos and qvel have added noise from uniform distribution [-1e-2, 1e-2]
+
+**Fail Conditions:**
+- Humanoid robot torso link leaves z range [0.7, 1.0]
+:::

examples/baselines/ppo/examples.sh

@@ -49,6 +49,17 @@ python ppo.py --env_id="MS-CartpoleSwingUp-v1" \
    --total_timesteps=10_000_000 --num-steps=250 --num-eval-steps=1000 \
    --gamma=0.99 --gae_lambda=0.95 \
    --eval_freq=5
+python ppo.py --env_id="MS-HumanoidStand-v1" --num_envs=2048 \
+  --update_epochs=8 --num_minibatches=32 --total_timesteps=40_000_000 \
+  --eval_freq=10 --num_eval_steps=1000 --num_steps=200 --gamma=0.95
+python ppo.py --env_id="MS-HumanoidWalk-v1" --num_envs=2048 \
+  --update_epochs=8 --num_minibatches=32 --total_timesteps=80_000_000 \
+  --eval_freq=10 --num_eval_steps=1000 --num_steps=200 --gamma=0.97 \
+  --ent_coef=1e-3
+python ppo.py --env_id="MS-HumanoidRun-v1" --num_envs=2048 \
+  --update_epochs=8 --num_minibatches=32 --total_timesteps=60_000_000 \
+  --eval_freq=10 --num_eval_steps=1000 --num_steps=200 --gamma=0.97 \
+  --ent_coef=1e-3
 python ppo.py --env_id="UnitreeG1PlaceAppleInBowl-v1" \
   --num_envs=512 --update_epochs=8 --num_minibatches=32 \
   --total_timesteps=50_000_000 --num-steps=100 --num-eval-steps=100
@@ -98,3 +109,8 @@ python ppo_rgb.py --env_id="PickSingleYCB-v1" \
 python ppo_rgb.py --env_id="PushT-v1" \
   --num_envs=256 --update_epochs=8 --num_minibatches=8 \
   --total_timesteps=25_000_000 --num-steps=100 --num_eval_steps=100 --gamma=0.99
+python ppo_rgb.py --env_id="MS-HumanoidRun-v1" \
+  --num_envs=256 --update_epochs=8 --num_minibatches=32 \
+  --total_timesteps=80_000_000 --eval_freq=15 --num_eval_steps=1000 \
+  --num_steps=200 --gamma=0.98 --no-include-state --render_mode="rgb_array" \
+  --ent_coef=1e-3

examples/baselines/ppo/ppo_rgb.py

@@ -45,6 +45,8 @@ class Args:
     """if toggled, only runs evaluation with the given model checkpoint and saves the evaluation trajectories"""
     checkpoint: str = None
     """path to a pretrained checkpoint file to start evaluation/training from"""
+    render_mode: str = "all"
+    """the environment rendering mode"""
 
     # Algorithm specific arguments
     env_id: str = "PickCube-v1"
@@ -288,7 +290,7 @@ def close(self):
     device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
 
     # env setup
-    env_kwargs = dict(obs_mode="rgb", control_mode="pd_joint_delta_pos", render_mode="all", sim_backend="gpu")
+    env_kwargs = dict(obs_mode="rgbd", control_mode="pd_joint_delta_pos", render_mode=args.render_mode, sim_backend="gpu")
     eval_envs = gym.make(args.env_id, num_envs=args.num_eval_envs, **env_kwargs)
     envs = gym.make(args.env_id, num_envs=args.num_envs if not args.evaluate else 1, **env_kwargs)
 

mani_skill/agents/robots/humanoid/humanoid.py

@@ -59,15 +59,47 @@ def _controller_configs(self):
             damping=10,
             normalize_action=False,
         )
+
+        # for pd_joint_delta_pos control
+        joints_dict = {
+            "abdomen_y": {"damping": 5, "stiffness": 40},
+            "abdomen_z": {"damping": 5, "stiffness": 40},
+            "abdomen_x": {"damping": 5, "stiffness": 40},
+            "right_hip_x": {"damping": 5, "stiffness": 40},
+            "right_hip_z": {"damping": 5, "stiffness": 40},
+            "right_hip_y": {"damping": 5, "stiffness": 120},
+            "right_knee": {"damping": 1, "stiffness": 80},
+            "right_ankle_x": {"damping": 3, "stiffness": 20},
+            "right_ankle_y": {"damping": 3, "stiffness": 40},
+            "left_hip_x": {"damping": 5, "stiffness": 40},
+            "left_hip_z": {"damping": 5, "stiffness": 40},
+            "left_hip_y": {"damping": 5, "stiffness": 120},
+            "left_knee": {"damping": 1, "stiffness": 80},
+            "left_ankle_x": {"damping": 3, "stiffness": 20},
+            "left_ankle_y": {"damping": 3, "stiffness": 40},
+            "right_shoulder1": {"damping": 1, "stiffness": 20},
+            "right_shoulder2": {"damping": 1, "stiffness": 20},
+            "right_elbow": {"damping": 0, "stiffness": 40},
+            "left_shoulder1": {"damping": 1, "stiffness": 20},
+            "left_shoulder2": {"damping": 1, "stiffness": 20},
+            "left_elbow": {"damping": 0, "stiffness": 40},
+        }
+
+        joint_names = list(joints_dict.keys())
+        assert sorted(joint_names) == sorted([x.name for x in self.robot.active_joints])
+
+        damping = np.array([joint["damping"] for joint in joints_dict.values()])
+        stiffness = np.array([joint["stiffness"] for joint in joints_dict.values()])
+
         pd_joint_delta_pos = PDJointPosControllerConfig(
-            [j.name for j in self.robot.active_joints],
-            -1,
-            1,
-            damping=5,
-            stiffness=20,
-            force_limit=100,
+            joint_names,
+            -2,
+            2,
+            damping=damping,
+            stiffness=stiffness,
             use_delta=True,
         )
+
         return deepcopy_dict(
             dict(
                 pd_joint_pos=dict(body=pd_joint_pos, balance_passive_force=False),

mani_skill/assets/robots/humanoid/humanoid.xml

@@ -30,7 +30,8 @@
 
   <worldbody>
     <geom name="floor" type="plane" conaffinity="1" size="100 100 .2" material="grid"/>
-    <body name="torso" pos="0 0 1.5" childclass="body">
+    <!-- body pos changed from pos="0 0 1.5" for compatability with maniskill articulation set_root_pose -->
+    <body name="torso" pos="0 0 0" childclass="body">
       <light name="top" pos="0 0 2" mode="trackcom"/>
       <camera name="back" pos="-3 0 1" xyaxes="0 -1 0 1 0 2" mode="trackcom"/>
       <camera name="side" pos="0 -3 1" xyaxes="1 0 0 0 1 2" mode="trackcom"/>

mani_skill/envs/tasks/control/__init__.py

@@ -1,2 +1,3 @@
 from .cartpole import CartpoleBalanceEnv, CartpoleSwingUpEnv
 from .hopper import HopperHopEnv, HopperStandEnv
+from .humanoid import HumanoidRun, HumanoidStand, HumanoidWalk