env with competition tasks

Author: wenzhangliu

.gitignore

@@ -1,6 +1,7 @@
 .idea
 .vscode
 .DS_Store
+docs/.DS_Store
 **/.DS_Store
 logs/
 videos/

docs/.DS_Store

@@ -118,14 +118,25 @@ def __init__(self, *args, **kwargs):
         self.num_agents: Optional[int] = None  # Number of all agents, e.g., 4.
         self.max_episode_steps: Optional[int] = None
 
-    def get_env_info(self):
+    def get_env_info(self) -> Dict[str, Any]:
         return {'state_space': self.state_space,
                 'observation_space': self.observation_space,
                 'action_space': self.action_space,
                 'agents': self.agents,
                 'num_agents': self.num_agents,
                 'max_episode_steps': self.max_episode_steps}
 
+    def get_groups_info(self) -> Dict[str, Any]:
+        agent_groups: List[AgentKeys] = []  # e.g., [['red_0', 'red_1'], ['blue_0', 'blue_1']]. Default is empty.
+        num_groups: int = 1  # The number of groups.
+        return {'num_groups': num_groups,
+                'agent_groups': agent_groups,
+                'observation_space_groups': [{k: self.observation_space[k] for i, k in enumerate(group)}
+                                             for group in agent_groups],
+                'action_space_groups': [{k: self.action_space[k] for i, k in enumerate(group)}
+                                        for group in agent_groups],
+                'num_agents_groups': [len(group) for group in agent_groups]}
+
     def agent_mask(self):
         """Returns boolean mask variables indicating which agents are currently alive."""
         return {agent: True for agent in self.agents}

xuance/environment/utils/base.py

@@ -131,6 +131,7 @@ def __init__(self, env, **kwargs):
         self.num_agents = self.env.num_agents  # Number of all agents, e.g., 4.
         self._episode_score = {agent: 0.0 for agent in self.agents}
         self.env_info = self.env.get_env_info()
+        self.groups_info = self.env.get_groups_info()
 
     def reset(self, **kwargs) -> Tuple[dict, dict]:
         """Resets the environment with kwargs."""

xuance/environment/utils/wrapper.py

@@ -21,6 +21,7 @@ def __init__(self, env_fns, env_seed):
         VecEnv.__init__(self, len(env_fns), env.observation_space, env.action_space)
 
         self.env_info = env.env_info
+        self.groups_info = env.groups_info
         self.agents = env.agents
         self.num_agents = env.num_agents
         self.state_space = env.state_space  # Type: Box

xuance/environment/vector_envs/dummy/dummy_vec_maenv.py

@@ -37,6 +37,9 @@ def step_env(env, action):
             elif cmd == 'get_env_info':
                 env_info = envs[0].env_info
                 remote.send(CloudpickleWrapper(env_info))
+            elif cmd == 'get_groups_info':
+                env_info = envs[0].groups_info
+                remote.send(CloudpickleWrapper(env_info))
             else:
                 raise NotImplementedError
     except KeyboardInterrupt:
@@ -99,6 +102,8 @@ def __init__(self, env_fns, env_seed, context='spawn', in_series=1):
 
         self.actions = None
         self.max_episode_steps = self.env_info['max_episode_steps']
+        self.remotes[0].send(('get_groups_info', None))
+        self.groups_info = self.remotes[0].recv().x
 
     def reset(self):
         self._assert_not_closed()

xuance/environment/vector_envs/subprocess/subproc_vec_maenv.py

@@ -52,7 +52,10 @@ def __init__(self,
 
         # Environment attributes.
         self.envs = envs
-        self.envs.reset()
+        try:
+            self.envs.reset()
+        except:
+            pass
         self.n_agents = self.config.n_agents = envs.num_agents
         self.render = config.render
         self.fps = config.fps

xuance/torch/agents/base/agents_marl.py

@@ -1,71 +1,124 @@
 import os
-import copy
+import argparse
+from copy import deepcopy
 import numpy as np
-from xuance.torch.runners import RunnerBase
 from xuance.torch.agents import REGISTRY_Agents
 from xuance.environment import make_envs
+from xuance.torch.utils.operations import set_seed
 
 
-class RunnerMARL(object):
-    def __init__(self, config):
-        super().__init__()
-        
-        self.agents = REGISTRY_Agents[config.agent](config, self.envs)
-        self.config = config
+class RunnerCompetition(object):
+    def __init__(self, configs):
+        self.configs = configs
+        # set random seeds
+        set_seed(configs.seed)
 
-        if self.agents.distributed_training:
+        # build environments
+        self.envs = make_envs(self.configs[0])
+        self.envs.reset()
+        self.group_info = self.envs.groups_infos
+        self.groups = self.group_info['agent_groups']
+        self.num_groups = self.group_info['num_groups']
+        self.obs_space_groups = self.group_info['observation_space_groups']
+        self.act_space_groups = self.group_info['action_space_groups']
+        assert len(configs) == self.num_groups, "Number of groups must be equal to the number of methods."
+        self.agents = []
+        for group in range(self.num_groups):
+            _env_info = dict(num_agents=len(self.groups[group]),
+                             num_envs=self.envs.num_envs,
+                             agents=self.groups[group],
+                             state_space=self.envs.state_space,
+                             observation_space=self.obs_space_groups[group],
+                             action_space=self.act_space_groups[group],
+                             max_episode_steps=self.envs.max_episode_steps)
+            _env = argparse.Namespace(**_env_info)
+            self.agents.append(REGISTRY_Agents[self.configs[group].agent](self.configs[group], _env))
+
+        self.observation_space = self.envs.observation_space
+        self.n_envs = self.envs.num_envs
+        self.rank = 0
+        if self.agents[0].distributed_training:
             self.rank = int(os.environ['RANK'])
 
+    def rprint(self, info: str):
+        if self.rank == 0:
+            print(info)
+
     def run(self):
-        if self.config.test_mode:
+        if self.configs[0].test_mode:
             def env_fn():
-                config_test = copy.deepcopy(self.config)
+                config_test = deepcopy(self.configs[0])
                 config_test.parallels = 1
                 config_test.render = True
                 return make_envs(config_test)
-            self.agents.render = True
-            self.agents.load_model(self.agents.model_dir_load)
+
+            for agent in self.agents:
+                agent.render = True
+                agent.load_model(agent.model_dir_load)
+
+            # ... Here is test ...
             scores = self.agents.test(env_fn, self.config.test_episode)
+
             print(f"Mean Score: {np.mean(scores)}, Std: {np.std(scores)}")
             print("Finish testing.")
         else:
-            n_train_steps = self.config.running_steps // self.n_envs
+            n_train_steps = self.configs[0].running_steps // self.n_envs
+
+            # ... Here is train ...
             self.agents.train(n_train_steps)
+
             print("Finish training.")
-            self.agents.save_model("final_train_model.pth")
+            for agent in self.agents:
+                agent.save_model("final_train_model.pth")
 
-        self.envs.close()
-        self.agents.finish()
+        for agent in self.agents:
+            agent.finish()
 
     def benchmark(self):
         def env_fn():
-            config_test = copy.deepcopy(self.config)
+            config_test = deepcopy(self.configs[0])
             config_test.parallels = 1  # config_test.test_episode
             return make_envs(config_test)
 
-        train_steps = self.config.running_steps // self.n_envs
-        eval_interval = self.config.eval_interval // self.n_envs
-        test_episode = self.config.test_episode
+        train_steps = self.configs[0].running_steps // self.n_envs
+        eval_interval = self.configs[0].eval_interval // self.n_envs
+        test_episode = self.configs[0].test_episode
         num_epoch = int(train_steps / eval_interval)
 
+        # ... Here is test ...
         test_scores = self.agents.test(env_fn, test_episode) if self.rank == 0 else 0.0
+
         best_scores_info = {"mean": np.mean(test_scores),
                             "std": np.std(test_scores),
-                            "step": self.agents.current_step}
+                            "step": self.agents[0].current_step}
+
         for i_epoch in range(num_epoch):
             print("Epoch: %d/%d:" % (i_epoch, num_epoch))
+
+            # ... Here is train ...
             self.agents.train(eval_interval)
+
             if self.rank == 0:
+
+                # ... Here is test ...
                 test_scores = self.agents.test(env_fn, test_episode)
 
                 if np.mean(test_scores) > best_scores_info["mean"]:
                     best_scores_info = {"mean": np.mean(test_scores),
                                         "std": np.std(test_scores),
                                         "step": self.agents.current_step}
                     # save best model
-                    self.agents.save_model(model_name="best_model.pth")
+                    for agent in self.agents:
+                        agent.save_model(model_name="best_model.pth")
 
         # end benchmarking
         print("Best Model Score: %.2f, std=%.2f" % (best_scores_info["mean"], best_scores_info["std"]))
-        self.envs.close()
-        self.agents.finish()
+        for agent in self.agents:
+            agent.finish()
+
+    def train(self, eval_interval):
+        return
+
+    def test(self, env_fn, test_episode):
+        scores = [None for handel in self.handles]
+        return scores