VDN

.gitignore

@@ -108,4 +108,8 @@ venv.bak/
 .vscode
 
 /nn
-/runs
+/runs
+db_config.private.yaml
+exp_scripts/
+.idea/
+analysis/

3rdparty/gym_0_10_8

@@ -0,0 +1 @@
+/home/cs/Documents/gym_0_10_8/gym

3rdparty/multiagent_mujoco

@@ -0,0 +1 @@
+/home/cs/Documents/multiagent_mujoco/src/multiagent_mujoco/

3rdparty/multiagent_particle_envs

@@ -0,0 +1 @@
+/home/cs/Documents/multiagent-particle-envs-schroederdewitt/multiagent

algos_tf14/a2c_discrete.py

@@ -21,8 +21,14 @@ def swap_and_flatten01(arr):
     return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:])
 
 class A2CAgent:
-    def __init__(self, sess, base_name, observation_space, action_space, config):
+    def __init__(self, sess, base_name, observation_space, action_space, config, logger, central_state_space=None):
         observation_shape = observation_space.shape
+
+        self.use_central_states = False
+        if central_state_space is not None:
+            self.use_central_states = True
+            central_state_shape = central_state_space.shape
+
         self.use_action_masks = config.get('use_action_masks', False)
         self.is_train = config.get('is_train', True)
         self.self_play = config.get('self_play', False)
@@ -67,6 +73,8 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
         self.game_lengths = deque([], maxlen=self.games_to_log)
         self.game_scores = deque([], maxlen=self.games_to_log)
         self.obs_ph = tf.placeholder(observation_space.dtype, (None, ) + observation_shape, name = 'obs')
+        if self.use_central_states:
+            self.central_states_ph = tf.placeholder(central_state_space.dtype, (None, ) + central_state_shape, name = 'central_state')
         self.target_obs_ph = tf.placeholder(observation_space.dtype, (None, ) + observation_shape, name = 'target_obs') 
         self.actions_num = action_space.n   
         self.actions_ph = tf.placeholder('int32', (None,), name = 'actions')       
@@ -84,6 +92,9 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
         self.update_epoch_op = self.epoch_num.assign(self.epoch_num + 1)
         self.current_lr = self.learning_rate_ph
 
+        #if self.use_central_states:
+        #    self.input_obs = self.central_states_ph
+        #else:
         self.input_obs = self.obs_ph
         self.input_target_obs = self.target_obs_ph
 
@@ -114,6 +125,9 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
             'action_mask_ph' : None
         }
 
+        if self.use_central_states:
+            self.train_dict["central_states"] = self.central_states_ph
+
         self.run_dict = {
             'name' : 'agent',
             'inputs' : self.input_target_obs,
@@ -124,11 +138,14 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
             'action_mask_ph' : self.action_mask_ph
         }
 
-        self.states = None
+        if self.use_central_states:
+            self.train_dict["central_states"] = self.central_states_ph
+
+        self.rnn_states = None
         if self.network.is_rnn():
-            self.logp_actions ,self.state_values, self.action, self.entropy, self.states_ph, self.masks_ph, self.lstm_state, self.initial_state = self.network(self.train_dict, reuse=False)
+            self.logp_actions, self.state_values, self.action, self.entropy, self.rnn_states_ph, self.masks_ph, self.lstm_state, self.initial_state = self.network(self.train_dict, reuse=False)
             self.target_neglogp, self.target_state_values, self.target_action, _,  self.target_states_ph, self.target_masks_ph, self.target_lstm_state, self.target_initial_state, self.logits = self.network(self.run_dict, reuse=True)
-            self.states = self.target_initial_state
+            self.rnn_states = self.target_initial_state
 
         else:
             self.logp_actions ,self.state_values, self.action, self.entropy = self.network(self.train_dict, reuse=False)
@@ -142,6 +159,10 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
 
         self.sess.run(tf.global_variables_initializer())
 
+        self.logger = logger
+
+        self.num_env_steps_train = 0
+
     def setup_losses(self):
         curr_e_clip = self.e_clip * self.lr_multiplier
         if (self.ppo):
@@ -192,22 +213,22 @@ def get_action_values(self, obs):
         run_ops = [self.target_action, self.target_state_values, self.target_neglogp]
         if self.network.is_rnn():
             run_ops.append(self.target_lstm_state)
-            return self.sess.run(run_ops, {self.target_obs_ph : obs, self.target_states_ph : self.states, self.target_masks_ph : self.dones})
+            return self.sess.run(run_ops, {self.target_obs_ph : obs, self.target_states_ph : self.rnn_states, self.target_masks_ph : self.dones})
         else:
             return (*self.sess.run(run_ops, {self.target_obs_ph : obs}), None)
 
     def get_masked_action_values(self, obs, action_masks):
         run_ops = [self.target_action, self.target_state_values, self.target_neglogp, self.logits]
         if self.network.is_rnn():
             run_ops.append(self.target_lstm_state)
-            return self.sess.run(run_ops, {self.action_mask_ph: action_masks, self.target_obs_ph : obs, self.target_states_ph : self.states, self.target_masks_ph : self.dones})
+            return self.sess.run(run_ops, {self.action_mask_ph: action_masks, self.target_obs_ph : obs, self.target_states_ph : self.rnn_states, self.target_masks_ph : self.dones})
         else:
             return (*self.sess.run(run_ops, {self.action_mask_ph: action_masks, self.target_obs_ph : obs}), None)
 
 
     def get_values(self, obs):
         if self.network.is_rnn():
-            return self.sess.run([self.target_state_values], {self.target_obs_ph : obs, self.target_states_ph : self.states, self.target_masks_ph : self.dones})
+            return self.sess.run([self.target_state_values], {self.target_obs_ph : obs, self.target_states_ph : self.rnn_states, self.target_masks_ph : self.dones})
         else:
             return self.sess.run([self.target_state_values], {self.target_obs_ph : obs})
 
@@ -222,33 +243,44 @@ def set_weights(self, weights):
     def play_steps(self):
         # here, we init the lists that will contain the mb of experiences
         mb_obs, mb_rewards, mb_actions, mb_values, mb_dones, mb_neglogpacs = [],[],[],[],[],[]
-
-        mb_states = []
+
+        if self.use_central_states:
+            mb_central_states = []
+
+        mb_rnn_states = []
         epinfos = []
 
         # for n in range number of steps
         for _ in range(self.steps_num):
             if self.network.is_rnn():
-                mb_states.append(self.states)
+                mb_rnn_states.append(self.rnn_states)
 
             if self.use_action_masks:
                 masks = self.vec_env.get_action_masks()
 
             if self.use_action_masks:
-                actions, values, neglogpacs, _, self.states = self.get_masked_action_values(self.obs, masks)
+                actions, values, neglogpacs, _, self.rnn_states = self.get_masked_action_values(self.obs, masks)
             else:
-                actions, values, neglogpacs, self.states = self.get_action_values(self.obs)
+                actions, values, neglogpacs, self.rnn_states = self.get_action_values(self.obs)
 
             actions = np.squeeze(actions)
             values = np.squeeze(values)
             neglogpacs = np.squeeze(neglogpacs)
             mb_obs.append(self.obs.copy())
+            if self.use_central_states:
+                mb_central_states.append(self.central_states.copy())
             mb_actions.append(actions)
             mb_values.append(values)
             mb_neglogpacs.append(neglogpacs)
             mb_dones.append(self.dones.copy())
 
             self.obs[:], rewards, self.dones, infos = self.vec_env.step(actions)
+            if self.use_central_states:
+                self.central_states[:] = self.vec_env.get_states()
+
+            # Increase step count by self.num_actors (WHIRL)
+            self.num_env_steps_train += self.num_actors
+
             self.current_rewards += rewards
 
             self.current_lengths += 1
@@ -268,12 +300,14 @@ def play_steps(self):
 
         #using openai baseline approach
         mb_obs = np.asarray(mb_obs, dtype=self.obs.dtype)
+        if self.use_central_states:
+            mb_central_states = np.asarray(mb_central_states, dtype=self.obs.dtype)
         mb_rewards = np.asarray(mb_rewards, dtype=np.float32)
         mb_actions = np.asarray(mb_actions, dtype=np.float32)
         mb_values = np.asarray(mb_values, dtype=np.float32)
         mb_neglogpacs = np.asarray(mb_neglogpacs, dtype=np.float32)
         mb_dones = np.asarray(mb_dones, dtype=np.bool)
-        mb_states = np.asarray(mb_states, dtype=np.float32)
+        mb_rnn_states = np.asarray(mb_rnn_states, dtype=np.float32)
         last_values = self.get_values(self.obs)
         last_values = np.squeeze(last_values)
 
@@ -294,9 +328,19 @@ def play_steps(self):
 
         mb_returns = mb_advs + mb_values
         if self.network.is_rnn():
-            result = (*map(swap_and_flatten01, (mb_obs, mb_returns, mb_dones, mb_actions, mb_values, mb_neglogpacs, mb_states  )), epinfos)
+            if self.use_central_states:
+                result = (*map(swap_and_flatten01,
+                               (mb_central_states, mb_obs, mb_returns, mb_dones, mb_actions, mb_values, mb_neglogpacs, mb_rnn_states)),
+                          epinfos)
+            else:
+                result = (*map(swap_and_flatten01, (mb_obs, mb_returns, mb_dones, mb_actions, mb_values, mb_neglogpacs, mb_rnn_states  )), epinfos)
         else:
-            result = (*map(swap_and_flatten01, (mb_obs, mb_returns, mb_dones, mb_actions, mb_values, mb_neglogpacs)), None, epinfos)
+            if self.use_central_states:
+                result = (
+                *map(swap_and_flatten01, (mb_central_states, mb_obs, mb_returns, mb_dones, mb_actions, mb_values, mb_neglogpacs)), None,
+                epinfos)
+            else:
+                result = (*map(swap_and_flatten01, (mb_obs, mb_returns, mb_dones, mb_actions, mb_values, mb_neglogpacs)), None, epinfos)
         return result
 
     def save(self, fn):
@@ -307,6 +351,8 @@ def restore(self, fn):
 
     def train(self):
         self.obs = self.vec_env.reset()
+        if self.use_central_states:
+            self.central_states = self.vec_env.get_states()
         batch_size = self.steps_num * self.num_actors * self.num_agents
         batch_size_envs = self.steps_num * self.num_actors
         minibatch_size = self.config['minibatch_size']
@@ -327,7 +373,10 @@ def train(self):
             play_time_start = time.time()
             epoch_num = self.update_epoch()
             frame += batch_size_envs
-            obses, returns, dones, actions, values, neglogpacs, lstm_states, _ = self.play_steps()
+            if self.use_central_states:
+                central_states, obses, returns, dones, actions, values, neglogpacs, lstm_states, _ = self.play_steps()
+            else:
+                obses, returns, dones, actions, values, neglogpacs, lstm_states, _ = self.play_steps()
             advantages = returns - values
             if self.normalize_advantage:
                 advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
@@ -363,7 +412,7 @@ def train(self):
                         dict[self.obs_ph] = obses[mbatch]
                         dict[self.masks_ph] = dones[mbatch]
 
-                        dict[self.states_ph] = lstm_states[batch]
+                        dict[self.rnn_states_ph] = lstm_states[batch]
 
                         dict[self.learning_rate_ph] = last_lr
                         run_ops = [self.actor_loss, self.critic_loss, self.entropy, self.kl_approx, self.current_lr, self.lr_multiplier,  self.train_op]
@@ -383,11 +432,13 @@ def train(self):
                     values = values[permutation]
                     neglogpacs = neglogpacs[permutation]
                     advantages = advantages[permutation]
+                    central_states = central_states[permutation]
 
                     for i in range(0, num_minibatches):
                         batch = range(i * minibatch_size, (i + 1) * minibatch_size)
                         dict = {self.obs_ph: obses[batch], self.actions_ph : actions[batch], self.rewards_ph : returns[batch], 
-                                self.advantages_ph : advantages[batch], self.old_logp_actions_ph : neglogpacs[batch], self.old_values_ph : values[batch]}
+                                self.advantages_ph : advantages[batch], self.old_logp_actions_ph : neglogpacs[batch], self.old_values_ph : values[batch],
+                                self.central_states_ph: central_states[batch]}
                         dict[self.learning_rate_ph] = last_lr
                         run_ops = [self.actor_loss, self.critic_loss, self.entropy, self.kl_approx, self.current_lr, self.lr_multiplier, self.train_op]
 
@@ -417,6 +468,18 @@ def train(self):
                 self.writer.add_scalar('info/e_clip', self.e_clip * lr_mul, frame)
                 self.writer.add_scalar('info/kl', np.mean(kls), frame)
                 self.writer.add_scalar('epochs', epoch_num, frame)
+
+                self.logger.log_stat("whirl/performance/fps", batch_size / scaled_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/performance/upd_time", update_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/performance/play_time", play_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/losses/a_loss", np.asscalar(np.mean(a_losses)), self.num_env_steps_train)
+                self.logger.log_stat("whirl/losses/c_loss", np.asscalar(np.mean(c_losses)), self.num_env_steps_train)
+                self.logger.log_stat("whirl/losses/entropy", np.asscalar(np.mean(entropies)), self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/last_lr", last_lr * lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/lr_mul", lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/e_clip", self.e_clip * lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/kl", np.asscalar(np.mean(kls)), self.num_env_steps_train)
+                self.logger.log_stat("whirl/epochs", epoch_num, self.num_env_steps_train)
 
                 if len(self.game_rewards) > 0:
                     mean_rewards = np.mean(self.game_rewards)
@@ -429,6 +492,14 @@ def train(self):
                     self.writer.add_scalar('win_rate/mean', mean_scores, frame)
                     self.writer.add_scalar('win_rate/time', mean_scores, total_time)
 
+                    self.logger.log_stat("whirl/rewards/mean", np.asscalar(mean_rewards), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/rewards/time", mean_rewards, total_time)
+                    self.logger.log_stat("whirl/episode_lengths/mean", np.asscalar(mean_lengths), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/episode_lengths/time", mean_lengths, total_time)
+                    self.logger.log_stat("whirl/win_rate/mean", np.asscalar(mean_scores), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/win_rate/time", mean_scores, total_time)
+
+
                     if rep_count % 10 == 0:
                         self.save("./nn/" + 'last_' + self.config['name'] + 'ep=' + str(epoch_num) + 'rew=' + str(mean_rewards))
                         rep_count += 1

algos_tf14/dqnagent.py

@@ -11,7 +11,7 @@
 from common.categorical import CategoricalQ
 
 class DQNAgent:
-    def __init__(self, sess, base_name, observation_space, action_space, config):
+    def __init__(self, sess, base_name, observation_space, action_space, config, logger):
         observation_shape = observation_space.shape
         actions_num = action_space.n
         self.config = config
@@ -47,7 +47,7 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
         self.epsilon_processor = tr_helpers.LinearValueProcessor(self.config['epsilon'], self.config['min_epsilon'], self.config['epsilon_decay_frames'])
         self.beta_processor = tr_helpers.LinearValueProcessor(self.config['priority_beta'], self.config['max_beta'], self.config['beta_decay_frames'])
         if self.env_name:
-            self.env = env_configurations.configurations[self.env_name]['env_creator']()
+            self.env = env_configurations.configurations[self.env_name]['env_creator'](name=config['name'])
         self.sess = sess
         self.steps_num = self.config['steps_num']
         self.states = deque([], maxlen=self.steps_num)
@@ -402,7 +402,16 @@ def train(self):
                 self.writer.add_scalar('info/epsilon', self.epsilon, frame)
                 if self.is_prioritized:
                     self.writer.add_scalar('beta', self.beta, frame)
-
+
+                self.logger.log_stat("whirl/performance/fps", 1000 / sum_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/performance/upd_time", update_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/performance/play_time", play_time, self.num_env_steps_train)
+                self.logger.log_stat("losses/td_loss", np.mean(losses), self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/last_lr", self.learning_rate*lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/lr_mul", lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/epochs", epoch_num, self.num_env_steps_train)
+                self.logger.log_stat("whirl/epsilon", self.epsilon, self.num_env_steps_train)
+
                 update_time = 0
                 play_time = 0
                 num_games = len(self.game_rewards)
@@ -415,6 +424,11 @@ def train(self):
                     self.writer.add_scalar('episode_lengths/mean', mean_lengths, frame)
                     self.writer.add_scalar('episode_lengths/time', mean_lengths, total_time)
 
+                    self.logger.log_stat("whirl/rewards/mean", np.asscalar(mean_rewards), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/rewards/time", mean_rewards, total_time)
+                    self.logger.log_stat("whirl/episode_lengths/mean", np.asscalar(mean_lengths), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/episode_lengths/time", mean_lengths, total_time)
+
                     if mean_rewards > last_mean_rewards:
                         print('saving next best rewards: ', mean_rewards)
                         last_mean_rewards = mean_rewards

algos_tf14/model_builder.py

@@ -13,6 +13,7 @@ def __init__(self):
         self.model_factory.register_builder('continuous_a2c_lstm', lambda network, **kwargs : models.LSTMModelA2CContinuous(network))
         self.model_factory.register_builder('continuous_a2c_lstm_logstd', lambda network, **kwargs : models.LSTMModelA2CContinuousLogStd(network))
         self.model_factory.register_builder('dqn', lambda network, **kwargs : models.AtariDQN(network))
+        self.model_factory.register_builder('vdn', lambda network, **kwargs : models.VDN_DQN(network))
 
 
         self.network_factory = object_factory.ObjectFactory()