VDN

.gitignore

@@ -108,4 +108,8 @@ venv.bak/
 .vscode
 
 /nn
-/runs
+/runs
+db_config.private.yaml
+exp_scripts/
+.idea/
+analysis/

3rdparty/gym_0_10_8

@@ -0,0 +1 @@
+/home/cs/Documents/gym_0_10_8/gym

3rdparty/multiagent_mujoco

@@ -0,0 +1 @@
+/home/cs/Documents/multiagent_mujoco/src/multiagent_mujoco/

3rdparty/multiagent_particle_envs

@@ -0,0 +1 @@
+/home/cs/Documents/multiagent-particle-envs-schroederdewitt/multiagent

algos_tf14/dqnagent.py

@@ -11,7 +11,7 @@
 from common.categorical import CategoricalQ
 
 class DQNAgent:
-    def __init__(self, sess, base_name, observation_space, action_space, config):
+    def __init__(self, sess, base_name, observation_space, action_space, config, logger, central_state_space=None):
         observation_shape = observation_space.shape
         actions_num = action_space.n
         self.config = config
@@ -47,7 +47,7 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
         self.epsilon_processor = tr_helpers.LinearValueProcessor(self.config['epsilon'], self.config['min_epsilon'], self.config['epsilon_decay_frames'])
         self.beta_processor = tr_helpers.LinearValueProcessor(self.config['priority_beta'], self.config['max_beta'], self.config['beta_decay_frames'])
         if self.env_name:
-            self.env = env_configurations.configurations[self.env_name]['env_creator']()
+            self.env = env_configurations.configurations[self.env_name]['env_creator'](name=config['name'])
         self.sess = sess
         self.steps_num = self.config['steps_num']
         self.states = deque([], maxlen=self.steps_num)
@@ -60,12 +60,14 @@ def __init__(self, sess, base_name, observation_space, action_space, config):
             self.v_max = self.config['v_max']
             self.delta_z = (self.v_max - self.v_min) / (self.atoms_num - 1)
             self.all_z = tf.range(self.v_min, self.v_max + self.delta_z, self.delta_z)
-            self.categorical = CategoricalQ(self.atoms_num, self.v_min, self.v_max)     
+            self.categorical = CategoricalQ(self.atoms_num, self.v_min, self.v_max)
+
+        self.n_agents = self.env.env_info['n_agents']
 
         if not self.is_prioritized:
-            self.exp_buffer = experience.ReplayBuffer(config['replay_buffer_size'])
+            self.exp_buffer = experience.ReplayBuffer(config['replay_buffer_size'], observation_space, self.n_agents)
         else: 
-            self.exp_buffer = experience.PrioritizedReplayBuffer(config['replay_buffer_size'], config['priority_alpha'])
+            self.exp_buffer = experience.PrioritizedReplayBuffer(config['replay_buffer_size'], config['priority_alpha'], observation_space, self.n_agents)
             self.sample_weights_ph = tf.placeholder(tf.float32, shape= [None,] , name='sample_weights')
 
         self.obs_ph = tf.placeholder(observation_space.dtype, shape=(None,) + self.state_shape , name = 'obs_ph')
@@ -402,7 +404,16 @@ def train(self):
                 self.writer.add_scalar('info/epsilon', self.epsilon, frame)
                 if self.is_prioritized:
                     self.writer.add_scalar('beta', self.beta, frame)
-
+
+                self.logger.log_stat("whirl/performance/fps", 1000 / sum_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/performance/upd_time", update_time, self.num_env_steps_train)
+                self.logger.log_stat("whirl/performance/play_time", play_time, self.num_env_steps_train)
+                self.logger.log_stat("losses/td_loss", np.mean(losses), self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/last_lr", self.learning_rate*lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/info/lr_mul", lr_mul, self.num_env_steps_train)
+                self.logger.log_stat("whirl/epochs", epoch_num, self.num_env_steps_train)
+                self.logger.log_stat("whirl/epsilon", self.epsilon, self.num_env_steps_train)
+
                 update_time = 0
                 play_time = 0
                 num_games = len(self.game_rewards)
@@ -415,6 +426,11 @@ def train(self):
                     self.writer.add_scalar('episode_lengths/mean', mean_lengths, frame)
                     self.writer.add_scalar('episode_lengths/time', mean_lengths, total_time)
 
+                    self.logger.log_stat("whirl/rewards/mean", np.asscalar(mean_rewards), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/rewards/time", mean_rewards, total_time)
+                    self.logger.log_stat("whirl/episode_lengths/mean", np.asscalar(mean_lengths), self.num_env_steps_train)
+                    self.logger.log_stat("whirl/episode_lengths/time", mean_lengths, total_time)
+
                     if mean_rewards > last_mean_rewards:
                         print('saving next best rewards: ', mean_rewards)
                         last_mean_rewards = mean_rewards