Differentiate summaries for train and eval.

tensor2tensor/models/research/rl.py

@@ -178,28 +178,6 @@ def ppo_pong_ae_base():
   return hparams
 
 
-@registry.register_hparams
-def pong_model_free():
-  """TODO(piotrmilos): Document this."""
-  hparams = mfrl_base()
-  hparams.batch_size = 2
-  hparams.ppo_eval_every_epochs = 2
-  hparams.ppo_epochs_num = 4
-  hparams.add_hparam("ppo_optimization_epochs", 3)
-  hparams.add_hparam("ppo_epoch_length", 30)
-  hparams.add_hparam("ppo_learning_rate", 8e-05)
-  hparams.add_hparam("ppo_optimizer", "Adam")
-  hparams.add_hparam("ppo_optimization_batch_size", 4)
-  hparams.add_hparam("ppo_save_models_every_epochs", 1000000)
-  env = gym_env.T2TGymEnv("PongNoFrameskip-v4", batch_size=2)
-  env.start_new_epoch(0)
-  hparams.add_hparam("env_fn", make_real_env_fn(env))
-  eval_env = gym_env.T2TGymEnv("PongNoFrameskip-v4", batch_size=2)
-  eval_env.start_new_epoch(0)
-  hparams.add_hparam("eval_env_fn", make_real_env_fn(eval_env))
-  return hparams
-
-
 @registry.register_hparams
 def dqn_atari_base():
   # These params are based on agents/dqn/configs/dqn.gin
@@ -242,7 +220,7 @@ def dqn_original_params():
 @registry.register_hparams
 def mfrl_original():
   return tf.contrib.training.HParams(
-      game="",
+      game="pong",
       base_algo="ppo",
       base_algo_params="ppo_original_params",
       batch_size=16,

tensor2tensor/rl/ppo_learner.py

@@ -460,11 +460,12 @@ def stop_condition(i, _, resets):
         new_memory.append(mem)
       memory = new_memory
 
-    mean_score_summary = tf.cond(
-        tf.greater(scores_num, 0),
-        lambda: tf.summary.scalar("mean_score_this_iter", mean_score), str)
-    summaries = tf.summary.merge([
-        mean_score_summary,
-        tf.summary.scalar("episodes_finished_this_iter", scores_num)
-    ])
-    return memory, summaries, initialization_lambda
+    with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
+      mean_score_summary = tf.cond(
+          tf.greater(scores_num, 0),
+          lambda: tf.summary.scalar("mean_score_this_iter", mean_score), str)
+      summaries = tf.summary.merge([
+          mean_score_summary,
+          tf.summary.scalar("episodes_finished_this_iter", scores_num)
+      ])
+      return memory, summaries, initialization_lambda