fix ssl models ptl monitor val through logging (#7608) (#7614)

Signed-off-by: Nithin Rao Koluguri <nithinraok> Co-authored-by: Nithin Rao <[email protected]> Co-authored-by: Eric Harper <[email protected]> Co-authored-by: Xuesong Yang <[email protected]>
NVIDIA · Oct 5, 2023 · e109c6e · e109c6e
1 parent 329bd3c
commit e109c6e
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/nemo/collections/asr/models/ssl_models.py b/nemo/collections/asr/models/ssl_models.py
@@ -527,7 +527,7 @@ def training_step(self, batch, batch_nb):
 
         return {'loss': loss_value, 'log': tensorboard_logs}
 
-    def validation_step(self, batch, batch_idx, dataloader_idx=0):
+    def validation_pass(self, batch, batch_idx, dataloader_idx=0):
         # Set flag to register tensors
         self._in_validation_step = True
 
@@ -554,9 +554,17 @@ def validation_step(self, batch, batch_idx, dataloader_idx=0):
         self.reset_registry()
         del self._in_validation_step
 
-        return {
-            'val_loss': loss_value,
-        }
+        metrics = {'val_loss': loss_value}
+
+        return metrics
+
+    def validation_step(self, batch, batch_idx, dataloader_idx=0):
+        metrics = self.validation_pass(batch, batch_idx, dataloader_idx)
+        if type(self.trainer.val_dataloaders) == list and len(self.trainer.val_dataloaders) > 1:
+            self.validation_step_outputs[dataloader_idx].append(metrics)
+        else:
+            self.validation_step_outputs.append(metrics)
+        return metrics
 
     def multi_validation_epoch_end(self, outputs, dataloader_idx: int = 0):
         val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean()

diff --git a/tutorials/asr/Self_Supervised_Pre_Training.ipynb b/tutorials/asr/Self_Supervised_Pre_Training.ipynb
@@ -215,7 +215,7 @@
         "                    file_id[file_id.find('-')+1 : file_id.rfind('-')],\n",
         "                    file_id + '.wav')\n",
         "\n",
-        "                duration = librosa.core.get_duration(filename=audio_path)\n",
+        "                duration = librosa.core.get_duration(path=audio_path)\n",
         "\n",
         "                # Write the metadata to the manifest\n",
         "                metadata = {\n",
@@ -331,7 +331,7 @@
         "\n",
         "cfg.model.optim.sched.name = \"CosineAnnealing\"\n",
         "cfg.model.optim.sched.warmup_steps = 1000\n",
-        "cfg.model.optim.sched.max_steps = 5000\n",
+        "cfg.model.optim.sched.max_steps = 2000\n",
         "#in practice you will usually want a much larger amount of pre-training steps\n",
         "cfg.model.optim.sched.min_lr = 0\n",
         "cfg.model.optim.lr = 0.015\n",
@@ -554,7 +554,7 @@
         "\n",
         "cfg.model.optim.sched.name = \"CosineAnnealing\"\n",
         "cfg.model.optim.sched.warmup_steps = 500\n",
-        "cfg.model.optim.sched.max_steps = 2000\n",
+        "cfg.model.optim.sched.max_steps = 1000\n",
         "cfg.model.optim.sched.min_lr = 0\n",
         "cfg.model.optim.lr = 0.015 #if encoder is frozen, lr can be much higher\n",
         "cfg.model.optim.weight_decay = 0\n",