Add ddp_find_unused_parameters=True and change accelerator to auto (N…

…VIDIA#7623) (NVIDIA#7644) * Add ddp_find_unused_parameters=True and change acclerator to auto * Add ddp_find_unused_parameters True for normalization_as_tagging_train.py --------- Signed-off-by: Abhishree <[email protected]> Co-authored-by: Abhishree Thittenamane <[email protected]>
rohitrango · Oct 5, 2023 · 25ecb34 · 25ecb34
1 parent 1f712b5
commit 25ecb34
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 4 deletions.
diff --git a/examples/nlp/intent_slot_classification/intent_slot_classification.py b/examples/nlp/intent_slot_classification/intent_slot_classification.py
@@ -23,6 +23,10 @@
 
 @hydra_runner(config_path="conf", config_name="intent_slot_classification_config")
 def main(cfg: DictConfig) -> None:
+    # PTL 2.0 has find_unused_parameters as False by default, so its required to set it to True
+    # when there are unused parameters like here
+    if cfg.trainer.strategy == 'ddp':
+        cfg.trainer.strategy = "ddp_find_unused_parameters_true"
     logging.info(f'Config Params:\n {OmegaConf.to_yaml(cfg)}')
     trainer = pl.Trainer(**cfg.trainer)
     exp_manager(trainer, cfg.get("exp_manager", None))

diff --git a/examples/nlp/question_answering/question_answering.py b/examples/nlp/question_answering/question_answering.py
@@ -28,7 +28,10 @@
 @hydra_runner(config_path="conf", config_name="qa_conf")
 def main(cfg: DictConfig) -> None:
     pl.seed_everything(42)
-
+    # PTL 2.0 has find_unused_parameters as False by default, so its required to set it to True
+    # when there are unused parameters like here
+    if cfg.trainer.strategy == 'ddp':
+        cfg.trainer.strategy = "ddp_find_unused_parameters_true"
     logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
     trainer = pl.Trainer(**cfg.trainer)
     exp_dir = exp_manager(trainer, cfg.get("exp_manager", None))

diff --git a/examples/nlp/text_normalization_as_tagging/normalization_as_tagging_train.py b/examples/nlp/text_normalization_as_tagging/normalization_as_tagging_train.py
@@ -62,6 +62,10 @@
 
 @hydra_runner(config_path="conf", config_name="thutmose_tagger_itn_config")
 def main(cfg: DictConfig) -> None:
+    # PTL 2.0 has find_unused_parameters as False by default, so its required to set it to True
+    # when there are unused parameters like here
+    if cfg.trainer.strategy == 'ddp':
+        cfg.trainer.strategy = "ddp_find_unused_parameters_true"
     logging.info(f'Config Params: {OmegaConf.to_yaml(cfg)}')
 
     # Train the model

diff --git a/examples/nlp/token_classification/token_classification_train.py b/examples/nlp/token_classification/token_classification_train.py
@@ -103,7 +103,7 @@
 @hydra_runner(config_path="conf", config_name="token_classification_config")
 def main(cfg: DictConfig) -> None:
     try:
-        strategy = NLPDDPStrategy()
+        strategy = NLPDDPStrategy(find_unused_parameters=True)
     except (ImportError, ModuleNotFoundError):
         strategy = None
 

diff --git a/nemo/collections/nlp/models/text_normalization_as_tagging/thutmose_tagger.py b/nemo/collections/nlp/models/text_normalization_as_tagging/thutmose_tagger.py
@@ -236,7 +236,7 @@ def validation_step(self, batch, batch_idx):
         val_loss_tag = self.loss_fn(logits=tag_logits, labels=tag_labels, loss_mask=labels_mask)
         val_loss_semiotic = self.loss_fn(logits=semiotic_logits, labels=semiotic_labels, loss_mask=labels_mask)
         val_loss = val_loss_tag + val_loss_semiotic
-        self.validation_step_outputs.append(val_loss)
+        self.validation_step_outputs.append({'val_loss': val_loss})
         return {'val_loss': val_loss}
 
     def on_validation_epoch_end(self):

diff --git a/tutorials/nlp/Entity_Linking_Medical.ipynb b/tutorials/nlp/Entity_Linking_Medical.ipynb
@@ -188,7 +188,7 @@
     "\n",
     "# remove distributed training flags\n",
     "cfg.trainer.strategy = 'auto'\n",
-    "cfg.trainer.accelerator = None"
+    "cfg.trainer.accelerator = 'auto'"
    ]
   },
   {