Fix _cache_founf_inf (PaddlePaddle#7997)

* Fix * Fix * Fix
DesmonDay · Mar 4, 2024 · 77a23d1 · 77a23d1
1 parent a574900
commit 77a23d1
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/examples/RLHF/ppo_trainer.py b/examples/RLHF/ppo_trainer.py
@@ -415,7 +415,11 @@ def full_training_step(self: Trainer, inputs: Dict[str, paddle.Tensor], **kwargs
             self.scaler.step(self.optimizer)
             self.scaler.update()
             scale_after = self.scaler._scale
-            optimizer_was_run = not self.scaler._cache_founf_inf
+            # Compatible with paddlepaddle 2.6.0 using typo word.
+            if hasattr(self.scaler, "_cache_founf_inf"):
+                optimizer_was_run = not self.scaler._cache_founf_inf
+            else:
+                optimizer_was_run = not self.scaler._cache_found_inf
             if not optimizer_was_run:
                 scale_before_value = scale_before.cpu().numpy()
                 scale_after_value = scale_after.cpu().numpy()

diff --git a/paddlenlp/trainer/auto_trainer.py b/paddlenlp/trainer/auto_trainer.py
@@ -438,7 +438,11 @@ def optimizer_step(self):
                 self.scaler.step(self.optimizer)
                 self.scaler.update()
                 scale_after = self.scaler._scale
-                optimizer_was_run = not self.scaler._cache_founf_inf
+                # Compatible with paddlepaddle 2.6.0 using typo word.
+                if hasattr(self.scaler, "_cache_founf_inf"):
+                    optimizer_was_run = not self.scaler._cache_founf_inf
+                else:
+                    optimizer_was_run = not self.scaler._cache_found_inf
                 if not optimizer_was_run:
                     scale_before_value = scale_before.cpu().numpy()
                     scale_after_value = scale_after.cpu().numpy()

diff --git a/paddlenlp/trainer/trainer.py b/paddlenlp/trainer/trainer.py
@@ -1031,7 +1031,11 @@ def _inner_training_loop(
                         self.scaler.step(self.optimizer)
                         self.scaler.update()
                         scale_after = self.scaler._scale
-                        optimizer_was_run = not self.scaler._cache_founf_inf
+                        # Compatible with paddlepaddle 2.6.0 using typo word.
+                        if hasattr(self.scaler, "_cache_founf_inf"):
+                            optimizer_was_run = not self.scaler._cache_founf_inf
+                        else:
+                            optimizer_was_run = not self.scaler._cache_found_inf
                         if not optimizer_was_run:
                             scale_before_value = scale_before.cpu().numpy()
                             scale_after_value = scale_after.cpu().numpy()