From d6678885c9bfe8070917d23c68ff913177e524da Mon Sep 17 00:00:00 2001 From: Greg Heinrich Date: Wed, 11 Sep 2024 07:37:04 -0700 Subject: [PATCH] Catch ConfigError when attempting to set num_parameters in WANDB The current version of the code catches AttributeError when attempting to set the model's number of parameters in the Weights & Biases config. ConfigError is another exception that needs catching in the case when the current number of model parameters differs from that of the config. This happens, for example, on resuming from checkpoints if extra layers have been added. Example error message: ``` wandb.sdk.lib.config_util.ConfigError: Attempted to change value of key "model/num_parameters" from 0 to 700416 ``` Since the current code already includes support for ignoring AttributeError, it feels safe to add this extra catch. --- src/transformers/integrations/integration_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/integrations/integration_utils.py b/src/transformers/integrations/integration_utils.py index 9172f9599f77..66ec93acbb9b 100755 --- a/src/transformers/integrations/integration_utils.py +++ b/src/transformers/integrations/integration_utils.py @@ -853,6 +853,10 @@ def setup(self, args, state, model, **kwargs): self._wandb.config["model/num_parameters"] = model.num_parameters() except AttributeError: logger.info("Could not log the number of model parameters in Weights & Biases.") + except self._wandb.sdk.lib.config_util.ConfigError: + logger.warning( + "A ConfigError was raised whilst setting the number of model parameters in Weights & Biases config." + ) # log the initial model architecture to an artifact if self._log_model.is_enabled: