From c97ff87e4cf9d0ff80ca7858122e2002b99b2527 Mon Sep 17 00:00:00 2001 From: jianan-gu Date: Tue, 10 May 2022 13:21:35 +0800 Subject: [PATCH 1/2] add torch SGD and Adagrad optimizer bits --- src/transformers/trainer.py | 4 ++++ src/transformers/training_args.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index aa54f2af1bb5..f3a574ac30e8 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -978,6 +978,10 @@ def get_optimizer_cls_and_kwargs(args: TrainingArguments) -> Tuple[Any, Any]: optimizer_kwargs.update(adam_kwargs) except ImportError: raise ValueError("Trainer tried to instantiate bnb Adam8bit but bnb is not installed!") + elif args.optim == OptimizerNames.SGD: + optimizer_cls = torch.optim.SGD + elif args.optim == OptimizerNames.ADAGRAD: + optimizer_cls = torch.optim.Adagrad else: raise ValueError(f"Trainer cannot instantiate unsupported optimizer: {args.optim}") return optimizer_cls, optimizer_kwargs diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index 631fe0b3ec57..d7a29a80c861 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -87,6 +87,8 @@ class OptimizerNames(ExplicitEnum): ADAMW_APEX_FUSED = "adamw_apex_fused" ADAFACTOR = "adafactor" ADAMW_BNB = "adamw_bnb_8bit" + SGD = "SGD" + ADAGRAD = "Adagrad" @dataclass From 363fa42a138ea979ddd7652f2b39e49f4adaa0c6 Mon Sep 17 00:00:00 2001 From: jianan-gu Date: Wed, 11 May 2022 09:13:19 +0800 Subject: [PATCH 2/2] refine naming Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/training_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index d7a29a80c861..e3b760081f84 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -87,8 +87,8 @@ class OptimizerNames(ExplicitEnum): ADAMW_APEX_FUSED = "adamw_apex_fused" ADAFACTOR = "adafactor" ADAMW_BNB = "adamw_bnb_8bit" - SGD = "SGD" - ADAGRAD = "Adagrad" + SGD = "sgd" + ADAGRAD = "adagrad" @dataclass