From 81c6b3ac6262c90981071165730382d255a6d540 Mon Sep 17 00:00:00 2001
From: Mohit Deopujari <mdeopujari@habana.ai>
Date: Fri, 8 Apr 2022 09:38:24 -0700
Subject: [PATCH] Updating FusedAdamW epsilon to Habana default 1e-6

---
 optimum/habana/training_args.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/optimum/habana/training_args.py b/optimum/habana/training_args.py
index d7d851faa4..230777d44f 100644
--- a/optimum/habana/training_args.py
+++ b/optimum/habana/training_args.py
@@ -66,6 +66,8 @@ class GaudiTrainingArguments(TrainingArguments):
         metadata={"help": "Whether to use lazy mode for training the model."},
     )
 
+    adam_epsilon: float = field(default=1e-6, metadata={"help": "Epsilon for AdamW optimizer."})
+
     @cached_property
     @torch_required
     def _setup_devices(self) -> "torch.device":