Merge pull request #862 from bghira/main

schedulefree: return to previous stable settings and add a new preset for aggressive training
bghira · Aug 24, 2024 · e704678 · e704678
2 parents 413f774 + 76b377c
commit e704678
Showing 1 changed file with 25 additions and 2 deletions.
diff --git a/helpers/training/optimizer_param.py b/helpers/training/optimizer_param.py
@@ -41,9 +41,32 @@
         "override_lr_scheduler": True,
         "can_warmup": True,
         "default_settings": {
-            "betas": (0.9, 0.990),
-            "weight_decay": 1e-4,
+            "betas": (0.9, 0.999),
+            "weight_decay": 1e-2,
+            "eps": 1e-8,
+        },
+        "class": AdamWScheduleFreeKahan,
+    },
+    "adamw_schedulefree+aggressive": {
+        "precision": "any",
+        "override_lr_scheduler": True,
+        "can_warmup": True,
+        "default_settings": {
+            "betas": (0.9, 0.999),
+            "weight_decay": 1e-3,
+            "eps": 1e-6,
+        },
+        "class": AdamWScheduleFreeKahan,
+    },
+    "adamw_schedulefree+no_kahan": {
+        "precision": "any",
+        "override_lr_scheduler": True,
+        "can_warmup": True,
+        "default_settings": {
+            "betas": (0.9, 0.999),
+            "weight_decay": 1e-3,
             "eps": 1e-6,
+            "use_kahan": False,
         },
         "class": AdamWScheduleFreeKahan,
     },