Merge pull request #662 from allenai/tiny-olmo-config-fix

updated config for olmo tiny suite of models
allenai · Jul 16, 2024 · b55fb5f · b55fb5f
2 parents 56d1fe0 + 43bc92c
commit b55fb5f
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 11 deletions.
diff --git a/configs/tiny/OLMo-150M.yaml b/configs/tiny/OLMo-150M.yaml
@@ -46,11 +46,11 @@ compile: null
 
 optimizer:
   name: adamw
-  learning_rate: 1.0e-3
+  learning_rate: 6.0e-4
   weight_decay: 0.1
   eps: 1e-8
   decay_norm_and_bias: true
-  decay_embeddings: false
+  decay_embeddings: true
   betas:
   - 0.9
   - 0.95

diff --git a/configs/tiny/OLMo-20M.yaml b/configs/tiny/OLMo-20M.yaml
@@ -46,11 +46,11 @@ compile: null
 
 optimizer:
   name: adamw
-  learning_rate: 1.0e-3
+  learning_rate: 6.0e-4
   weight_decay: 0.1
   eps: 1e-8
   decay_norm_and_bias: true
-  decay_embeddings: false
+  decay_embeddings: true
   betas:
   - 0.9
   - 0.95

diff --git a/configs/tiny/OLMo-300M.yaml b/configs/tiny/OLMo-300M.yaml
@@ -50,7 +50,7 @@ optimizer:
   weight_decay: 0.1
   eps: 1e-8
   decay_norm_and_bias: true
-  decay_embeddings: false
+  decay_embeddings: true
   betas:
   - 0.9
   - 0.95

diff --git a/configs/tiny/OLMo-60M.yaml b/configs/tiny/OLMo-60M.yaml
@@ -46,11 +46,11 @@ compile: null
 
 optimizer:
   name: adamw
-  learning_rate: 1.0e-3
+  learning_rate: 6.0e-4
   weight_decay: 0.1
   eps: 1e-8
   decay_norm_and_bias: true
-  decay_embeddings: false
+  decay_embeddings: true
   betas:
   - 0.9
   - 0.95

diff --git a/configs/tiny/OLMo-750M.yaml → configs/tiny/OLMo-700M.yaml b/configs/tiny/OLMo-750M.yaml → configs/tiny/OLMo-700M.yaml
@@ -1,4 +1,4 @@
-run_name: OLMo-750M
+run_name: OLMo-700M
 seed: 6198
 dry_run: false
 
@@ -8,8 +8,8 @@ wandb:
 
 model:
   d_model: 1536
-  n_heads: 24
-  n_layers: 24
+  n_heads: 16
+  n_layers: 16
   mlp_ratio: 8
   weight_tying: false
   alibi: false
@@ -50,7 +50,7 @@ optimizer:
   weight_decay: 0.1
   eps: 1e-8
   decay_norm_and_bias: true
-  decay_embeddings: false
+  decay_embeddings: true
   betas:
   - 0.9
   - 0.95