From d8aa2b0d52233661303dae486db41d876b4462d7 Mon Sep 17 00:00:00 2001 From: ananyahjha93 Date: Tue, 16 Jul 2024 13:23:38 -0700 Subject: [PATCH] updaated config for olmo tiny suite of models --- configs/tiny/OLMo-150M.yaml | 4 ++-- configs/tiny/OLMo-20M.yaml | 4 ++-- configs/tiny/OLMo-300M.yaml | 2 +- configs/tiny/OLMo-60M.yaml | 4 ++-- configs/tiny/{OLMo-750M.yaml => OLMo-700M.yaml} | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) rename configs/tiny/{OLMo-750M.yaml => OLMo-700M.yaml} (99%) diff --git a/configs/tiny/OLMo-150M.yaml b/configs/tiny/OLMo-150M.yaml index 3536c3727..3ef70e2d0 100644 --- a/configs/tiny/OLMo-150M.yaml +++ b/configs/tiny/OLMo-150M.yaml @@ -46,11 +46,11 @@ compile: null optimizer: name: adamw - learning_rate: 1.0e-3 + learning_rate: 6.0e-4 weight_decay: 0.1 eps: 1e-8 decay_norm_and_bias: true - decay_embeddings: false + decay_embeddings: true betas: - 0.9 - 0.95 diff --git a/configs/tiny/OLMo-20M.yaml b/configs/tiny/OLMo-20M.yaml index 838a8c11a..a82fab7aa 100644 --- a/configs/tiny/OLMo-20M.yaml +++ b/configs/tiny/OLMo-20M.yaml @@ -46,11 +46,11 @@ compile: null optimizer: name: adamw - learning_rate: 1.0e-3 + learning_rate: 6.0e-4 weight_decay: 0.1 eps: 1e-8 decay_norm_and_bias: true - decay_embeddings: false + decay_embeddings: true betas: - 0.9 - 0.95 diff --git a/configs/tiny/OLMo-300M.yaml b/configs/tiny/OLMo-300M.yaml index c6dd7f4eb..9dfd9941c 100644 --- a/configs/tiny/OLMo-300M.yaml +++ b/configs/tiny/OLMo-300M.yaml @@ -50,7 +50,7 @@ optimizer: weight_decay: 0.1 eps: 1e-8 decay_norm_and_bias: true - decay_embeddings: false + decay_embeddings: true betas: - 0.9 - 0.95 diff --git a/configs/tiny/OLMo-60M.yaml b/configs/tiny/OLMo-60M.yaml index 7f615ce45..88c096347 100644 --- a/configs/tiny/OLMo-60M.yaml +++ b/configs/tiny/OLMo-60M.yaml @@ -46,11 +46,11 @@ compile: null optimizer: name: adamw - learning_rate: 1.0e-3 + learning_rate: 6.0e-4 weight_decay: 0.1 eps: 1e-8 decay_norm_and_bias: true - decay_embeddings: false + decay_embeddings: true betas: - 0.9 - 0.95 diff --git a/configs/tiny/OLMo-750M.yaml b/configs/tiny/OLMo-700M.yaml similarity index 99% rename from configs/tiny/OLMo-750M.yaml rename to configs/tiny/OLMo-700M.yaml index dbf6cea8e..0db9ebe3c 100644 --- a/configs/tiny/OLMo-750M.yaml +++ b/configs/tiny/OLMo-700M.yaml @@ -1,4 +1,4 @@ -run_name: OLMo-750M +run_name: OLMo-700M seed: 6198 dry_run: false @@ -8,8 +8,8 @@ wandb: model: d_model: 1536 - n_heads: 24 - n_layers: 24 + n_heads: 16 + n_layers: 16 mlp_ratio: 8 weight_tying: false alibi: false @@ -50,7 +50,7 @@ optimizer: weight_decay: 0.1 eps: 1e-8 decay_norm_and_bias: true - decay_embeddings: false + decay_embeddings: true betas: - 0.9 - 0.95