llama-7b.yaml

# Model

archive: ./results/models/MEND/llama-2-7b-ZsRE
alg_name: "MEND"
device: 1
model_name: ./hugging_cache/llama-2-7b

model_class: LlamaForCausalLM
tokenizer_class: LlamaTokenizer
tokenizer_name: ./hugging_cache/llama-2-7b
model_parallel: false
inner_params:
- model.layers.29.mlp.gate_proj.weight
- model.layers.29.mlp.up_proj.weight
- model.layers.29.mlp.down_proj.weight
- model.layers.30.mlp.gate_proj.weight
- model.layers.30.mlp.up_proj.weight
- model.layers.30.mlp.down_proj.weight
- model.layers.31.mlp.gate_proj.weight
- model.layers.31.mlp.up_proj.weight
- model.layers.31.mlp.down_proj.weight

# Method
alg: MEND
lr: 1e-6
edit_lr: 1e-4
lr_lr: 1e-4
lr_scale: 1.0
seed: 42
cedit: 0.1
cloc: 1.0
cbase: 1.0
dropout: 0.0
train_base: False
no_grad_layers: null
one_sided: False
n_hidden: 1
hidden_dim: null
init: id
norm: True
combine: True
x_only: False
delta_only: False
act: relu
rank: 1920
mlp_class: IDMLP
shared: True

# Train
batch_size: 1
model_save_pt: 5000
silent: False
#max_epochs: 1
max_iters: 100000
log_interval: 1000
eval_log_interval: 1000
final_eval: True
val_interval: 1000
early_stop_patience: 20000
early_stop_key: "loss/total_edit_val"
eval_only: True
half: False
debug: False
save: False
verbose: True

val_batch_size: 5
accumulate_bs: 10
val_steps: 500 # only for debug
opt: Adam
grad_clip: 100.

# Output

results_dir: ./results