diff --git a/slime/scripts/models/qwen3.5-4B.sh b/slime/scripts/models/qwen3.5-4B.sh new file mode 100644 index 0000000000..18bea47c9c --- /dev/null +++ b/slime/scripts/models/qwen3.5-4B.sh @@ -0,0 +1,27 @@ +MODEL_ARGS=( + --spec "slime_plugins.models.qwen3_5" "get_qwen3_5_spec" + + --disable-bias-linear + --qk-layernorm + --group-query-attention + --num-attention-heads 16 + --num-query-groups 4 + --kv-channels 256 + --num-layers 32 + --hidden-size 2560 + --ffn-hidden-size 9216 + --use-gated-attention + + --normalization RMSNorm + --apply-layernorm-1p + --position-embedding-type rope + --norm-epsilon 1e-6 + --rotary-percent 0.25 + --swiglu + --vocab-size 248320 + + --rotary-base 10000000 + + # qwen3.5 specific + --attention-output-gate +)