From b72bdad7f13d53b57b888c1d2cc9626943713722 Mon Sep 17 00:00:00 2001 From: shihaohou Date: Fri, 13 Mar 2026 11:45:56 +0800 Subject: [PATCH] feat: add Qwen3.5-4B model support --- slime/scripts/models/qwen3.5-4B.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 slime/scripts/models/qwen3.5-4B.sh diff --git a/slime/scripts/models/qwen3.5-4B.sh b/slime/scripts/models/qwen3.5-4B.sh new file mode 100644 index 0000000000..18bea47c9c --- /dev/null +++ b/slime/scripts/models/qwen3.5-4B.sh @@ -0,0 +1,27 @@ +MODEL_ARGS=( + --spec "slime_plugins.models.qwen3_5" "get_qwen3_5_spec" + + --disable-bias-linear + --qk-layernorm + --group-query-attention + --num-attention-heads 16 + --num-query-groups 4 + --kv-channels 256 + --num-layers 32 + --hidden-size 2560 + --ffn-hidden-size 9216 + --use-gated-attention + + --normalization RMSNorm + --apply-layernorm-1p + --position-embedding-type rope + --norm-epsilon 1e-6 + --rotary-percent 0.25 + --swiglu + --vocab-size 248320 + + --rotary-base 10000000 + + # qwen3.5 specific + --attention-output-gate +)