Skip to content

llama : make tensor-split regex patterns static to avoid per-token recompilation #24710

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JohannesGaessler merged 1 commit into from
Jul 10, 2026
Merged

llama : make tensor-split regex patterns static to avoid per-token recompilation #24710

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 32 additions & 32 deletions src/llama-model.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,38 +336,38 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
const llama_hparams & hparams = ud->model->hparams;
const std::string tensor_name = tensor->name;

const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");

const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
const std::regex pattern_r_cache ("cache_r_l\\d*");
const std::regex pattern_s_cache ("cache_s_l\\d*");
const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");

const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");

const std::regex pattern_output_weight("output\\.weight");
const std::regex pattern_output_bias ("output\\.bias");
static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
static const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
static const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
static const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
static const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");

static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
static const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
static const std::regex pattern_r_cache ("cache_r_l\\d*");
static const std::regex pattern_s_cache ("cache_s_l\\d*");
static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");

static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
static const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");

static const std::regex pattern_output_weight("output\\.weight");
static const std::regex pattern_output_bias ("output\\.bias");

struct tensor_config {
ggml_backend_meta_split_axis axis;
Expand Down