Repository navigation
llama: add llama_batch_ext #24669
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
llama: add llama_batch_ext #24669
Changes from 19 commits
0dfa5e1
c7c5468
c0fb071
3132371
32343b2
bf372b3
901ed68
231af77
4cd8c26
e4c474f
4ba39e5
7db50e9
7d626f5
3b86110
05a8c11
51b471f
95956ee
e640410
6d00861
69258fb
2b50dab
4bcbdff
de88a4a
4dd6864
e67da54
d48a14d
12691fb
82a63e3
4070433
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -286,6 +286,11 @@ extern "C" { | |||||
| LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, | ||||||
| }; | ||||||
|
|
||||||
| enum llama_process_type { | ||||||
| LLAMA_PROCESS_TYPE_ENCODE, | ||||||
| LLAMA_PROCESS_TYPE_DECODE, | ||||||
| }; | ||||||
|
|
||||||
| struct llama_model_kv_override { | ||||||
| enum llama_model_kv_override_type tag; | ||||||
|
|
||||||
|
|
@@ -982,6 +987,84 @@ extern "C" { | |||||
| struct llama_context * ctx, | ||||||
| struct llama_batch batch); | ||||||
|
|
||||||
| // | ||||||
| // Extended batch API | ||||||
| // | ||||||
|
|
||||||
| struct llama_batch_ext; | ||||||
|
|
||||||
| struct llama_embd { | ||||||
| const float * data; | ||||||
| size_t n_rows; // number of embedding rows in data | ||||||
| size_t n_embd; // size of one row | ||||||
| }; | ||||||
|
|
||||||
| LLAMA_API struct llama_batch_ext * llama_batch_ext_init (struct llama_context * ctx); | ||||||
| LLAMA_API void llama_batch_ext_free (struct llama_batch_ext * batch); | ||||||
| LLAMA_API void llama_batch_ext_clear(struct llama_batch_ext * batch); | ||||||
|
|
||||||
| // Add an input token to the batch, with default values: | ||||||
| // id = LLAMA_TOKEN_NULL | ||||||
| // embd = nullptr | ||||||
| // pos = next available position for the sequence id | ||||||
| // Returns the batch index (>= 0) | ||||||
| // On error: | ||||||
| // -1: batch is full | ||||||
| // -2: token is invalid (id == LLAMA_TOKEN_NULL or invalid embd) | ||||||
| // -3: invalid sequence id | ||||||
| LLAMA_API int32_t llama_batch_ext_add (struct llama_batch_ext * batch, llama_seq_id seq_id); | ||||||
|
|
||||||
| // Add an input token to the batch, with a specified token ID or token embedding | ||||||
| LLAMA_API int32_t llama_batch_ext_add_token (struct llama_batch_ext * batch, llama_seq_id seq_id, llama_token id); | ||||||
| LLAMA_API int32_t llama_batch_ext_add_embd_token(struct llama_batch_ext * batch, llama_seq_id seq_id, struct llama_embd embd); | ||||||
|
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok I just realized that maybe better to explicitly define the behavior here: if
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sounds good. I would rather call this |
||||||
|
|
||||||
| // Add the token at index idx in the batch to another sequence id. The position will stays the same. | ||||||
| // Note: this should be called before other _set() functions | ||||||
| LLAMA_API bool llama_batch_ext_add_seq( | ||||||
| struct llama_batch_ext * batch, | ||||||
| int32_t idx, | ||||||
| llama_seq_id seq_id); | ||||||
|
|
||||||
| // Set the "state" embedding for the token at index idx in the batch | ||||||
| // "state" here means extra hidden state carried over from a previous stage, e.g.: | ||||||
| // - MTP: state from N layers of the target model | ||||||
| // - Qwen3 VL (deepstack): state from N layers of the vision encoder | ||||||
| // LLAMA_API bool llama_batch_ext_set_embd_state( | ||||||
| // struct llama_batch_ext * batch, | ||||||
| // int32_t idx, | ||||||
| // struct llama_embd embd); | ||||||
|
|
||||||
|
Comment on lines
+1052
to
+1060
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. note: |
||||||
| // Set output embedding for the token at index idx in the batch | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Similar change for the |
||||||
| // Note: for now, this is equivalent to setting the output logits | ||||||
| LLAMA_API bool llama_batch_ext_set_output_embd( | ||||||
| struct llama_batch_ext * batch, | ||||||
| int32_t idx, | ||||||
| bool value); | ||||||
|
|
||||||
| // Set output logits for the token at index idx in the batch | ||||||
| // Note: for now, this is equivalent to setting the output embd | ||||||
| LLAMA_API bool llama_batch_ext_set_output_logits( | ||||||
| struct llama_batch_ext * batch, | ||||||
| int32_t idx, | ||||||
| bool value); | ||||||
|
|
||||||
| // Set custom position for the token at index idx in the batch | ||||||
| // For M-RoPE models: | ||||||
| // - Embedding tokens must have multiple positions per token | ||||||
| // - Text token only requires one single position per token | ||||||
| LLAMA_API bool llama_batch_ext_set_pos( | ||||||
| struct llama_batch_ext * batch, | ||||||
| int32_t idx, | ||||||
| llama_pos * pos); | ||||||
|
|
||||||
| // TODO: implement get_embeddings() and get_logits() for llama_batch_ext | ||||||
|
|
||||||
| // Return values are the same as llama_decode() | ||||||
| LLAMA_API int32_t llama_process( | ||||||
| struct llama_context * ctx, | ||||||
| enum llama_process_type type, | ||||||
| struct llama_batch_ext * batch); | ||||||
|
|
||||||
| // Set the number of threads used for decoding | ||||||
| // n_threads is the number of threads used for generation (single token) | ||||||
| // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) | ||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
to be discussed:
n_rowsorn_tokens?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Rows seems better.