From 363b198ad43251def8335d45b101dbb43f37e949 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 17 Aug 2026 19:39:51 +0200 Subject: [PATCH] server: save processed mtmd chunks as placeholder --- tools/mtmd/mtmd.cpp | 46 +++++++++++++++++++++++---------- tools/mtmd/mtmd.h | 3 +++ tools/server/server-common.cpp | 17 ++++++++++++ tools/server/server-common.h | 4 +++ tools/server/server-context.cpp | 3 ++- 5 files changed, 58 insertions(+), 15 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 6596b2785f5f..4063d28e0751 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2322,23 +2322,12 @@ void mtmd_input_chunk_free(mtmd_input_chunk * chunk) { } } -int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) { +// returns 0 on success +static int32_t mtmd_input_chunk_save_impl(const mtmd_input_chunk * chunk, std::vector & out_buf) { try { mtmd_serialization ser(MTMD_SERIALIZATION_VERSION); chunk->serialize(ser); - - if (expected_out_len) { - *expected_out_len = ser.data.size(); - } - if (!out_buf) { - // caller is only querying the required size - return 0; - } - if (out_len < ser.data.size()) { - LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, ser.data.size(), out_len); - return -1; - } - std::memcpy(out_buf, ser.data.data(), ser.data.size()); + out_buf = std::move(ser.data); return 0; } catch (const std::exception & e) { LOG_ERR("%s: %s\n", __func__, e.what()); @@ -2346,6 +2335,35 @@ int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, si } } +mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk) { + // this is hacky, but still faster than copy the whole batch data + std::vector buf; + if (mtmd_input_chunk_save_impl(chunk, buf) != 0) { + return nullptr; + } + return mtmd_input_chunk_load(buf.data(), buf.size()); +} + +int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) { + std::vector buf; + if (mtmd_input_chunk_save_impl(chunk, buf) != 0) { + return -1; + } + if (expected_out_len) { + *expected_out_len = buf.size(); + } + if (!out_buf) { + // caller is only querying the required size + return 0; + } + if (out_len < buf.size()) { + LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, buf.size(), out_len); + return -1; + } + std::memcpy(out_buf, buf.data(), buf.size()); + return 0; +} + mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len) { try { mtmd_serialization ser(MTMD_SERIALIZATION_VERSION, buf, len); diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index c1a5921db2f3..78587f3fed55 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -233,6 +233,9 @@ MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk); MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); +// similar to mtmd_input_chunk_copy, but returns a placeholder chunk +MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk); + // save/load an input chunk to/from a buffer (useful for KV save/load) // important: only chunk's metadata will be saved, the actual image/audio data will not be saved // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 7ed486528cab..585f65e83c65 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -507,6 +507,23 @@ void server_tokens::push_back(const mtmd_input_chunk * chunk) { } } +void server_tokens::push_back_placeholder(const mtmd_input_chunk * chunk) { + auto type = mtmd_input_chunk_get_type(chunk); + if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { + GGML_ASSERT(has_mtmd); + mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_get_placeholder(chunk)); + GGML_ASSERT(new_chunk != nullptr && "failed to create placeholder chunk"); + const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk); + size_t start_idx = tokens.size(); + for (size_t i = 0; i < n_tokens; ++i) { + tokens.emplace_back(LLAMA_TOKEN_NULL); + } + map_idx_to_media[start_idx] = std::move(new_chunk); + } else { + push_back(chunk); + } +} + void server_tokens::push_back(server_tokens & tokens) { size_t start_idx = size(); for (size_t i = 0; i < tokens.size(); i++) { diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 7082abdd91e8..6488be344c6a 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -195,6 +195,10 @@ struct server_tokens { // will create a copy of the chunk if it contains non-text data void push_back(const mtmd_input_chunk * chunk); + // same as push_back, but media chunks are stored as placeholders (no image/audio data) + // only use this if the chunk will never be encoded again (e.g. it is already in the KV cache) + void push_back_placeholder(const mtmd_input_chunk * chunk); + // appends server tokens, updates the media map. copies media chunks. void push_back(server_tokens & tokens); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fdc8f2b805fd..842e4203cd2a 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3416,7 +3416,8 @@ struct server_context_impl { // add the mtmd chunk to cache { const auto & chunk = input_tokens.find_chunk(cur_token_idx); - slot.prompt.tokens.push_back(chunk.get()); // copy + // the chunk is already in the KV cache at this point, so we don't need to keep its data around + slot.prompt.tokens.push_back_placeholder(chunk.get()); } has_mtmd = true;