From bb0aafccdd6211c58f290ce64aa8535448580a43 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 15 Apr 2026 18:57:58 +0200 Subject: [PATCH 1/4] server: use random media marker --- tools/server/server-common.cpp | 27 +++++++++++++++++++++------ tools/server/server-common.h | 5 ++++- tools/server/server-context.cpp | 1 + 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index e3f24390233b..40220100b745 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -61,8 +61,15 @@ json format_error_response(const std::string & message, const enum error_type ty // random string / id // -std::string random_string() { - static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); +std::string random_string(bool include_special) { + static const std::string str_no_special( + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + ); + static const std::string str_with_special( + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!@#$%^&*()-_=+[]{}|;:,.<>?/~`" + ); + + auto & str = include_special ? str_with_special : str_no_special; std::random_device rd; std::mt19937 generator(rd()); @@ -81,7 +88,15 @@ std::string gen_chatcmplid() { } std::string gen_tool_call_id() { - return random_string(); + return random_string(true); +} + +static std::string media_marker = ""; +const char * get_media_marker() { + if (media_marker.empty()) { + media_marker = "<__media_" + random_string() + "__>"; + } + return media_marker.c_str(); } // @@ -975,7 +990,7 @@ json oaicompat_chat_params_parse( handle_media(out_files, image_url, opt.media_path); p["type"] = "media_marker"; - p["text"] = mtmd_default_marker(); + p["text"] = get_media_marker(); p.erase("image_url"); } else if (type == "input_audio") { @@ -996,7 +1011,7 @@ json oaicompat_chat_params_parse( // TODO: add audio_url support by reusing handle_media() p["type"] = "media_marker"; - p["text"] = mtmd_default_marker(); + p["text"] = get_media_marker(); p.erase("input_audio"); } else if (type != "text") { @@ -1460,7 +1475,7 @@ json convert_transcriptions_to_chatcmpl( if (!language.empty()) { prompt += string_format(" (language: %s)", language.c_str()); } - prompt += mtmd_default_marker(); + prompt += get_media_marker(); json chatcmpl_body = inp_body; // copy all fields chatcmpl_body["messages"] = json::array({ diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 440ebc597af7..fed921e8c364 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -88,10 +88,13 @@ json format_error_response(const std::string & message, const enum error_type ty // random string / id // -std::string random_string(); +std::string random_string(bool include_special = false); std::string gen_chatcmplid(); std::string gen_tool_call_id(); +// get a random marker; note: each time the server restarts, the marker will be different +const char * get_media_marker(); + // // lora utils // diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e134b3cfb260..41bdad6f8784 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -708,6 +708,7 @@ struct server_context_impl { mparams.warmup = params_base.warmup; mparams.image_min_tokens = params_base.image_min_tokens; mparams.image_max_tokens = params_base.image_max_tokens; + mparams.media_marker = get_media_marker(); mctx = mtmd_init_from_file(mmproj_path.c_str(), model, mparams); if (mctx == nullptr) { From 4ec46e2caf201f2d68076abe64e88a431b01c543 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 15 Apr 2026 19:01:31 +0200 Subject: [PATCH 2/4] nits --- tools/server/server-common.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 40220100b745..a6702525a3e2 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -88,13 +88,13 @@ std::string gen_chatcmplid() { } std::string gen_tool_call_id() { - return random_string(true); + return random_string(); } static std::string media_marker = ""; const char * get_media_marker() { if (media_marker.empty()) { - media_marker = "<__media_" + random_string() + "__>"; + media_marker = "<__media_" + random_string(true) + "__>"; } return media_marker.c_str(); } From b5e9adbdc1456390ebd8263f64f14bfef4aff267 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 15 Apr 2026 19:36:38 +0200 Subject: [PATCH 3/4] remove legacy <__image__> token --- tools/mtmd/mtmd.cpp | 7 ++----- tools/mtmd/mtmd.h | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index a56d3b35b484..d0a0a4865ef8 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -109,7 +109,7 @@ mtmd_context_params mtmd_context_params_default() { /* use_gpu */ true, /* print_timings */ true, /* n_threads */ 4, - /* image_marker */ MTMD_DEFAULT_IMAGE_MARKER, + /* image_marker */ nullptr, /* media_marker */ mtmd_default_marker(), /* flash_attn_type */ LLAMA_FLASH_ATTN_TYPE_AUTO, /* warmup */ true, @@ -169,7 +169,7 @@ struct mtmd_context { media_marker (ctx_params.media_marker), n_embd_text (llama_model_n_embd_inp(text_model)) { - if (std::string(ctx_params.image_marker) != MTMD_DEFAULT_IMAGE_MARKER) { + if (ctx_params.image_marker != nullptr) { throw std::runtime_error("custom image_marker is not supported anymore, use media_marker instead"); } @@ -584,9 +584,6 @@ struct mtmd_tokenizer { parse_special = text->parse_special; input_text = text->text; vocab = llama_model_get_vocab(ctx->text_model); - - // for compatibility, we convert image marker to media marker - string_replace_all(input_text, MTMD_DEFAULT_IMAGE_MARKER, ctx->media_marker); } int32_t tokenize(mtmd_input_chunks * output) { diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index c91bc0810543..a6fd8efa5d0b 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -46,9 +46,6 @@ # define MTMD_API #endif -// deprecated marker, use mtmd_default_marker() instead -#define MTMD_DEFAULT_IMAGE_MARKER "<__image__>" - #ifdef __cplusplus extern "C" { #endif From 953f36ed8aeeeac599d993b207d9414847e6474d Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 15 Apr 2026 20:10:13 +0200 Subject: [PATCH 4/4] revert special char in random --- tools/server/server-common.cpp | 13 +++---------- tools/server/server-common.h | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index a6702525a3e2..fd417393f8cc 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -61,15 +61,8 @@ json format_error_response(const std::string & message, const enum error_type ty // random string / id // -std::string random_string(bool include_special) { - static const std::string str_no_special( - "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - ); - static const std::string str_with_special( - "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!@#$%^&*()-_=+[]{}|;:,.<>?/~`" - ); - - auto & str = include_special ? str_with_special : str_no_special; +std::string random_string() { + static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); std::random_device rd; std::mt19937 generator(rd()); @@ -94,7 +87,7 @@ std::string gen_tool_call_id() { static std::string media_marker = ""; const char * get_media_marker() { if (media_marker.empty()) { - media_marker = "<__media_" + random_string(true) + "__>"; + media_marker = "<__media_" + random_string() + "__>"; } return media_marker.c_str(); } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index fed921e8c364..57545aa53edf 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -88,7 +88,7 @@ json format_error_response(const std::string & message, const enum error_type ty // random string / id // -std::string random_string(bool include_special = false); +std::string random_string(); std::string gen_chatcmplid(); std::string gen_tool_call_id();