From 7cdfc14e996cc5aaddbb9bafe91a5d7c57843e7d Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 10 Jun 2026 19:07:33 -0700 Subject: [PATCH 01/15] add initial tool and reasoning tag migration --- src/config.cpp | 32 ++++++++++++++++++ src/config.h | 10 ++++++ src/models/model.cpp | 61 ++++++++++++++++++++++++++++++++++ src/models/model.h | 6 ++++ src/ort_genai.h | 24 ++++++++++++++ src/ort_genai_c.cpp | 28 ++++++++++++++++ src/ort_genai_c.h | 32 ++++++++++++++++++ test/c_api_tests.cpp | 78 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 271 insertions(+) diff --git a/src/config.cpp b/src/config.cpp index a32a986c19..6a53e31ad6 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1532,6 +1532,34 @@ void ClearDecoderProviderOptionsHardwareVendorId(Config& config, std::string_vie } } +struct ToolCalling_Element : JSON::Element { + explicit ToolCalling_Element(Config::ToolCalling& v) : v_{v} {} + + void OnValue(std::string_view name, JSON::Value value) override { + if (name == "tool_call_start_token") { + v_.tool_call_start_token = JSON::Get(value); + } else if (name == "tool_call_end_token") { + v_.tool_call_end_token = JSON::Get(value); + } + } + + Config::ToolCalling& v_; +}; + +struct Reasoning_Element : JSON::Element { + explicit Reasoning_Element(Config::Reasoning& v) : v_{v} {} + + void OnValue(std::string_view name, JSON::Value value) override { + if (name == "reasoning_start_token") { + v_.reasoning_start_token = JSON::Get(value); + } else if (name == "reasoning_end_token") { + v_.reasoning_end_token = JSON::Get(value); + } + } + + Config::Reasoning& v_; +}; + struct Root_Element : JSON::Element { explicit Root_Element(Config& config) : config_{config} {} @@ -1543,6 +1571,8 @@ struct Root_Element : JSON::Element { if (name == "model") return model_element_; if (name == "search") return search_element_; if (name == "engine") return engine_element_; + if (name == "tool_calling") return tool_calling_element_; + if (name == "reasoning") return reasoning_element_; throw JSON::unknown_value_error{}; } @@ -1550,6 +1580,8 @@ struct Root_Element : JSON::Element { Model_Element model_element_{config_.model}; Search_Element search_element_{config_.search}; Engine_Element engine_element_{config_.engine}; + ToolCalling_Element tool_calling_element_{config_.tool_calling}; + Reasoning_Element reasoning_element_{config_.reasoning}; }; struct RootObject_Element : JSON::Element { diff --git a/src/config.h b/src/config.h index 8be9de3ecb..25976115a7 100644 --- a/src/config.h +++ b/src/config.h @@ -443,6 +443,16 @@ struct Config { std::optional static_batching; // Static batching settings } engine; // Engine settings + struct ToolCalling { + std::string tool_call_start_token; // e.g., "" + std::string tool_call_end_token; // e.g., "" + } tool_calling; + + struct Reasoning { + std::string reasoning_start_token; // e.g., "" + std::string reasoning_end_token; // e.g., "" + } reasoning; + void AddMapping(const std::string& nominal_name, const std::string& graph_name); // Returns graph name and true if the nominal name is found in the mapping // otherwise returns the nominal name and false diff --git a/src/models/model.cpp b/src/models/model.cpp index 06e0dda316..2682bdcd7c 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "../generators.h" #include "../search.h" @@ -773,6 +774,66 @@ bool Model::IsPruned() const { return logits_shape[1] == 1; } +namespace { + +struct FallbackTokens { + std::string tool_call_start; + std::string tool_call_end; + std::string reasoning_start; + std::string reasoning_end; +}; + +// Fallback map for models whose genai_config.json doesn't yet have tool_calling/reasoning sections. +// Keyed by model.type string from genai_config.json. +const FallbackTokens* GetFallbackTokens(const std::string& model_type) { + static const std::unordered_map fallback_map = { + {"qwen2", {"", "", "", ""}}, + {"qwen3", {"", "", "", ""}}, + {"phi3", {"", "", "", ""}}, + {"gptoss", {"<|start|>", "<|call|>", "", ""}}, + }; + auto it = fallback_map.find(model_type); + return it != fallback_map.end() ? &it->second : nullptr; +} + +} // namespace + +const std::string& Model::GetToolCallStartToken() const { + if (!config_->tool_calling.tool_call_start_token.empty()) + return config_->tool_calling.tool_call_start_token; + const auto* fallback = GetFallbackTokens(config_->model.type); + if (fallback) return fallback->tool_call_start; + static const std::string empty; + return empty; +} + +const std::string& Model::GetToolCallEndToken() const { + if (!config_->tool_calling.tool_call_end_token.empty()) + return config_->tool_calling.tool_call_end_token; + const auto* fallback = GetFallbackTokens(config_->model.type); + if (fallback) return fallback->tool_call_end; + static const std::string empty; + return empty; +} + +const std::string& Model::GetReasoningStartToken() const { + if (!config_->reasoning.reasoning_start_token.empty()) + return config_->reasoning.reasoning_start_token; + const auto* fallback = GetFallbackTokens(config_->model.type); + if (fallback && !fallback->reasoning_start.empty()) return fallback->reasoning_start; + static const std::string empty; + return empty; +} + +const std::string& Model::GetReasoningEndToken() const { + if (!config_->reasoning.reasoning_end_token.empty()) + return config_->reasoning.reasoning_end_token; + const auto* fallback = GetFallbackTokens(config_->model.type); + if (fallback && !fallback->reasoning_end.empty()) return fallback->reasoning_end; + static const std::string empty; + return empty; +} + std::shared_ptr CreateModel(OrtEnv& ort_env, const char* config_path, const RuntimeSettings* settings /*= nullptr*/) { std::string config_overlay; if (settings) { diff --git a/src/models/model.h b/src/models/model.h index bf4a1f6a04..a3546e463b 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -163,6 +163,12 @@ struct Model : std::enable_shared_from_this, LeakChecked, External bool IsPruned() const; + // Tool calling and reasoning token accessors (reads from config with fallback map) + const std::string& GetToolCallStartToken() const; + const std::string& GetToolCallEndToken() const; + const std::string& GetReasoningStartToken() const; + const std::string& GetReasoningEndToken() const; + std::unique_ptr config_; std::unique_ptr session_options_; diff --git a/src/ort_genai.h b/src/ort_genai.h index 3424287f1d..1c23cb745c 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -253,6 +253,30 @@ struct OgaModel : OgaAbstract { return p; } + OgaString GetToolCallStartToken() const { + const char* p; + OgaCheckResult(OgaModelGetToolCallStartToken(this, &p)); + return p; + } + + OgaString GetToolCallEndToken() const { + const char* p; + OgaCheckResult(OgaModelGetToolCallEndToken(this, &p)); + return p; + } + + OgaString GetReasoningStartToken() const { + const char* p; + OgaCheckResult(OgaModelGetReasoningStartToken(this, &p)); + return p; + } + + OgaString GetReasoningEndToken() const { + const char* p; + OgaCheckResult(OgaModelGetReasoningEndToken(this, &p)); + return p; + } + static void operator delete(void* p) { OgaDestroyModel(reinterpret_cast(p)); } }; diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index 3ea17eaca7..9458969d9d 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -364,6 +364,34 @@ OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char* OGA_CATCH } +OgaResult* OGA_API_CALL OgaModelGetToolCallStartToken(const OgaModel* model, const char** out) { + OGA_TRY + *out = AllocOgaString(model->GetToolCallStartToken().c_str()); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaModelGetToolCallEndToken(const OgaModel* model, const char** out) { + OGA_TRY + *out = AllocOgaString(model->GetToolCallEndToken().c_str()); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaModelGetReasoningStartToken(const OgaModel* model, const char** out) { + OGA_TRY + *out = AllocOgaString(model->GetReasoningStartToken().c_str()); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaModelGetReasoningEndToken(const OgaModel* model, const char** out) { + OGA_TRY + *out = AllocOgaString(model->GetReasoningEndToken().c_str()); + return nullptr; + OGA_CATCH +} + OgaResult* OGA_API_CALL OgaCreateGeneratorParams(const OgaModel* model, OgaGeneratorParams** out) { OGA_TRY auto params = std::make_shared(*model); diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index 62f71c690d..cbfa2d63c5 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -386,6 +386,38 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetType(const OgaModel* model, const */ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char** out); +/** + * \brief Returns the tool call start token for this model (empty string if model doesn't support tool calling). + * \param[in] model The model to query. + * \param[out] out The token string. Must be destroyed with OgaDestroyString. + * \return OgaResult containing the error message if the call failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetToolCallStartToken(const OgaModel* model, const char** out); + +/** + * \brief Returns the tool call end token for this model (empty string if model doesn't support tool calling). + * \param[in] model The model to query. + * \param[out] out The token string. Must be destroyed with OgaDestroyString. + * \return OgaResult containing the error message if the call failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetToolCallEndToken(const OgaModel* model, const char** out); + +/** + * \brief Returns the reasoning start token for this model (empty string if model doesn't support reasoning). + * \param[in] model The model to query. + * \param[out] out The token string. Must be destroyed with OgaDestroyString. + * \return OgaResult containing the error message if the call failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetReasoningStartToken(const OgaModel* model, const char** out); + +/** + * \brief Returns the reasoning end token for this model (empty string if model doesn't support reasoning). + * \param[in] model The model to query. + * \param[out] out The token string. Must be destroyed with OgaDestroyString. + * \return OgaResult containing the error message if the call failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetReasoningEndToken(const OgaModel* model, const char** out); + /** * \brief Destroys the given config * \param[in] config The config to be destroyed. diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index f6e952a9bd..3bf882e503 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1801,3 +1801,81 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { auto transcription = RunParakeetTdt(PARAKEET_TDT_AUDIO_TEDLIUM); EXPECT_FALSE(transcription.empty()); } + +// Test tool_calling and reasoning config parsing and fallback map +TEST(CAPITests, ToolCallAndReasoningTokens_Fallback) { + // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → empty tokens + auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); + + auto tool_start = model->GetToolCallStartToken(); + auto tool_end = model->GetToolCallEndToken(); + auto reasoning_start = model->GetReasoningStartToken(); + auto reasoning_end = model->GetReasoningEndToken(); + + EXPECT_STREQ(static_cast(tool_start), ""); + EXPECT_STREQ(static_cast(tool_end), ""); + EXPECT_STREQ(static_cast(reasoning_start), ""); + EXPECT_STREQ(static_cast(reasoning_end), ""); +} + +TEST(CAPITests, ToolCallAndReasoningTokens_FromConfig) { + // Create a temporary model directory with tool_calling and reasoning sections + auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; + std::filesystem::create_directories(temp_dir); + + // Copy minimal model files from tiny-random-gpt2 + std::string src_dir = MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"; + for (const auto& entry : std::filesystem::directory_iterator(src_dir)) { + if (entry.path().filename() != "genai_config.json") { + std::filesystem::copy_file(entry.path(), temp_dir / entry.path().filename(), + std::filesystem::copy_options::overwrite_existing); + } + } + + // Write genai_config.json with tool_calling and reasoning sections + { + std::ofstream f((temp_dir / "genai_config.json").string()); + f << R"({ + "model": { + "type": "gpt2", + "pad_token_id": 98, + "bos_token_id": 98, + "eos_token_id": 98, + "vocab_size": 1000, + "context_length": 512, + "decoder": { + "session_options": { "provider_options": [] }, + "filename": "past.onnx", + "num_key_value_heads": 4, + "head_size": 8, + "num_hidden_layers": 5, + "inputs": { "past_names": "past_%d" }, + "outputs": { "present_names": "present_%d" } + } + }, + "tool_calling": { + "tool_call_start_token": "", + "tool_call_end_token": "" + }, + "reasoning": { + "reasoning_start_token": "", + "reasoning_end_token": "" + } +})"; + } + + auto model = OgaModel::Create(temp_dir.string().c_str()); + + auto tool_start = model->GetToolCallStartToken(); + auto tool_end = model->GetToolCallEndToken(); + auto reasoning_start = model->GetReasoningStartToken(); + auto reasoning_end = model->GetReasoningEndToken(); + + EXPECT_STREQ(static_cast(tool_start), ""); + EXPECT_STREQ(static_cast(tool_end), ""); + EXPECT_STREQ(static_cast(reasoning_start), ""); + EXPECT_STREQ(static_cast(reasoning_end), ""); + + // Cleanup + std::filesystem::remove_all(temp_dir); +} From 476e71e7c25e98da2291618e56be07c34ad6a05d Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Tue, 23 Jun 2026 20:46:43 -0700 Subject: [PATCH 02/15] consolidate APIs --- src/models/model.cpp | 79 ++++++++++++++++++++------------------------ src/models/model.h | 9 +++-- src/ort_genai.h | 22 ++---------- src/ort_genai_c.cpp | 25 ++------------ src/ort_genai_c.h | 35 +++++--------------- test/c_api_tests.cpp | 22 ++++++------ 6 files changed, 62 insertions(+), 130 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index f28955ba4f..42e1c36f4b 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -286,6 +287,11 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, // Resolve tokenizer_dir (may be empty, relative, absolute, or "package:"-scheme). const fs::path tokenizer_dir = config.ResolvePath(config.model.tokenizer_dir); CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); + + // TODO: Once ORT Extensions supports an "additional_special_tokens" option, pass the generation + // tags (tool_calling/reasoning tokens) here so that models which don't already mark them as + // special in their tokenizer_config.json will still get correct skip_special_tokens behavior. + // This is needed for the FL SDK's dual-stream special token detection in OnnxChatGenerator::Decode(). } std::unique_ptr Tokenizer::CreateStream() const { @@ -820,60 +826,45 @@ bool Model::IsPruned() const { namespace { -struct FallbackTokens { - std::string tool_call_start; - std::string tool_call_end; - std::string reasoning_start; - std::string reasoning_end; -}; - // Fallback map for models whose genai_config.json doesn't yet have tool_calling/reasoning sections. // Keyed by model.type string from genai_config.json. -const FallbackTokens* GetFallbackTokens(const std::string& model_type) { - static const std::unordered_map fallback_map = { - {"qwen2", {"", "", "", ""}}, - {"qwen3", {"", "", "", ""}}, - {"phi3", {"", "", "", ""}}, - {"gptoss", {"<|start|>", "<|call|>", "", ""}}, +// Inner map: tag_name -> value +const std::string* GetFallbackTag(const std::string& model_type, const std::string& tag_name) { + static const std::unordered_map> fallback_map = { + {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, + {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, + {"phi3", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, + {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, }; - auto it = fallback_map.find(model_type); - return it != fallback_map.end() ? &it->second : nullptr; + auto type_it = fallback_map.find(model_type); + if (type_it == fallback_map.end()) return nullptr; + auto tag_it = type_it->second.find(tag_name); + if (tag_it == type_it->second.end()) return nullptr; + return &tag_it->second; } } // namespace -const std::string& Model::GetToolCallStartToken() const { - if (!config_->tool_calling.tool_call_start_token.empty()) - return config_->tool_calling.tool_call_start_token; - const auto* fallback = GetFallbackTokens(config_->model.type); - if (fallback) return fallback->tool_call_start; - static const std::string empty; - return empty; -} +const std::string& Model::GetGenerationTag(const std::string& tag_name) const { + // Check config first (tool_calling and reasoning sections) + static const std::unordered_map> config_accessors = { + {"tool_call_start", [](const Config& c) -> const std::string& { return c.tool_calling.tool_call_start_token; }}, + {"tool_call_end", [](const Config& c) -> const std::string& { return c.tool_calling.tool_call_end_token; }}, + {"reasoning_start", [](const Config& c) -> const std::string& { return c.reasoning.reasoning_start_token; }}, + {"reasoning_end", [](const Config& c) -> const std::string& { return c.reasoning.reasoning_end_token; }}, + }; -const std::string& Model::GetToolCallEndToken() const { - if (!config_->tool_calling.tool_call_end_token.empty()) - return config_->tool_calling.tool_call_end_token; - const auto* fallback = GetFallbackTokens(config_->model.type); - if (fallback) return fallback->tool_call_end; - static const std::string empty; - return empty; -} + auto accessor_it = config_accessors.find(tag_name); + if (accessor_it != config_accessors.end()) { + const std::string& config_val = accessor_it->second(*config_); + if (!config_val.empty()) + return config_val; + } -const std::string& Model::GetReasoningStartToken() const { - if (!config_->reasoning.reasoning_start_token.empty()) - return config_->reasoning.reasoning_start_token; - const auto* fallback = GetFallbackTokens(config_->model.type); - if (fallback && !fallback->reasoning_start.empty()) return fallback->reasoning_start; - static const std::string empty; - return empty; -} + // Fallback to model-type-based map + const auto* fallback = GetFallbackTag(config_->model.type, tag_name); + if (fallback) return *fallback; -const std::string& Model::GetReasoningEndToken() const { - if (!config_->reasoning.reasoning_end_token.empty()) - return config_->reasoning.reasoning_end_token; - const auto* fallback = GetFallbackTokens(config_->model.type); - if (fallback && !fallback->reasoning_end.empty()) return fallback->reasoning_end; static const std::string empty; return empty; } diff --git a/src/models/model.h b/src/models/model.h index a3546e463b..5a87c06e62 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -163,11 +163,10 @@ struct Model : std::enable_shared_from_this, LeakChecked, External bool IsPruned() const; - // Tool calling and reasoning token accessors (reads from config with fallback map) - const std::string& GetToolCallStartToken() const; - const std::string& GetToolCallEndToken() const; - const std::string& GetReasoningStartToken() const; - const std::string& GetReasoningEndToken() const; + // Generic generation tag accessor (reads from config with model-type fallback). + // Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". + // Returns the tag value, or an empty string if the model doesn't define the tag. + const std::string& GetGenerationTag(const std::string& tag_name) const; std::unique_ptr config_; std::unique_ptr session_options_; diff --git a/src/ort_genai.h b/src/ort_genai.h index ed1715dfea..f0c73b66aa 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -258,27 +258,9 @@ struct OgaModel : OgaAbstract { return p; } - OgaString GetToolCallStartToken() const { + OgaString GetGenerationTag(const char* tag_name) const { const char* p; - OgaCheckResult(OgaModelGetToolCallStartToken(this, &p)); - return p; - } - - OgaString GetToolCallEndToken() const { - const char* p; - OgaCheckResult(OgaModelGetToolCallEndToken(this, &p)); - return p; - } - - OgaString GetReasoningStartToken() const { - const char* p; - OgaCheckResult(OgaModelGetReasoningStartToken(this, &p)); - return p; - } - - OgaString GetReasoningEndToken() const { - const char* p; - OgaCheckResult(OgaModelGetReasoningEndToken(this, &p)); + OgaCheckResult(OgaModelGetGenerationTag(this, tag_name, &p)); return p; } diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index b62c741499..1acfe27990 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -378,30 +378,9 @@ OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char* OGA_CATCH } -OgaResult* OGA_API_CALL OgaModelGetToolCallStartToken(const OgaModel* model, const char** out) { +OgaResult* OGA_API_CALL OgaModelGetGenerationTag(const OgaModel* model, const char* tag_name, const char** out) { OGA_TRY - *out = AllocOgaString(model->GetToolCallStartToken().c_str()); - return nullptr; - OGA_CATCH -} - -OgaResult* OGA_API_CALL OgaModelGetToolCallEndToken(const OgaModel* model, const char** out) { - OGA_TRY - *out = AllocOgaString(model->GetToolCallEndToken().c_str()); - return nullptr; - OGA_CATCH -} - -OgaResult* OGA_API_CALL OgaModelGetReasoningStartToken(const OgaModel* model, const char** out) { - OGA_TRY - *out = AllocOgaString(model->GetReasoningStartToken().c_str()); - return nullptr; - OGA_CATCH -} - -OgaResult* OGA_API_CALL OgaModelGetReasoningEndToken(const OgaModel* model, const char** out) { - OGA_TRY - *out = AllocOgaString(model->GetReasoningEndToken().c_str()); + *out = AllocOgaString(model->GetGenerationTag(tag_name).c_str()); return nullptr; OGA_CATCH } diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index 671bd8c1e2..d3877cb85d 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -407,36 +407,17 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetType(const OgaModel* model, const OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char** out); /** - * \brief Returns the tool call start token for this model (empty string if model doesn't support tool calling). - * \param[in] model The model to query. - * \param[out] out The token string. Must be destroyed with OgaDestroyString. - * \return OgaResult containing the error message if the call failed. - */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetToolCallStartToken(const OgaModel* model, const char** out); - -/** - * \brief Returns the tool call end token for this model (empty string if model doesn't support tool calling). - * \param[in] model The model to query. - * \param[out] out The token string. Must be destroyed with OgaDestroyString. - * \return OgaResult containing the error message if the call failed. - */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetToolCallEndToken(const OgaModel* model, const char** out); - -/** - * \brief Returns the reasoning start token for this model (empty string if model doesn't support reasoning). - * \param[in] model The model to query. - * \param[out] out The token string. Must be destroyed with OgaDestroyString. - * \return OgaResult containing the error message if the call failed. - */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetReasoningStartToken(const OgaModel* model, const char** out); - -/** - * \brief Returns the reasoning end token for this model (empty string if model doesn't support reasoning). + * \brief Returns a generation tag value for this model by name. + * + * Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". + * Returns an empty string if the model doesn't define the requested tag. + * * \param[in] model The model to query. - * \param[out] out The token string. Must be destroyed with OgaDestroyString. + * \param[in] tag_name The name of the generation tag to retrieve. + * \param[out] out The tag value string. Must be destroyed with OgaDestroyString. * \return OgaResult containing the error message if the call failed. */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetReasoningEndToken(const OgaModel* model, const char** out); +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetGenerationTag(const OgaModel* model, const char* tag_name, const char** out); /** * \brief Destroys the given config diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index b1758407cd..4065c8d52d 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1818,14 +1818,14 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { } // Test tool_calling and reasoning config parsing and fallback map -TEST(CAPITests, ToolCallAndReasoningTokens_Fallback) { - // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → empty tokens +TEST(CAPITests, GenerationTags_Fallback) { + // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → empty tags auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); - auto tool_start = model->GetToolCallStartToken(); - auto tool_end = model->GetToolCallEndToken(); - auto reasoning_start = model->GetReasoningStartToken(); - auto reasoning_end = model->GetReasoningEndToken(); + auto tool_start = model->GetGenerationTag("tool_call_start"); + auto tool_end = model->GetGenerationTag("tool_call_end"); + auto reasoning_start = model->GetGenerationTag("reasoning_start"); + auto reasoning_end = model->GetGenerationTag("reasoning_end"); EXPECT_STREQ(static_cast(tool_start), ""); EXPECT_STREQ(static_cast(tool_end), ""); @@ -1833,7 +1833,7 @@ TEST(CAPITests, ToolCallAndReasoningTokens_Fallback) { EXPECT_STREQ(static_cast(reasoning_end), ""); } -TEST(CAPITests, ToolCallAndReasoningTokens_FromConfig) { +TEST(CAPITests, GenerationTags_FromConfig) { // Create a temporary model directory with tool_calling and reasoning sections auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; std::filesystem::create_directories(temp_dir); @@ -1881,10 +1881,10 @@ TEST(CAPITests, ToolCallAndReasoningTokens_FromConfig) { auto model = OgaModel::Create(temp_dir.string().c_str()); - auto tool_start = model->GetToolCallStartToken(); - auto tool_end = model->GetToolCallEndToken(); - auto reasoning_start = model->GetReasoningStartToken(); - auto reasoning_end = model->GetReasoningEndToken(); + auto tool_start = model->GetGenerationTag("tool_call_start"); + auto tool_end = model->GetGenerationTag("tool_call_end"); + auto reasoning_start = model->GetGenerationTag("reasoning_start"); + auto reasoning_end = model->GetGenerationTag("reasoning_end"); EXPECT_STREQ(static_cast(tool_start), ""); EXPECT_STREQ(static_cast(tool_end), ""); From 5d1327039994c089bf454a827d1755885f3ce340 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Fri, 26 Jun 2026 15:20:11 -0700 Subject: [PATCH 03/15] Rename GetGenerationTag -> GetTag per review feedback from Scott --- src/models/model.cpp | 2 +- src/models/model.h | 4 ++-- src/ort_genai.h | 4 ++-- src/ort_genai_c.cpp | 4 ++-- src/ort_genai_c.h | 6 +++--- test/c_api_tests.cpp | 20 ++++++++++---------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index 42e1c36f4b..2314526b7c 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -845,7 +845,7 @@ const std::string* GetFallbackTag(const std::string& model_type, const std::stri } // namespace -const std::string& Model::GetGenerationTag(const std::string& tag_name) const { +const std::string& Model::GetTag(const std::string& tag_name) const { // Check config first (tool_calling and reasoning sections) static const std::unordered_map> config_accessors = { {"tool_call_start", [](const Config& c) -> const std::string& { return c.tool_calling.tool_call_start_token; }}, diff --git a/src/models/model.h b/src/models/model.h index 5a87c06e62..aeb0463875 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -163,10 +163,10 @@ struct Model : std::enable_shared_from_this, LeakChecked, External bool IsPruned() const; - // Generic generation tag accessor (reads from config with model-type fallback). + // Generic tag accessor (reads from config with model-type fallback). // Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". // Returns the tag value, or an empty string if the model doesn't define the tag. - const std::string& GetGenerationTag(const std::string& tag_name) const; + const std::string& GetTag(const std::string& tag_name) const; std::unique_ptr config_; std::unique_ptr session_options_; diff --git a/src/ort_genai.h b/src/ort_genai.h index f0c73b66aa..8fc2c64b4d 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -258,9 +258,9 @@ struct OgaModel : OgaAbstract { return p; } - OgaString GetGenerationTag(const char* tag_name) const { + OgaString GetTag(const char* tag_name) const { const char* p; - OgaCheckResult(OgaModelGetGenerationTag(this, tag_name, &p)); + OgaCheckResult(OgaModelGetTag(this, tag_name, &p)); return p; } diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index 1acfe27990..7362425caa 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -378,9 +378,9 @@ OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char* OGA_CATCH } -OgaResult* OGA_API_CALL OgaModelGetGenerationTag(const OgaModel* model, const char* tag_name, const char** out) { +OgaResult* OGA_API_CALL OgaModelGetTag(const OgaModel* model, const char* tag_name, const char** out) { OGA_TRY - *out = AllocOgaString(model->GetGenerationTag(tag_name).c_str()); + *out = AllocOgaString(model->GetTag(tag_name).c_str()); return nullptr; OGA_CATCH } diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index d3877cb85d..a9eb92a3e8 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -407,17 +407,17 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetType(const OgaModel* model, const OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char** out); /** - * \brief Returns a generation tag value for this model by name. + * \brief Returns a tag value for this model by name. * * Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". * Returns an empty string if the model doesn't define the requested tag. * * \param[in] model The model to query. - * \param[in] tag_name The name of the generation tag to retrieve. + * \param[in] tag_name The name of the tag to retrieve. * \param[out] out The tag value string. Must be destroyed with OgaDestroyString. * \return OgaResult containing the error message if the call failed. */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetGenerationTag(const OgaModel* model, const char* tag_name, const char** out); +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetTag(const OgaModel* model, const char* tag_name, const char** out); /** * \brief Destroys the given config diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index 4065c8d52d..ee2aee50bf 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1818,14 +1818,14 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { } // Test tool_calling and reasoning config parsing and fallback map -TEST(CAPITests, GenerationTags_Fallback) { +TEST(CAPITests, Tags_Fallback) { // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → empty tags auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); - auto tool_start = model->GetGenerationTag("tool_call_start"); - auto tool_end = model->GetGenerationTag("tool_call_end"); - auto reasoning_start = model->GetGenerationTag("reasoning_start"); - auto reasoning_end = model->GetGenerationTag("reasoning_end"); + auto tool_start = model->GetTag("tool_call_start"); + auto tool_end = model->GetTag("tool_call_end"); + auto reasoning_start = model->GetTag("reasoning_start"); + auto reasoning_end = model->GetTag("reasoning_end"); EXPECT_STREQ(static_cast(tool_start), ""); EXPECT_STREQ(static_cast(tool_end), ""); @@ -1833,7 +1833,7 @@ TEST(CAPITests, GenerationTags_Fallback) { EXPECT_STREQ(static_cast(reasoning_end), ""); } -TEST(CAPITests, GenerationTags_FromConfig) { +TEST(CAPITests, Tags_FromConfig) { // Create a temporary model directory with tool_calling and reasoning sections auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; std::filesystem::create_directories(temp_dir); @@ -1881,10 +1881,10 @@ TEST(CAPITests, GenerationTags_FromConfig) { auto model = OgaModel::Create(temp_dir.string().c_str()); - auto tool_start = model->GetGenerationTag("tool_call_start"); - auto tool_end = model->GetGenerationTag("tool_call_end"); - auto reasoning_start = model->GetGenerationTag("reasoning_start"); - auto reasoning_end = model->GetGenerationTag("reasoning_end"); + auto tool_start = model->GetTag("tool_call_start"); + auto tool_end = model->GetTag("tool_call_end"); + auto reasoning_start = model->GetTag("reasoning_start"); + auto reasoning_end = model->GetTag("reasoning_end"); EXPECT_STREQ(static_cast(tool_start), ""); EXPECT_STREQ(static_cast(tool_end), ""); From 4da60c43cb485b21d28bc01132d740197f8e5f69 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Mon, 29 Jun 2026 11:04:25 -0700 Subject: [PATCH 04/15] Clean up temp directory before test to avoid leftover collisions --- test/c_api_tests.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index ee2aee50bf..6a97796b87 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1836,6 +1836,7 @@ TEST(CAPITests, Tags_Fallback) { TEST(CAPITests, Tags_FromConfig) { // Create a temporary model directory with tool_calling and reasoning sections auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; + std::filesystem::remove_all(temp_dir); // Clean up any leftover from a previous failed run std::filesystem::create_directories(temp_dir); // Copy minimal model files from tiny-random-gpt2 From e1df10ce05cd60c62b30d4a59a6a8fddd0873207 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Tue, 30 Jun 2026 17:27:46 -0700 Subject: [PATCH 05/15] Refactor: store tag token IDs in model section, expose GetTagId API --- src/config.cpp | 40 +++++++-------------------------- src/config.h | 14 +++++------- src/models/model.cpp | 53 +++++++++++++++++++++++++++----------------- src/models/model.h | 11 ++++++--- src/ort_genai.h | 8 +++---- src/ort_genai_c.cpp | 4 ++-- src/ort_genai_c.h | 10 +++++---- test/c_api_tests.cpp | 51 ++++++++++++++++-------------------------- 8 files changed, 86 insertions(+), 105 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index c01d4ce7dd..12a74e9b1b 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1179,6 +1179,14 @@ struct Model_Element : JSON::Element { v_.left_context_samples = static_cast(JSON::Get(value)); } else if (name == "right_context_samples") { v_.right_context_samples = static_cast(JSON::Get(value)); + } else if (name == "tool_call_start_token_id") { + v_.tool_call_start_token_id = static_cast(JSON::Get(value)); + } else if (name == "tool_call_end_token_id") { + v_.tool_call_end_token_id = static_cast(JSON::Get(value)); + } else if (name == "reasoning_start_token_id") { + v_.reasoning_start_token_id = static_cast(JSON::Get(value)); + } else if (name == "reasoning_end_token_id") { + v_.reasoning_end_token_id = static_cast(JSON::Get(value)); } else { throw JSON::unknown_value_error{}; } @@ -1545,34 +1553,6 @@ void ClearDecoderProviderOptionsHardwareVendorId(Config& config, std::string_vie } } -struct ToolCalling_Element : JSON::Element { - explicit ToolCalling_Element(Config::ToolCalling& v) : v_{v} {} - - void OnValue(std::string_view name, JSON::Value value) override { - if (name == "tool_call_start_token") { - v_.tool_call_start_token = JSON::Get(value); - } else if (name == "tool_call_end_token") { - v_.tool_call_end_token = JSON::Get(value); - } - } - - Config::ToolCalling& v_; -}; - -struct Reasoning_Element : JSON::Element { - explicit Reasoning_Element(Config::Reasoning& v) : v_{v} {} - - void OnValue(std::string_view name, JSON::Value value) override { - if (name == "reasoning_start_token") { - v_.reasoning_start_token = JSON::Get(value); - } else if (name == "reasoning_end_token") { - v_.reasoning_end_token = JSON::Get(value); - } - } - - Config::Reasoning& v_; -}; - struct Root_Element : JSON::Element { explicit Root_Element(Config& config) : config_{config} {} @@ -1584,8 +1564,6 @@ struct Root_Element : JSON::Element { if (name == "model") return model_element_; if (name == "search") return search_element_; if (name == "engine") return engine_element_; - if (name == "tool_calling") return tool_calling_element_; - if (name == "reasoning") return reasoning_element_; throw JSON::unknown_value_error{}; } @@ -1593,8 +1571,6 @@ struct Root_Element : JSON::Element { Model_Element model_element_{config_.model}; Search_Element search_element_{config_.search}; Engine_Element engine_element_{config_.engine}; - ToolCalling_Element tool_calling_element_{config_.tool_calling}; - Reasoning_Element reasoning_element_{config_.reasoning}; }; struct RootObject_Element : JSON::Element { diff --git a/src/config.h b/src/config.h index 046ef698b6..ec4f92debb 100644 --- a/src/config.h +++ b/src/config.h @@ -145,6 +145,12 @@ struct Config { int video_token_id{}; int vision_start_token_id{}; + // Tool-calling and reasoning token IDs (used for efficient token-level detection) + int tool_call_start_token_id{-1}; + int tool_call_end_token_id{-1}; + int reasoning_start_token_id{-1}; + int reasoning_end_token_id{-1}; + int vocab_size{}; int context_length{}; @@ -451,15 +457,7 @@ struct Config { std::optional static_batching; // Static batching settings } engine; // Engine settings - struct ToolCalling { - std::string tool_call_start_token; // e.g., "" - std::string tool_call_end_token; // e.g., "" - } tool_calling; - struct Reasoning { - std::string reasoning_start_token; // e.g., "" - std::string reasoning_end_token; // e.g., "" - } reasoning; void AddMapping(const std::string& nominal_name, const std::string& graph_name); // Returns graph name and true if the nominal name is found in the mapping diff --git a/src/models/model.cpp b/src/models/model.cpp index 2314526b7c..dacdbcb7b7 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -826,9 +826,9 @@ bool Model::IsPruned() const { namespace { -// Fallback map for models whose genai_config.json doesn't yet have tool_calling/reasoning sections. +// Fallback map for models whose genai_config.json doesn't yet have token IDs in the model section. // Keyed by model.type string from genai_config.json. -// Inner map: tag_name -> value +// Inner map: tag_name -> token string (used for vocab lookup to get the ID). const std::string* GetFallbackTag(const std::string& model_type, const std::string& tag_name) { static const std::unordered_map> fallback_map = { {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, @@ -845,28 +845,41 @@ const std::string* GetFallbackTag(const std::string& model_type, const std::stri } // namespace -const std::string& Model::GetTag(const std::string& tag_name) const { - // Check config first (tool_calling and reasoning sections) - static const std::unordered_map> config_accessors = { - {"tool_call_start", [](const Config& c) -> const std::string& { return c.tool_calling.tool_call_start_token; }}, - {"tool_call_end", [](const Config& c) -> const std::string& { return c.tool_calling.tool_call_end_token; }}, - {"reasoning_start", [](const Config& c) -> const std::string& { return c.reasoning.reasoning_start_token; }}, - {"reasoning_end", [](const Config& c) -> const std::string& { return c.reasoning.reasoning_end_token; }}, +void Model::InitTagIdCache() const { + auto tokenizer = CreateTokenizer(); + + static const char* tag_names[] = {"tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end"}; + + auto get_config_id = [&](const std::string& name) -> int32_t { + if (name == "tool_call_start") return config_->model.tool_call_start_token_id; + if (name == "tool_call_end") return config_->model.tool_call_end_token_id; + if (name == "reasoning_start") return config_->model.reasoning_start_token_id; + if (name == "reasoning_end") return config_->model.reasoning_end_token_id; + return -1; }; - auto accessor_it = config_accessors.find(tag_name); - if (accessor_it != config_accessors.end()) { - const std::string& config_val = accessor_it->second(*config_); - if (!config_val.empty()) - return config_val; - } + for (const auto* tag_name : tag_names) { + int32_t id = get_config_id(tag_name); + if (id >= 0) { + tag_id_cache_[tag_name] = id; + continue; + } - // Fallback to model-type-based map - const auto* fallback = GetFallbackTag(config_->model.type, tag_name); - if (fallback) return *fallback; + // Fallback: look up the token string in the vocabulary to get its ID. + const auto* fallback_str = GetFallbackTag(config_->model.type, tag_name); + if (fallback_str && !fallback_str->empty()) { + int32_t fallback_id = tokenizer->TokenToTokenId(fallback_str->c_str()); + if (fallback_id >= 0) { + tag_id_cache_[tag_name] = fallback_id; + } + } + } +} - static const std::string empty; - return empty; +int32_t Model::GetTagId(const std::string& tag_name) const { + std::call_once(tag_id_cache_flag_, [this]() { InitTagIdCache(); }); + auto it = tag_id_cache_.find(tag_name); + return (it != tag_id_cache_.end()) ? it->second : -1; } std::shared_ptr CreateModel(OrtEnv& ort_env, const char* config_path, const RuntimeSettings* settings /*= nullptr*/) { diff --git a/src/models/model.h b/src/models/model.h index aeb0463875..8c69be1195 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -163,10 +163,10 @@ struct Model : std::enable_shared_from_this, LeakChecked, External bool IsPruned() const; - // Generic tag accessor (reads from config with model-type fallback). + // Returns the token ID for the given tag, or -1 if the model doesn't define the tag. + // Checks genai_config.json model section first, then encodes the model-type fallback string. // Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". - // Returns the tag value, or an empty string if the model doesn't define the tag. - const std::string& GetTag(const std::string& tag_name) const; + int32_t GetTagId(const std::string& tag_name) const; std::unique_ptr config_; std::unique_ptr session_options_; @@ -189,8 +189,13 @@ struct Model : std::enable_shared_from_this, LeakChecked, External protected: void CreateSessionOptions(); + void InitTagIdCache() const; std::map> pipeline_session_options_; + + // Cached tag token IDs (lazily populated on first GetTagId call). + mutable std::unordered_map tag_id_cache_; + mutable std::once_flag tag_id_cache_flag_; }; } // namespace Generators diff --git a/src/ort_genai.h b/src/ort_genai.h index 8fc2c64b4d..9df4b280ba 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -258,10 +258,10 @@ struct OgaModel : OgaAbstract { return p; } - OgaString GetTag(const char* tag_name) const { - const char* p; - OgaCheckResult(OgaModelGetTag(this, tag_name, &p)); - return p; + int32_t GetTagId(const char* tag_name) const { + int32_t id; + OgaCheckResult(OgaModelGetTagId(this, tag_name, &id)); + return id; } static void operator delete(void* p) { OgaDestroyModel(reinterpret_cast(p)); } diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index 7362425caa..ff1179835b 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -378,9 +378,9 @@ OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char* OGA_CATCH } -OgaResult* OGA_API_CALL OgaModelGetTag(const OgaModel* model, const char* tag_name, const char** out) { +OgaResult* OGA_API_CALL OgaModelGetTagId(const OgaModel* model, const char* tag_name, int32_t* out) { OGA_TRY - *out = AllocOgaString(model->GetTag(tag_name).c_str()); + *out = model->GetTagId(tag_name); return nullptr; OGA_CATCH } diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index a9eb92a3e8..3bee91295b 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -407,17 +407,19 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetType(const OgaModel* model, const OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char** out); /** - * \brief Returns a tag value for this model by name. + * \brief Returns a tag token ID for this model by name. * + * Checks the genai_config.json model section first, then falls back to encoding the + * model-type-specific fallback token string via the tokenizer vocabulary. * Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". - * Returns an empty string if the model doesn't define the requested tag. + * Returns -1 if the model doesn't define the requested tag. * * \param[in] model The model to query. * \param[in] tag_name The name of the tag to retrieve. - * \param[out] out The tag value string. Must be destroyed with OgaDestroyString. + * \param[out] out The tag token ID. * \return OgaResult containing the error message if the call failed. */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetTag(const OgaModel* model, const char* tag_name, const char** out); +OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetTagId(const OgaModel* model, const char* tag_name, int32_t* out); /** * \brief Destroys the given config diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index 6a97796b87..a72a661f40 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1817,24 +1817,19 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { EXPECT_FALSE(transcription.empty()); } -// Test tool_calling and reasoning config parsing and fallback map -TEST(CAPITests, Tags_Fallback) { - // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → empty tags +// Test that GetTagId returns -1 for unknown model types (not in config, not in fallback map) +TEST(CAPITests, TagId_Unknown) { + // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → -1 auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); - auto tool_start = model->GetTag("tool_call_start"); - auto tool_end = model->GetTag("tool_call_end"); - auto reasoning_start = model->GetTag("reasoning_start"); - auto reasoning_end = model->GetTag("reasoning_end"); - - EXPECT_STREQ(static_cast(tool_start), ""); - EXPECT_STREQ(static_cast(tool_end), ""); - EXPECT_STREQ(static_cast(reasoning_start), ""); - EXPECT_STREQ(static_cast(reasoning_end), ""); + EXPECT_EQ(model->GetTagId("tool_call_start"), -1); + EXPECT_EQ(model->GetTagId("tool_call_end"), -1); + EXPECT_EQ(model->GetTagId("reasoning_start"), -1); + EXPECT_EQ(model->GetTagId("reasoning_end"), -1); } -TEST(CAPITests, Tags_FromConfig) { - // Create a temporary model directory with tool_calling and reasoning sections +TEST(CAPITests, TagId_FromConfig) { + // Create a temporary model directory with tool_call/reasoning token IDs in model section auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; std::filesystem::remove_all(temp_dir); // Clean up any leftover from a previous failed run std::filesystem::create_directories(temp_dir); @@ -1848,7 +1843,7 @@ TEST(CAPITests, Tags_FromConfig) { } } - // Write genai_config.json with tool_calling and reasoning sections + // Write genai_config.json with token IDs in model section { std::ofstream f((temp_dir / "genai_config.json").string()); f << R"({ @@ -1859,6 +1854,10 @@ TEST(CAPITests, Tags_FromConfig) { "eos_token_id": 98, "vocab_size": 1000, "context_length": 512, + "tool_call_start_token_id": 151657, + "tool_call_end_token_id": 151658, + "reasoning_start_token_id": 151659, + "reasoning_end_token_id": 151660, "decoder": { "session_options": { "provider_options": [] }, "filename": "past.onnx", @@ -1868,29 +1867,17 @@ TEST(CAPITests, Tags_FromConfig) { "inputs": { "past_names": "past_%d" }, "outputs": { "present_names": "present_%d" } } - }, - "tool_calling": { - "tool_call_start_token": "", - "tool_call_end_token": "" - }, - "reasoning": { - "reasoning_start_token": "", - "reasoning_end_token": "" } })"; } auto model = OgaModel::Create(temp_dir.string().c_str()); - auto tool_start = model->GetTag("tool_call_start"); - auto tool_end = model->GetTag("tool_call_end"); - auto reasoning_start = model->GetTag("reasoning_start"); - auto reasoning_end = model->GetTag("reasoning_end"); - - EXPECT_STREQ(static_cast(tool_start), ""); - EXPECT_STREQ(static_cast(tool_end), ""); - EXPECT_STREQ(static_cast(reasoning_start), ""); - EXPECT_STREQ(static_cast(reasoning_end), ""); + // GetTagId returns configured IDs from model section + EXPECT_EQ(model->GetTagId("tool_call_start"), 151657); + EXPECT_EQ(model->GetTagId("tool_call_end"), 151658); + EXPECT_EQ(model->GetTagId("reasoning_start"), 151659); + EXPECT_EQ(model->GetTagId("reasoning_end"), 151660); // Cleanup std::filesystem::remove_all(temp_dir); From f49972563bffda8c1e3b2e1a4b40585114784c3e Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 8 Jul 2026 13:37:17 -0700 Subject: [PATCH 06/15] Refactor: move tag IDs to Tokenizer, rename to bot/eot/bor/eor --- src/config.cpp | 16 +++---- src/config.h | 14 ++++-- src/models/model.cpp | 110 ++++++++++++++++++------------------------- src/models/model.h | 24 ++++++---- src/ort_genai.h | 32 ++++++++++--- src/ort_genai_c.cpp | 35 +++++++++++--- src/ort_genai_c.h | 47 ++++++++++++------ test/c_api_tests.cpp | 32 +++++++------ 8 files changed, 181 insertions(+), 129 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 02b314a20c..9dbe42c51e 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1181,14 +1181,14 @@ struct Model_Element : JSON::Element { v_.left_context_samples = SafeDoubleToInt(JSON::Get(value), name); } else if (name == "right_context_samples") { v_.right_context_samples = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "tool_call_start_token_id") { - v_.tool_call_start_token_id = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "tool_call_end_token_id") { - v_.tool_call_end_token_id = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "reasoning_start_token_id") { - v_.reasoning_start_token_id = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "reasoning_end_token_id") { - v_.reasoning_end_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == "bot_token_id") { + v_.bot_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == "eot_token_id") { + v_.eot_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == "bor_token_id") { + v_.bor_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == "eor_token_id") { + v_.eor_token_id = SafeDoubleToInt(JSON::Get(value), name); } else { throw JSON::unknown_value_error{}; } diff --git a/src/config.h b/src/config.h index dc24b1e8cd..31ff4eb80e 100644 --- a/src/config.h +++ b/src/config.h @@ -137,11 +137,15 @@ struct Config { int video_token_id{}; int vision_start_token_id{}; - // Tool-calling and reasoning token IDs (used for efficient token-level detection) - int tool_call_start_token_id{-1}; - int tool_call_end_token_id{-1}; - int reasoning_start_token_id{-1}; - int reasoning_end_token_id{-1}; + // Tool-calling and reasoning token IDs. + // Follows the bos/eos/pad naming convention: + // bot = beginning of tool (call), eot = end of tool (call) + // bor = beginning of reasoning, eor = end of reasoning + // -1 means the model does not define this token. + int bot_token_id{-1}; + int eot_token_id{-1}; + int bor_token_id{-1}; + int eor_token_id{-1}; int vocab_size{}; int context_length{}; diff --git a/src/models/model.cpp b/src/models/model.cpp index 5ac815afcb..8cccced8bf 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -298,9 +298,37 @@ const std::string& TokenizerStream::Decode(int32_t token) { return chunk_; } +namespace { + +// Fallback token strings for models whose genai_config.json doesn't yet include +// bot/eot/bor/eor token IDs in the model section. This exists specifically for +// Foundry Local backward compatibility with older model packages that predate +// these config fields. +// Keyed by model.type string from genai_config.json. +// Inner map: tag_name -> token string (used for vocab lookup to get the ID). +const std::string* GetFallbackTag(const std::string& model_type, const std::string& tag_name) { + static const std::unordered_map> fallback_map = { + {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, + {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, + {"phi3", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, + {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, + }; + auto type_it = fallback_map.find(model_type); + if (type_it == fallback_map.end()) return nullptr; + auto tag_it = type_it->second.find(tag_name); + if (tag_it == type_it->second.end()) return nullptr; + return &tag_it->second; +} + +} // namespace + Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, eos_token_id_{config.model.eos_token_id}, - pad_token_id_{config.model.pad_token_id} { + pad_token_id_{config.model.pad_token_id}, + bot_token_id_{config.model.bot_token_id}, + eot_token_id_{config.model.eot_token_id}, + bor_token_id_{config.model.bor_token_id}, + eor_token_id_{config.model.eor_token_id} { // Default tokenizer options const char* keys[] = {"add_special_tokens", "skip_special_tokens"}; const char* values[] = {"false", "true"}; @@ -309,10 +337,24 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, const fs::path tokenizer_dir = config.ResolvePath(config.model.tokenizer_dir); CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); - // TODO: Once ORT Extensions supports an "additional_special_tokens" option, pass the generation - // tags (tool_calling/reasoning tokens) here so that models which don't already mark them as - // special in their tokenizer_config.json will still get correct skip_special_tokens behavior. - // This is needed for the FL SDK's dual-stream special token detection in OnnxChatGenerator::Decode(). + // Fallback: if bot/eot/bor/eor were not explicitly set in genai_config.json (-1), + // attempt to resolve them by encoding well-known token strings for the model type. + // This provides backward compatibility for Foundry Local when consuming older model + // packages that predate the bot/eot/bor/eor config fields. + if (bot_token_id_ < 0 || eot_token_id_ < 0 || bor_token_id_ < 0 || eor_token_id_ < 0) { + auto try_fallback = [&](int32_t& id, const std::string& tag_name) { + if (id >= 0) return; + const auto* fallback_str = GetFallbackTag(config.model.type, tag_name); + if (fallback_str && !fallback_str->empty()) { + int32_t resolved = TokenToTokenId(fallback_str->c_str()); + if (resolved >= 0) id = resolved; + } + }; + try_fallback(bot_token_id_, "tool_call_start"); + try_fallback(eot_token_id_, "tool_call_end"); + try_fallback(bor_token_id_, "reasoning_start"); + try_fallback(eor_token_id_, "reasoning_end"); + } } std::unique_ptr Tokenizer::CreateStream() const { @@ -827,64 +869,6 @@ bool Model::IsPruned() const { return logits_shape[1] == 1; } -namespace { - -// Fallback map for models whose genai_config.json doesn't yet have token IDs in the model section. -// Keyed by model.type string from genai_config.json. -// Inner map: tag_name -> token string (used for vocab lookup to get the ID). -const std::string* GetFallbackTag(const std::string& model_type, const std::string& tag_name) { - static const std::unordered_map> fallback_map = { - {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, - {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, - {"phi3", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, - {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, - }; - auto type_it = fallback_map.find(model_type); - if (type_it == fallback_map.end()) return nullptr; - auto tag_it = type_it->second.find(tag_name); - if (tag_it == type_it->second.end()) return nullptr; - return &tag_it->second; -} - -} // namespace - -void Model::InitTagIdCache() const { - auto tokenizer = CreateTokenizer(); - - static const char* tag_names[] = {"tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end"}; - - auto get_config_id = [&](const std::string& name) -> int32_t { - if (name == "tool_call_start") return config_->model.tool_call_start_token_id; - if (name == "tool_call_end") return config_->model.tool_call_end_token_id; - if (name == "reasoning_start") return config_->model.reasoning_start_token_id; - if (name == "reasoning_end") return config_->model.reasoning_end_token_id; - return -1; - }; - - for (const auto* tag_name : tag_names) { - int32_t id = get_config_id(tag_name); - if (id >= 0) { - tag_id_cache_[tag_name] = id; - continue; - } - - // Fallback: look up the token string in the vocabulary to get its ID. - const auto* fallback_str = GetFallbackTag(config_->model.type, tag_name); - if (fallback_str && !fallback_str->empty()) { - int32_t fallback_id = tokenizer->TokenToTokenId(fallback_str->c_str()); - if (fallback_id >= 0) { - tag_id_cache_[tag_name] = fallback_id; - } - } - } -} - -int32_t Model::GetTagId(const std::string& tag_name) const { - std::call_once(tag_id_cache_flag_, [this]() { InitTagIdCache(); }); - auto it = tag_id_cache_.find(tag_name); - return (it != tag_id_cache_.end()) ? it->second : -1; -} - std::shared_ptr CreateModel(OrtEnv& ort_env, const char* config_path, const RuntimeSettings* settings /*= nullptr*/) { std::string config_overlay; if (settings) { diff --git a/src/models/model.h b/src/models/model.h index 9067412dd3..0809f0bdf7 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -106,12 +106,26 @@ struct Tokenizer : std::enable_shared_from_this, LeakChecked& GetEosTokenIds() const { return eos_token_id_; } int32_t GetPadTokenId() const { return pad_token_id_; } + // Tool-calling and reasoning token IDs. + // Naming follows the bos/eos/pad convention: + // bot = beginning of tool (call), eot = end of tool (call) + // bor = beginning of reasoning, eor = end of reasoning + // Returns -1 if the model does not define the token. + int32_t GetBotTokenId() const { return bot_token_id_; } + int32_t GetEotTokenId() const { return eot_token_id_; } + int32_t GetBorTokenId() const { return bor_token_id_; } + int32_t GetEorTokenId() const { return eor_token_id_; } + OrtxPtr tokenizer_; private: int32_t bos_token_id_; std::vector eos_token_id_; int32_t pad_token_id_; + int32_t bot_token_id_; + int32_t eot_token_id_; + int32_t bor_token_id_; + int32_t eor_token_id_; }; struct MultiModalProcessor : std::enable_shared_from_this, ExternalRefCounted { @@ -168,11 +182,6 @@ struct Model : std::enable_shared_from_this, LeakChecked, External bool IsPruned() const; - // Returns the token ID for the given tag, or -1 if the model doesn't define the tag. - // Checks genai_config.json model section first, then encodes the model-type fallback string. - // Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". - int32_t GetTagId(const std::string& tag_name) const; - std::unique_ptr config_; std::unique_ptr session_options_; @@ -194,13 +203,8 @@ struct Model : std::enable_shared_from_this, LeakChecked, External protected: void CreateSessionOptions(); - void InitTagIdCache() const; std::map> pipeline_session_options_; - - // Cached tag token IDs (lazily populated on first GetTagId call). - mutable std::unordered_map tag_id_cache_; - mutable std::once_flag tag_id_cache_flag_; }; } // namespace Generators diff --git a/src/ort_genai.h b/src/ort_genai.h index a6869393da..f9f79b926f 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -258,12 +258,6 @@ struct OgaModel : OgaAbstract { return p; } - int32_t GetTagId(const char* tag_name) const { - int32_t id; - OgaCheckResult(OgaModelGetTagId(this, tag_name, &id)); - return id; - } - static void operator delete(void* p) { OgaDestroyModel(reinterpret_cast(p)); } }; @@ -348,6 +342,32 @@ struct OgaTokenizer : OgaAbstract { return token_id; } + // Tool-calling and reasoning token IDs (bot/eot/bor/eor). + // Returns -1 if the model does not define the token. + int32_t GetBotTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetBotTokenId(this, &token_id)); + return token_id; + } + + int32_t GetEotTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetEotTokenId(this, &token_id)); + return token_id; + } + + int32_t GetBorTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetBorTokenId(this, &token_id)); + return token_id; + } + + int32_t GetEorTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetEorTokenId(this, &token_id)); + return token_id; + } + void Encode(const char* str, OgaSequences& sequences) const { OgaCheckResult(OgaTokenizerEncode(this, str, &sequences)); } diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index ff1179835b..6a275f2f55 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -378,13 +378,6 @@ OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char* OGA_CATCH } -OgaResult* OGA_API_CALL OgaModelGetTagId(const OgaModel* model, const char* tag_name, int32_t* out) { - OGA_TRY - *out = model->GetTagId(tag_name); - return nullptr; - OGA_CATCH -} - OgaResult* OGA_API_CALL OgaCreateGeneratorParams(const OgaModel* model, OgaGeneratorParams** out) { OGA_TRY auto params = std::make_shared(*model); @@ -674,6 +667,34 @@ OgaResult* OGA_API_CALL OgaTokenizerGetPadTokenId(const OgaTokenizer* tokenizer, OGA_CATCH } +OgaResult* OGA_API_CALL OgaTokenizerGetBotTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetBotTokenId(); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaTokenizerGetEotTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetEotTokenId(); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaTokenizerGetBorTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetBorTokenId(); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaTokenizerGetEorTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetEorTokenId(); + return nullptr; + OGA_CATCH +} + OgaResult* OGA_API_CALL OgaTokenizerEncode(const OgaTokenizer* tokenizer, const char* str, OgaSequences* sequences) { OGA_TRY sequences->emplace_back(tokenizer->Encode(str)); diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index e3535194e7..6462fec983 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -415,21 +415,6 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetType(const OgaModel* model, const */ OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetDeviceType(const OgaModel* model, const char** out); -/** - * \brief Returns a tag token ID for this model by name. - * - * Checks the genai_config.json model section first, then falls back to encoding the - * model-type-specific fallback token string via the tokenizer vocabulary. - * Known tag names: "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". - * Returns -1 if the model doesn't define the requested tag. - * - * \param[in] model The model to query. - * \param[in] tag_name The name of the tag to retrieve. - * \param[out] out The tag token ID. - * \return OgaResult containing the error message if the call failed. - */ -OGA_EXPORT OgaResult* OGA_API_CALL OgaModelGetTagId(const OgaModel* model, const char* tag_name, int32_t* out); - /** * \brief Destroys the given config * \param[in] config The config to be destroyed. @@ -732,6 +717,38 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEosTokenIds(const OgaTokenizer */ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetPadTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); +/** + * \brief Return the BOT (beginning of tool call) token id, or -1 if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The BOT token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBotTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + +/** + * \brief Return the EOT (end of tool call) token id, or -1 if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The EOT token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEotTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + +/** + * \brief Return the BOR (beginning of reasoning) token id, or -1 if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The BOR token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBorTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + +/** + * \brief Return the EOR (end of reasoning) token id, or -1 if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The EOR token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEorTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + /** * Encodes a single string and adds the encoded sequence of tokens to the OgaSequences. The OgaSequences must be freed with OgaDestroySequences * when it is no longer needed. diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index c1a444636e..0aa2b7dfc7 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1817,19 +1817,20 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { EXPECT_FALSE(transcription.empty()); } -// Test that GetTagId returns -1 for unknown model types (not in config, not in fallback map) +// Test that bot/eot/bor/eor return -1 for models without these tokens configured TEST(CAPITests, TagId_Unknown) { // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → -1 auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); + auto tokenizer = OgaTokenizer::Create(*model); - EXPECT_EQ(model->GetTagId("tool_call_start"), -1); - EXPECT_EQ(model->GetTagId("tool_call_end"), -1); - EXPECT_EQ(model->GetTagId("reasoning_start"), -1); - EXPECT_EQ(model->GetTagId("reasoning_end"), -1); + EXPECT_EQ(tokenizer->GetBotTokenId(), -1); + EXPECT_EQ(tokenizer->GetEotTokenId(), -1); + EXPECT_EQ(tokenizer->GetBorTokenId(), -1); + EXPECT_EQ(tokenizer->GetEorTokenId(), -1); } TEST(CAPITests, TagId_FromConfig) { - // Create a temporary model directory with tool_call/reasoning token IDs in model section + // Create a temporary model directory with bot/eot/bor/eor token IDs in model section auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; std::filesystem::remove_all(temp_dir); // Clean up any leftover from a previous failed run std::filesystem::create_directories(temp_dir); @@ -1854,10 +1855,10 @@ TEST(CAPITests, TagId_FromConfig) { "eos_token_id": 98, "vocab_size": 1000, "context_length": 512, - "tool_call_start_token_id": 151657, - "tool_call_end_token_id": 151658, - "reasoning_start_token_id": 151659, - "reasoning_end_token_id": 151660, + "bot_token_id": 151657, + "eot_token_id": 151658, + "bor_token_id": 151659, + "eor_token_id": 151660, "decoder": { "session_options": { "provider_options": [] }, "filename": "past.onnx", @@ -1872,12 +1873,13 @@ TEST(CAPITests, TagId_FromConfig) { } auto model = OgaModel::Create(temp_dir.string().c_str()); + auto tokenizer = OgaTokenizer::Create(*model); - // GetTagId returns configured IDs from model section - EXPECT_EQ(model->GetTagId("tool_call_start"), 151657); - EXPECT_EQ(model->GetTagId("tool_call_end"), 151658); - EXPECT_EQ(model->GetTagId("reasoning_start"), 151659); - EXPECT_EQ(model->GetTagId("reasoning_end"), 151660); + // Tokenizer returns configured IDs from model section + EXPECT_EQ(tokenizer->GetBotTokenId(), 151657); + EXPECT_EQ(tokenizer->GetEotTokenId(), 151658); + EXPECT_EQ(tokenizer->GetBorTokenId(), 151659); + EXPECT_EQ(tokenizer->GetEorTokenId(), 151660); // Cleanup std::filesystem::remove_all(temp_dir); From 6276e87a99473b31ba6152833ba4eb3c839a4812 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Tue, 14 Jul 2026 18:24:38 -0700 Subject: [PATCH 07/15] Address review: simplify fallback, add C#/Java/ObjC/Python bindings --- src/csharp/NativeMethods.cs | 16 +++++ src/csharp/Tokenizer.cs | 36 +++++++++++ .../java/ai/onnxruntime/genai/Tokenizer.java | 64 +++++++++++++++++++ .../native/ai_onnxruntime_genai_Tokenizer.cpp | 48 ++++++++++++++ src/models/model.cpp | 48 ++++++-------- src/objectivec/include/ort_genai_objc.h | 20 ++++++ src/objectivec/oga_tokenizer.mm | 28 ++++++++ src/python/python.cpp | 4 ++ 8 files changed, 235 insertions(+), 29 deletions(-) diff --git a/src/csharp/NativeMethods.cs b/src/csharp/NativeMethods.cs index a7b30fa9c0..9227b3749e 100644 --- a/src/csharp/NativeMethods.cs +++ b/src/csharp/NativeMethods.cs @@ -268,6 +268,22 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq public static extern IntPtr /* OgaResult* */ OgaTokenizerGetPadTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, out int /* const int32_t* */ outPadTokenId); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetBotTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outBotTokenId); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetEotTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outEotTokenId); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetBorTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outBorTokenId); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetEorTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outEorTokenId); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] public static extern IntPtr /* OgaResult* */ OgaTokenizerEncode(IntPtr /* const OgaTokenizer* */ tokenizer, byte[] /* const char* */ strings, diff --git a/src/csharp/Tokenizer.cs b/src/csharp/Tokenizer.cs index 0c11aee389..d9f9373f2e 100644 --- a/src/csharp/Tokenizer.cs +++ b/src/csharp/Tokenizer.cs @@ -141,6 +141,42 @@ public int GetPadTokenId() return padTokenId; } + /// + /// Returns the BOT (beginning of tool call) token ID, or -1 if not defined. + /// + public int GetBotTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetBotTokenId(_tokenizerHandle, out int botTokenId)); + return botTokenId; + } + + /// + /// Returns the EOT (end of tool call) token ID, or -1 if not defined. + /// + public int GetEotTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetEotTokenId(_tokenizerHandle, out int eotTokenId)); + return eotTokenId; + } + + /// + /// Returns the BOR (beginning of reasoning) token ID, or -1 if not defined. + /// + public int GetBorTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetBorTokenId(_tokenizerHandle, out int borTokenId)); + return borTokenId; + } + + /// + /// Returns the EOR (end of reasoning) token ID, or -1 if not defined. + /// + public int GetEorTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetEorTokenId(_tokenizerHandle, out int eorTokenId)); + return eorTokenId; + } + public TokenizerStream CreateStream() { IntPtr tokenizerStreamHandle = IntPtr.Zero; diff --git a/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java b/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java index 1634c2c2d5..a404cf04c4 100644 --- a/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java +++ b/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java @@ -107,6 +107,62 @@ public int getPadTokenId() throws GenAIException { return tokenizerGetPadTokenId(nativeHandle); } + /** + * Gets the BOT (beginning of tool call) token ID, or -1 if the model does not define one. + * + * @return The BOT token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getBotTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetBotTokenId(nativeHandle); + } + + /** + * Gets the EOT (end of tool call) token ID, or -1 if the model does not define one. + * + * @return The EOT token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getEotTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetEotTokenId(nativeHandle); + } + + /** + * Gets the BOR (beginning of reasoning) token ID, or -1 if the model does not define one. + * + * @return The BOR token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getBorTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetBorTokenId(nativeHandle); + } + + /** + * Gets the EOR (end of reasoning) token ID, or -1 if the model does not define one. + * + * @return The EOR token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getEorTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetEorTokenId(nativeHandle); + } + /** * Gets the end of sentence token IDs. * @@ -229,6 +285,14 @@ public void close() { private native int[] tokenizerGetEosTokenIds(long tokenizerHandle) throws GenAIException; + private native int tokenizerGetBotTokenId(long tokenizerHandle) throws GenAIException; + + private native int tokenizerGetEotTokenId(long tokenizerHandle) throws GenAIException; + + private native int tokenizerGetBorTokenId(long tokenizerHandle) throws GenAIException; + + private native int tokenizerGetEorTokenId(long tokenizerHandle) throws GenAIException; + private native int tokenizerToTokenId(long tokenizerHandle, String str) throws GenAIException; private native String tokenizerApplyChatTemplate( diff --git a/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp b/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp index 8eba53e0a5..1ae8ea9582 100644 --- a/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp +++ b/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp @@ -110,6 +110,54 @@ Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetPadTokenId(JNIEnv* env, jobject return static_cast(token_id); } +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetBotTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetBotTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetEotTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetEotTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetBorTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetBorTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetEorTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetEorTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + JNIEXPORT jintArray JNICALL Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetEosTokenIds(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); diff --git a/src/models/model.cpp b/src/models/model.cpp index 9e87eb0808..a90d9845bf 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -299,29 +299,30 @@ const std::string& TokenizerStream::Decode(int32_t token) { return chunk_; } -namespace { - -// Fallback token strings for models whose genai_config.json doesn't yet include -// bot/eot/bor/eor token IDs in the model section. This exists specifically for -// Foundry Local backward compatibility with older model packages that predate -// these config fields. +// Fallback: if the given token ID is unset (-1), attempt to resolve it by looking up +// a well-known token string for the model type in the tokenizer vocabulary. +// This provides backward compatibility for Foundry Local when consuming older model +// packages that predate the bot/eot/bor/eor config fields. // Keyed by model.type string from genai_config.json. -// Inner map: tag_name -> token string (used for vocab lookup to get the ID). -const std::string* GetFallbackTag(const std::string& model_type, const std::string& tag_name) { +static void ResolveFallbackTagId(int32_t& id, const std::string& model_type, + const std::string& tag_name, const Tokenizer& tokenizer) { + if (id >= 0) return; + static const std::unordered_map> fallback_map = { {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, {"phi3", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, }; + auto type_it = fallback_map.find(model_type); - if (type_it == fallback_map.end()) return nullptr; + if (type_it == fallback_map.end()) return; auto tag_it = type_it->second.find(tag_name); - if (tag_it == type_it->second.end()) return nullptr; - return &tag_it->second; -} + if (tag_it == type_it->second.end()) return; -} // namespace + int32_t resolved = tokenizer.TokenToTokenId(tag_it->second.c_str()); + if (resolved >= 0) id = resolved; +} Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, eos_token_id_{config.model.eos_token_id}, @@ -338,23 +339,12 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, const fs::path tokenizer_dir = config.ResolvePath(config.model.tokenizer_dir); CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); - // Fallback: if bot/eot/bor/eor were not explicitly set in genai_config.json (-1), - // attempt to resolve them by encoding well-known token strings for the model type. - // This provides backward compatibility for Foundry Local when consuming older model - // packages that predate the bot/eot/bor/eor config fields. + // Resolve any unset bot/eot/bor/eor IDs via model-type fallback strings. if (bot_token_id_ < 0 || eot_token_id_ < 0 || bor_token_id_ < 0 || eor_token_id_ < 0) { - auto try_fallback = [&](int32_t& id, const std::string& tag_name) { - if (id >= 0) return; - const auto* fallback_str = GetFallbackTag(config.model.type, tag_name); - if (fallback_str && !fallback_str->empty()) { - int32_t resolved = TokenToTokenId(fallback_str->c_str()); - if (resolved >= 0) id = resolved; - } - }; - try_fallback(bot_token_id_, "tool_call_start"); - try_fallback(eot_token_id_, "tool_call_end"); - try_fallback(bor_token_id_, "reasoning_start"); - try_fallback(eor_token_id_, "reasoning_end"); + ResolveFallbackTagId(bot_token_id_, config.model.type, "tool_call_start", *this); + ResolveFallbackTagId(eot_token_id_, config.model.type, "tool_call_end", *this); + ResolveFallbackTagId(bor_token_id_, config.model.type, "reasoning_start", *this); + ResolveFallbackTagId(eor_token_id_, config.model.type, "reasoning_end", *this); } } diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index c3f23655c7..129bbff3db 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -167,6 +167,26 @@ typedef NS_ENUM(NSInteger, OGAElementType) { */ - (int32_t)getPadTokenId:(NSError**)error; +/** + * Return the BOT (beginning of tool call) token ID, or -1 if not defined. + */ +- (int32_t)getBotTokenId:(NSError**)error; + +/** + * Return the EOT (end of tool call) token ID, or -1 if not defined. + */ +- (int32_t)getEotTokenId:(NSError**)error; + +/** + * Return the BOR (beginning of reasoning) token ID, or -1 if not defined. + */ +- (int32_t)getBorTokenId:(NSError**)error; + +/** + * Return the EOR (end of reasoning) token ID, or -1 if not defined. + */ +- (int32_t)getEorTokenId:(NSError**)error; + /** * Encode text to sequences * diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index a3eb27ffd2..08fa35e0da 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -50,6 +50,34 @@ - (int32_t)getPadTokenId:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) } +- (int32_t)getBotTokenId:(NSError**)error { + try { + return _tokenizer->GetBotTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + +- (int32_t)getEotTokenId:(NSError**)error { + try { + return _tokenizer->GetEotTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + +- (int32_t)getBorTokenId:(NSError**)error { + try { + return _tokenizer->GetBorTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + +- (int32_t)getEorTokenId:(NSError**)error { + try { + return _tokenizer->GetEorTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { OGASequences* sequences = [[OGASequences alloc] initWithError:error]; if (!sequences) { diff --git a/src/python/python.cpp b/src/python/python.cpp index 40a6a0d0cd..2c6fefc625 100644 --- a/src/python/python.cpp +++ b/src/python/python.cpp @@ -394,6 +394,10 @@ PYBIND11_MODULE(onnxruntime_genai, m) { return ToPython(t.GetEosTokenIds()); }) .def_property_readonly("pad_token_id", &OgaTokenizer::GetPadTokenId) + .def_property_readonly("bot_token_id", &OgaTokenizer::GetBotTokenId) + .def_property_readonly("eot_token_id", &OgaTokenizer::GetEotTokenId) + .def_property_readonly("bor_token_id", &OgaTokenizer::GetBorTokenId) + .def_property_readonly("eor_token_id", &OgaTokenizer::GetEorTokenId) .def("update_options", [](OgaTokenizer& t, pybind11::kwargs kwargs) { std::vector key_storage; std::vector value_storage; From 1cb59fb9b32e12e53eb50e876779ee4869dbfe91 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Tue, 14 Jul 2026 19:07:21 -0700 Subject: [PATCH 08/15] Fix clang-format in config.cpp switch/case block from main (unrelated) --- src/config.cpp | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 994998a079..6d16509de3 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1424,13 +1424,27 @@ static std::string EscapeJsonString(std::string_view s) { result.reserve(s.size()); for (char c : s) { switch (c) { - case '"': result += "\\\""; break; - case '\\': result += "\\\\"; break; - case '\b': result += "\\b"; break; - case '\f': result += "\\f"; break; - case '\n': result += "\\n"; break; - case '\r': result += "\\r"; break; - case '\t': result += "\\t"; break; + case '"': + result += "\\\""; + break; + case '\\': + result += "\\\\"; + break; + case '\b': + result += "\\b"; + break; + case '\f': + result += "\\f"; + break; + case '\n': + result += "\\n"; + break; + case '\r': + result += "\\r"; + break; + case '\t': + result += "\\t"; + break; default: if (static_cast(c) < 0x20) { throw std::runtime_error( From e418c6284e1724bdd33e9fcea096ef0e3af8f769 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 15 Jul 2026 09:34:47 -0700 Subject: [PATCH 09/15] Fix phi3 fallback: use <|tool_call|>/<|/tool_call|> (with pipes) --- src/models/model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index 99cf2072a6..f17ca1fe25 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -320,7 +320,7 @@ static void ResolveFallbackTagId(int32_t& id, const std::string& model_type, static const std::unordered_map> fallback_map = { {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, - {"phi3", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, + {"phi3", {{"tool_call_start", "<|tool_call|>"}, {"tool_call_end", "<|/tool_call|>"}}}, {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, }; From 03bce7bc9e8345aad1fb8b7ed9158b909759ba9e Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 15 Jul 2026 14:30:53 -0700 Subject: [PATCH 10/15] Add reasoning fallback entries to qwen2 (covers DeepSeek think tags) --- src/models/model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index f17ca1fe25..61c50fadd1 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -318,7 +318,7 @@ static void ResolveFallbackTagId(int32_t& id, const std::string& model_type, if (id >= 0) return; static const std::unordered_map> fallback_map = { - {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}}}, + {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, {"phi3", {{"tool_call_start", "<|tool_call|>"}, {"tool_call_end", "<|/tool_call|>"}}}, {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, From 4270829fb94f5f3f2ed9d25b7d4e48bef10743de Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 22 Jul 2026 12:27:35 -0700 Subject: [PATCH 11/15] Use std::optional for tag IDs, extract fallback to tokenizer_tag_utils, rename tests --- src/config.h | 9 +++-- src/models/model.cpp | 56 +++++++++++++----------------- src/models/model.h | 19 +++++----- src/models/tokenizer_tag_utils.cpp | 29 ++++++++++++++++ src/models/tokenizer_tag_utils.h | 25 +++++++++++++ test/c_api_tests.cpp | 16 ++++----- 6 files changed, 101 insertions(+), 53 deletions(-) create mode 100644 src/models/tokenizer_tag_utils.cpp create mode 100644 src/models/tokenizer_tag_utils.h diff --git a/src/config.h b/src/config.h index cfdcbc129b..006502f63a 100644 --- a/src/config.h +++ b/src/config.h @@ -149,11 +149,10 @@ struct Config { // Follows the bos/eos/pad naming convention: // bot = beginning of tool (call), eot = end of tool (call) // bor = beginning of reasoning, eor = end of reasoning - // -1 means the model does not define this token. - int bot_token_id{-1}; - int eot_token_id{-1}; - int bor_token_id{-1}; - int eor_token_id{-1}; + std::optional bot_token_id; + std::optional eot_token_id; + std::optional bor_token_id; + std::optional eor_token_id; int vocab_size{}; int context_length{}; diff --git a/src/models/model.cpp b/src/models/model.cpp index 61c50fadd1..ca81308d42 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -19,6 +19,7 @@ #include "../tracing.h" #include "model.h" #include "model_package.h" +#include "tokenizer_tag_utils.h" #include "gpt.h" #include "decoder_only.h" #include "whisper.h" @@ -308,31 +309,6 @@ const std::string& TokenizerStream::Decode(int32_t token) { return chunk_; } -// Fallback: if the given token ID is unset (-1), attempt to resolve it by looking up -// a well-known token string for the model type in the tokenizer vocabulary. -// This provides backward compatibility for Foundry Local when consuming older model -// packages that predate the bot/eot/bor/eor config fields. -// Keyed by model.type string from genai_config.json. -static void ResolveFallbackTagId(int32_t& id, const std::string& model_type, - const std::string& tag_name, const Tokenizer& tokenizer) { - if (id >= 0) return; - - static const std::unordered_map> fallback_map = { - {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, - {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, - {"phi3", {{"tool_call_start", "<|tool_call|>"}, {"tool_call_end", "<|/tool_call|>"}}}, - {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, - }; - - auto type_it = fallback_map.find(model_type); - if (type_it == fallback_map.end()) return; - auto tag_it = type_it->second.find(tag_name); - if (tag_it == type_it->second.end()) return; - - int32_t resolved = tokenizer.TokenToTokenId(tag_it->second.c_str()); - if (resolved >= 0) id = resolved; -} - Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, eos_token_id_{config.model.eos_token_id}, pad_token_id_{config.model.pad_token_id}, @@ -349,12 +325,30 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); // Resolve any unset bot/eot/bor/eor IDs via model-type fallback strings. - if (bot_token_id_ < 0 || eot_token_id_ < 0 || bor_token_id_ < 0 || eor_token_id_ < 0) { - ResolveFallbackTagId(bot_token_id_, config.model.type, "tool_call_start", *this); - ResolveFallbackTagId(eot_token_id_, config.model.type, "tool_call_end", *this); - ResolveFallbackTagId(bor_token_id_, config.model.type, "reasoning_start", *this); - ResolveFallbackTagId(eor_token_id_, config.model.type, "reasoning_end", *this); - } + if (!bot_token_id_) bot_token_id_ = ResolveFallbackTokenId(config.model.type, "tool_call_start", *this); + if (!eot_token_id_) eot_token_id_ = ResolveFallbackTokenId(config.model.type, "tool_call_end", *this); + if (!bor_token_id_) bor_token_id_ = ResolveFallbackTokenId(config.model.type, "reasoning_start", *this); + if (!eor_token_id_) eor_token_id_ = ResolveFallbackTokenId(config.model.type, "reasoning_end", *this); +} + +int32_t Tokenizer::GetBotTokenId() const { + if (!bot_token_id_) throw std::runtime_error("bot_token_id is not defined for this model"); + return *bot_token_id_; +} + +int32_t Tokenizer::GetEotTokenId() const { + if (!eot_token_id_) throw std::runtime_error("eot_token_id is not defined for this model"); + return *eot_token_id_; +} + +int32_t Tokenizer::GetBorTokenId() const { + if (!bor_token_id_) throw std::runtime_error("bor_token_id is not defined for this model"); + return *bor_token_id_; +} + +int32_t Tokenizer::GetEorTokenId() const { + if (!eor_token_id_) throw std::runtime_error("eor_token_id is not defined for this model"); + return *eor_token_id_; } std::unique_ptr Tokenizer::CreateStream() const { diff --git a/src/models/model.h b/src/models/model.h index 0809f0bdf7..022421ce2d 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -8,6 +8,7 @@ #include "ortx_tokenizer.h" #include "../generators.h" #include "utils.h" +#include #include "phi_image_processor.h" #include "whisper_processor.h" #include "parakeet_processor.h" @@ -110,11 +111,11 @@ struct Tokenizer : std::enable_shared_from_this, LeakChecked tokenizer_; @@ -122,10 +123,10 @@ struct Tokenizer : std::enable_shared_from_this, LeakChecked eos_token_id_; int32_t pad_token_id_; - int32_t bot_token_id_; - int32_t eot_token_id_; - int32_t bor_token_id_; - int32_t eor_token_id_; + std::optional bot_token_id_; + std::optional eot_token_id_; + std::optional bor_token_id_; + std::optional eor_token_id_; }; struct MultiModalProcessor : std::enable_shared_from_this, ExternalRefCounted { diff --git a/src/models/tokenizer_tag_utils.cpp b/src/models/tokenizer_tag_utils.cpp new file mode 100644 index 0000000000..caee64b844 --- /dev/null +++ b/src/models/tokenizer_tag_utils.cpp @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "tokenizer_tag_utils.h" +#include "model.h" + +namespace Generators { + +std::optional ResolveFallbackTokenId(const std::string& model_type, + const std::string& tag_name, + const Tokenizer& tokenizer) { + static const std::unordered_map> fallback_map = { + {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, + {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, + {"phi3", {{"tool_call_start", "<|tool_call|>"}, {"tool_call_end", "<|/tool_call|>"}}}, + {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, + }; + + auto type_it = fallback_map.find(model_type); + if (type_it == fallback_map.end()) return std::nullopt; + auto tag_it = type_it->second.find(tag_name); + if (tag_it == type_it->second.end()) return std::nullopt; + + int32_t resolved = tokenizer.TokenToTokenId(tag_it->second.c_str()); + if (resolved >= 0) return resolved; + return std::nullopt; +} + +} // namespace Generators diff --git a/src/models/tokenizer_tag_utils.h b/src/models/tokenizer_tag_utils.h new file mode 100644 index 0000000000..b87900f795 --- /dev/null +++ b/src/models/tokenizer_tag_utils.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include + +namespace Generators { + +struct Tokenizer; + +// Resolves a fallback token ID for models whose genai_config.json doesn't yet include +// bot/eot/bor/eor token IDs in the model section. This exists specifically for +// Foundry Local backward compatibility with older model packages that predate +// these config fields. +// +// Returns the resolved token ID if found in the fallback map and tokenizer vocabulary, +// or std::nullopt if the model type/tag name is not in the map or the token string +// doesn't resolve in the vocabulary. +std::optional ResolveFallbackTokenId(const std::string& model_type, + const std::string& tag_name, + const Tokenizer& tokenizer); + +} // namespace Generators diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index 37ba614e53..35d03600a1 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1969,19 +1969,19 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { EXPECT_FALSE(transcription.empty()); } -// Test that bot/eot/bor/eor return -1 for models without these tokens configured -TEST(CAPITests, TagId_Unknown) { - // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → -1 +// Test that bot/eot/bor/eor throw for models without these tokens configured +TEST(CAPITests, TokenId_Unsupported) { + // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → throws auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); auto tokenizer = OgaTokenizer::Create(*model); - EXPECT_EQ(tokenizer->GetBotTokenId(), -1); - EXPECT_EQ(tokenizer->GetEotTokenId(), -1); - EXPECT_EQ(tokenizer->GetBorTokenId(), -1); - EXPECT_EQ(tokenizer->GetEorTokenId(), -1); + EXPECT_THROW(tokenizer->GetBotTokenId(), std::runtime_error); + EXPECT_THROW(tokenizer->GetEotTokenId(), std::runtime_error); + EXPECT_THROW(tokenizer->GetBorTokenId(), std::runtime_error); + EXPECT_THROW(tokenizer->GetEorTokenId(), std::runtime_error); } -TEST(CAPITests, TagId_FromConfig) { +TEST(CAPITests, TokenId_FromConfig) { // Create a temporary model directory with bot/eot/bor/eor token IDs in model section auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; std::filesystem::remove_all(temp_dir); // Clean up any leftover from a previous failed run From c7f6ddfcaba36955f4cd53c7700b15c8f7b95e9b Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Thu, 23 Jul 2026 11:05:37 -0700 Subject: [PATCH 12/15] Update comments: getters throw on unsupported, not return -1 --- src/objectivec/include/ort_genai_objc.h | 8 ++++---- src/ort_genai.h | 2 +- src/ort_genai_c.h | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index 129bbff3db..94458159c8 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -168,22 +168,22 @@ typedef NS_ENUM(NSInteger, OGAElementType) { - (int32_t)getPadTokenId:(NSError**)error; /** - * Return the BOT (beginning of tool call) token ID, or -1 if not defined. + * Return the BOT (beginning of tool call) token ID. Returns an error if not defined. */ - (int32_t)getBotTokenId:(NSError**)error; /** - * Return the EOT (end of tool call) token ID, or -1 if not defined. + * Return the EOT (end of tool call) token ID. Returns an error if not defined. */ - (int32_t)getEotTokenId:(NSError**)error; /** - * Return the BOR (beginning of reasoning) token ID, or -1 if not defined. + * Return the BOR (beginning of reasoning) token ID. Returns an error if not defined. */ - (int32_t)getBorTokenId:(NSError**)error; /** - * Return the EOR (end of reasoning) token ID, or -1 if not defined. + * Return the EOR (end of reasoning) token ID. Returns an error if not defined. */ - (int32_t)getEorTokenId:(NSError**)error; diff --git a/src/ort_genai.h b/src/ort_genai.h index f9f79b926f..e09aa96ef9 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -343,7 +343,7 @@ struct OgaTokenizer : OgaAbstract { } // Tool-calling and reasoning token IDs (bot/eot/bor/eor). - // Returns -1 if the model does not define the token. + // Throws if the model does not define the token. int32_t GetBotTokenId() const { int32_t token_id; OgaCheckResult(OgaTokenizerGetBotTokenId(this, &token_id)); diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index ca46ab4b65..962f8b6b2f 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -729,7 +729,7 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEosTokenIds(const OgaTokenizer OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetPadTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); /** - * \brief Return the BOT (beginning of tool call) token id, or -1 if the model does not define one. + * \brief Return the BOT (beginning of tool call) token id. Returns an error if the model does not define one. * \param[in] tokenizer The tokenizer to read from * \param[out] token_id The BOT token id * \return OgaResult containing the error message if the call fails. @@ -737,7 +737,7 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetPadTokenId(const OgaTokenizer* OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBotTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); /** - * \brief Return the EOT (end of tool call) token id, or -1 if the model does not define one. + * \brief Return the EOT (end of tool call) token id. Returns an error if the model does not define one. * \param[in] tokenizer The tokenizer to read from * \param[out] token_id The EOT token id * \return OgaResult containing the error message if the call fails. @@ -745,7 +745,7 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBotTokenId(const OgaTokenizer* OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEotTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); /** - * \brief Return the BOR (beginning of reasoning) token id, or -1 if the model does not define one. + * \brief Return the BOR (beginning of reasoning) token id. Returns an error if the model does not define one. * \param[in] tokenizer The tokenizer to read from * \param[out] token_id The BOR token id * \return OgaResult containing the error message if the call fails. @@ -753,7 +753,7 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEotTokenId(const OgaTokenizer* OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBorTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); /** - * \brief Return the EOR (end of reasoning) token id, or -1 if the model does not define one. + * \brief Return the EOR (end of reasoning) token id. Returns an error if the model does not define one. * \param[in] tokenizer The tokenizer to read from * \param[out] token_id The EOR token id * \return OgaResult containing the error message if the call fails. From b1a5aff5cc2b78f9e7454ec036eb476525d4d967 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Fri, 24 Jul 2026 12:05:47 -0700 Subject: [PATCH 13/15] Standardize fallback map keys to bot/eot/bor/eor --- src/models/model.cpp | 8 ++++---- src/models/tokenizer_tag_utils.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index 71a904c230..a98e254c65 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -325,10 +325,10 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); // Resolve any unset bot/eot/bor/eor IDs via model-type fallback strings. - if (!bot_token_id_) bot_token_id_ = ResolveFallbackTokenId(config.model.type, "tool_call_start", *this); - if (!eot_token_id_) eot_token_id_ = ResolveFallbackTokenId(config.model.type, "tool_call_end", *this); - if (!bor_token_id_) bor_token_id_ = ResolveFallbackTokenId(config.model.type, "reasoning_start", *this); - if (!eor_token_id_) eor_token_id_ = ResolveFallbackTokenId(config.model.type, "reasoning_end", *this); + if (!bot_token_id_) bot_token_id_ = ResolveFallbackTokenId(config.model.type, "bot", *this); + if (!eot_token_id_) eot_token_id_ = ResolveFallbackTokenId(config.model.type, "eot", *this); + if (!bor_token_id_) bor_token_id_ = ResolveFallbackTokenId(config.model.type, "bor", *this); + if (!eor_token_id_) eor_token_id_ = ResolveFallbackTokenId(config.model.type, "eor", *this); } int32_t Tokenizer::GetBotTokenId() const { diff --git a/src/models/tokenizer_tag_utils.cpp b/src/models/tokenizer_tag_utils.cpp index caee64b844..50688a73ee 100644 --- a/src/models/tokenizer_tag_utils.cpp +++ b/src/models/tokenizer_tag_utils.cpp @@ -10,10 +10,10 @@ std::optional ResolveFallbackTokenId(const std::string& model_type, const std::string& tag_name, const Tokenizer& tokenizer) { static const std::unordered_map> fallback_map = { - {"qwen2", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, - {"qwen3", {{"tool_call_start", ""}, {"tool_call_end", ""}, {"reasoning_start", ""}, {"reasoning_end", ""}}}, - {"phi3", {{"tool_call_start", "<|tool_call|>"}, {"tool_call_end", "<|/tool_call|>"}}}, - {"gptoss", {{"tool_call_start", "<|start|>"}, {"tool_call_end", "<|call|>"}}}, + {"qwen2", {{"bot", ""}, {"eot", ""}, {"bor", ""}, {"eor", ""}}}, + {"qwen3", {{"bot", ""}, {"eot", ""}, {"bor", ""}, {"eor", ""}}}, + {"phi3", {{"bot", "<|tool_call|>"}, {"eot", "<|/tool_call|>"}}}, + {"gptoss", {{"bot", "<|start|>"}, {"eot", "<|call|>"}}}, }; auto type_it = fallback_map.find(model_type); From b9b8b83b19a5f3863bff6af36d7c7c2ffa9f0473 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Fri, 24 Jul 2026 15:30:47 -0700 Subject: [PATCH 14/15] use fallback IDs, remove TokenToTokenId call, add Config::Defaults constants, clean up table format --- src/config.cpp | 8 ++++---- src/config.h | 8 ++++++++ src/models/tokenizer_tag_utils.cpp | 29 ++++++++++++++++++++--------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 2625f17e70..1b4f68ceec 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1181,13 +1181,13 @@ struct Model_Element : JSON::Element { v_.left_context_samples = SafeDoubleToInt(JSON::Get(value), name); } else if (name == "right_context_samples") { v_.right_context_samples = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "bot_token_id") { + } else if (name == Config::Defaults::BotTokenIdName) { v_.bot_token_id = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "eot_token_id") { + } else if (name == Config::Defaults::EotTokenIdName) { v_.eot_token_id = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "bor_token_id") { + } else if (name == Config::Defaults::BorTokenIdName) { v_.bor_token_id = SafeDoubleToInt(JSON::Get(value), name); - } else if (name == "eor_token_id") { + } else if (name == Config::Defaults::EorTokenIdName) { v_.eor_token_id = SafeDoubleToInt(JSON::Get(value), name); } else { throw JSON::unknown_value_error{}; diff --git a/src/config.h b/src/config.h index a8af9110f4..21f27e5ed6 100644 --- a/src/config.h +++ b/src/config.h @@ -85,6 +85,14 @@ struct Config { static constexpr std::string_view JoinerEncoderOutputsName = "encoder_outputs"; static constexpr std::string_view JoinerDecoderOutputsName = "decoder_outputs"; static constexpr std::string_view JoinerLogitsName = "outputs"; + + // Tool-calling and reasoning token ID config field names. + // bot = beginning of tool (call), eot = end of tool (call) + // bor = beginning of reasoning, eor = end of reasoning + static constexpr std::string_view BotTokenIdName = "bot_token_id"; + static constexpr std::string_view EotTokenIdName = "eot_token_id"; + static constexpr std::string_view BorTokenIdName = "bor_token_id"; + static constexpr std::string_view EorTokenIdName = "eor_token_id"; }; fs::path config_path; // Path of the config directory diff --git a/src/models/tokenizer_tag_utils.cpp b/src/models/tokenizer_tag_utils.cpp index 50688a73ee..ecdc2aa6e3 100644 --- a/src/models/tokenizer_tag_utils.cpp +++ b/src/models/tokenizer_tag_utils.cpp @@ -8,22 +8,33 @@ namespace Generators { std::optional ResolveFallbackTokenId(const std::string& model_type, const std::string& tag_name, - const Tokenizer& tokenizer) { - static const std::unordered_map> fallback_map = { - {"qwen2", {{"bot", ""}, {"eot", ""}, {"bor", ""}, {"eor", ""}}}, - {"qwen3", {{"bot", ""}, {"eot", ""}, {"bor", ""}, {"eor", ""}}}, - {"phi3", {{"bot", "<|tool_call|>"}, {"eot", "<|/tool_call|>"}}}, - {"gptoss", {{"bot", "<|start|>"}, {"eot", "<|call|>"}}}, + const Tokenizer& /*tokenizer*/) { + // Hardcoded fallback token IDs for models whose genai_config.json doesn't yet include + // bot/eot/bor/eor fields. Provides backward compatibility for Foundry Local when + // consuming older model packages that predate these config fields. + // + // Model type | Tag | Token string | Token ID + // ------------|-----|-----------------|-------- + // qwen2/qwen3| bot | | 151657 + // qwen2/qwen3| eot | | 151658 + // qwen2/qwen3| bor | | 151667 + // qwen2/qwen3| eor | | 151668 + // phi3 | bot | <|tool_call|> | 200025 + // phi3 | eot | <|/tool_call|> | 200026 + // clang-format off + static const std::unordered_map> fallback_map = { + {"qwen2", {{"bot", 151657}, {"eot", 151658}, {"bor", 151667}, {"eor", 151668}}}, + {"qwen3", {{"bot", 151657}, {"eot", 151658}, {"bor", 151667}, {"eor", 151668}}}, + {"phi3", {{"bot", 200025}, {"eot", 200026}}}, }; + // clang-format on auto type_it = fallback_map.find(model_type); if (type_it == fallback_map.end()) return std::nullopt; auto tag_it = type_it->second.find(tag_name); if (tag_it == type_it->second.end()) return std::nullopt; - int32_t resolved = tokenizer.TokenToTokenId(tag_it->second.c_str()); - if (resolved >= 0) return resolved; - return std::nullopt; + return tag_it->second; } } // namespace Generators From 46677b9744f4f3fb25ceca74152a72db4768e286 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Mon, 27 Jul 2026 14:21:46 -0700 Subject: [PATCH 15/15] Use Config::Defaults constants in fallback map keys for consistency --- src/models/model.cpp | 9 +++++---- src/models/tokenizer_tag_utils.cpp | 27 +++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/models/model.cpp b/src/models/model.cpp index e043c3338b..65d1dd1e3e 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -316,10 +316,11 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); // Resolve any unset bot/eot/bor/eor IDs via model-type fallback strings. - if (!bot_token_id_) bot_token_id_ = ResolveFallbackTokenId(config.model.type, "bot", *this); - if (!eot_token_id_) eot_token_id_ = ResolveFallbackTokenId(config.model.type, "eot", *this); - if (!bor_token_id_) bor_token_id_ = ResolveFallbackTokenId(config.model.type, "bor", *this); - if (!eor_token_id_) eor_token_id_ = ResolveFallbackTokenId(config.model.type, "eor", *this); + // Resolve any unset bot/eot/bor/eor IDs via model-type fallback. + if (!bot_token_id_) bot_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::BotTokenIdName), *this); + if (!eot_token_id_) eot_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::EotTokenIdName), *this); + if (!bor_token_id_) bor_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::BorTokenIdName), *this); + if (!eor_token_id_) eor_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::EorTokenIdName), *this); } int32_t Tokenizer::GetBotTokenId() const { diff --git a/src/models/tokenizer_tag_utils.cpp b/src/models/tokenizer_tag_utils.cpp index ecdc2aa6e3..c982ab5111 100644 --- a/src/models/tokenizer_tag_utils.cpp +++ b/src/models/tokenizer_tag_utils.cpp @@ -3,6 +3,7 @@ #include "tokenizer_tag_utils.h" #include "model.h" +#include "../config.h" namespace Generators { @@ -13,19 +14,21 @@ std::optional ResolveFallbackTokenId(const std::string& model_type, // bot/eot/bor/eor fields. Provides backward compatibility for Foundry Local when // consuming older model packages that predate these config fields. // - // Model type | Tag | Token string | Token ID - // ------------|-----|-----------------|-------- - // qwen2/qwen3| bot | | 151657 - // qwen2/qwen3| eot | | 151658 - // qwen2/qwen3| bor | | 151667 - // qwen2/qwen3| eor | | 151668 - // phi3 | bot | <|tool_call|> | 200025 - // phi3 | eot | <|/tool_call|> | 200026 + // Model type | Tag | Token string | Token ID + // ------------|----------------|-----------------|-------- + // qwen2/qwen3| bot_token_id | | 151657 + // qwen2/qwen3| eot_token_id | | 151658 + // qwen2/qwen3| bor_token_id | | 151667 + // qwen2/qwen3| eor_token_id | | 151668 + // phi3 | bot_token_id | <|tool_call|> | 200025 + // phi3 | eot_token_id | <|/tool_call|> | 200026 + + using D = Config::Defaults; // clang-format off - static const std::unordered_map> fallback_map = { - {"qwen2", {{"bot", 151657}, {"eot", 151658}, {"bor", 151667}, {"eor", 151668}}}, - {"qwen3", {{"bot", 151657}, {"eot", 151658}, {"bor", 151667}, {"eor", 151668}}}, - {"phi3", {{"bot", 200025}, {"eot", 200026}}}, + static const std::unordered_map> fallback_map = { + {"qwen2", {{D::BotTokenIdName, 151657}, {D::EotTokenIdName, 151658}, {D::BorTokenIdName, 151667}, {D::EorTokenIdName, 151668}}}, + {"qwen3", {{D::BotTokenIdName, 151657}, {D::EotTokenIdName, 151658}, {D::BorTokenIdName, 151667}, {D::EorTokenIdName, 151668}}}, + {"phi3", {{D::BotTokenIdName, 200025}, {D::EotTokenIdName, 200026}}}, }; // clang-format on