diff --git a/src/config.cpp b/src/config.cpp index ffd6ecbdfd..be4e72d16b 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1181,6 +1181,14 @@ struct Model_Element : JSON::Element { v_.left_context_samples = SafeDoubleToInt(JSON::Get(value), name); } else if (name == "right_context_samples") { v_.right_context_samples = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == Config::Defaults::BotTokenIdName) { + v_.bot_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == Config::Defaults::EotTokenIdName) { + v_.eot_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == Config::Defaults::BorTokenIdName) { + v_.bor_token_id = SafeDoubleToInt(JSON::Get(value), name); + } else if (name == Config::Defaults::EorTokenIdName) { + v_.eor_token_id = SafeDoubleToInt(JSON::Get(value), name); } else { throw JSON::unknown_value_error{}; } diff --git a/src/config.h b/src/config.h index bcb547315a..dbecf546af 100644 --- a/src/config.h +++ b/src/config.h @@ -88,6 +88,14 @@ struct Config { static constexpr std::string_view JoinerEncoderOutputsName = "encoder_outputs"; static constexpr std::string_view JoinerDecoderOutputsName = "decoder_outputs"; static constexpr std::string_view JoinerLogitsName = "outputs"; + + // Tool-calling and reasoning token ID config field names. + // bot = beginning of tool (call), eot = end of tool (call) + // bor = beginning of reasoning, eor = end of reasoning + static constexpr std::string_view BotTokenIdName = "bot_token_id"; + static constexpr std::string_view EotTokenIdName = "eot_token_id"; + static constexpr std::string_view BorTokenIdName = "bor_token_id"; + static constexpr std::string_view EorTokenIdName = "eor_token_id"; }; fs::path config_path; // Path of the config directory @@ -188,6 +196,15 @@ struct Config { int video_token_id{}; int vision_start_token_id{}; + // Tool-calling and reasoning token IDs. + // Follows the bos/eos/pad naming convention: + // bot = beginning of tool (call), eot = end of tool (call) + // bor = beginning of reasoning, eor = end of reasoning + std::optional bot_token_id; + std::optional eot_token_id; + std::optional bor_token_id; + std::optional eor_token_id; + int vocab_size{}; int context_length{}; diff --git a/src/csharp/NativeMethods.cs b/src/csharp/NativeMethods.cs index a7b30fa9c0..9227b3749e 100644 --- a/src/csharp/NativeMethods.cs +++ b/src/csharp/NativeMethods.cs @@ -268,6 +268,22 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq public static extern IntPtr /* OgaResult* */ OgaTokenizerGetPadTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, out int /* const int32_t* */ outPadTokenId); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetBotTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outBotTokenId); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetEotTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outEotTokenId); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetBorTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outBorTokenId); + + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OgaResult* */ OgaTokenizerGetEorTokenId(IntPtr /* const OgaTokenizer* */ tokenizer, + out int /* const int32_t* */ outEorTokenId); + [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] public static extern IntPtr /* OgaResult* */ OgaTokenizerEncode(IntPtr /* const OgaTokenizer* */ tokenizer, byte[] /* const char* */ strings, diff --git a/src/csharp/Tokenizer.cs b/src/csharp/Tokenizer.cs index 0c11aee389..d9f9373f2e 100644 --- a/src/csharp/Tokenizer.cs +++ b/src/csharp/Tokenizer.cs @@ -141,6 +141,42 @@ public int GetPadTokenId() return padTokenId; } + /// + /// Returns the BOT (beginning of tool call) token ID, or -1 if not defined. + /// + public int GetBotTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetBotTokenId(_tokenizerHandle, out int botTokenId)); + return botTokenId; + } + + /// + /// Returns the EOT (end of tool call) token ID, or -1 if not defined. + /// + public int GetEotTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetEotTokenId(_tokenizerHandle, out int eotTokenId)); + return eotTokenId; + } + + /// + /// Returns the BOR (beginning of reasoning) token ID, or -1 if not defined. + /// + public int GetBorTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetBorTokenId(_tokenizerHandle, out int borTokenId)); + return borTokenId; + } + + /// + /// Returns the EOR (end of reasoning) token ID, or -1 if not defined. + /// + public int GetEorTokenId() + { + Result.VerifySuccess(NativeMethods.OgaTokenizerGetEorTokenId(_tokenizerHandle, out int eorTokenId)); + return eorTokenId; + } + public TokenizerStream CreateStream() { IntPtr tokenizerStreamHandle = IntPtr.Zero; diff --git a/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java b/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java index 1634c2c2d5..a404cf04c4 100644 --- a/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java +++ b/src/java/src/main/java/ai/onnxruntime/genai/Tokenizer.java @@ -107,6 +107,62 @@ public int getPadTokenId() throws GenAIException { return tokenizerGetPadTokenId(nativeHandle); } + /** + * Gets the BOT (beginning of tool call) token ID, or -1 if the model does not define one. + * + * @return The BOT token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getBotTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetBotTokenId(nativeHandle); + } + + /** + * Gets the EOT (end of tool call) token ID, or -1 if the model does not define one. + * + * @return The EOT token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getEotTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetEotTokenId(nativeHandle); + } + + /** + * Gets the BOR (beginning of reasoning) token ID, or -1 if the model does not define one. + * + * @return The BOR token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getBorTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetBorTokenId(nativeHandle); + } + + /** + * Gets the EOR (end of reasoning) token ID, or -1 if the model does not define one. + * + * @return The EOR token ID. + * @throws GenAIException If the call to the GenAI native API fails. + */ + public int getEorTokenId() throws GenAIException { + if (nativeHandle == 0) { + throw new IllegalStateException("Instance has been freed and is invalid"); + } + + return tokenizerGetEorTokenId(nativeHandle); + } + /** * Gets the end of sentence token IDs. * @@ -229,6 +285,14 @@ public void close() { private native int[] tokenizerGetEosTokenIds(long tokenizerHandle) throws GenAIException; + private native int tokenizerGetBotTokenId(long tokenizerHandle) throws GenAIException; + + private native int tokenizerGetEotTokenId(long tokenizerHandle) throws GenAIException; + + private native int tokenizerGetBorTokenId(long tokenizerHandle) throws GenAIException; + + private native int tokenizerGetEorTokenId(long tokenizerHandle) throws GenAIException; + private native int tokenizerToTokenId(long tokenizerHandle, String str) throws GenAIException; private native String tokenizerApplyChatTemplate( diff --git a/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp b/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp index 8eba53e0a5..1ae8ea9582 100644 --- a/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp +++ b/src/java/src/main/native/ai_onnxruntime_genai_Tokenizer.cpp @@ -110,6 +110,54 @@ Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetPadTokenId(JNIEnv* env, jobject return static_cast(token_id); } +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetBotTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetBotTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetEotTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetEotTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetBorTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetBorTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + +JNIEXPORT jint JNICALL +Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetEorTokenId(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { + const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); + int32_t token_id = 0; + + if (ThrowIfError(env, OgaTokenizerGetEorTokenId(tokenizer, &token_id))) { + return 0; + } + + return static_cast(token_id); +} + JNIEXPORT jintArray JNICALL Java_ai_onnxruntime_genai_Tokenizer_tokenizerGetEosTokenIds(JNIEnv* env, jobject thiz, jlong tokenizer_handle) { const OgaTokenizer* tokenizer = reinterpret_cast(tokenizer_handle); diff --git a/src/models/model.cpp b/src/models/model.cpp index 24e4c597a8..65d1dd1e3e 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -6,17 +6,20 @@ #include #include #include +#include #include #include #include #include #include +#include #include "../generators.h" #include "../search.h" #include "../tracing.h" #include "model.h" #include "model_package.h" +#include "tokenizer_tag_utils.h" #include "gpt.h" #include "decoder_only.h" #include "whisper.h" @@ -299,7 +302,11 @@ const std::string& TokenizerStream::Decode(int32_t token) { Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, eos_token_id_{config.model.eos_token_id}, - pad_token_id_{config.model.pad_token_id} { + pad_token_id_{config.model.pad_token_id}, + bot_token_id_{config.model.bot_token_id}, + eot_token_id_{config.model.eot_token_id}, + bor_token_id_{config.model.bor_token_id}, + eor_token_id_{config.model.eor_token_id} { // Default tokenizer options const char* keys[] = {"add_special_tokens", "skip_special_tokens"}; const char* values[] = {"false", "true"}; @@ -307,6 +314,33 @@ Tokenizer::Tokenizer(Config& config) : bos_token_id_{config.model.bos_token_id}, // Resolve tokenizer_dir (may be empty, relative, absolute, or a "sha256:" shared-asset reference). const fs::path tokenizer_dir = config.ResolvePath(config.model.tokenizer_dir); CheckResult(OrtxCreateTokenizerWithOptions(tokenizer_.Address(), tokenizer_dir.string().c_str(), keys, values, 2)); + + // Resolve any unset bot/eot/bor/eor IDs via model-type fallback strings. + // Resolve any unset bot/eot/bor/eor IDs via model-type fallback. + if (!bot_token_id_) bot_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::BotTokenIdName), *this); + if (!eot_token_id_) eot_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::EotTokenIdName), *this); + if (!bor_token_id_) bor_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::BorTokenIdName), *this); + if (!eor_token_id_) eor_token_id_ = ResolveFallbackTokenId(config.model.type, std::string(Config::Defaults::EorTokenIdName), *this); +} + +int32_t Tokenizer::GetBotTokenId() const { + if (!bot_token_id_) throw std::runtime_error("bot_token_id is not defined for this model"); + return *bot_token_id_; +} + +int32_t Tokenizer::GetEotTokenId() const { + if (!eot_token_id_) throw std::runtime_error("eot_token_id is not defined for this model"); + return *eot_token_id_; +} + +int32_t Tokenizer::GetBorTokenId() const { + if (!bor_token_id_) throw std::runtime_error("bor_token_id is not defined for this model"); + return *bor_token_id_; +} + +int32_t Tokenizer::GetEorTokenId() const { + if (!eor_token_id_) throw std::runtime_error("eor_token_id is not defined for this model"); + return *eor_token_id_; } std::unique_ptr Tokenizer::CreateStream() const { diff --git a/src/models/model.h b/src/models/model.h index 680197a1ef..022421ce2d 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -8,6 +8,7 @@ #include "ortx_tokenizer.h" #include "../generators.h" #include "utils.h" +#include #include "phi_image_processor.h" #include "whisper_processor.h" #include "parakeet_processor.h" @@ -106,12 +107,26 @@ struct Tokenizer : std::enable_shared_from_this, LeakChecked& GetEosTokenIds() const { return eos_token_id_; } int32_t GetPadTokenId() const { return pad_token_id_; } + // Tool-calling and reasoning token IDs. + // Naming follows the bos/eos/pad convention: + // bot = beginning of tool (call), eot = end of tool (call) + // bor = beginning of reasoning, eor = end of reasoning + // Throws if the model does not define the requested token. + int32_t GetBotTokenId() const; + int32_t GetEotTokenId() const; + int32_t GetBorTokenId() const; + int32_t GetEorTokenId() const; + OrtxPtr tokenizer_; private: int32_t bos_token_id_; std::vector eos_token_id_; int32_t pad_token_id_; + std::optional bot_token_id_; + std::optional eot_token_id_; + std::optional bor_token_id_; + std::optional eor_token_id_; }; struct MultiModalProcessor : std::enable_shared_from_this, ExternalRefCounted { diff --git a/src/models/tokenizer_tag_utils.cpp b/src/models/tokenizer_tag_utils.cpp new file mode 100644 index 0000000000..c982ab5111 --- /dev/null +++ b/src/models/tokenizer_tag_utils.cpp @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "tokenizer_tag_utils.h" +#include "model.h" +#include "../config.h" + +namespace Generators { + +std::optional ResolveFallbackTokenId(const std::string& model_type, + const std::string& tag_name, + const Tokenizer& /*tokenizer*/) { + // Hardcoded fallback token IDs for models whose genai_config.json doesn't yet include + // bot/eot/bor/eor fields. Provides backward compatibility for Foundry Local when + // consuming older model packages that predate these config fields. + // + // Model type | Tag | Token string | Token ID + // ------------|----------------|-----------------|-------- + // qwen2/qwen3| bot_token_id | | 151657 + // qwen2/qwen3| eot_token_id | | 151658 + // qwen2/qwen3| bor_token_id | | 151667 + // qwen2/qwen3| eor_token_id | | 151668 + // phi3 | bot_token_id | <|tool_call|> | 200025 + // phi3 | eot_token_id | <|/tool_call|> | 200026 + + using D = Config::Defaults; + // clang-format off + static const std::unordered_map> fallback_map = { + {"qwen2", {{D::BotTokenIdName, 151657}, {D::EotTokenIdName, 151658}, {D::BorTokenIdName, 151667}, {D::EorTokenIdName, 151668}}}, + {"qwen3", {{D::BotTokenIdName, 151657}, {D::EotTokenIdName, 151658}, {D::BorTokenIdName, 151667}, {D::EorTokenIdName, 151668}}}, + {"phi3", {{D::BotTokenIdName, 200025}, {D::EotTokenIdName, 200026}}}, + }; + // clang-format on + + auto type_it = fallback_map.find(model_type); + if (type_it == fallback_map.end()) return std::nullopt; + auto tag_it = type_it->second.find(tag_name); + if (tag_it == type_it->second.end()) return std::nullopt; + + return tag_it->second; +} + +} // namespace Generators diff --git a/src/models/tokenizer_tag_utils.h b/src/models/tokenizer_tag_utils.h new file mode 100644 index 0000000000..b87900f795 --- /dev/null +++ b/src/models/tokenizer_tag_utils.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include + +namespace Generators { + +struct Tokenizer; + +// Resolves a fallback token ID for models whose genai_config.json doesn't yet include +// bot/eot/bor/eor token IDs in the model section. This exists specifically for +// Foundry Local backward compatibility with older model packages that predate +// these config fields. +// +// Returns the resolved token ID if found in the fallback map and tokenizer vocabulary, +// or std::nullopt if the model type/tag name is not in the map or the token string +// doesn't resolve in the vocabulary. +std::optional ResolveFallbackTokenId(const std::string& model_type, + const std::string& tag_name, + const Tokenizer& tokenizer); + +} // namespace Generators diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h index c3f23655c7..94458159c8 100644 --- a/src/objectivec/include/ort_genai_objc.h +++ b/src/objectivec/include/ort_genai_objc.h @@ -167,6 +167,26 @@ typedef NS_ENUM(NSInteger, OGAElementType) { */ - (int32_t)getPadTokenId:(NSError**)error; +/** + * Return the BOT (beginning of tool call) token ID. Returns an error if not defined. + */ +- (int32_t)getBotTokenId:(NSError**)error; + +/** + * Return the EOT (end of tool call) token ID. Returns an error if not defined. + */ +- (int32_t)getEotTokenId:(NSError**)error; + +/** + * Return the BOR (beginning of reasoning) token ID. Returns an error if not defined. + */ +- (int32_t)getBorTokenId:(NSError**)error; + +/** + * Return the EOR (end of reasoning) token ID. Returns an error if not defined. + */ +- (int32_t)getEorTokenId:(NSError**)error; + /** * Encode text to sequences * diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm index a3eb27ffd2..08fa35e0da 100644 --- a/src/objectivec/oga_tokenizer.mm +++ b/src/objectivec/oga_tokenizer.mm @@ -50,6 +50,34 @@ - (int32_t)getPadTokenId:(NSError**)error { OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) } +- (int32_t)getBotTokenId:(NSError**)error { + try { + return _tokenizer->GetBotTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + +- (int32_t)getEotTokenId:(NSError**)error { + try { + return _tokenizer->GetEotTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + +- (int32_t)getBorTokenId:(NSError**)error { + try { + return _tokenizer->GetBorTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + +- (int32_t)getEorTokenId:(NSError**)error { + try { + return _tokenizer->GetEorTokenId(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_INT32_T(error) +} + - (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { OGASequences* sequences = [[OGASequences alloc] initWithError:error]; if (!sequences) { diff --git a/src/ort_genai.h b/src/ort_genai.h index 15d9af9d0a..e09aa96ef9 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -342,6 +342,32 @@ struct OgaTokenizer : OgaAbstract { return token_id; } + // Tool-calling and reasoning token IDs (bot/eot/bor/eor). + // Throws if the model does not define the token. + int32_t GetBotTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetBotTokenId(this, &token_id)); + return token_id; + } + + int32_t GetEotTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetEotTokenId(this, &token_id)); + return token_id; + } + + int32_t GetBorTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetBorTokenId(this, &token_id)); + return token_id; + } + + int32_t GetEorTokenId() const { + int32_t token_id; + OgaCheckResult(OgaTokenizerGetEorTokenId(this, &token_id)); + return token_id; + } + void Encode(const char* str, OgaSequences& sequences) const { OgaCheckResult(OgaTokenizerEncode(this, str, &sequences)); } diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index 7c1379ca1e..3dd40a179e 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -684,6 +684,34 @@ OgaResult* OGA_API_CALL OgaTokenizerGetPadTokenId(const OgaTokenizer* tokenizer, OGA_CATCH } +OgaResult* OGA_API_CALL OgaTokenizerGetBotTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetBotTokenId(); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaTokenizerGetEotTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetEotTokenId(); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaTokenizerGetBorTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetBorTokenId(); + return nullptr; + OGA_CATCH +} + +OgaResult* OGA_API_CALL OgaTokenizerGetEorTokenId(const OgaTokenizer* tokenizer, int32_t* out) { + OGA_TRY + *out = tokenizer->GetEorTokenId(); + return nullptr; + OGA_CATCH +} + OgaResult* OGA_API_CALL OgaTokenizerEncode(const OgaTokenizer* tokenizer, const char* str, OgaSequences* sequences) { OGA_TRY sequences->emplace_back(tokenizer->Encode(str)); diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index c77fcffb05..962f8b6b2f 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -728,6 +728,38 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEosTokenIds(const OgaTokenizer */ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetPadTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); +/** + * \brief Return the BOT (beginning of tool call) token id. Returns an error if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The BOT token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBotTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + +/** + * \brief Return the EOT (end of tool call) token id. Returns an error if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The EOT token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEotTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + +/** + * \brief Return the BOR (beginning of reasoning) token id. Returns an error if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The BOR token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetBorTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + +/** + * \brief Return the EOR (end of reasoning) token id. Returns an error if the model does not define one. + * \param[in] tokenizer The tokenizer to read from + * \param[out] token_id The EOR token id + * \return OgaResult containing the error message if the call fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerGetEorTokenId(const OgaTokenizer* tokenizer, int32_t* token_id); + /** * Encodes a single string and adds the encoded sequence of tokens to the OgaSequences. The OgaSequences must be freed with OgaDestroySequences * when it is no longer needed. diff --git a/src/python/python.cpp b/src/python/python.cpp index c1f42b52d6..22be160170 100644 --- a/src/python/python.cpp +++ b/src/python/python.cpp @@ -394,6 +394,10 @@ PYBIND11_MODULE(onnxruntime_genai, m) { return ToPython(t.GetEosTokenIds()); }) .def_property_readonly("pad_token_id", &OgaTokenizer::GetPadTokenId) + .def_property_readonly("bot_token_id", &OgaTokenizer::GetBotTokenId) + .def_property_readonly("eot_token_id", &OgaTokenizer::GetEotTokenId) + .def_property_readonly("bor_token_id", &OgaTokenizer::GetBorTokenId) + .def_property_readonly("eor_token_id", &OgaTokenizer::GetEorTokenId) .def("update_options", [](OgaTokenizer& t, pybind11::kwargs kwargs) { std::vector key_storage; std::vector value_storage; diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index ff57b4003d..1391d1aeb1 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -1999,6 +1999,74 @@ TEST(CAPITests, ParakeetTdtTranscribeLong) { EXPECT_FALSE(transcription.empty()); } +// Test that bot/eot/bor/eor throw for models without these tokens configured +TEST(CAPITests, TokenId_Unsupported) { + // tiny-random-gpt2 model has type "gpt2" which is NOT in the fallback map → throws + auto model = OgaModel::Create(MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"); + auto tokenizer = OgaTokenizer::Create(*model); + + EXPECT_THROW(tokenizer->GetBotTokenId(), std::runtime_error); + EXPECT_THROW(tokenizer->GetEotTokenId(), std::runtime_error); + EXPECT_THROW(tokenizer->GetBorTokenId(), std::runtime_error); + EXPECT_THROW(tokenizer->GetEorTokenId(), std::runtime_error); +} + +TEST(CAPITests, TokenId_FromConfig) { + // Create a temporary model directory with bot/eot/bor/eor token IDs in model section + auto temp_dir = std::filesystem::temp_directory_path() / "oga_test_tool_tags"; + std::filesystem::remove_all(temp_dir); // Clean up any leftover from a previous failed run + std::filesystem::create_directories(temp_dir); + + // Copy minimal model files from tiny-random-gpt2 + std::string src_dir = MODEL_PATH "hf-internal-testing/tiny-random-gpt2-fp32"; + for (const auto& entry : std::filesystem::directory_iterator(src_dir)) { + if (entry.path().filename() != "genai_config.json") { + std::filesystem::copy_file(entry.path(), temp_dir / entry.path().filename(), + std::filesystem::copy_options::overwrite_existing); + } + } + + // Write genai_config.json with token IDs in model section + { + std::ofstream f((temp_dir / "genai_config.json").string()); + f << R"({ + "model": { + "type": "gpt2", + "pad_token_id": 98, + "bos_token_id": 98, + "eos_token_id": 98, + "vocab_size": 1000, + "context_length": 512, + "bot_token_id": 151657, + "eot_token_id": 151658, + "bor_token_id": 151659, + "eor_token_id": 151660, + "decoder": { + "session_options": { "provider_options": [] }, + "filename": "past.onnx", + "num_key_value_heads": 4, + "head_size": 8, + "num_hidden_layers": 5, + "inputs": { "past_names": "past_%d" }, + "outputs": { "present_names": "present_%d" } + } + } +})"; + } + + auto model = OgaModel::Create(temp_dir.string().c_str()); + auto tokenizer = OgaTokenizer::Create(*model); + + // Tokenizer returns configured IDs from model section + EXPECT_EQ(tokenizer->GetBotTokenId(), 151657); + EXPECT_EQ(tokenizer->GetEotTokenId(), 151658); + EXPECT_EQ(tokenizer->GetBorTokenId(), 151659); + EXPECT_EQ(tokenizer->GetEorTokenId(), 151660); + + // Cleanup + std::filesystem::remove_all(temp_dir); +} + // Regression test for MSRC: malformed audio buffers smaller than the minimum valid // audio header size must be rejected with an error, not cause a crash. TEST(CAPITests, LoadAudiosFromBuffersRejectsEmptyBuffer) {