diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index eb76ef94d..055a51954 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -524,7 +524,10 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // tool-call markers configured, both marker strings are empty and the accumulator degrades to passthrough. // REASONING segments bypass the accumulator entirely — tool-call-shaped text inside ... is the // model's scratchpad and is not a real tool call. - ToolCallStreamAccumulator tool_accumulator(cached_tool_ctx_.tool_call_start, cached_tool_ctx_.tool_call_end); + auto tool_call_config = ToolCallConfig::FromModelType( + model_.GetGenAIConfig().model ? model_.GetGenAIConfig().model->type : ""); + ToolCallStreamAccumulator tool_accumulator(cached_tool_ctx_.tool_call_start, cached_tool_ctx_.tool_call_end, + std::move(tool_call_config)); // Tool calls parsed during streaming. Reused by ProcessGeneratedOutput so call_ids stay stable across stream // deltas and the final response (OpenAI Chat Completions contract). Populated even when there is no streaming @@ -719,7 +722,10 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co // qwen tokenizer happens to emit `` as a single special token. Tokenizers that split the marker across // multiple tokens (or chat templates that produce marker-shaped text gradually) would silently fail. The shared // accumulator buffers across tokens and is verified by unit tests. - ToolCallStreamAccumulator tool_accumulator(tool_ctx.tool_call_start, tool_ctx.tool_call_end); + auto tool_call_config = ToolCallConfig::FromModelType( + model_.GetGenAIConfig().model ? model_.GetGenAIConfig().model->type : ""); + ToolCallStreamAccumulator tool_accumulator(tool_ctx.tool_call_start, tool_ctx.tool_call_end, + std::move(tool_call_config)); // Tool calls parsed during streaming. Reused by ProcessGeneratedOutput so call_ids stay stable across stream // deltas and the final ChatCompletionResponse (OpenAI Chat Completions contract). diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h new file mode 100644 index 000000000..bbd0fdf36 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_config.h @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include + +namespace fl { + +/// Data-driven configuration for tool-call parsing. +/// All format-specific knowledge is captured as data properties — no model-specific +/// code branches. A single accumulator interprets the config at runtime. +struct ToolCallConfig { + enum class Mode { + kSimple, // ChatML: bot → JSON body (has name+args) → eot + kHeaderInspection // Harmony: bot → header → message_token → args JSON → end_token + }; + + Mode mode = Mode::kSimple; + + // --- Header-inspection mode properties --- + + /// Token that separates the header from the message body (e.g., "<|message|>") + std::string message_token; + + /// Regex to match the header and extract the function name. + /// Capture group 1 = function name (e.g., "to=functions\\.(.+)" captures "get_weather") + std::string header_regex; + + /// Token between header routing and channel info (e.g., "<|channel|>") + /// Used to trim the header before regex matching. + std::string channel_token; + + /// All tokens that terminate a tool-call block. + /// For Harmony: both <|end|> (intermediate) and <|call|> (final/EOS) terminate. + /// For Simple mode: this is just {eot_marker} (populated from context). + std::vector end_tokens; + + // --- Factory methods --- + + /// Default config for ChatML-style models (Phi, Qwen). + /// Markers come from ToolCallContext; this just sets mode = kSimple. + static ToolCallConfig Simple() { + ToolCallConfig cfg; + cfg.mode = Mode::kSimple; + return cfg; + } + + /// Config for GPT-OSS (Harmony) models. + static ToolCallConfig Harmony() { + ToolCallConfig cfg; + cfg.mode = Mode::kHeaderInspection; + cfg.message_token = "<|message|>"; + cfg.header_regex = R"(to=functions\.(.+))"; + cfg.channel_token = "<|channel|>"; + cfg.end_tokens = {"<|end|>", "<|call|>"}; + return cfg; + } + + /// Infer config from model type string (from genai_config.json "model.type"). + static ToolCallConfig FromModelType(const std::string& model_type) { + if (model_type == "gptoss") return Harmony(); + return Simple(); + } +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h index d8233ab15..b0b6a4c35 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h @@ -2,9 +2,11 @@ // Licensed under the MIT License. #pragma once +#include "inferencing/generative/toolcalling/tool_call_config.h" #include "inferencing/generative/toolcalling/tool_call_utils.h" #include +#include #include #include #include @@ -48,6 +50,13 @@ class ToolCallStreamAccumulator { ToolCallStreamAccumulator(std::string start_marker, std::string end_marker) : start_marker_(std::move(start_marker)), end_marker_(std::move(end_marker)) {} + /// Construct with a ToolCallConfig for header-inspection (Harmony) or simple (ChatML) mode. + /// In header-inspection mode, start_marker is the bot marker and end_tokens come from config. + ToolCallStreamAccumulator(std::string start_marker, std::string end_marker, ToolCallConfig config) + : start_marker_(std::move(start_marker)), + end_marker_(std::move(end_marker)), + config_(std::move(config)) {} + /// Feed a chunk into the accumulator. Returns visible text and any tool calls completed by this chunk. Output Push(const std::string& chunk) { Output out; @@ -63,7 +72,12 @@ class ToolCallStreamAccumulator { } buffer_ += chunk; - Drain(out, /*flushing=*/false); + + if (config_.mode == ToolCallConfig::Mode::kHeaderInspection) { + DrainHeaderInspection(out, /*flushing=*/false); + } else { + Drain(out, /*flushing=*/false); + } return out; } @@ -77,7 +91,11 @@ class ToolCallStreamAccumulator { return out; } - Drain(out, /*flushing=*/true); + if (config_.mode == ToolCallConfig::Mode::kHeaderInspection) { + DrainHeaderInspection(out, /*flushing=*/true); + } else { + Drain(out, /*flushing=*/true); + } return out; } @@ -180,6 +198,162 @@ class ToolCallStreamAccumulator { std::string buffer_; // pending bytes from Push() that haven't yet been routed std::string tool_call_buffer_; // accumulated bytes of the in-progress tool-call block (incl. start marker) bool inside_tool_call_ = false; + + ToolCallConfig config_; // default = Simple mode + + // --- Header-inspection mode state --- + enum class HState { Idle, InHeader, InBody }; + HState hstate_ = HState::Idle; + std::string header_buffer_; // header text between bot marker and message_token + std::string body_buffer_; // body text between message_token and end_token + std::string extracted_name_; // function name extracted from header regex + + /// Header-inspection state machine for Harmony-style models. + /// States: Idle → InHeader → InBody → (emit) → Idle + void DrainHeaderInspection(Output& out, bool flushing) { + while (true) { + switch (hstate_) { + case HState::Idle: { + // Look for the start marker (bot token) + size_t found = buffer_.find(start_marker_); + if (found != std::string::npos) { + // Emit everything before the marker as visible text + if (found > 0) { + out.visible_text.append(buffer_, 0, found); + } + buffer_.erase(0, found + start_marker_.size()); + hstate_ = HState::InHeader; + header_buffer_.clear(); + continue; + } + + if (flushing) { + out.visible_text.append(buffer_); + buffer_.clear(); + return; + } + + // Hold back suffix that could grow into start_marker + size_t hold = LongestSuffixThatIsPrefixOf(buffer_, start_marker_); + size_t safe = buffer_.size() - hold; + if (safe > 0) { + out.visible_text.append(buffer_, 0, safe); + buffer_.erase(0, safe); + } + return; + } + + case HState::InHeader: { + // Accumulate until we see the message_token + size_t msg_pos = buffer_.find(config_.message_token); + if (msg_pos != std::string::npos) { + header_buffer_.append(buffer_, 0, msg_pos); + buffer_.erase(0, msg_pos + config_.message_token.size()); + + // Trim header at channel_token if present (e.g., "assistant to=functions.foo<|channel|>default") + std::string header_to_match = header_buffer_; + if (!config_.channel_token.empty()) { + size_t ch_pos = header_to_match.find(config_.channel_token); + if (ch_pos != std::string::npos) { + header_to_match = header_to_match.substr(0, ch_pos); + } + } + + // Try to extract function name from header + std::smatch match; + std::regex re(config_.header_regex); + if (std::regex_search(header_to_match, match, re) && match.size() > 1) { + // This is a tool call — extract name, transition to InBody + extracted_name_ = match[1].str(); + body_buffer_.clear(); + hstate_ = HState::InBody; + } else { + // Not a tool call — this is regular assistant text. + // Emit the header content as visible text and look for end token to return to Idle. + out.visible_text.append(header_buffer_); + out.visible_text.append(config_.message_token); + header_buffer_.clear(); + hstate_ = HState::Idle; + } + continue; + } + + if (flushing) { + // Unterminated header — emit as visible text + out.visible_text.append(start_marker_); + out.visible_text.append(header_buffer_); + out.visible_text.append(buffer_); + header_buffer_.clear(); + buffer_.clear(); + hstate_ = HState::Idle; + return; + } + + // Buffer everything — header not complete yet + header_buffer_.append(buffer_); + buffer_.clear(); + return; + } + + case HState::InBody: { + // Accumulate until we see any end token + size_t best_pos = std::string::npos; + size_t best_len = 0; + for (const auto& end_tok : config_.end_tokens) { + size_t pos = buffer_.find(end_tok); + if (pos != std::string::npos && (best_pos == std::string::npos || pos < best_pos)) { + best_pos = pos; + best_len = end_tok.size(); + } + } + + if (best_pos != std::string::npos) { + // Found end token — finalize the tool call + body_buffer_.append(buffer_, 0, best_pos); + buffer_.erase(0, best_pos + best_len); + + // Emit as a parsed tool call + ParsedToolCall call; + call.name = extracted_name_; + call.arguments = body_buffer_; + call.id = GenerateToolCallId(); + out.ready_calls.push_back(std::move(call)); + + body_buffer_.clear(); + extracted_name_.clear(); + hstate_ = HState::Idle; + continue; + } + + if (flushing) { + // Unterminated body — emit everything as visible text (not a valid tool call) + out.visible_text.append(start_marker_); + out.visible_text.append(header_buffer_); + out.visible_text.append(config_.message_token); + out.visible_text.append(body_buffer_); + out.visible_text.append(buffer_); + header_buffer_.clear(); + body_buffer_.clear(); + buffer_.clear(); + hstate_ = HState::Idle; + return; + } + + // Buffer body content, hold back suffix that could be an end token prefix + size_t hold = 0; + for (const auto& end_tok : config_.end_tokens) { + hold = std::max(hold, LongestSuffixThatIsPrefixOf(buffer_, end_tok)); + } + size_t safe = buffer_.size() - hold; + if (safe > 0) { + body_buffer_.append(buffer_, 0, safe); + buffer_.erase(0, safe); + } + return; + } + } + } + } }; } // namespace fl diff --git a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc index 7c607723f..2b3a2a0fb 100644 --- a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc +++ b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc @@ -236,3 +236,110 @@ TEST(ToolCallStreamAccumulatorTest, FalseStartPrefixReleasesAfterDisambiguation) EXPECT_EQ(out2.visible_text, ""); EXPECT_TRUE(out2.ready_calls.empty()); } + +// ======================================================================== +// Header-inspection mode (Harmony / GPT-OSS protocol) +// ======================================================================== + +TEST(ToolCallStreamAccumulatorTest, HarmonyBasicToolCall) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + auto out = acc.Push("<|start|>assistant to=functions.get_weather<|channel|>default<|message|>{\"city\": \"Seattle\"}<|call|>"); + EXPECT_EQ(out.visible_text, ""); + ASSERT_EQ(out.ready_calls.size(), 1u); + EXPECT_EQ(out.ready_calls[0].name, "get_weather"); + EXPECT_EQ(out.ready_calls[0].arguments, "{\"city\": \"Seattle\"}"); + EXPECT_FALSE(out.ready_calls[0].id.empty()); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyToolCallSplitAcrossChunks) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + std::vector chunks = { + "<|start|>", + "assistant to=functions.multiply", + "<|channel|>default", + "<|message|>", + "{\"a\": 7, \"b\": 6}", + "<|call|>"}; + + auto outs = RunChunks(acc, chunks); + std::string visible = CollectVisible(outs); + EXPECT_EQ(visible, ""); + + // Collect all ready_calls across all outputs + std::vector all_calls; + for (const auto& o : outs) { + for (const auto& c : o.ready_calls) { + all_calls.push_back(c); + } + } + ASSERT_EQ(all_calls.size(), 1u); + EXPECT_EQ(all_calls[0].name, "multiply"); + EXPECT_EQ(all_calls[0].arguments, "{\"a\": 7, \"b\": 6}"); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyRegularTextNotToolCall) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + // Regular assistant text: header doesn't match "to=functions.X" pattern + auto out = acc.Push("<|start|>assistant<|channel|>default<|message|>Hello, how can I help?<|end|>"); + // The header doesn't match → emitted as visible text (header + message_token + body until we see end) + // Note: <|end|> is an end_token in config, so body terminates there + // But since header didn't match, it went back to Idle and this is all visible + EXPECT_FALSE(out.visible_text.empty()); + EXPECT_TRUE(out.ready_calls.empty()); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyParallelToolCalls) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + // Two tool calls: first ends with <|end|> (intermediate), second with <|call|> (final) + std::string input = + "<|start|>assistant to=functions.func1<|channel|>default<|message|>{\"x\": 1}<|end|>" + "<|start|>assistant to=functions.func2<|channel|>default<|message|>{\"x\": 2}<|call|>"; + + auto out = acc.Push(input); + EXPECT_EQ(out.visible_text, ""); + ASSERT_EQ(out.ready_calls.size(), 2u); + EXPECT_EQ(out.ready_calls[0].name, "func1"); + EXPECT_EQ(out.ready_calls[0].arguments, "{\"x\": 1}"); + EXPECT_EQ(out.ready_calls[1].name, "func2"); + EXPECT_EQ(out.ready_calls[1].arguments, "{\"x\": 2}"); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyTextBeforeToolCall) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + auto out = acc.Push("Some preamble text<|start|>assistant to=functions.foo<|channel|>default<|message|>{}<|call|>"); + EXPECT_EQ(out.visible_text, "Some preamble text"); + ASSERT_EQ(out.ready_calls.size(), 1u); + EXPECT_EQ(out.ready_calls[0].name, "foo"); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyFlushUnterminatedHeader) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + acc.Push("<|start|>assistant to=functions.bar"); + auto flush_out = acc.Flush(); + // Unterminated → surface as visible text + EXPECT_FALSE(flush_out.visible_text.empty()); + EXPECT_TRUE(flush_out.ready_calls.empty()); +} + +TEST(ToolCallStreamAccumulatorTest, HarmonyFlushUnterminatedBody) { + auto config = ToolCallConfig::Harmony(); + ToolCallStreamAccumulator acc("<|start|>", "<|call|>", config); + + acc.Push("<|start|>assistant to=functions.bar<|channel|>default<|message|>{\"partial\": true"); + auto flush_out = acc.Flush(); + // Unterminated body → surface everything as visible text + EXPECT_FALSE(flush_out.visible_text.empty()); + EXPECT_TRUE(flush_out.ready_calls.empty()); +}