diff --git a/.github/workflows/linux-cpu-arm64-build.yml b/.github/workflows/linux-cpu-arm64-build.yml index ac454b9eb1..f49ffc8704 100644 --- a/.github/workflows/linux-cpu-arm64-build.yml +++ b/.github/workflows/linux-cpu-arm64-build.yml @@ -72,7 +72,7 @@ jobs: - name: Download Docker Image run: | set -e -x - az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4 + az login --identity --object-id c90de106-42dc-405a-8bad-2438f4279448 az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87 python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile \ --context tools/ci_build/github/linux/docker/inference/aarch64/default/cpu \ diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index b7a66b0e10..4f79979f51 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -103,7 +103,7 @@ jobs: - name: Get Docker Image run: | set -e -x - az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4 + az login --identity --object-id c90de106-42dc-405a-8bad-2438f4279448 az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87 python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/manylinux/Dockerfile.manylinux2_28_cuda_12.2 \ --context tools/ci_build/github/linux/docker/manylinux \ diff --git a/.pipelines/stages/jobs/steps/capi-linux-step.yml b/.pipelines/stages/jobs/steps/capi-linux-step.yml index 2e297ab82d..81bb76108a 100644 --- a/.pipelines/stages/jobs/steps/capi-linux-step.yml +++ b/.pipelines/stages/jobs/steps/capi-linux-step.yml @@ -43,7 +43,7 @@ steps: - bash: | set -e -x - az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4 + az login --identity --object-id c90de106-42dc-405a-8bad-2438f4279448 az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87 python3 -m pip install requests python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/manylinux/Dockerfile.manylinux2_28_$(ep)_$(cuda_version) \ @@ -59,7 +59,7 @@ steps: - bash: | set -e -x - az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4 + az login --identity --object-id c90de106-42dc-405a-8bad-2438f4279448 az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87 python3 -m pip install requests python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/manylinux/Dockerfile.manylinux2_28_$(ep) \ diff --git a/.pipelines/stages/jobs/steps/nuget-validation-step.yml b/.pipelines/stages/jobs/steps/nuget-validation-step.yml index ebf60dedc8..e0205566d3 100644 --- a/.pipelines/stages/jobs/steps/nuget-validation-step.yml +++ b/.pipelines/stages/jobs/steps/nuget-validation-step.yml @@ -58,7 +58,7 @@ steps: NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180 - bash: | set -e -x - az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4 + az login --identity --object-id c90de106-42dc-405a-8bad-2438f4279448 az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87 docker pull $(cuda_docker_image) diff --git a/.pipelines/stages/jobs/steps/python-validation-step.yml b/.pipelines/stages/jobs/steps/python-validation-step.yml index dd0e4cb007..b5c26fb7f4 100644 --- a/.pipelines/stages/jobs/steps/python-validation-step.yml +++ b/.pipelines/stages/jobs/steps/python-validation-step.yml @@ -56,7 +56,7 @@ steps: - bash: | set -e -x - az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4 + az login --identity --object-id c90de106-42dc-405a-8bad-2438f4279448 az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87 docker pull $(cuda_docker_image) python_exe=/opt/python/cp310-cp310/bin/python3.10 diff --git a/src/config.cpp b/src/config.cpp index 3a61120b8a..8018e5b5cf 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -723,20 +723,28 @@ void SetProviderOption(Config& config, std::string_view provider_name, std::stri } bool IsGraphCaptureEnabled(Config::SessionOptions& session_options) { - for (const auto& provider_options : session_options.provider_options) { - if (provider_options.name == "cuda") { - // Graph Capture is currently broken for CUDA - for (const auto& value : provider_options.options) { - if (value.first == "enable_cuda_graph" && value.second == "1") { - throw std::runtime_error("Graph Capture is currently unsupported for CUDA"); + for (const auto& provider : session_options.providers) { + const auto provider_options = std::find_if(session_options.provider_options.begin(), + session_options.provider_options.end(), + [&provider](const Config::ProviderOptions& po) { + return po.name == provider; + }); + if (provider_options != session_options.provider_options.end()) { + if (provider_options->name == "cuda") { + // Graph Capture is currently broken for CUDA + for (const auto& value : provider_options->options) { + if (value.first == "enable_cuda_graph" && value.second == "1") { + throw std::runtime_error("Graph Capture is currently unsupported for CUDA"); + } } + } else if (provider_options->name == "DML") { + return true; + } else if (provider_options->name == "NvTensorRtRtx") { + return true; } - } else if (provider_options.name == "DML") { - return true; - } else if (provider_options.name == "NvTensorRtRtx") { - return true; } } + return false; } diff --git a/test/python/test_onnxruntime_genai_api.py b/test/python/test_onnxruntime_genai_api.py index 4c8b559149..c17a9c2636 100644 --- a/test/python/test_onnxruntime_genai_api.py +++ b/test/python/test_onnxruntime_genai_api.py @@ -32,8 +32,11 @@ if og.is_openvino_available(): devices.append("openvino") + def test_config(test_data_path): - model_path = os.fspath(Path(test_data_path) / "hf-internal-testing" / "tiny-random-gpt2-fp32") + model_path = os.fspath( + Path(test_data_path) / "hf-internal-testing" / "tiny-random-gpt2-fp32" + ) config = og.Config(model_path) config.clear_providers() config.append_provider("cuda") @@ -42,16 +45,21 @@ def test_config(test_data_path): config.set_provider_option("quantum", "break_universe", "true") config.append_provider("slide rule") + def test_NamedTensors(): named_tensors = og.NamedTensors() - named_tensors["input_ids"] = np.array([[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32) - named_tensors["attention_mask"] = np.array([[1, 1, 1, 1], [1, 1, 1, 1]], dtype=np.int32) + named_tensors["input_ids"] = np.array( + [[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32 + ) + named_tensors["attention_mask"] = np.array( + [[1, 1, 1, 1], [1, 1, 1, 1]], dtype=np.int32 + ) named_tensors["test1"] = og.Tensor(np.random.rand(2, 2).astype(np.float32)) named_tensors["test2"] = og.Tensor(np.random.rand(2, 2).astype(np.float32)) # List out the tensors: - names = named_tensors.keys(); - print() # To not print on the same line as the test name + names = named_tensors.keys() + print() # To not print on the same line as the test name for name in names: print(name) # Assert that the named tensors contains the name @@ -62,6 +70,7 @@ def test_NamedTensors(): # Test that the named tensors is empty assert len(named_tensors) == 0 + @pytest.mark.parametrize( "relative_model_path", ( @@ -77,14 +86,16 @@ def test_NamedTensors(): def test_greedy_search(test_data_path, relative_model_path): model_path = os.fspath(Path(test_data_path) / relative_model_path) - config = og.Config(model_path) # Test using config vs path directly + config = og.Config(model_path) # Test using config vs path directly model = og.Model(config) search_params = og.GeneratorParams(model) input_ids_shape = [2, 4] batch_size = input_ids_shape[0] search_params = og.GeneratorParams(model) - search_params.set_search_options(do_sample=False, max_length=10, batch_size=batch_size) + search_params.set_search_options( + do_sample=False, max_length=10, batch_size=batch_size + ) generator = og.Generator(model, search_params) generator.append_tokens(np.array([[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32)) @@ -92,7 +103,7 @@ def test_greedy_search(test_data_path, relative_model_path): # Test getting/setting logits logits = generator.get_logits() generator.set_logits(logits) - generator.set_logits(logits) # twice just to be sure buffer is still valid + generator.set_logits(logits) # twice just to be sure buffer is still valid generator.generate_next_token() @@ -128,7 +139,9 @@ def test_rewind_cuda(test_data_path, relative_model_path): input_ids_shape = [1, 4] batch_size = input_ids_shape[0] search_params = og.GeneratorParams(model) - search_params.set_search_options(do_sample=False, max_length=10, batch_size=batch_size) + search_params.set_search_options( + do_sample=False, max_length=10, batch_size=batch_size + ) generator = og.Generator(model, search_params) generator.append_tokens(np.array([[0, 0, 195, 731]], dtype=np.int32)) @@ -142,26 +155,35 @@ def test_rewind_cuda(test_data_path, relative_model_path): generator.append_tokens(np.array([[731, 731]], dtype=np.int32)) while not generator.is_done(): generator.generate_next_token() - + assert generator.get_sequence(0) is not None # Batch size > 1 case input_ids_shape = [3, 4] batch_size = input_ids_shape[0] search_params = og.GeneratorParams(model) - search_params.set_search_options(do_sample=False, max_length=10, batch_size=batch_size) + search_params.set_search_options( + do_sample=False, max_length=10, batch_size=batch_size + ) generator = og.Generator(model, search_params) - generator.append_tokens(np.array([[0, 0, 0, 52], [0, 0, 195, 731], [64, 65, 66, 67]], dtype=np.int32)) + generator.append_tokens( + np.array([[0, 0, 0, 52], [0, 0, 195, 731], [64, 65, 66, 67]], dtype=np.int32) + ) while not generator.is_done(): generator.generate_next_token() - + for i in range(batch_size): assert generator.get_sequence(i) is not None - + generator.rewind_to(0) - generator.append_tokens(np.array([[52, 204, 204, 204], [731, 731, 114, 114], [67, 68, 69, 70]], dtype=np.int32)) + generator.append_tokens( + np.array( + [[52, 204, 204, 204], [731, 731, 114, 114], [67, 68, 69, 70]], + dtype=np.int32, + ) + ) while not generator.is_done(): generator.generate_next_token() @@ -171,9 +193,7 @@ def test_rewind_cuda(test_data_path, relative_model_path): @pytest.mark.parametrize( "relative_model_path", - ( - [Path("hf-internal-testing") / "tiny-random-gpt2-fp32"] - ), + ([Path("hf-internal-testing") / "tiny-random-gpt2-fp32"]), ) def test_rewind(test_data_path, relative_model_path): model_path = os.fspath(Path(test_data_path) / relative_model_path) @@ -184,11 +204,13 @@ def test_rewind(test_data_path, relative_model_path): [0, 0, 195, 731, 731, 114, 114, 114, 114, 114], dtype=np.int32, ) - + input_ids_shape = [1, 4] batch_size = input_ids_shape[0] search_params = og.GeneratorParams(model) - search_params.set_search_options(do_sample=False, max_length=10, batch_size=batch_size) + search_params.set_search_options( + do_sample=False, max_length=10, batch_size=batch_size + ) generator = og.Generator(model, search_params) generator.append_tokens(np.array([[0, 0, 195, 731]], dtype=np.int32)) @@ -202,12 +224,13 @@ def test_rewind(test_data_path, relative_model_path): generator.append_tokens(np.array([[731, 731]], dtype=np.int32)) while not generator.is_done(): generator.generate_next_token() - + assert np.array_equal(expected_sequence, generator.get_sequence(0)) - + # Test Model Loading with No Chat Template + # TODO: CUDA pipelines use python3.6 and do not have a way to download models since downloading models # requires pytorch and hf transformers. This test should be re-enabled once the pipeline is updated. @pytest.mark.skipif( @@ -238,6 +261,7 @@ def test_tokenizer_encode_decode(device, phi2_for, batch): decoded_string = tokenizer.decode(sequence) assert prompt == decoded_string + # Test Chat Template Supported Model @pytest.mark.skipif( sysconfig.get_platform().endswith("arm64") or sys.version_info.minor < 8, @@ -251,12 +275,16 @@ def test_qwen_chat_template(device, qwen_for): tokenizer = og.Tokenizer(model) messages = f"""[{{"role": "system", "content": "This is a test."}}, {{"role": "user", "content": "Hi, how are you?"}}]""" - + template = "'{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n'" + try: - tokenizer.apply_chat_template(messages = messages, add_generation_prompt=True) + tokenizer.apply_chat_template( + template_str=template, messages=messages, add_generation_prompt=True + ) except Exception as e: assert False, f"Error while trying to apply chat template: {e}" + # Test Chat Template Unsupported Model with Template String Override @pytest.mark.skipif( sysconfig.get_platform().endswith("arm64") or sys.version_info.minor < 8, @@ -275,10 +303,13 @@ def test_phi2_chat_template(device, phi2_for): template = """{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}""" template_string = f"""{template}""" try: - tokenizer.apply_chat_template(template_str = template_string, messages = messages, add_generation_prompt=True) + tokenizer.apply_chat_template( + template_str=template_string, messages=messages, add_generation_prompt=True + ) except Exception as e: assert False, f"Error while trying to override chat template: {e}" + @pytest.mark.skipif( sysconfig.get_platform().endswith("arm64") or sys.version_info.minor < 8, reason="Python 3.8 is required for downloading models.", @@ -405,10 +436,10 @@ def test_get_output(test_data_path, relative_model_path): model = og.Model(model_path) search_params = og.GeneratorParams(model) - input_ids = np.array( - [[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32 + input_ids = np.array([[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32) + search_params.set_search_options( + do_sample=False, max_length=10, batch_size=input_ids.shape[0] ) - search_params.set_search_options(do_sample=False, max_length=10, batch_size=input_ids.shape[0]) generator = og.Generator(model, search_params) generator.append_tokens(input_ids) @@ -449,6 +480,7 @@ def test_get_output(test_data_path, relative_model_path): logits[:, :, ::200], expected_sampled_logits_token_gen, atol=1e-3 ) + @pytest.mark.skipif( sysconfig.get_platform().endswith("arm64") or sys.version_info.minor < 8, reason="Python 3.8 is required for downloading models.", @@ -458,10 +490,10 @@ def test_hidden_states(qwen_for, device): model = og.Model(qwen_for(device)) search_params = og.GeneratorParams(model) - input_ids = np.array( - [[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32 + input_ids = np.array([[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32) + search_params.set_search_options( + do_sample=False, max_length=10, batch_size=input_ids.shape[0] ) - search_params.set_search_options(do_sample=False, max_length=10, batch_size=input_ids.shape[0]) generator = og.Generator(model, search_params) generator.append_tokens(input_ids) @@ -472,6 +504,7 @@ def test_hidden_states(qwen_for, device): hidden_states = generator.get_output("hidden_states") assert hidden_states.shape == (2, 1, 896) + @pytest.mark.skipif( not og.is_cuda_available(), reason="Pipeline model uses a mix of CPU and CUDA EP." ) @@ -552,7 +585,7 @@ def _split(onnx_model_path: os.PathLike, output_dir: os.PathLike): generator.generate_next_token() expected_output = [ - 'This is a test.\n # TOD import * doct proofingrad', + "This is a test.\n # TOD import * doct proofingrad", 'Rats are awesome pets!\n """\n\n', 'The quick brown fox jumps over the lazy dog.\n """\n\n', ] @@ -566,6 +599,7 @@ def _split(onnx_model_path: os.PathLike, output_dir: os.PathLike): print(f"actual = {repr(actual_output)}", flush=True) assert equal + @pytest.mark.parametrize("relative_model_path", [Path("vision-preprocessing")]) @pytest.mark.parametrize("relative_image_path", [Path("images") / "sheet.png"]) def test_vision_preprocessing(test_data_path, relative_model_path, relative_image_path): @@ -667,12 +701,14 @@ def _prepare_adapter_model(test_data_path): model.graph.input.extend([adapter_a, adapter_b]) for adapter_name in ["adapter_a", "adapter_b"]: - adapter_weight = np.zeros([vocab_size], dtype=(np.float32 if device == "cpu" else np.float16)) + adapter_weight = np.zeros( + [vocab_size], dtype=(np.float32 if device == "cpu" else np.float16) + ) adapter_weight_tensor = onnx.helper.make_tensor( adapter_name, onnx.TensorProto.FLOAT if device == "cpu" else onnx.TensorProto.FLOAT16, [vocab_size], - adapter_weight.flatten() + adapter_weight.flatten(), ) model.graph.initializer.append(adapter_weight_tensor) @@ -722,7 +758,9 @@ def _export_adapter(adapter, adapter_file_name): adapter_paths = [] if multiple_adapters: for i, adapter in enumerate(adapters): - adapter_file_name = str(Path(adapter_model_path) / f"adapter_{i}.onnx_adapter") + adapter_file_name = str( + Path(adapter_model_path) / f"adapter_{i}.onnx_adapter" + ) _export_adapter(adapter, adapter_file_name) adapter_paths.append(adapter_file_name) else: @@ -731,7 +769,7 @@ def _export_adapter(adapter, adapter_file_name): adapter_paths.append(adapter_file_name) return adapter_model_path, adapter_paths - + if device == "dml": pytest.skip("EP DML does not support adapters") @@ -754,7 +792,7 @@ def _export_adapter(adapter, adapter_file_name): generator = og.Generator(model, params) for i in range(len(adapter_paths)): generator.set_active_adapter(adapters, f"adapter_{i}") - + generator.append_tokens(tokenizer.encode_batch(prompts)) while not generator.is_done(): generator.generate_next_token() @@ -765,7 +803,10 @@ def _export_adapter(adapter, adapter_file_name): sysconfig.get_platform().endswith("arm64"), reason="ONNX is not available on ARM64", ) -@pytest.mark.parametrize("extra_inputs", [("num_logits_to_keep", True), ("onnx::Neg_67", True), ("abcde", False)]) +@pytest.mark.parametrize( + "extra_inputs", + [("num_logits_to_keep", True), ("onnx::Neg_67", True), ("abcde", False)], +) def test_preset_extra_inputs(test_data_path, device, phi2_for, extra_inputs): def _prepare_model(test_data_path): phi2_model_path = phi2_for(device) @@ -792,10 +833,16 @@ def _prepare_model(test_data_path): model.graph.input.append(extra_input) cast_node = onnx.helper.make_node( - "Cast", [extra_input_name], [f"{extra_input_name}_cast"], to=onnx.TensorProto.FLOAT if device == "cpu" else onnx.TensorProto.FLOAT16 + "Cast", + [extra_input_name], + [f"{extra_input_name}_cast"], + to=onnx.TensorProto.FLOAT if device == "cpu" else onnx.TensorProto.FLOAT16, ) add_node = onnx.helper.make_node( - "Add", [f"{extra_input_name}_cast", "logits_0"], ["logits"], name="add_to_logits" + "Add", + [f"{extra_input_name}_cast", "logits_0"], + ["logits"], + name="add_to_logits", ) model.graph.node.extend([cast_node, add_node])