From b5a9d3ae969f551666ab5a5497d44f544ae68d7f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 16:59:01 +0000 Subject: [PATCH 1/7] Initial plan From ec645928d468bec5325cf900cc5453e8b1578b2e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 17:03:41 +0000 Subject: [PATCH 2/7] Add update_inplace overload accepting OrtValue for GPU-to-GPU copy Add a second overload of update_inplace that accepts an OrtValue as source, enabling device-to-device memory copy (e.g., GPU to GPU) without requiring data to go through CPU. This is useful for CUDA graph scenarios where inputs need to be updated in fixed memory addresses. Changes: - onnxruntime_pybind_ortvalue.cc: Add pybind overload for OrtValue source - onnxruntime_inference_collection.py: Update Python wrapper to accept both numpy arrays and OrtValues - onnxruntime_test_python_cudagraph.py: Add tests for OrtValue-to-OrtValue update_inplace across different device combinations Agent-Logs-Url: https://github.com/microsoft/onnxruntime/sessions/a05648d1-3616-4d6e-a448-b8d506c55854 Co-authored-by: tianleiwu <30328909+tianleiwu@users.noreply.github.com> --- .../onnxruntime_inference_collection.py | 20 ++-- .../python/onnxruntime_pybind_ortvalue.cc | 91 +++++++++++++++++++ .../onnxruntime_test_python_cudagraph.py | 29 ++++++ 3 files changed, 133 insertions(+), 7 deletions(-) diff --git a/onnxruntime/python/onnxruntime_inference_collection.py b/onnxruntime/python/onnxruntime_inference_collection.py index def2240358c10..39e04b9a2ea25 100644 --- a/onnxruntime/python/onnxruntime_inference_collection.py +++ b/onnxruntime/python/onnxruntime_inference_collection.py @@ -1302,15 +1302,21 @@ def from_dlpack(cls, data, /) -> OrtValue: return cls(C.OrtValue.from_dlpack(capsule, is_bool)) - def update_inplace(self, np_arr) -> None: + def update_inplace(self, data) -> None: """ - Update the OrtValue in place with a new Numpy array. The numpy contents - are copied over to the device memory backing the OrtValue. It can be used - to update the input valuess for an InferenceSession with CUDA graph - enabled or other scenarios where the OrtValue needs to be updated while - the memory address can not be changed. + Update the OrtValue in place. The source data is copied over to the device + memory backing the OrtValue. It can be used to update the input values for + an InferenceSession with CUDA graph enabled or other scenarios where the + OrtValue needs to be updated while the memory address can not be changed. + + :param data: The source data, which can be a Numpy array or another OrtValue. + When an OrtValue is provided, data can be copied between devices (e.g., + GPU to GPU) without going through the CPU. """ - self._ortvalue.update_inplace(np_arr) + if isinstance(data, OrtValue): + self._ortvalue.update_inplace(data._ortvalue) + else: + self._ortvalue.update_inplace(data) def copy_tensors(src: Sequence[OrtValue], dst: Sequence[OrtValue], stream=None) -> None: diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc index eb966ac5fc314..db563f5f6b3ee 100644 --- a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -237,6 +237,97 @@ void addOrtValueMethods(pybind11::module& m) { throw std::runtime_error("Unsupported device: Cannot update the OrtValue on this device"); } }) + .def("update_inplace", [](OrtValue* ml_value, const OrtValue* source) { + if (!ml_value->IsTensor()) { + throw std::runtime_error("Inplace update of OrtValues is only supported for Tensors"); + } + if (!source->IsTensor()) { + throw std::runtime_error("The source OrtValue must contain a Tensor"); + } + + const auto& dst_tensor = ml_value->Get(); + const auto& src_tensor = source->Get(); + + if (dst_tensor.DataType() != src_tensor.DataType()) { + throw std::runtime_error("The source and destination OrtValues must have the same data type"); + } + + if (dst_tensor.Shape().Size() != src_tensor.Shape().Size()) { + throw std::runtime_error("The source and destination OrtValues must have the same size"); + } + + if (dst_tensor.IsDataTypeString()) { + throw std::runtime_error("Inplace update of string tensors is not supported"); + } + + size_t bytes = 0; + auto status = Tensor::CalculateTensorStorageSize(dst_tensor.DataType(), dst_tensor.Shape(), 0, bytes); + if (!status.IsOK()) { + throw std::runtime_error(status.ErrorMessage()); + } + + const auto src_device = src_tensor.Location().device; + const auto dst_device = dst_tensor.Location().device; + + void* dst = ml_value->GetMutable()->MutableDataRaw(); + const void* src = src_tensor.DataRaw(); + + if (src_device.UsesCpuMemory() && dst_device.UsesCpuMemory()) { + memcpy(dst, src, bytes); + } else { + auto copy_fn = CreateDataTransferMemCpy(src_device, dst_device); + if (!copy_fn) { + // Fall back to built-in EP copy functions +#ifdef USE_CUDA + if (src_device.Type() == OrtDevice::GPU && dst_device.Type() == OrtDevice::GPU) { + auto data_transfer = GetGPUDataTransfer(); + ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *(ml_value->GetMutable()))); + return; + } + if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { + CpuToCudaMemCpy(dst, src, bytes); + return; + } + if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { + CudaToCpuMemCpy(dst, src, bytes); + return; + } +#endif +#if USE_MIGRAPHX + if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { + CpuToMIGraphXMemCpy(dst, src, bytes); + return; + } + if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { + MIGraphXToCpuMemCpy(dst, src, bytes); + return; + } +#endif +#if USE_DML + if (src_device.UsesCpuMemory() && (dst_device.Type() == OrtDevice::GPU || dst_device.Type() == OrtDevice::DML)) { + CpuToDmlMemCpy(dst, src, bytes); + return; + } + if ((src_device.Type() == OrtDevice::GPU || src_device.Type() == OrtDevice::DML) && dst_device.UsesCpuMemory()) { + DmlToCpuMemCpy(dst, src, bytes); + return; + } +#endif +#ifdef USE_CANN + if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::NPU) { + CpuToCannMemCpy(dst, src, bytes); + return; + } + if (src_device.Type() == OrtDevice::NPU && dst_device.UsesCpuMemory()) { + CannToCpuMemCpy(dst, src, bytes); + return; + } +#endif + throw std::runtime_error("Unable to copy data between the source and destination devices"); + } + copy_fn(dst, src, bytes); + } + }) // Create an ortvalue value on top of the numpy array, but interpret the data // as a different type with the same element size. .def_static("ortvalue_from_numpy_with_onnx_type", [](py::array& data, int32_t onnx_element_type) -> std::unique_ptr { diff --git a/onnxruntime/test/python/onnxruntime_test_python_cudagraph.py b/onnxruntime/test/python/onnxruntime_test_python_cudagraph.py index d6c1dd9cff3f3..987efd5af5e8e 100644 --- a/onnxruntime/test/python/onnxruntime_test_python_cudagraph.py +++ b/onnxruntime/test/python/onnxruntime_test_python_cudagraph.py @@ -76,6 +76,35 @@ def test_ort_value_update_in_place(self): ortvalue_gpu.update_inplace(x1) np.testing.assert_allclose(ortvalue_gpu.numpy(), x1) + def test_ort_value_update_in_place_from_ortvalue(self): + # Test CPU to CPU copy via OrtValue + x0 = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + x1 = np.array([[10.0, 20.0], [30.0, 40.0], [50.0, 60.0]], dtype=np.float32) + + ortvalue_dst = onnxrt.OrtValue.ortvalue_from_numpy(x0) + ortvalue_src = onnxrt.OrtValue.ortvalue_from_numpy(x1) + ortvalue_dst.update_inplace(ortvalue_src) + np.testing.assert_allclose(ortvalue_dst.numpy(), x1) + + if "CUDAExecutionProvider" in onnxrt.get_available_providers(): + # Test GPU to GPU copy via OrtValue + ortvalue_gpu_dst = onnxrt.OrtValue.ortvalue_from_numpy(x0, "cuda", 0) + ortvalue_gpu_src = onnxrt.OrtValue.ortvalue_from_numpy(x1, "cuda", 0) + ortvalue_gpu_dst.update_inplace(ortvalue_gpu_src) + np.testing.assert_allclose(ortvalue_gpu_dst.numpy(), x1) + + # Test CPU OrtValue to GPU OrtValue copy + ortvalue_gpu_dst2 = onnxrt.OrtValue.ortvalue_from_numpy(x0, "cuda", 0) + ortvalue_cpu_src = onnxrt.OrtValue.ortvalue_from_numpy(x1) + ortvalue_gpu_dst2.update_inplace(ortvalue_cpu_src) + np.testing.assert_allclose(ortvalue_gpu_dst2.numpy(), x1) + + # Test GPU OrtValue to CPU OrtValue copy + ortvalue_cpu_dst = onnxrt.OrtValue.ortvalue_from_numpy(x0) + ortvalue_gpu_src2 = onnxrt.OrtValue.ortvalue_from_numpy(x1, "cuda", 0) + ortvalue_cpu_dst.update_inplace(ortvalue_gpu_src2) + np.testing.assert_allclose(ortvalue_cpu_dst.numpy(), x1) + def test_select_ep_to_run_cuda_graph(self): if "TensorrtExecutionProvider" in onnxrt.get_available_providers(): providers = [("TensorrtExecutionProvider", {"trt_cuda_graph_enable": True})] From 18a2bac26a37b3d280ae6e680d44f062239c08ab Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Tue, 28 Apr 2026 11:19:47 -0700 Subject: [PATCH 3/7] refactor: extract update_inplace logic into reusable helper function - Create UpdateOrtValueInplace() helper in onnxruntime_pybind_mlvalue for code reuse - Simplify Python binding to call helper instead of duplicating logic - Supports device-to-device transfers (CPU, GPU, DML, CANN, MIGraphX) - Addresses review feedback on PR #28256 --- .../python/onnxruntime_pybind_mlvalue.cc | 92 +++++++++++++++++++ .../python/onnxruntime_pybind_mlvalue.h | 5 + .../python/onnxruntime_pybind_ortvalue.cc | 90 +----------------- 3 files changed, 98 insertions(+), 89 deletions(-) diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 89651c2d955de..ff18c726fe339 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -1071,5 +1071,97 @@ void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, const } } +void UpdateOrtValueInplace(OrtValue* dst, const OrtValue* src) { + if (!dst->IsTensor()) { + throw std::runtime_error("Inplace update of OrtValues is only supported for Tensors"); + } + if (!src->IsTensor()) { + throw std::runtime_error("The source OrtValue must contain a Tensor"); + } + + const auto& dst_tensor = dst->Get(); + const auto& src_tensor = src->Get(); + + if (dst_tensor.DataType() != src_tensor.DataType()) { + throw std::runtime_error("The source and destination OrtValues must have the same data type"); + } + + if (dst_tensor.Shape().Size() != src_tensor.Shape().Size()) { + throw std::runtime_error("The source and destination OrtValues must have the same size"); + } + + if (dst_tensor.IsDataTypeString()) { + throw std::runtime_error("Inplace update of string tensors is not supported"); + } + + size_t bytes = 0; + auto status = Tensor::CalculateTensorStorageSize(dst_tensor.DataType(), dst_tensor.Shape(), 0, bytes); + if (!status.IsOK()) { + throw std::runtime_error(status.ErrorMessage()); + } + + const auto src_device = src_tensor.Location().device; + const auto dst_device = dst_tensor.Location().device; + + void* dst_ptr = dst->GetMutable()->MutableDataRaw(); + const void* src_ptr = src_tensor.DataRaw(); + + if (src_device.UsesCpuMemory() && dst_device.UsesCpuMemory()) { + memcpy(dst_ptr, src_ptr, bytes); + } else { + auto copy_fn = CreateDataTransferMemCpy(src_device, dst_device); + if (!copy_fn) { + // Fall back to built-in EP copy functions +#ifdef USE_CUDA + if (src_device.Type() == OrtDevice::GPU && dst_device.Type() == OrtDevice::GPU) { + auto data_transfer = GetGPUDataTransfer(); + ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *(dst->GetMutable()))); + return; + } + if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { + CpuToCudaMemCpy(dst_ptr, src_ptr, bytes); + return; + } + if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { + CudaToCpuMemCpy(dst_ptr, src_ptr, bytes); + return; + } +#endif +#if USE_MIGRAPHX + if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { + CpuToMIGraphXMemCpy(dst_ptr, src_ptr, bytes); + return; + } + if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { + MIGraphXToCpuMemCpy(dst_ptr, src_ptr, bytes); + return; + } +#endif +#if USE_DML + if (src_device.UsesCpuMemory() && (dst_device.Type() == OrtDevice::GPU || dst_device.Type() == OrtDevice::DML)) { + CpuToDmlMemCpy(dst_ptr, src_ptr, bytes); + return; + } + if ((src_device.Type() == OrtDevice::GPU || src_device.Type() == OrtDevice::DML) && dst_device.UsesCpuMemory()) { + DmlToCpuMemCpy(dst_ptr, src_ptr, bytes); + return; + } +#endif +#ifdef USE_CANN + if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::NPU) { + CpuToCannMemCpy(dst_ptr, src_ptr, bytes); + return; + } + if (src_device.Type() == OrtDevice::NPU && dst_device.UsesCpuMemory()) { + CannToCpuMemCpy(dst_ptr, src_ptr, bytes); + return; + } +#endif + throw std::runtime_error("Unable to copy data between the source and destination devices"); + } + copy_fn(dst_ptr, src_ptr, bytes); + } +} + } // namespace python } // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.h b/onnxruntime/python/onnxruntime_pybind_mlvalue.h index 144b3edcad404..080b82842b7c5 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.h +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.h @@ -138,6 +138,11 @@ pybind11::object GetPyObjFromTensor(const OrtValue& rtensor, const std::unordered_map* mem_cpy_to_host_functions = nullptr, bool zero_copy_non_owning = false); +// Update the tensor data in an OrtValue in-place from another OrtValue. +// Both OrtValues must contain tensors of the same data type and size. +// This function supports various device-to-device transfers. +void UpdateOrtValueInplace(OrtValue* dst, const OrtValue* src); + // The below two functions are used to convert OrtValue to numpy arrays /// diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc index db563f5f6b3ee..cb59b92d35851 100644 --- a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -238,95 +238,7 @@ void addOrtValueMethods(pybind11::module& m) { } }) .def("update_inplace", [](OrtValue* ml_value, const OrtValue* source) { - if (!ml_value->IsTensor()) { - throw std::runtime_error("Inplace update of OrtValues is only supported for Tensors"); - } - if (!source->IsTensor()) { - throw std::runtime_error("The source OrtValue must contain a Tensor"); - } - - const auto& dst_tensor = ml_value->Get(); - const auto& src_tensor = source->Get(); - - if (dst_tensor.DataType() != src_tensor.DataType()) { - throw std::runtime_error("The source and destination OrtValues must have the same data type"); - } - - if (dst_tensor.Shape().Size() != src_tensor.Shape().Size()) { - throw std::runtime_error("The source and destination OrtValues must have the same size"); - } - - if (dst_tensor.IsDataTypeString()) { - throw std::runtime_error("Inplace update of string tensors is not supported"); - } - - size_t bytes = 0; - auto status = Tensor::CalculateTensorStorageSize(dst_tensor.DataType(), dst_tensor.Shape(), 0, bytes); - if (!status.IsOK()) { - throw std::runtime_error(status.ErrorMessage()); - } - - const auto src_device = src_tensor.Location().device; - const auto dst_device = dst_tensor.Location().device; - - void* dst = ml_value->GetMutable()->MutableDataRaw(); - const void* src = src_tensor.DataRaw(); - - if (src_device.UsesCpuMemory() && dst_device.UsesCpuMemory()) { - memcpy(dst, src, bytes); - } else { - auto copy_fn = CreateDataTransferMemCpy(src_device, dst_device); - if (!copy_fn) { - // Fall back to built-in EP copy functions -#ifdef USE_CUDA - if (src_device.Type() == OrtDevice::GPU && dst_device.Type() == OrtDevice::GPU) { - auto data_transfer = GetGPUDataTransfer(); - ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *(ml_value->GetMutable()))); - return; - } - if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { - CpuToCudaMemCpy(dst, src, bytes); - return; - } - if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { - CudaToCpuMemCpy(dst, src, bytes); - return; - } -#endif -#if USE_MIGRAPHX - if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { - CpuToMIGraphXMemCpy(dst, src, bytes); - return; - } - if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { - MIGraphXToCpuMemCpy(dst, src, bytes); - return; - } -#endif -#if USE_DML - if (src_device.UsesCpuMemory() && (dst_device.Type() == OrtDevice::GPU || dst_device.Type() == OrtDevice::DML)) { - CpuToDmlMemCpy(dst, src, bytes); - return; - } - if ((src_device.Type() == OrtDevice::GPU || src_device.Type() == OrtDevice::DML) && dst_device.UsesCpuMemory()) { - DmlToCpuMemCpy(dst, src, bytes); - return; - } -#endif -#ifdef USE_CANN - if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::NPU) { - CpuToCannMemCpy(dst, src, bytes); - return; - } - if (src_device.Type() == OrtDevice::NPU && dst_device.UsesCpuMemory()) { - CannToCpuMemCpy(dst, src, bytes); - return; - } -#endif - throw std::runtime_error("Unable to copy data between the source and destination devices"); - } - copy_fn(dst, src, bytes); - } + python::UpdateOrtValueInplace(ml_value, source); }) // Create an ortvalue value on top of the numpy array, but interpret the data // as a different type with the same element size. From cfb825af37c12264da0cc016bbf614ad5234b44b Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Tue, 28 Apr 2026 11:38:20 -0700 Subject: [PATCH 4/7] use reference in helper --- onnxruntime/python/onnxruntime_pybind_mlvalue.cc | 14 +++++++------- onnxruntime/python/onnxruntime_pybind_mlvalue.h | 2 +- onnxruntime/python/onnxruntime_pybind_ortvalue.cc | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index ff18c726fe339..2ccf7508a8098 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -1071,16 +1071,16 @@ void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, const } } -void UpdateOrtValueInplace(OrtValue* dst, const OrtValue* src) { - if (!dst->IsTensor()) { +void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { + if (!dst.IsTensor()) { throw std::runtime_error("Inplace update of OrtValues is only supported for Tensors"); } - if (!src->IsTensor()) { + if (!src.IsTensor()) { throw std::runtime_error("The source OrtValue must contain a Tensor"); } - const auto& dst_tensor = dst->Get(); - const auto& src_tensor = src->Get(); + const auto& dst_tensor = dst.Get(); + const auto& src_tensor = src.Get(); if (dst_tensor.DataType() != src_tensor.DataType()) { throw std::runtime_error("The source and destination OrtValues must have the same data type"); @@ -1103,7 +1103,7 @@ void UpdateOrtValueInplace(OrtValue* dst, const OrtValue* src) { const auto src_device = src_tensor.Location().device; const auto dst_device = dst_tensor.Location().device; - void* dst_ptr = dst->GetMutable()->MutableDataRaw(); + void* dst_ptr = dst.GetMutable()->MutableDataRaw(); const void* src_ptr = src_tensor.DataRaw(); if (src_device.UsesCpuMemory() && dst_device.UsesCpuMemory()) { @@ -1115,7 +1115,7 @@ void UpdateOrtValueInplace(OrtValue* dst, const OrtValue* src) { #ifdef USE_CUDA if (src_device.Type() == OrtDevice::GPU && dst_device.Type() == OrtDevice::GPU) { auto data_transfer = GetGPUDataTransfer(); - ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *(dst->GetMutable()))); + ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *dst.GetMutable())); return; } if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.h b/onnxruntime/python/onnxruntime_pybind_mlvalue.h index 080b82842b7c5..097c5b4d20d65 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.h +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.h @@ -141,7 +141,7 @@ pybind11::object GetPyObjFromTensor(const OrtValue& rtensor, // Update the tensor data in an OrtValue in-place from another OrtValue. // Both OrtValues must contain tensors of the same data type and size. // This function supports various device-to-device transfers. -void UpdateOrtValueInplace(OrtValue* dst, const OrtValue* src); +void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src); // The below two functions are used to convert OrtValue to numpy arrays diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc index cb59b92d35851..578f0cfdfea2c 100644 --- a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -238,7 +238,7 @@ void addOrtValueMethods(pybind11::module& m) { } }) .def("update_inplace", [](OrtValue* ml_value, const OrtValue* source) { - python::UpdateOrtValueInplace(ml_value, source); + python::UpdateOrtValueInplace(*ml_value, *source); }) // Create an ortvalue value on top of the numpy array, but interpret the data // as a different type with the same element size. From e1f90a6d10df799283aee9baa7d38a4ba561fac6 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 28 Apr 2026 17:58:21 -0700 Subject: [PATCH 5/7] address feedback --- .../onnxruntime_inference_collection.py | 13 ++++--- .../python/onnxruntime_pybind_mlvalue.cc | 34 +++++++++++++------ .../python/onnxruntime_pybind_ortvalue.cc | 4 +-- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/onnxruntime/python/onnxruntime_inference_collection.py b/onnxruntime/python/onnxruntime_inference_collection.py index 39e04b9a2ea25..e35e3c5753d36 100644 --- a/onnxruntime/python/onnxruntime_inference_collection.py +++ b/onnxruntime/python/onnxruntime_inference_collection.py @@ -13,10 +13,11 @@ from enum import IntEnum from typing import Any +import numpy as np + from onnxruntime.capi import _pybind_state as C if typing.TYPE_CHECKING: - import numpy as np import numpy.typing as npt import onnxruntime @@ -1212,8 +1213,6 @@ def __array__(self, dtype=None, copy=None) -> np.ndarray: If ``None`` (default), a copy will be made only if needed. :return: A numpy array with the same data as the OrtValue. """ - import numpy as np # noqa: PLC0415 - arr = self.numpy() if copy is not None: @@ -1315,8 +1314,12 @@ def update_inplace(self, data) -> None: """ if isinstance(data, OrtValue): self._ortvalue.update_inplace(data._ortvalue) - else: - self._ortvalue.update_inplace(data) + return + + if not isinstance(data, np.ndarray): + raise TypeError("data must be a numpy.ndarray or an OrtValue.") + + self._ortvalue.update_inplace(data) def copy_tensors(src: Sequence[OrtValue], dst: Sequence[OrtValue], stream=None) -> None: diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 2ccf7508a8098..1aecece086084 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -1111,48 +1111,62 @@ void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { } else { auto copy_fn = CreateDataTransferMemCpy(src_device, dst_device); if (!copy_fn) { - // Fall back to built-in EP copy functions + // Fall back to built-in EP copy functions. + // Gate each path on (Type, VendorId) so that builds with multiple GPU EPs + // (e.g. CUDA + DML) route through the correct backend. + const auto is_cuda_device = [](const OrtDevice& device) { + return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::NVIDIA; + }; + const auto is_migraphx_device = [](const OrtDevice& device) { + return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::AMD; + }; + const auto is_dml_device = [](const OrtDevice& device) { + return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::MICROSOFT; + }; + const auto is_cann_device = [](const OrtDevice& device) { + return device.Type() == OrtDevice::NPU && device.Vendor() == OrtDevice::VendorIds::HUAWEI; + }; #ifdef USE_CUDA - if (src_device.Type() == OrtDevice::GPU && dst_device.Type() == OrtDevice::GPU) { + if (is_cuda_device(src_device) && is_cuda_device(dst_device)) { auto data_transfer = GetGPUDataTransfer(); ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *dst.GetMutable())); return; } - if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { + if (src_device.UsesCpuMemory() && is_cuda_device(dst_device)) { CpuToCudaMemCpy(dst_ptr, src_ptr, bytes); return; } - if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { + if (is_cuda_device(src_device) && dst_device.UsesCpuMemory()) { CudaToCpuMemCpy(dst_ptr, src_ptr, bytes); return; } #endif #if USE_MIGRAPHX - if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::GPU) { + if (src_device.UsesCpuMemory() && is_migraphx_device(dst_device)) { CpuToMIGraphXMemCpy(dst_ptr, src_ptr, bytes); return; } - if (src_device.Type() == OrtDevice::GPU && dst_device.UsesCpuMemory()) { + if (is_migraphx_device(src_device) && dst_device.UsesCpuMemory()) { MIGraphXToCpuMemCpy(dst_ptr, src_ptr, bytes); return; } #endif #if USE_DML - if (src_device.UsesCpuMemory() && (dst_device.Type() == OrtDevice::GPU || dst_device.Type() == OrtDevice::DML)) { + if (src_device.UsesCpuMemory() && is_dml_device(dst_device)) { CpuToDmlMemCpy(dst_ptr, src_ptr, bytes); return; } - if ((src_device.Type() == OrtDevice::GPU || src_device.Type() == OrtDevice::DML) && dst_device.UsesCpuMemory()) { + if (is_dml_device(src_device) && dst_device.UsesCpuMemory()) { DmlToCpuMemCpy(dst_ptr, src_ptr, bytes); return; } #endif #ifdef USE_CANN - if (src_device.UsesCpuMemory() && dst_device.Type() == OrtDevice::NPU) { + if (src_device.UsesCpuMemory() && is_cann_device(dst_device)) { CpuToCannMemCpy(dst_ptr, src_ptr, bytes); return; } - if (src_device.Type() == OrtDevice::NPU && dst_device.UsesCpuMemory()) { + if (is_cann_device(src_device) && dst_device.UsesCpuMemory()) { CannToCpuMemCpy(dst_ptr, src_ptr, bytes); return; } diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc index 578f0cfdfea2c..168d57fc0827b 100644 --- a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -237,8 +237,8 @@ void addOrtValueMethods(pybind11::module& m) { throw std::runtime_error("Unsupported device: Cannot update the OrtValue on this device"); } }) - .def("update_inplace", [](OrtValue* ml_value, const OrtValue* source) { - python::UpdateOrtValueInplace(*ml_value, *source); + .def("update_inplace", [](OrtValue* ml_value, const OrtValue& source) { + python::UpdateOrtValueInplace(*ml_value, source); }) // Create an ortvalue value on top of the numpy array, but interpret the data // as a different type with the same element size. From 2a277f9725bf5c9d8d5ab9b57f90464fe61ffa5a Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 28 Apr 2026 18:22:21 -0700 Subject: [PATCH 6/7] fix(python): extend DML device predicate to match OrtDevice::DML type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The is_dml_device lambda only checked OrtDevice::GPU with Microsoft vendor, but DML tensors can also use the dedicated OrtDevice::DML device type. This caused CPU↔DML copies through the OrtValue overload to incorrectly fall through and throw 'Unable to copy...'. --- onnxruntime/python/onnxruntime_pybind_mlvalue.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 1aecece086084..719df697a7c14 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -1121,7 +1121,8 @@ void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::AMD; }; const auto is_dml_device = [](const OrtDevice& device) { - return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::MICROSOFT; + return (device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::MICROSOFT) || + device.Type() == OrtDevice::DML; }; const auto is_cann_device = [](const OrtDevice& device) { return device.Type() == OrtDevice::NPU && device.Vendor() == OrtDevice::VendorIds::HUAWEI; From 8d2e14d0ecc7ab296ed4a2616a336fd1a293cd13 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 28 Apr 2026 18:35:03 -0700 Subject: [PATCH 7/7] refine Co-authored-by: Copilot --- .../python/onnxruntime_pybind_mlvalue.cc | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 719df697a7c14..fa609fe6ea83d 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -1114,20 +1114,11 @@ void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { // Fall back to built-in EP copy functions. // Gate each path on (Type, VendorId) so that builds with multiple GPU EPs // (e.g. CUDA + DML) route through the correct backend. +#ifdef USE_CUDA const auto is_cuda_device = [](const OrtDevice& device) { return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::NVIDIA; }; - const auto is_migraphx_device = [](const OrtDevice& device) { - return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::AMD; - }; - const auto is_dml_device = [](const OrtDevice& device) { - return (device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::MICROSOFT) || - device.Type() == OrtDevice::DML; - }; - const auto is_cann_device = [](const OrtDevice& device) { - return device.Type() == OrtDevice::NPU && device.Vendor() == OrtDevice::VendorIds::HUAWEI; - }; -#ifdef USE_CUDA + if (is_cuda_device(src_device) && is_cuda_device(dst_device)) { auto data_transfer = GetGPUDataTransfer(); ORT_THROW_IF_ERROR(data_transfer->CopyTensor(src_tensor, *dst.GetMutable())); @@ -1143,6 +1134,10 @@ void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { } #endif #if USE_MIGRAPHX + const auto is_migraphx_device = [](const OrtDevice& device) { + return device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::AMD; + }; + if (src_device.UsesCpuMemory() && is_migraphx_device(dst_device)) { CpuToMIGraphXMemCpy(dst_ptr, src_ptr, bytes); return; @@ -1153,6 +1148,11 @@ void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { } #endif #if USE_DML + const auto is_dml_device = [](const OrtDevice& device) { + return (device.Type() == OrtDevice::GPU && device.Vendor() == OrtDevice::VendorIds::MICROSOFT) || + device.Type() == OrtDevice::DML; + }; + if (src_device.UsesCpuMemory() && is_dml_device(dst_device)) { CpuToDmlMemCpy(dst_ptr, src_ptr, bytes); return; @@ -1163,6 +1163,10 @@ void UpdateOrtValueInplace(OrtValue& dst, const OrtValue& src) { } #endif #ifdef USE_CANN + const auto is_cann_device = [](const OrtDevice& device) { + return device.Type() == OrtDevice::NPU && device.Vendor() == OrtDevice::VendorIds::HUAWEI; + }; + if (src_device.UsesCpuMemory() && is_cann_device(dst_device)) { CpuToCannMemCpy(dst_ptr, src_ptr, bytes); return;