From 2091c6d33144f5b67d74bc9ae4144bd24be7f648 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 23 Jul 2025 16:39:11 +1000 Subject: [PATCH 1/3] Update OrtEpFactory in new EPs to add allocator, data transfer and stream stubs. --- .../nv_tensorrt_rtx/nv_provider_factory.cc | 47 +++++++++++++++++++ .../core/providers/openvino/ov_factory.cc | 9 +++- .../core/providers/openvino/ov_factory.h | 38 ++++++++++++++- .../vitisai/vitisai_provider_factory.cc | 47 +++++++++++++++++++ 4 files changed, 138 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc index 1555f47efd8ed..045ae44443416 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc @@ -166,6 +166,13 @@ struct NvTensorRtRtxEpFactory : OrtEpFactory { GetSupportedDevices = GetSupportedDevicesImpl; CreateEp = CreateEpImpl; ReleaseEp = ReleaseEpImpl; + + CreateAllocator = CreateAllocatorImpl; + ReleaseAllocator = ReleaseAllocatorImpl; + CreateDataTransfer = CreateDataTransferImpl; + + IsStreamAware = IsStreamAwareImpl; + CreateSyncStreamForDevice = CreateSyncStreamForDeviceImpl; } // Returns the name for the EP. Each unique factory configuration must have a unique name. @@ -233,6 +240,46 @@ struct NvTensorRtRtxEpFactory : OrtEpFactory { // no-op as we never create an EP here. } + static OrtStatus* ORT_API_CALL CreateAllocatorImpl(OrtEpFactory* this_ptr, + const OrtMemoryInfo* /*memory_info*/, + const OrtKeyValuePairs* /*allocator_options*/, + OrtAllocator** allocator) noexcept { + auto* factory = static_cast(this_ptr); + + *allocator = nullptr; + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, + "CreateAllocator should not be called as we did not add OrtMemoryInfo to our OrtEpDevice."); + } + + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this_ptr, OrtAllocator* /*allocator*/) noexcept { + auto* factory = static_cast(this_ptr); + + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, "ReleaseAllocator should not be called as we did not implement CreateAllocator."); + } + + static OrtStatus* ORT_API_CALL CreateDataTransferImpl(OrtEpFactory* /*this_ptr*/, + OrtDataTransferImpl** data_transfer) noexcept { + *data_transfer = nullptr; // not implemented + return nullptr; + } + + static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return false; + } + + static OrtStatus* ORT_API_CALL CreateSyncStreamForDeviceImpl(OrtEpFactory* this_ptr, + const OrtMemoryDevice* /*memory_device*/, + const OrtKeyValuePairs* /*stream_options*/, + OrtSyncStreamImpl** stream) noexcept { + auto* factory = static_cast(this_ptr); + + *stream = nullptr; + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, "CreateSyncStreamForDevice should not be called as IsStreamAware returned false."); + } + const OrtApi& ort_api; const OrtLogger& default_logger; const std::string ep_name{kNvTensorRTRTXExecutionProvider}; diff --git a/onnxruntime/core/providers/openvino/ov_factory.cc b/onnxruntime/core/providers/openvino/ov_factory.cc index 94e8cbfeca8d1..9f0871b14e92f 100644 --- a/onnxruntime/core/providers/openvino/ov_factory.cc +++ b/onnxruntime/core/providers/openvino/ov_factory.cc @@ -37,10 +37,17 @@ OpenVINOEpPluginFactory::OpenVINOEpPluginFactory(ApiPtrs apis, const std::string OrtEpFactory::GetName = GetNameImpl; OrtEpFactory::GetVendor = GetVendorImpl; OrtEpFactory::GetVendorId = GetVendorIdImpl; - OrtEpFactory::GetSupportedDevices = GetSupportedDevicesImpl; OrtEpFactory::GetVersion = GetVersionImpl; + + OrtEpFactory::GetSupportedDevices = GetSupportedDevicesImpl; + + OrtEpFactory::CreateAllocator = CreateAllocatorImpl; + OrtEpFactory::ReleaseAllocator = ReleaseAllocatorImpl; OrtEpFactory::CreateDataTransfer = CreateDataTransferImpl; + OrtEpFactory::IsStreamAware = IsStreamAwareImpl; + OrtEpFactory::CreateSyncStreamForDevice = CreateSyncStreamForDeviceImpl; + ort_version_supported = ORT_API_VERSION; // Set to the ORT version we were compiled with. } diff --git a/onnxruntime/core/providers/openvino/ov_factory.h b/onnxruntime/core/providers/openvino/ov_factory.h index 37739f67323c1..67d7d8b18e6d9 100644 --- a/onnxruntime/core/providers/openvino/ov_factory.h +++ b/onnxruntime/core/providers/openvino/ov_factory.h @@ -131,6 +131,10 @@ class OpenVINOEpPluginFactory : public OrtEpFactory, public ApiPtrs { return OpenVINOEpPluginFactory::vendor_id_; } + static const char* ORT_API_CALL GetVersionImpl(const OrtEpFactory*) noexcept { + return ORT_VERSION; + } + static OrtStatus* ORT_API_CALL GetSupportedDevicesImpl(OrtEpFactory* this_ptr, const OrtHardwareDevice* const* devices, size_t num_devices, @@ -141,14 +145,44 @@ class OpenVINOEpPluginFactory : public OrtEpFactory, public ApiPtrs { return ApiEntry([&]() { return factory->GetSupportedDevices(devices, num_devices, ep_devices, max_ep_devices, p_num_ep_devices); }); } + static OrtStatus* ORT_API_CALL CreateAllocatorImpl(OrtEpFactory* this_ptr, + const OrtMemoryInfo* /*memory_info*/, + const OrtKeyValuePairs* /*allocator_options*/, + OrtAllocator** allocator) noexcept { + auto* factory = static_cast(this_ptr); + + *allocator = nullptr; + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, + "CreateAllocator should not be called as we did not add OrtMemoryInfo to our OrtEpDevice."); + } + + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this, OrtAllocator* allocator) noexcept { + auto* factory = static_cast(this_ptr); + + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, "ReleaseAllocator should not be called as we did not implement CreateAllocator."); + } + static OrtStatus* ORT_API_CALL CreateDataTransferImpl(OrtEpFactory* /*this_ptr*/, OrtDataTransferImpl** data_transfer) noexcept { *data_transfer = nullptr; // return nullptr to indicate that this EP does not support data transfer. return nullptr; } - static const char* ORT_API_CALL GetVersionImpl(const OrtEpFactory*) noexcept { - return ORT_VERSION; + static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* this_ptr) noexcept { + return false; + } + + static OrtStatus* ORT_API_CALL CreateSyncStreamForDeviceImpl(OrtEpFactory* this_ptr, + const OrtMemoryDevice* memory_device, + const OrtKeyValuePairs* stream_options, + OrtSyncStreamImpl** stream) noexcept { + auto* factory = static_cast(this_ptr); + + *stream = nullptr; + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, "CreateSyncStreamForDevice should not be called as IsStreamAware returned false."); } }; diff --git a/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc index cf4060e103c6d..87aa812c37d8a 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc @@ -111,6 +111,13 @@ struct VitisAIEpFactory : OrtEpFactory { GetSupportedDevices = GetSupportedDevicesImpl; CreateEp = CreateEpImpl; ReleaseEp = ReleaseEpImpl; + + CreateAllocator = CreateAllocatorImpl; + ReleaseAllocator = ReleaseAllocatorImpl; + CreateDataTransfer = CreateDataTransferImpl; + + IsStreamAware = IsStreamAwareImpl; + CreateSyncStreamForDevice = CreateSyncStreamForDeviceImpl; } static const char* GetNameImpl(const OrtEpFactory* /*this_ptr*/) noexcept { @@ -175,6 +182,46 @@ struct VitisAIEpFactory : OrtEpFactory { // no-op as we never create an EP here. } + static OrtStatus* ORT_API_CALL CreateAllocatorImpl(OrtEpFactory* this_ptr, + const OrtMemoryInfo* /*memory_info*/, + const OrtKeyValuePairs* /*allocator_options*/, + OrtAllocator** allocator) noexcept { + auto* factory = static_cast(this_ptr); + + *allocator = nullptr; + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, + "CreateAllocator should not be called as we did not add OrtMemoryInfo to our OrtEpDevice."); + } + + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this_ptr, OrtAllocator* /*allocator*/) noexcept { + auto* factory = static_cast(this_ptr); + + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, "ReleaseAllocator should not be called as we did not implement CreateAllocator."); + } + + static OrtStatus* ORT_API_CALL CreateDataTransferImpl(OrtEpFactory* /*this_ptr*/, + OrtDataTransferImpl** data_transfer) noexcept { + *data_transfer = nullptr; // not implemented + return nullptr; + } + + static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* /*this_ptr*/) noexcept { + return false; + } + + static OrtStatus* ORT_API_CALL CreateSyncStreamForDeviceImpl(OrtEpFactory* this_ptr, + const OrtMemoryDevice* /*memory_device*/, + const OrtKeyValuePairs* /*stream_options*/, + OrtSyncStreamImpl** stream) noexcept { + auto* factory = static_cast(this_ptr); + + *stream = nullptr; + return factory->ort_api.CreateStatus( + ORT_INVALID_ARGUMENT, "CreateSyncStreamForDevice should not be called as IsStreamAware returned false."); + } + const OrtApi& ort_api; const OrtLogger& default_logger; static constexpr const char* const ep_name{kVitisAIExecutionProvider}; From 8f673f4d06b0244036fff13f7712ff023be0bd02 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 23 Jul 2025 16:43:48 +1000 Subject: [PATCH 2/3] Fix release allocator impl. --- .../core/providers/nv_tensorrt_rtx/nv_provider_factory.cc | 7 ++----- onnxruntime/core/providers/openvino/ov_factory.h | 7 ++----- .../core/providers/vitisai/vitisai_provider_factory.cc | 7 ++----- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc index 045ae44443416..428d24f2f3df8 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc @@ -252,11 +252,8 @@ struct NvTensorRtRtxEpFactory : OrtEpFactory { "CreateAllocator should not be called as we did not add OrtMemoryInfo to our OrtEpDevice."); } - static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this_ptr, OrtAllocator* /*allocator*/) noexcept { - auto* factory = static_cast(this_ptr); - - return factory->ort_api.CreateStatus( - ORT_INVALID_ARGUMENT, "ReleaseAllocator should not be called as we did not implement CreateAllocator."); + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtAllocator* /*allocator*/) noexcept { + // should never be called as we don't implement CreateAllocator } static OrtStatus* ORT_API_CALL CreateDataTransferImpl(OrtEpFactory* /*this_ptr*/, diff --git a/onnxruntime/core/providers/openvino/ov_factory.h b/onnxruntime/core/providers/openvino/ov_factory.h index 67d7d8b18e6d9..6f43824aa7cde 100644 --- a/onnxruntime/core/providers/openvino/ov_factory.h +++ b/onnxruntime/core/providers/openvino/ov_factory.h @@ -157,11 +157,8 @@ class OpenVINOEpPluginFactory : public OrtEpFactory, public ApiPtrs { "CreateAllocator should not be called as we did not add OrtMemoryInfo to our OrtEpDevice."); } - static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this, OrtAllocator* allocator) noexcept { - auto* factory = static_cast(this_ptr); - - return factory->ort_api.CreateStatus( - ORT_INVALID_ARGUMENT, "ReleaseAllocator should not be called as we did not implement CreateAllocator."); + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtAllocator* /*allocator*/) noexcept { + // should never be called as we don't implement CreateAllocator } static OrtStatus* ORT_API_CALL CreateDataTransferImpl(OrtEpFactory* /*this_ptr*/, diff --git a/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc index 87aa812c37d8a..50f924e468ed0 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc @@ -194,11 +194,8 @@ struct VitisAIEpFactory : OrtEpFactory { "CreateAllocator should not be called as we did not add OrtMemoryInfo to our OrtEpDevice."); } - static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* this_ptr, OrtAllocator* /*allocator*/) noexcept { - auto* factory = static_cast(this_ptr); - - return factory->ort_api.CreateStatus( - ORT_INVALID_ARGUMENT, "ReleaseAllocator should not be called as we did not implement CreateAllocator."); + static void ORT_API_CALL ReleaseAllocatorImpl(OrtEpFactory* /*this_ptr*/, OrtAllocator* /*allocator*/) noexcept { + // should never be called as we don't implement CreateAllocator } static OrtStatus* ORT_API_CALL CreateDataTransferImpl(OrtEpFactory* /*this_ptr*/, From 647c133f8463b55f7fb325da93ffea0960dc784c Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 23 Jul 2025 18:29:52 +1000 Subject: [PATCH 3/3] Fix OV unused pararms --- onnxruntime/core/providers/openvino/ov_factory.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/openvino/ov_factory.h b/onnxruntime/core/providers/openvino/ov_factory.h index 6f43824aa7cde..d860725a4c173 100644 --- a/onnxruntime/core/providers/openvino/ov_factory.h +++ b/onnxruntime/core/providers/openvino/ov_factory.h @@ -167,13 +167,13 @@ class OpenVINOEpPluginFactory : public OrtEpFactory, public ApiPtrs { return nullptr; } - static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* this_ptr) noexcept { + static bool ORT_API_CALL IsStreamAwareImpl(const OrtEpFactory* /*this_ptr*/) noexcept { return false; } static OrtStatus* ORT_API_CALL CreateSyncStreamForDeviceImpl(OrtEpFactory* this_ptr, - const OrtMemoryDevice* memory_device, - const OrtKeyValuePairs* stream_options, + const OrtMemoryDevice* /*memory_device*/, + const OrtKeyValuePairs* /*stream_options*/, OrtSyncStreamImpl** stream) noexcept { auto* factory = static_cast(this_ptr);