From a80c97b3e017d94e9fe4160cb44da041439217c7 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 6 May 2026 14:23:32 -0700 Subject: [PATCH 01/12] update --- .../core/session/onnxruntime_ep_c_api.h | 32 +++++++++ .../session/plugin_ep/ep_factory_internal.cc | 1 + .../session/plugin_ep/ep_factory_internal.h | 10 +++ .../plugin_ep/ep_factory_internal_impl.h | 18 +++++ .../plugin_ep/forward_to_factory_impl.h | 11 +++ .../library/example_plugin_ep/ep_factory.cc | 71 +++++++++++++++++++ .../library/example_plugin_ep/ep_factory.h | 8 +++ onnxruntime/test/autoep/test_model_package.cc | 48 +++++++++++++ .../test/framework/ep_compatibility_test.cc | 1 + 9 files changed, 200 insertions(+) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 62757812cd6e3..4b4cc4601fc13 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2965,6 +2965,38 @@ struct OrtEpFactory { */ ORT_API2_STATUS(DeinitGraphicsInterop, _In_ OrtEpFactory* this_ptr, _In_ const OrtEpDevice* ep_device); + + /** \brief Select the best compiled model compatibility info from candidate strings. + * + * Evaluates each candidate compatibility string against the given hardware devices and returns the selected index. + * + * Context about having this function: + * The existed ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best + * compatible model when there are multiple candidates. For example, an EP may support multiple compilation modes + * (e.g., "speed optimized" vs "memory optimized") that produce different compatibility strings. The EP can implement + * this function to evaluate the candidate strings and select the best compatible one based on its own criteria and + * the target devices. + * + * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] devices Array of OrtHardwareDevice pointers that the EP would run on. All must map to this EP. + * \param[in] num_devices Number of entries in `devices`. + * \param[in] compatibility_infos Array of candidate compatibility strings. + * \param[in] num_compatibility_infos Number of entries in `compatibility_infos`. + * \param[out] selected_index Index of selected candidate, or SIZE_MAX if all candidates are unsupported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.27. + */ + ORT_API2_STATUS(SelectBestCompiledModelCompatibilityInfo, + _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, + _In_ size_t num_compatibility_infos, + _Out_ size_t* selected_index); }; #ifdef __cplusplus diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc index ca39d6e750088..77ef684db7b1f 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc @@ -36,6 +36,7 @@ EpFactoryInternal::EpFactoryInternal(std::unique_ptr impl OrtEpFactory::GetHardwareDeviceIncompatibilityDetails = Forward::GetHardwareDeviceIncompatibilityDetails; OrtEpFactory::InitGraphicsInterop = Forward::InitGraphicsInterop; OrtEpFactory::DeinitGraphicsInterop = Forward::DeinitGraphicsInterop; + OrtEpFactory::SelectBestCompiledModelCompatibilityInfo = Forward::SelectBestCompiledModelCompatibilityInfo; } InternalExecutionProviderFactory::InternalExecutionProviderFactory(EpFactoryInternal& ep_factory, diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h index 9ac883da06465..17bbfcc4282ef 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h @@ -106,6 +106,16 @@ class EpFactoryInternal : public OrtEpFactory { return impl_->DeinitGraphicsInterop(ep_device); } + OrtStatus* SelectBestCompiledModelCompatibilityInfo(_In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, + _In_ size_t num_compatibility_infos, + _Out_ size_t* selected_index) noexcept { + return impl_->SelectBestCompiledModelCompatibilityInfo(devices, num_devices, + compatibility_infos, num_compatibility_infos, + selected_index); + } + // Function ORT calls to release an EP instance. void ReleaseEp(OrtEp* /*ep*/) noexcept { // we never create an OrtEp so we should never be trying to release one diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h index a6140b2ac260f..d86b8daeb8981 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h @@ -122,6 +122,24 @@ class EpFactoryInternalImpl { return nullptr; } + virtual OrtStatus* SelectBestCompiledModelCompatibilityInfo( + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, + _In_ size_t num_compatibility_infos, + _Out_ size_t* selected_index) noexcept { + ORT_UNUSED_PARAMETER(devices); + ORT_UNUSED_PARAMETER(num_devices); + if (compatibility_infos == nullptr || num_compatibility_infos == 0 || selected_index == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "Invalid arguments to SelectBestCompiledModelCompatibilityInfo."); + } + + // Default implementation: all candidates are unsupported, sets `selected_index` to SIZE_MAX. + *selected_index = SIZE_MAX; + return nullptr; + } + // Function ORT calls to release an EP instance. void ReleaseEp(OrtEp* ep); diff --git a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h index d4d93152c4f80..623a2a4ae2c0f 100644 --- a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h +++ b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h @@ -106,6 +106,17 @@ struct ForwardToFactoryImpl { return static_cast(this_ptr)->DeinitGraphicsInterop(ep_device); } + static OrtStatus* ORT_API_CALL SelectBestCompiledModelCompatibilityInfo( + OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + size_t num_devices, + _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, + size_t num_compatibility_infos, + size_t* selected_index) noexcept { + return static_cast(this_ptr)->SelectBestCompiledModelCompatibilityInfo( + devices, num_devices, compatibility_infos, num_compatibility_infos, selected_index); + } + static void ORT_API_CALL ReleaseEp(OrtEpFactory* this_ptr, OrtEp* ep) noexcept { static_cast(this_ptr)->ReleaseEp(ep); } diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc index 6137b23111bf9..4defe82ac7172 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc @@ -4,6 +4,7 @@ #include "ep_factory.h" #include +#include #include "ep.h" #include "ep_allocator.h" @@ -13,6 +14,22 @@ #include "core/session/onnxruntime_session_options_config_keys.h" +namespace { +int CompatibilityRank(OrtCompiledModelCompatibility c) { + switch (c) { + case OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL: + return 3; + case OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION: + return 2; + case OrtCompiledModelCompatibility_EP_NOT_APPLICABLE: + return 1; + case OrtCompiledModelCompatibility_EP_UNSUPPORTED: + default: + return 0; + } +} +} // namespace + ExampleEpFactory::ExampleEpFactory(const char* ep_name, ApiPtrs apis, const OrtLogger& default_logger) : OrtEpFactory{}, ApiPtrs(apis), @@ -45,6 +62,7 @@ ExampleEpFactory::ExampleEpFactory(const char* ep_name, ApiPtrs apis, const OrtL GetNumCustomOpDomains = GetNumCustomOpDomainsImpl; GetCustomOpDomains = GetCustomOpDomainsImpl; ValidateCompiledModelCompatibilityInfo = ValidateCompiledModelCompatibilityInfoImpl; + SelectBestCompiledModelCompatibilityInfo = SelectBestCompiledModelCompatibilityInfoImpl; // setup the OrtMemoryInfo instances required by the EP. // We pretend the device the EP is running on is GPU. @@ -514,3 +532,56 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::ValidateCompiledModelCompatibilityInfo *model_compatibility = OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL; return nullptr; } + +OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCompatibilityInfoImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + const char* const* compatibility_infos, + size_t num_compatibility_infos, + size_t* selected_index) noexcept { + auto& factory = *static_cast(this_ptr); + + if (selected_index == nullptr) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, "selected_index cannot be nullptr"); + } + + *selected_index = std::numeric_limits::max(); + + if (compatibility_infos == nullptr || num_compatibility_infos == 0) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, + "compatibility_infos cannot be nullptr or empty"); + } + + int best_rank = -1; + size_t best_idx = std::numeric_limits::max(); + + for (size_t i = 0; i < num_compatibility_infos; ++i) { + if (compatibility_infos[i] == nullptr) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, + "compatibility_infos contains a nullptr entry"); + } + + OrtCompiledModelCompatibility compatibility = OrtCompiledModelCompatibility_EP_UNSUPPORTED; + OrtStatus* status = ValidateCompiledModelCompatibilityInfoImpl( + this_ptr, devices, num_devices, compatibility_infos[i], &compatibility); + if (status != nullptr) { + return status; + } + + const int rank = CompatibilityRank(compatibility); + if (rank > best_rank) { + best_rank = rank; + best_idx = i; + } + } + + // all unsupported => SIZE_MAX + if (best_rank <= CompatibilityRank(OrtCompiledModelCompatibility_EP_UNSUPPORTED)) { + *selected_index = std::numeric_limits::max(); + } else { + *selected_index = best_idx; + } + + return nullptr; +} diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h index 91478047afb0a..fa3d5460f18aa 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h @@ -106,6 +106,14 @@ class ExampleEpFactory : public OrtEpFactory, public ApiPtrs { const char* compatibility_info, OrtCompiledModelCompatibility* model_compatibility) noexcept; + static OrtStatus* ORT_API_CALL SelectBestCompiledModelCompatibilityInfoImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* const* devices, + size_t num_devices, + const char* const* compatibility_infos, + size_t num_compatibility_infos, + size_t* selected_index) noexcept; + const std::string ep_name_; // EP name const std::string vendor_{"Contoso"}; // EP vendor name const uint32_t vendor_id_{0xB357}; // EP vendor ID diff --git a/onnxruntime/test/autoep/test_model_package.cc b/onnxruntime/test/autoep/test_model_package.cc index e3cf87b5f83d3..71974dc388b7a 100644 --- a/onnxruntime/test/autoep/test_model_package.cc +++ b/onnxruntime/test/autoep/test_model_package.cc @@ -753,5 +753,53 @@ TEST(ModelPackageTest, ParseVariantsFromRoot_UsesModelIdForModelDirectory) { std::filesystem::remove_all(package_root, ec); } + +TEST(EpCompatibilityCapiTest, SelectBestCompiledModelCompatibilityInfo_UsesHardwareDevices) { + // Use C++ Env helper to register example plugin EP and get its device. + Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatSelectBestExampleEp"}; + onnxruntime::test::RegisteredEpDeviceUniquePtr example_ep; + onnxruntime::test::Utils::RegisterAndGetExampleEp( + env, onnxruntime::test::Utils::example_ep_info, example_ep); + + ASSERT_NE(example_ep.get(), nullptr); + + const OrtEpDevice* ep_device = example_ep.get(); + OrtEpFactory* factory = ep_device->GetMutableFactory(); + ASSERT_NE(factory, nullptr); + ASSERT_NE(factory->SelectBestCompiledModelCompatibilityInfo, nullptr); + + // API requires OrtHardwareDevice[] + const OrtHardwareDevice* devices[] = {ep_device->device}; + + // Build strings in the exact format expected by ExampleEpFactory::ValidateCompiledModelCompatibilityInfoImpl. + const std::string ep_name = factory->GetName(factory); // expected prefix ";" + const std::string optimal = + ep_name + ";version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + const std::string prefer_recompile = + ep_name + ";version=9.9.9;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + const std::string unsupported = "SomeOtherEp;version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION); + + const char* compatibility_infos[] = { + unsupported.c_str(), + prefer_recompile.c_str(), + optimal.c_str(), + }; + + size_t selected_index = std::numeric_limits::max(); + OrtStatus* st = factory->SelectBestCompiledModelCompatibilityInfo( + factory, + devices, + 1, + compatibility_infos, + 3, + &selected_index); + + const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + ASSERT_NE(api, nullptr); + ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); + + // best should be the OPTIMAL candidate + EXPECT_EQ(selected_index, 2u); +} } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/framework/ep_compatibility_test.cc b/onnxruntime/test/framework/ep_compatibility_test.cc index 288023a130529..5b8371fb5ea96 100644 --- a/onnxruntime/test/framework/ep_compatibility_test.cc +++ b/onnxruntime/test/framework/ep_compatibility_test.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include "gtest/gtest.h" #include "gmock/gmock.h" From 3fc4c52e3235160ffa7727b73f7e43f334097730 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 6 May 2026 14:32:33 -0700 Subject: [PATCH 02/12] update commnet --- include/onnxruntime/core/session/onnxruntime_ep_c_api.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 4b4cc4601fc13..2478ce23125b2 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2971,7 +2971,7 @@ struct OrtEpFactory { * Evaluates each candidate compatibility string against the given hardware devices and returns the selected index. * * Context about having this function: - * The existed ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best + * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best * compatible model when there are multiple candidates. For example, an EP may support multiple compilation modes * (e.g., "speed optimized" vs "memory optimized") that produce different compatibility strings. The EP can implement * this function to evaluate the candidate strings and select the best compatible one based on its own criteria and From b4fa35e54830dd68f7c6a5c300d64a0486dac20f Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 6 May 2026 14:44:14 -0700 Subject: [PATCH 03/12] revert --- onnxruntime/test/framework/ep_compatibility_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/onnxruntime/test/framework/ep_compatibility_test.cc b/onnxruntime/test/framework/ep_compatibility_test.cc index 5b8371fb5ea96..288023a130529 100644 --- a/onnxruntime/test/framework/ep_compatibility_test.cc +++ b/onnxruntime/test/framework/ep_compatibility_test.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include -#include #include "gtest/gtest.h" #include "gmock/gmock.h" From b58f0e35e0dc5efacbccc469e146755720de4478 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 6 May 2026 15:02:28 -0700 Subject: [PATCH 04/12] move test to another file --- .../test/autoep/test_ep_compatibility.cc | 67 +++++++++++++++++++ onnxruntime/test/autoep/test_model_package.cc | 48 ------------- 2 files changed, 67 insertions(+), 48 deletions(-) create mode 100644 onnxruntime/test/autoep/test_ep_compatibility.cc diff --git a/onnxruntime/test/autoep/test_ep_compatibility.cc b/onnxruntime/test/autoep/test_ep_compatibility.cc new file mode 100644 index 0000000000000..7fe5335d6a512 --- /dev/null +++ b/onnxruntime/test/autoep/test_ep_compatibility.cc @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include "gtest/gtest.h" + +#include "core/session/onnxruntime_c_api.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/session/abi_devices.h" +#include "test/autoep/test_autoep_utils.h" + +using namespace onnxruntime; + +TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCompatibilityInfo_UsesHardwareDevices) { + Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatSelectBestExampleEp"}; + + onnxruntime::test::RegisteredEpDeviceUniquePtr example_ep; + onnxruntime::test::Utils::RegisterAndGetExampleEp( + env, onnxruntime::test::Utils::example_ep_info, example_ep); + + ASSERT_NE(example_ep.get(), nullptr); + + const OrtEpDevice* ep_device = example_ep.get(); + OrtEpFactory* factory = ep_device->GetMutableFactory(); + ASSERT_NE(factory, nullptr); + ASSERT_NE(factory->SelectBestCompiledModelCompatibilityInfo, nullptr); + + // API takes OrtHardwareDevice[]. + const OrtHardwareDevice* devices[] = {ep_device->device}; + + // Match ExampleEpFactory::ValidateCompiledModelCompatibilityInfoImpl format: + // ";version=X;ort_api_version=Y;hardware_architecture=Z" + const std::string ep_name = factory->GetName(factory); + + const std::string optimal = + ep_name + ";version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + + const std::string prefer_recompile = + ep_name + ";version=9.9.9;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + + const std::string unsupported = + "SomeOtherEp;version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + + const char* compatibility_infos[] = { + unsupported.c_str(), + prefer_recompile.c_str(), + optimal.c_str(), + }; + + size_t selected_index = std::numeric_limits::max(); + OrtStatus* st = factory->SelectBestCompiledModelCompatibilityInfo( + factory, + devices, + 1, + compatibility_infos, + 3, + &selected_index); + + const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + ASSERT_NE(api, nullptr); + ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); + + // Best candidate should be the fully optimal one. + EXPECT_EQ(selected_index, 2u); +} \ No newline at end of file diff --git a/onnxruntime/test/autoep/test_model_package.cc b/onnxruntime/test/autoep/test_model_package.cc index 71974dc388b7a..e3cf87b5f83d3 100644 --- a/onnxruntime/test/autoep/test_model_package.cc +++ b/onnxruntime/test/autoep/test_model_package.cc @@ -753,53 +753,5 @@ TEST(ModelPackageTest, ParseVariantsFromRoot_UsesModelIdForModelDirectory) { std::filesystem::remove_all(package_root, ec); } - -TEST(EpCompatibilityCapiTest, SelectBestCompiledModelCompatibilityInfo_UsesHardwareDevices) { - // Use C++ Env helper to register example plugin EP and get its device. - Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatSelectBestExampleEp"}; - onnxruntime::test::RegisteredEpDeviceUniquePtr example_ep; - onnxruntime::test::Utils::RegisterAndGetExampleEp( - env, onnxruntime::test::Utils::example_ep_info, example_ep); - - ASSERT_NE(example_ep.get(), nullptr); - - const OrtEpDevice* ep_device = example_ep.get(); - OrtEpFactory* factory = ep_device->GetMutableFactory(); - ASSERT_NE(factory, nullptr); - ASSERT_NE(factory->SelectBestCompiledModelCompatibilityInfo, nullptr); - - // API requires OrtHardwareDevice[] - const OrtHardwareDevice* devices[] = {ep_device->device}; - - // Build strings in the exact format expected by ExampleEpFactory::ValidateCompiledModelCompatibilityInfoImpl. - const std::string ep_name = factory->GetName(factory); // expected prefix ";" - const std::string optimal = - ep_name + ";version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; - const std::string prefer_recompile = - ep_name + ";version=9.9.9;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; - const std::string unsupported = "SomeOtherEp;version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION); - - const char* compatibility_infos[] = { - unsupported.c_str(), - prefer_recompile.c_str(), - optimal.c_str(), - }; - - size_t selected_index = std::numeric_limits::max(); - OrtStatus* st = factory->SelectBestCompiledModelCompatibilityInfo( - factory, - devices, - 1, - compatibility_infos, - 3, - &selected_index); - - const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); - ASSERT_NE(api, nullptr); - ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); - - // best should be the OPTIMAL candidate - EXPECT_EQ(selected_index, 2u); -} } // namespace test } // namespace onnxruntime From ad43b3c6c4c8b37804db8d0990ef34c23326a81e Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 8 May 2026 11:29:55 -0700 Subject: [PATCH 05/12] address reviewer's comment --- .../core/session/onnxruntime_ep_c_api.h | 37 +++++++++++++------ .../session/plugin_ep/ep_factory_internal.cc | 2 +- .../session/plugin_ep/ep_factory_internal.h | 16 ++++---- .../plugin_ep/ep_factory_internal_impl.h | 10 ++--- .../plugin_ep/forward_to_factory_impl.h | 10 ++--- .../library/example_plugin_ep/ep_factory.cc | 36 ++++++++++++------ .../library/example_plugin_ep/ep_factory.h | 6 +-- .../test/autoep/test_ep_compatibility.cc | 30 +++++++-------- 8 files changed, 85 insertions(+), 62 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 2478ce23125b2..1875f053b8790 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2111,6 +2111,19 @@ typedef enum OrtGraphCaptureNodeAssignmentPolicy { OrtGraphCaptureNodeAssignmentPolicy_ALLOW_CPU_FOR_SHAPES = 1, } OrtGraphCaptureNodeAssignmentPolicy; +/** \brief Per-candidate metadata passed to SelectBestCompiledModelCandidate. + * + * Required key: + * - "ep_compatibility_info" + * + * \since Version 1.27. + */ +typedef struct OrtCompiledModelCandidateMetadata { + _In_reads_(num_entries) const char* const* keys; + _In_reads_(num_entries) const char* const* values; + size_t num_entries; +} OrtCompiledModelCandidateMetadata; + /** * \brief The OrtEp struct provides functions to implement for an execution provider. * \since Version 1.22. @@ -2966,9 +2979,12 @@ struct OrtEpFactory { ORT_API2_STATUS(DeinitGraphicsInterop, _In_ OrtEpFactory* this_ptr, _In_ const OrtEpDevice* ep_device); - /** \brief Select the best compiled model compatibility info from candidate strings. + /** \brief Select the best compiled model candidate from metadata lists. * - * Evaluates each candidate compatibility string against the given hardware devices and returns the selected index. + * Evaluates each candidate metadata against the given hardware devices and returns the selected index. + * + * Each candidate metadata can have multiple entrys and the entry with "ep_compatibility_info" key is required, + * it's associated value is the compatibility information stored in onnx model metadata. * * Context about having this function: * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best @@ -2980,22 +2996,21 @@ struct OrtEpFactory { * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. * * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] devices Array of OrtHardwareDevice pointers that the EP would run on. All must map to this EP. - * \param[in] num_devices Number of entries in `devices`. - * \param[in] compatibility_infos Array of candidate compatibility strings. - * \param[in] num_compatibility_infos Number of entries in `compatibility_infos`. - * \param[out] selected_index Index of selected candidate, or SIZE_MAX if all candidates are unsupported. + * \param[in] devices Target hardware devices. + * \param[in] num_devices Number of devices. + * \param[in] candidates Array of candidate metadata entries (one per model variant). + * \param[in] num_candidates Number of candidates. + * \param[out] selected_index Selected candidate index, or SIZE_MAX if all unsupported. * * \snippet{doc} snippets.dox OrtStatus Return Value * * \since Version 1.27. */ - ORT_API2_STATUS(SelectBestCompiledModelCompatibilityInfo, - _In_ OrtEpFactory* this_ptr, + ORT_API2_STATUS(SelectBestCompiledModelCandidate, _In_ OrtEpFactory* this_ptr, _In_reads_(num_devices) const OrtHardwareDevice* const* devices, _In_ size_t num_devices, - _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, - _In_ size_t num_compatibility_infos, + _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + _In_ size_t num_candidates, _Out_ size_t* selected_index); }; diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc index 77ef684db7b1f..52f38a2465abb 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc @@ -36,7 +36,7 @@ EpFactoryInternal::EpFactoryInternal(std::unique_ptr impl OrtEpFactory::GetHardwareDeviceIncompatibilityDetails = Forward::GetHardwareDeviceIncompatibilityDetails; OrtEpFactory::InitGraphicsInterop = Forward::InitGraphicsInterop; OrtEpFactory::DeinitGraphicsInterop = Forward::DeinitGraphicsInterop; - OrtEpFactory::SelectBestCompiledModelCompatibilityInfo = Forward::SelectBestCompiledModelCompatibilityInfo; + OrtEpFactory::SelectBestCompiledModelCandidate = Forward::SelectBestCompiledModelCandidate; } InternalExecutionProviderFactory::InternalExecutionProviderFactory(EpFactoryInternal& ep_factory, diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h index 17bbfcc4282ef..e928b6c1461d3 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h @@ -106,14 +106,14 @@ class EpFactoryInternal : public OrtEpFactory { return impl_->DeinitGraphicsInterop(ep_device); } - OrtStatus* SelectBestCompiledModelCompatibilityInfo(_In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, - _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, - _In_ size_t num_compatibility_infos, - _Out_ size_t* selected_index) noexcept { - return impl_->SelectBestCompiledModelCompatibilityInfo(devices, num_devices, - compatibility_infos, num_compatibility_infos, - selected_index); + OrtStatus* SelectBestCompiledModelCandidate(_In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + _In_ size_t num_candidates, + _Out_ size_t* selected_index) noexcept { + return impl_->SelectBestCompiledModelCandidate(devices, num_devices, + candidates, num_candidates, + selected_index); } // Function ORT calls to release an EP instance. diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h index d86b8daeb8981..8cbd3f46d5a04 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h @@ -122,17 +122,17 @@ class EpFactoryInternalImpl { return nullptr; } - virtual OrtStatus* SelectBestCompiledModelCompatibilityInfo( + virtual OrtStatus* SelectBestCompiledModelCandidate( _In_reads_(num_devices) const OrtHardwareDevice* const* devices, _In_ size_t num_devices, - _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, - _In_ size_t num_compatibility_infos, + _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + _In_ size_t num_candidates, _Out_ size_t* selected_index) noexcept { ORT_UNUSED_PARAMETER(devices); ORT_UNUSED_PARAMETER(num_devices); - if (compatibility_infos == nullptr || num_compatibility_infos == 0 || selected_index == nullptr) { + if (candidates == nullptr || num_candidates == 0 || selected_index == nullptr) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, - "Invalid arguments to SelectBestCompiledModelCompatibilityInfo."); + "Invalid arguments to SelectBestCompiledModelCandidate."); } // Default implementation: all candidates are unsupported, sets `selected_index` to SIZE_MAX. diff --git a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h index 623a2a4ae2c0f..64ca0ecdb3f36 100644 --- a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h +++ b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h @@ -106,15 +106,15 @@ struct ForwardToFactoryImpl { return static_cast(this_ptr)->DeinitGraphicsInterop(ep_device); } - static OrtStatus* ORT_API_CALL SelectBestCompiledModelCompatibilityInfo( + static OrtStatus* ORT_API_CALL SelectBestCompiledModelCandidate( OrtEpFactory* this_ptr, _In_reads_(num_devices) const OrtHardwareDevice* const* devices, size_t num_devices, - _In_reads_(num_compatibility_infos) const char* const* compatibility_infos, - size_t num_compatibility_infos, + _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + size_t num_candidates, size_t* selected_index) noexcept { - return static_cast(this_ptr)->SelectBestCompiledModelCompatibilityInfo( - devices, num_devices, compatibility_infos, num_compatibility_infos, selected_index); + return static_cast(this_ptr)->SelectBestCompiledModelCandidate( + devices, num_devices, candidates, num_candidates, selected_index); } static void ORT_API_CALL ReleaseEp(OrtEpFactory* this_ptr, OrtEp* ep) noexcept { diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc index 4defe82ac7172..09a3aef4b11fa 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc @@ -62,7 +62,7 @@ ExampleEpFactory::ExampleEpFactory(const char* ep_name, ApiPtrs apis, const OrtL GetNumCustomOpDomains = GetNumCustomOpDomainsImpl; GetCustomOpDomains = GetCustomOpDomainsImpl; ValidateCompiledModelCompatibilityInfo = ValidateCompiledModelCompatibilityInfoImpl; - SelectBestCompiledModelCompatibilityInfo = SelectBestCompiledModelCompatibilityInfoImpl; + SelectBestCompiledModelCandidate = SelectBestCompiledModelCandidateImpl; // setup the OrtMemoryInfo instances required by the EP. // We pretend the device the EP is running on is GPU. @@ -533,12 +533,12 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::ValidateCompiledModelCompatibilityInfo return nullptr; } -OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCompatibilityInfoImpl( +OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCandidateImpl( OrtEpFactory* this_ptr, const OrtHardwareDevice* const* devices, size_t num_devices, - const char* const* compatibility_infos, - size_t num_compatibility_infos, + const OrtCompiledModelCandidateMetadata* candidates, + size_t num_candidates, size_t* selected_index) noexcept { auto& factory = *static_cast(this_ptr); @@ -548,23 +548,36 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCompatibilityIn *selected_index = std::numeric_limits::max(); - if (compatibility_infos == nullptr || num_compatibility_infos == 0) { - return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, - "compatibility_infos cannot be nullptr or empty"); + if (candidates == nullptr || num_candidates == 0) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, "candidates cannot be nullptr or empty"); } + auto find_value_for_key = [](const OrtCompiledModelCandidateMetadata& candidate, const char* key) -> const char* { + if (candidate.keys == nullptr || candidate.values == nullptr) return nullptr; + for (size_t i = 0; i < candidate.num_entries; ++i) { + if (candidate.keys[i] != nullptr && candidate.values[i] != nullptr && + std::strcmp(candidate.keys[i], key) == 0) { + return candidate.values[i]; + } + } + return nullptr; + }; + int best_rank = -1; size_t best_idx = std::numeric_limits::max(); - for (size_t i = 0; i < num_compatibility_infos; ++i) { - if (compatibility_infos[i] == nullptr) { + for (size_t i = 0; i < num_candidates; ++i) { + const char* compatibility_info = + find_value_for_key(candidates[i], "ep_compatibility_info"); + + if (compatibility_info == nullptr) { return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, - "compatibility_infos contains a nullptr entry"); + "candidate metadata is missing required key: ep_compatibility_info"); } OrtCompiledModelCompatibility compatibility = OrtCompiledModelCompatibility_EP_UNSUPPORTED; OrtStatus* status = ValidateCompiledModelCompatibilityInfoImpl( - this_ptr, devices, num_devices, compatibility_infos[i], &compatibility); + this_ptr, devices, num_devices, compatibility_info, &compatibility); if (status != nullptr) { return status; } @@ -576,7 +589,6 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCompatibilityIn } } - // all unsupported => SIZE_MAX if (best_rank <= CompatibilityRank(OrtCompiledModelCompatibility_EP_UNSUPPORTED)) { *selected_index = std::numeric_limits::max(); } else { diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h index fa3d5460f18aa..82410a307c8fc 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h @@ -106,12 +106,12 @@ class ExampleEpFactory : public OrtEpFactory, public ApiPtrs { const char* compatibility_info, OrtCompiledModelCompatibility* model_compatibility) noexcept; - static OrtStatus* ORT_API_CALL SelectBestCompiledModelCompatibilityInfoImpl( + static OrtStatus* ORT_API_CALL SelectBestCompiledModelCandidateImpl( OrtEpFactory* this_ptr, const OrtHardwareDevice* const* devices, size_t num_devices, - const char* const* compatibility_infos, - size_t num_compatibility_infos, + const OrtCompiledModelCandidateMetadata* candidates, + size_t num_candidates, size_t* selected_index) noexcept; const std::string ep_name_; // EP name diff --git a/onnxruntime/test/autoep/test_ep_compatibility.cc b/onnxruntime/test/autoep/test_ep_compatibility.cc index 7fe5335d6a512..be436b8de01bc 100644 --- a/onnxruntime/test/autoep/test_ep_compatibility.cc +++ b/onnxruntime/test/autoep/test_ep_compatibility.cc @@ -13,7 +13,7 @@ using namespace onnxruntime; -TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCompatibilityInfo_UsesHardwareDevices) { +TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCandidate_UsesHardwareDevices) { Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatSelectBestExampleEp"}; onnxruntime::test::RegisteredEpDeviceUniquePtr example_ep; @@ -25,13 +25,10 @@ TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCompatibilityInfo_Use const OrtEpDevice* ep_device = example_ep.get(); OrtEpFactory* factory = ep_device->GetMutableFactory(); ASSERT_NE(factory, nullptr); - ASSERT_NE(factory->SelectBestCompiledModelCompatibilityInfo, nullptr); + ASSERT_NE(factory->SelectBestCompiledModelCandidate, nullptr); - // API takes OrtHardwareDevice[]. const OrtHardwareDevice* devices[] = {ep_device->device}; - // Match ExampleEpFactory::ValidateCompiledModelCompatibilityInfoImpl format: - // ";version=X;ort_api_version=Y;hardware_architecture=Z" const std::string ep_name = factory->GetName(factory); const std::string optimal = @@ -43,25 +40,24 @@ TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCompatibilityInfo_Use const std::string unsupported = "SomeOtherEp;version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; - const char* compatibility_infos[] = { - unsupported.c_str(), - prefer_recompile.c_str(), - optimal.c_str(), + const char* keys[] = {"ep_compatibility_info"}; + const char* values0[] = {unsupported.c_str()}; + const char* values1[] = {prefer_recompile.c_str()}; + const char* values2[] = {optimal.c_str()}; + + const OrtCompiledModelCandidateMetadata candidates[] = { + {keys, values0, 1}, + {keys, values1, 1}, + {keys, values2, 1}, }; size_t selected_index = std::numeric_limits::max(); - OrtStatus* st = factory->SelectBestCompiledModelCompatibilityInfo( - factory, - devices, - 1, - compatibility_infos, - 3, - &selected_index); + OrtStatus* st = factory->SelectBestCompiledModelCandidate( + factory, devices, 1, candidates, 3, &selected_index); const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); ASSERT_NE(api, nullptr); ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); - // Best candidate should be the fully optimal one. EXPECT_EQ(selected_index, 2u); } \ No newline at end of file From 5a8a47cacbb5bd5c0804a957008b57e9fe682c7d Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 29 May 2026 11:15:40 -0700 Subject: [PATCH 06/12] address reviewer's comments --- .../core/session/onnxruntime_ep_c_api.h | 26 +++++--------- .../session/plugin_ep/ep_factory_internal.cc | 2 +- .../session/plugin_ep/ep_factory_internal.h | 16 ++++----- .../plugin_ep/ep_factory_internal_impl.h | 6 ++-- .../plugin_ep/forward_to_factory_impl.h | 6 ++-- .../library/example_plugin_ep/ep_factory.cc | 19 +++-------- .../library/example_plugin_ep/ep_factory.h | 4 +-- .../test/autoep/test_ep_compatibility.cc | 34 +++++++++++-------- 8 files changed, 49 insertions(+), 64 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 1875f053b8790..094dea77a97b9 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2111,19 +2111,6 @@ typedef enum OrtGraphCaptureNodeAssignmentPolicy { OrtGraphCaptureNodeAssignmentPolicy_ALLOW_CPU_FOR_SHAPES = 1, } OrtGraphCaptureNodeAssignmentPolicy; -/** \brief Per-candidate metadata passed to SelectBestCompiledModelCandidate. - * - * Required key: - * - "ep_compatibility_info" - * - * \since Version 1.27. - */ -typedef struct OrtCompiledModelCandidateMetadata { - _In_reads_(num_entries) const char* const* keys; - _In_reads_(num_entries) const char* const* values; - size_t num_entries; -} OrtCompiledModelCandidateMetadata; - /** * \brief The OrtEp struct provides functions to implement for an execution provider. * \since Version 1.22. @@ -2983,8 +2970,8 @@ struct OrtEpFactory { * * Evaluates each candidate metadata against the given hardware devices and returns the selected index. * - * Each candidate metadata can have multiple entrys and the entry with "ep_compatibility_info" key is required, - * it's associated value is the compatibility information stored in onnx model metadata. + * Each candidate's OrtKeyValuePairs can have multiple entries and the entry with "ep_compatibility_info" key is + * required; its associated value is the compatibility information stored in onnx model metadata. * * Context about having this function: * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best @@ -2995,10 +2982,13 @@ struct OrtEpFactory { * * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. * + * \note The implementer should validate the compatibility of each candidate (e.g., by calling + * ValidateCompiledModelCompatibilityInfo for each one) before determining the best match. + * * \param[in] this_ptr The OrtEpFactory instance. * \param[in] devices Target hardware devices. * \param[in] num_devices Number of devices. - * \param[in] candidates Array of candidate metadata entries (one per model variant). + * \param[in] candidates Array of OrtKeyValuePairs pointers (one per model variant). * \param[in] num_candidates Number of candidates. * \param[out] selected_index Selected candidate index, or SIZE_MAX if all unsupported. * @@ -3006,10 +2996,10 @@ struct OrtEpFactory { * * \since Version 1.27. */ - ORT_API2_STATUS(SelectBestCompiledModelCandidate, _In_ OrtEpFactory* this_ptr, + ORT_API2_STATUS(SelectBestModelCandidate, _In_ OrtEpFactory* this_ptr, _In_reads_(num_devices) const OrtHardwareDevice* const* devices, _In_ size_t num_devices, - _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, _In_ size_t num_candidates, _Out_ size_t* selected_index); }; diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc index 52f38a2465abb..fb8a90f25ce13 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc @@ -36,7 +36,7 @@ EpFactoryInternal::EpFactoryInternal(std::unique_ptr impl OrtEpFactory::GetHardwareDeviceIncompatibilityDetails = Forward::GetHardwareDeviceIncompatibilityDetails; OrtEpFactory::InitGraphicsInterop = Forward::InitGraphicsInterop; OrtEpFactory::DeinitGraphicsInterop = Forward::DeinitGraphicsInterop; - OrtEpFactory::SelectBestCompiledModelCandidate = Forward::SelectBestCompiledModelCandidate; + OrtEpFactory::SelectBestModelCandidate = Forward::SelectBestModelCandidate; } InternalExecutionProviderFactory::InternalExecutionProviderFactory(EpFactoryInternal& ep_factory, diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h index e928b6c1461d3..8ad426ae1523b 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h @@ -106,14 +106,14 @@ class EpFactoryInternal : public OrtEpFactory { return impl_->DeinitGraphicsInterop(ep_device); } - OrtStatus* SelectBestCompiledModelCandidate(_In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, - _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, - _In_ size_t num_candidates, - _Out_ size_t* selected_index) noexcept { - return impl_->SelectBestCompiledModelCandidate(devices, num_devices, - candidates, num_candidates, - selected_index); + OrtStatus* SelectBestModelCandidate(_In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, + _In_ size_t num_candidates, + _Out_ size_t* selected_index) noexcept { + return impl_->SelectBestModelCandidate(devices, num_devices, + candidates, num_candidates, + selected_index); } // Function ORT calls to release an EP instance. diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h index 8cbd3f46d5a04..9b2b67871d230 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h @@ -122,17 +122,17 @@ class EpFactoryInternalImpl { return nullptr; } - virtual OrtStatus* SelectBestCompiledModelCandidate( + virtual OrtStatus* SelectBestModelCandidate( _In_reads_(num_devices) const OrtHardwareDevice* const* devices, _In_ size_t num_devices, - _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, _In_ size_t num_candidates, _Out_ size_t* selected_index) noexcept { ORT_UNUSED_PARAMETER(devices); ORT_UNUSED_PARAMETER(num_devices); if (candidates == nullptr || num_candidates == 0 || selected_index == nullptr) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, - "Invalid arguments to SelectBestCompiledModelCandidate."); + "Invalid arguments to SelectBestModelCandidate."); } // Default implementation: all candidates are unsupported, sets `selected_index` to SIZE_MAX. diff --git a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h index 64ca0ecdb3f36..390a94fef0b54 100644 --- a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h +++ b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h @@ -106,14 +106,14 @@ struct ForwardToFactoryImpl { return static_cast(this_ptr)->DeinitGraphicsInterop(ep_device); } - static OrtStatus* ORT_API_CALL SelectBestCompiledModelCandidate( + static OrtStatus* ORT_API_CALL SelectBestModelCandidate( OrtEpFactory* this_ptr, _In_reads_(num_devices) const OrtHardwareDevice* const* devices, size_t num_devices, - _In_reads_(num_candidates) const OrtCompiledModelCandidateMetadata* candidates, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, size_t num_candidates, size_t* selected_index) noexcept { - return static_cast(this_ptr)->SelectBestCompiledModelCandidate( + return static_cast(this_ptr)->SelectBestModelCandidate( devices, num_devices, candidates, num_candidates, selected_index); } diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc index 09a3aef4b11fa..566e617a48511 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc @@ -62,7 +62,7 @@ ExampleEpFactory::ExampleEpFactory(const char* ep_name, ApiPtrs apis, const OrtL GetNumCustomOpDomains = GetNumCustomOpDomainsImpl; GetCustomOpDomains = GetCustomOpDomainsImpl; ValidateCompiledModelCompatibilityInfo = ValidateCompiledModelCompatibilityInfoImpl; - SelectBestCompiledModelCandidate = SelectBestCompiledModelCandidateImpl; + SelectBestModelCandidate = SelectBestModelCandidateImpl; // setup the OrtMemoryInfo instances required by the EP. // We pretend the device the EP is running on is GPU. @@ -533,11 +533,11 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::ValidateCompiledModelCompatibilityInfo return nullptr; } -OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCandidateImpl( +OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestModelCandidateImpl( OrtEpFactory* this_ptr, const OrtHardwareDevice* const* devices, size_t num_devices, - const OrtCompiledModelCandidateMetadata* candidates, + const OrtKeyValuePairs* const* candidates, size_t num_candidates, size_t* selected_index) noexcept { auto& factory = *static_cast(this_ptr); @@ -552,23 +552,12 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestCompiledModelCandidateImpl( return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, "candidates cannot be nullptr or empty"); } - auto find_value_for_key = [](const OrtCompiledModelCandidateMetadata& candidate, const char* key) -> const char* { - if (candidate.keys == nullptr || candidate.values == nullptr) return nullptr; - for (size_t i = 0; i < candidate.num_entries; ++i) { - if (candidate.keys[i] != nullptr && candidate.values[i] != nullptr && - std::strcmp(candidate.keys[i], key) == 0) { - return candidate.values[i]; - } - } - return nullptr; - }; - int best_rank = -1; size_t best_idx = std::numeric_limits::max(); for (size_t i = 0; i < num_candidates; ++i) { const char* compatibility_info = - find_value_for_key(candidates[i], "ep_compatibility_info"); + factory.ort_api.GetKeyValue(candidates[i], "ep_compatibility_info"); if (compatibility_info == nullptr) { return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h index 82410a307c8fc..3acd5a565e4fc 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h @@ -106,11 +106,11 @@ class ExampleEpFactory : public OrtEpFactory, public ApiPtrs { const char* compatibility_info, OrtCompiledModelCompatibility* model_compatibility) noexcept; - static OrtStatus* ORT_API_CALL SelectBestCompiledModelCandidateImpl( + static OrtStatus* ORT_API_CALL SelectBestModelCandidateImpl( OrtEpFactory* this_ptr, const OrtHardwareDevice* const* devices, size_t num_devices, - const OrtCompiledModelCandidateMetadata* candidates, + const OrtKeyValuePairs* const* candidates, size_t num_candidates, size_t* selected_index) noexcept; diff --git a/onnxruntime/test/autoep/test_ep_compatibility.cc b/onnxruntime/test/autoep/test_ep_compatibility.cc index be436b8de01bc..7c2526c4a46f5 100644 --- a/onnxruntime/test/autoep/test_ep_compatibility.cc +++ b/onnxruntime/test/autoep/test_ep_compatibility.cc @@ -13,7 +13,7 @@ using namespace onnxruntime; -TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCandidate_UsesHardwareDevices) { +TEST(EpCompatibilitySelectBestTest, SelectBestModelCandidate_UsesHardwareDevices) { Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatSelectBestExampleEp"}; onnxruntime::test::RegisteredEpDeviceUniquePtr example_ep; @@ -25,7 +25,7 @@ TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCandidate_UsesHardwar const OrtEpDevice* ep_device = example_ep.get(); OrtEpFactory* factory = ep_device->GetMutableFactory(); ASSERT_NE(factory, nullptr); - ASSERT_NE(factory->SelectBestCompiledModelCandidate, nullptr); + ASSERT_NE(factory->SelectBestModelCandidate, nullptr); const OrtHardwareDevice* devices[] = {ep_device->device}; @@ -40,24 +40,30 @@ TEST(EpCompatibilitySelectBestTest, SelectBestCompiledModelCandidate_UsesHardwar const std::string unsupported = "SomeOtherEp;version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; - const char* keys[] = {"ep_compatibility_info"}; - const char* values0[] = {unsupported.c_str()}; - const char* values1[] = {prefer_recompile.c_str()}; - const char* values2[] = {optimal.c_str()}; + const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + ASSERT_NE(api, nullptr); + + OrtKeyValuePairs* kvp0 = nullptr; + OrtKeyValuePairs* kvp1 = nullptr; + OrtKeyValuePairs* kvp2 = nullptr; + api->CreateKeyValuePairs(&kvp0); + api->CreateKeyValuePairs(&kvp1); + api->CreateKeyValuePairs(&kvp2); + api->AddKeyValuePair(kvp0, "ep_compatibility_info", unsupported.c_str()); + api->AddKeyValuePair(kvp1, "ep_compatibility_info", prefer_recompile.c_str()); + api->AddKeyValuePair(kvp2, "ep_compatibility_info", optimal.c_str()); - const OrtCompiledModelCandidateMetadata candidates[] = { - {keys, values0, 1}, - {keys, values1, 1}, - {keys, values2, 1}, - }; + const OrtKeyValuePairs* candidates[] = {kvp0, kvp1, kvp2}; size_t selected_index = std::numeric_limits::max(); - OrtStatus* st = factory->SelectBestCompiledModelCandidate( + OrtStatus* st = factory->SelectBestModelCandidate( factory, devices, 1, candidates, 3, &selected_index); - const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); - ASSERT_NE(api, nullptr); ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); EXPECT_EQ(selected_index, 2u); + + api->ReleaseKeyValuePairs(kvp0); + api->ReleaseKeyValuePairs(kvp1); + api->ReleaseKeyValuePairs(kvp2); } \ No newline at end of file From 6bdf709087f1f1a23b8785a1fca6e0318434b11e Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 12 Jun 2026 10:37:06 -0700 Subject: [PATCH 07/12] address reviewer's comments --- .../core/session/onnxruntime_ep_c_api.h | 18 ++++++++++-------- .../session/plugin_ep/ep_factory_internal.h | 9 ++++----- .../plugin_ep/ep_factory_internal_impl.h | 8 ++++---- .../plugin_ep/forward_to_factory_impl.h | 6 +++--- .../library/example_plugin_ep/ep_factory.cc | 8 +++++--- .../library/example_plugin_ep/ep_factory.h | 4 ++-- .../test/autoep/test_ep_compatibility.cc | 4 +--- 7 files changed, 29 insertions(+), 28 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 094dea77a97b9..a34ad925589a5 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2966,9 +2966,10 @@ struct OrtEpFactory { ORT_API2_STATUS(DeinitGraphicsInterop, _In_ OrtEpFactory* this_ptr, _In_ const OrtEpDevice* ep_device); - /** \brief Select the best compiled model candidate from metadata lists. + /** \brief Select the best model candidate from metadata lists. * - * Evaluates each candidate metadata against the given hardware devices and returns the selected index. + * Evaluates each candidate metadata against the given hardware device and optional session options, + * and returns the index of the best match. * * Each candidate's OrtKeyValuePairs can have multiple entries and the entry with "ep_compatibility_info" key is * required; its associated value is the compatibility information stored in onnx model metadata. @@ -2977,8 +2978,8 @@ struct OrtEpFactory { * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best * compatible model when there are multiple candidates. For example, an EP may support multiple compilation modes * (e.g., "speed optimized" vs "memory optimized") that produce different compatibility strings. The EP can implement - * this function to evaluate the candidate strings and select the best compatible one based on its own criteria and - * the target devices. + * this function to evaluate the candidate strings and select the best compatible one based on its own criteria, + * the target device, and the session options. * * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. * @@ -2986,10 +2987,11 @@ struct OrtEpFactory { * ValidateCompiledModelCompatibilityInfo for each one) before determining the best match. * * \param[in] this_ptr The OrtEpFactory instance. - * \param[in] devices Target hardware devices. - * \param[in] num_devices Number of devices. + * \param[in] device The target hardware device that the EP would run on. Must map to this EP. * \param[in] candidates Array of OrtKeyValuePairs pointers (one per model variant). * \param[in] num_candidates Number of candidates. + * \param[in] session_options Optional session options to consider when selecting the best candidate. + * May be nullptr if no session-level preferences are relevant. * \param[out] selected_index Selected candidate index, or SIZE_MAX if all unsupported. * * \snippet{doc} snippets.dox OrtStatus Return Value @@ -2997,10 +2999,10 @@ struct OrtEpFactory { * \since Version 1.27. */ ORT_API2_STATUS(SelectBestModelCandidate, _In_ OrtEpFactory* this_ptr, - _In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, + _In_ const OrtHardwareDevice* device, _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, _Out_ size_t* selected_index); }; diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h index 8ad426ae1523b..cdfe5347185b7 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h @@ -106,14 +106,13 @@ class EpFactoryInternal : public OrtEpFactory { return impl_->DeinitGraphicsInterop(ep_device); } - OrtStatus* SelectBestModelCandidate(_In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, + OrtStatus* SelectBestModelCandidate(_In_ const OrtHardwareDevice* device, _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, _Out_ size_t* selected_index) noexcept { - return impl_->SelectBestModelCandidate(devices, num_devices, - candidates, num_candidates, - selected_index); + return impl_->SelectBestModelCandidate(device, candidates, num_candidates, + session_options, selected_index); } // Function ORT calls to release an EP instance. diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h index 9b2b67871d230..cbadabfd8c3eb 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h @@ -123,13 +123,13 @@ class EpFactoryInternalImpl { } virtual OrtStatus* SelectBestModelCandidate( - _In_reads_(num_devices) const OrtHardwareDevice* const* devices, - _In_ size_t num_devices, + _In_ const OrtHardwareDevice* device, _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, _Out_ size_t* selected_index) noexcept { - ORT_UNUSED_PARAMETER(devices); - ORT_UNUSED_PARAMETER(num_devices); + ORT_UNUSED_PARAMETER(device); + ORT_UNUSED_PARAMETER(session_options); if (candidates == nullptr || num_candidates == 0 || selected_index == nullptr) { return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Invalid arguments to SelectBestModelCandidate."); diff --git a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h index 390a94fef0b54..9dbf25a8251a4 100644 --- a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h +++ b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h @@ -108,13 +108,13 @@ struct ForwardToFactoryImpl { static OrtStatus* ORT_API_CALL SelectBestModelCandidate( OrtEpFactory* this_ptr, - _In_reads_(num_devices) const OrtHardwareDevice* const* devices, - size_t num_devices, + _In_ const OrtHardwareDevice* device, _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, size_t* selected_index) noexcept { return static_cast(this_ptr)->SelectBestModelCandidate( - devices, num_devices, candidates, num_candidates, selected_index); + device, candidates, num_candidates, session_options, selected_index); } static void ORT_API_CALL ReleaseEp(OrtEpFactory* this_ptr, OrtEp* ep) noexcept { diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc index 566e617a48511..77fe29d52e6e9 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc @@ -535,10 +535,10 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::ValidateCompiledModelCompatibilityInfo OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestModelCandidateImpl( OrtEpFactory* this_ptr, - const OrtHardwareDevice* const* devices, - size_t num_devices, + const OrtHardwareDevice* device, const OrtKeyValuePairs* const* candidates, size_t num_candidates, + const OrtSessionOptions* /*session_options*/, size_t* selected_index) noexcept { auto& factory = *static_cast(this_ptr); @@ -552,6 +552,8 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestModelCandidateImpl( return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, "candidates cannot be nullptr or empty"); } + const OrtHardwareDevice* devices[] = {device}; + int best_rank = -1; size_t best_idx = std::numeric_limits::max(); @@ -566,7 +568,7 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestModelCandidateImpl( OrtCompiledModelCompatibility compatibility = OrtCompiledModelCompatibility_EP_UNSUPPORTED; OrtStatus* status = ValidateCompiledModelCompatibilityInfoImpl( - this_ptr, devices, num_devices, compatibility_info, &compatibility); + this_ptr, devices, 1, compatibility_info, &compatibility); if (status != nullptr) { return status; } diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h index 3acd5a565e4fc..31bc4abeff887 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h @@ -108,10 +108,10 @@ class ExampleEpFactory : public OrtEpFactory, public ApiPtrs { static OrtStatus* ORT_API_CALL SelectBestModelCandidateImpl( OrtEpFactory* this_ptr, - const OrtHardwareDevice* const* devices, - size_t num_devices, + const OrtHardwareDevice* device, const OrtKeyValuePairs* const* candidates, size_t num_candidates, + const OrtSessionOptions* session_options, size_t* selected_index) noexcept; const std::string ep_name_; // EP name diff --git a/onnxruntime/test/autoep/test_ep_compatibility.cc b/onnxruntime/test/autoep/test_ep_compatibility.cc index 7c2526c4a46f5..437ebc70ef08f 100644 --- a/onnxruntime/test/autoep/test_ep_compatibility.cc +++ b/onnxruntime/test/autoep/test_ep_compatibility.cc @@ -27,8 +27,6 @@ TEST(EpCompatibilitySelectBestTest, SelectBestModelCandidate_UsesHardwareDevices ASSERT_NE(factory, nullptr); ASSERT_NE(factory->SelectBestModelCandidate, nullptr); - const OrtHardwareDevice* devices[] = {ep_device->device}; - const std::string ep_name = factory->GetName(factory); const std::string optimal = @@ -57,7 +55,7 @@ TEST(EpCompatibilitySelectBestTest, SelectBestModelCandidate_UsesHardwareDevices size_t selected_index = std::numeric_limits::max(); OrtStatus* st = factory->SelectBestModelCandidate( - factory, devices, 1, candidates, 3, &selected_index); + factory, ep_device->device, candidates, 3, nullptr, &selected_index); ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); From 6354b14b05099e2b7557a1a7bf266dc40c771fcd Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 30 Jun 2026 10:00:48 -0700 Subject: [PATCH 08/12] update api comment mentioning in ORT 1.28 --- include/onnxruntime/core/session/onnxruntime_ep_c_api.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index a34ad925589a5..0bfc5d532a54a 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2996,7 +2996,7 @@ struct OrtEpFactory { * * \snippet{doc} snippets.dox OrtStatus Return Value * - * \since Version 1.27. + * \since Version 1.28. */ ORT_API2_STATUS(SelectBestModelCandidate, _In_ OrtEpFactory* this_ptr, _In_ const OrtHardwareDevice* device, From 0b3c044b77d339b83635fec96ca21d6807fbafa7 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 1 Jul 2026 09:48:23 -0700 Subject: [PATCH 09/12] update api comment mentioning model package variant body fields --- include/onnxruntime/core/session/onnxruntime_ep_c_api.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 0bfc5d532a54a..4d9a9ed0e4a93 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2972,7 +2972,9 @@ struct OrtEpFactory { * and returns the index of the best match. * * Each candidate's OrtKeyValuePairs can have multiple entries and the entry with "ep_compatibility_info" key is - * required; its associated value is the compatibility information stored in onnx model metadata. + * required; its associated value is the compatibility information stored in onnx model metadata. When coming from + * a model package, these correspond to the fields in variant body. + * See model_package/README.md#variant-body for the full spec. * * Context about having this function: * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best From e8f33431a53ceec89b9e30b3c8934ae5275a3848 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 2 Jul 2026 17:38:52 -0700 Subject: [PATCH 10/12] Update SelectBestModelCandidate doc comment for multi-model variant support Document the indexed KVP convention for variants with multiple sub-models (e.g., prefill + decode). Single-model variants continue to use a single ep_compatibility_info key. Multi-model variants use indexed keys (num_models, .ep_compatibility_info, .role, .path) so EPs can inspect each sub-model independently. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../core/session/onnxruntime_ep_c_api.h | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 4d9a9ed0e4a93..0195cc94c90df 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2966,32 +2966,52 @@ struct OrtEpFactory { ORT_API2_STATUS(DeinitGraphicsInterop, _In_ OrtEpFactory* this_ptr, _In_ const OrtEpDevice* ep_device); - /** \brief Select the best model candidate from metadata lists. + /** \brief Select the best model variant candidate from metadata. * - * Evaluates each candidate metadata against the given hardware device and optional session options, + * Evaluates each candidate's metadata against the given hardware device and optional session options, * and returns the index of the best match. * - * Each candidate's OrtKeyValuePairs can have multiple entries and the entry with "ep_compatibility_info" key is - * required; its associated value is the compatibility information stored in onnx model metadata. When coming from - * a model package, these correspond to the fields in variant body. - * See model_package/README.md#variant-body for the full spec. + * Each candidate is an OrtKeyValuePairs representing one model variant. When coming from a model package, + * these correspond to the fields in the variant body of metadata.json. + * See model_package/README.md for the full spec. + * + * **Single-model variants (simple case):** + * + * The KVP contains a single "ep_compatibility_info" key whose value is the compatibility string + * stored in the ONNX model metadata (as produced by OrtEp::GetCompiledModelCompatibilityInfo()). + * + * **Multi-model variants:** + * + * When a variant contains multiple sub-models (e.g., prefill + decode in a GenAI scenario), + * the KVP uses indexed keys so that the EP can inspect each sub-model's metadata independently: + * + * - "num_models" — number of sub-models in this variant (e.g., "3") + * - ".ep_compatibility_info" — compatibility string for sub-model i (required per sub-model) + * - ".role" — role/purpose of sub-model i (e.g., "prefill", "decode") (optional) + * - ".path" — relative path to sub-model i (optional) + * + * where is a zero-based index (e.g., "0.ep_compatibility_info", "1.ep_compatibility_info"). + * + * A basic implementation can simply validate every ".ep_compatibility_info" entry. + * An advanced implementation may additionally consider "role" or other metadata when ranking candidates. + * + * **Why this function exists:** * - * Context about having this function: * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best * compatible model when there are multiple candidates. For example, an EP may support multiple compilation modes * (e.g., "speed optimized" vs "memory optimized") that produce different compatibility strings. The EP can implement - * this function to evaluate the candidate strings and select the best compatible one based on its own criteria, + * this function to evaluate the candidate metadata and select the best compatible variant based on its own criteria, * the target device, and the session options. * * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. * * \note The implementer should validate the compatibility of each candidate (e.g., by calling - * ValidateCompiledModelCompatibilityInfo for each one) before determining the best match. + * ValidateCompiledModelCompatibilityInfo for each sub-model) before determining the best match. * * \param[in] this_ptr The OrtEpFactory instance. * \param[in] device The target hardware device that the EP would run on. Must map to this EP. * \param[in] candidates Array of OrtKeyValuePairs pointers (one per model variant). - * \param[in] num_candidates Number of candidates. + * \param[in] num_candidates Number of candidates (i.e., number of model variants to evaluate). * \param[in] session_options Optional session options to consider when selecting the best candidate. * May be nullptr if no session-level preferences are relevant. * \param[out] selected_index Selected candidate index, or SIZE_MAX if all unsupported. From f5c3d52de50cb41a609f259f0ac346a791dd8c87 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 2 Jul 2026 17:48:36 -0700 Subject: [PATCH 11/12] update API comment --- .../onnxruntime/core/session/onnxruntime_ep_c_api.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 0195cc94c90df..4f0e032ca2e45 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2972,8 +2972,7 @@ struct OrtEpFactory { * and returns the index of the best match. * * Each candidate is an OrtKeyValuePairs representing one model variant. When coming from a model package, - * these correspond to the fields in the variant body of metadata.json. - * See model_package/README.md for the full spec. + * these correspond to the fields in the variant body. See model_package/README.md for the full spec. * * **Single-model variants (simple case):** * @@ -2985,10 +2984,10 @@ struct OrtEpFactory { * When a variant contains multiple sub-models (e.g., prefill + decode in a GenAI scenario), * the KVP uses indexed keys so that the EP can inspect each sub-model's metadata independently: * - * - "num_models" — number of sub-models in this variant (e.g., "3") - * - ".ep_compatibility_info" — compatibility string for sub-model i (required per sub-model) - * - ".role" — role/purpose of sub-model i (e.g., "prefill", "decode") (optional) - * - ".path" — relative path to sub-model i (optional) + * - "num_models" — number of sub-models in this variant (e.g., "3") + * - ".ep_compatibility_info" — compatibility string for sub-model i (required per sub-model) + * - ".role" — role/purpose of sub-model i (e.g., "prefill", "decode") (optional) + * - ".future_meaningful_info" — additional EP-meaningful metadata for sub-model i (optional) * * where is a zero-based index (e.g., "0.ep_compatibility_info", "1.ep_compatibility_info"). * From 37abbe7891acc59f1378cdb9a11f44be7570b9ac Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 2 Jul 2026 20:26:43 -0700 Subject: [PATCH 12/12] Update SelectBestModelCandidate doc: unified indexed format, clarify loop range - Remove single vs multi-model special case; always use indexed keys with num_models >= 1 - Fix loop range to 0..num_models-1 - Explicitly reference .ep_compatibility_info in validation note Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../core/session/onnxruntime_ep_c_api.h | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 4f0e032ca2e45..a53b03c13c834 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2971,27 +2971,18 @@ struct OrtEpFactory { * Evaluates each candidate's metadata against the given hardware device and optional session options, * and returns the index of the best match. * - * Each candidate is an OrtKeyValuePairs representing one model variant. When coming from a model package, - * these correspond to the fields in the variant body. See model_package/README.md for the full spec. + * Each candidate is an OrtKeyValuePairs representing one model variant. The KVP uses indexed keys + * so that the EP can inspect each model's metadata independently. A variant always has num_models >= 1. * - * **Single-model variants (simple case):** - * - * The KVP contains a single "ep_compatibility_info" key whose value is the compatibility string - * stored in the ONNX model metadata (as produced by OrtEp::GetCompiledModelCompatibilityInfo()). - * - * **Multi-model variants:** - * - * When a variant contains multiple sub-models (e.g., prefill + decode in a GenAI scenario), - * the KVP uses indexed keys so that the EP can inspect each sub-model's metadata independently: - * - * - "num_models" — number of sub-models in this variant (e.g., "3") - * - ".ep_compatibility_info" — compatibility string for sub-model i (required per sub-model) - * - ".role" — role/purpose of sub-model i (e.g., "prefill", "decode") (optional) - * - ".future_meaningful_info" — additional EP-meaningful metadata for sub-model i (optional) + * Required and optional keys: + * - "num_models" — number of models in this variant (>= 1) (required) + * - ".ep_compatibility_info" — compatibility string for model i (required per model) + * - ".role" — role/purpose of model i (e.g., "prefill", "decode") (optional) + * - ".future_meaningful_info" — additional EP-meaningful metadata for model i (optional) * * where is a zero-based index (e.g., "0.ep_compatibility_info", "1.ep_compatibility_info"). * - * A basic implementation can simply validate every ".ep_compatibility_info" entry. + * The implementer should loop from 0 to num_models - 1 and validate each ".ep_compatibility_info" entry. * An advanced implementation may additionally consider "role" or other metadata when ranking candidates. * * **Why this function exists:** @@ -3004,8 +2995,8 @@ struct OrtEpFactory { * * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. * - * \note The implementer should validate the compatibility of each candidate (e.g., by calling - * ValidateCompiledModelCompatibilityInfo for each sub-model) before determining the best match. + * \note The implementer should validate each ".ep_compatibility_info" in the candidate (e.g., by calling + * ValidateCompiledModelCompatibilityInfo for each one) before determining the best match. * * \param[in] this_ptr The OrtEpFactory instance. * \param[in] device The target hardware device that the EP would run on. Must map to this EP.