diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 62757812cd6e3..a53b03c13c834 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2965,6 +2965,57 @@ struct OrtEpFactory { */ ORT_API2_STATUS(DeinitGraphicsInterop, _In_ OrtEpFactory* this_ptr, _In_ const OrtEpDevice* ep_device); + + /** \brief Select the best model variant candidate from metadata. + * + * Evaluates each candidate's metadata against the given hardware device and optional session options, + * and returns the index of the best match. + * + * Each candidate is an OrtKeyValuePairs representing one model variant. The KVP uses indexed keys + * so that the EP can inspect each model's metadata independently. A variant always has num_models >= 1. + * + * Required and optional keys: + * - "num_models" — number of models in this variant (>= 1) (required) + * - ".ep_compatibility_info" — compatibility string for model i (required per model) + * - ".role" — role/purpose of model i (e.g., "prefill", "decode") (optional) + * - ".future_meaningful_info" — additional EP-meaningful metadata for model i (optional) + * + * where is a zero-based index (e.g., "0.ep_compatibility_info", "1.ep_compatibility_info"). + * + * The implementer should loop from 0 to num_models - 1 and validate each ".ep_compatibility_info" entry. + * An advanced implementation may additionally consider "role" or other metadata when ranking candidates. + * + * **Why this function exists:** + * + * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best + * compatible model when there are multiple candidates. For example, an EP may support multiple compilation modes + * (e.g., "speed optimized" vs "memory optimized") that produce different compatibility strings. The EP can implement + * this function to evaluate the candidate metadata and select the best compatible variant based on its own criteria, + * the target device, and the session options. + * + * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. + * + * \note The implementer should validate each ".ep_compatibility_info" in the candidate (e.g., by calling + * ValidateCompiledModelCompatibilityInfo for each one) before determining the best match. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] device The target hardware device that the EP would run on. Must map to this EP. + * \param[in] candidates Array of OrtKeyValuePairs pointers (one per model variant). + * \param[in] num_candidates Number of candidates (i.e., number of model variants to evaluate). + * \param[in] session_options Optional session options to consider when selecting the best candidate. + * May be nullptr if no session-level preferences are relevant. + * \param[out] selected_index Selected candidate index, or SIZE_MAX if all unsupported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.28. + */ + ORT_API2_STATUS(SelectBestModelCandidate, _In_ OrtEpFactory* this_ptr, + _In_ const OrtHardwareDevice* device, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, + _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, + _Out_ size_t* selected_index); }; #ifdef __cplusplus diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc index ca39d6e750088..fb8a90f25ce13 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.cc @@ -36,6 +36,7 @@ EpFactoryInternal::EpFactoryInternal(std::unique_ptr impl OrtEpFactory::GetHardwareDeviceIncompatibilityDetails = Forward::GetHardwareDeviceIncompatibilityDetails; OrtEpFactory::InitGraphicsInterop = Forward::InitGraphicsInterop; OrtEpFactory::DeinitGraphicsInterop = Forward::DeinitGraphicsInterop; + OrtEpFactory::SelectBestModelCandidate = Forward::SelectBestModelCandidate; } InternalExecutionProviderFactory::InternalExecutionProviderFactory(EpFactoryInternal& ep_factory, diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h index 9ac883da06465..cdfe5347185b7 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal.h @@ -106,6 +106,15 @@ class EpFactoryInternal : public OrtEpFactory { return impl_->DeinitGraphicsInterop(ep_device); } + OrtStatus* SelectBestModelCandidate(_In_ const OrtHardwareDevice* device, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, + _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, + _Out_ size_t* selected_index) noexcept { + return impl_->SelectBestModelCandidate(device, candidates, num_candidates, + session_options, selected_index); + } + // Function ORT calls to release an EP instance. void ReleaseEp(OrtEp* /*ep*/) noexcept { // we never create an OrtEp so we should never be trying to release one diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h index a6140b2ac260f..cbadabfd8c3eb 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_internal_impl.h @@ -122,6 +122,24 @@ class EpFactoryInternalImpl { return nullptr; } + virtual OrtStatus* SelectBestModelCandidate( + _In_ const OrtHardwareDevice* device, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, + _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, + _Out_ size_t* selected_index) noexcept { + ORT_UNUSED_PARAMETER(device); + ORT_UNUSED_PARAMETER(session_options); + if (candidates == nullptr || num_candidates == 0 || selected_index == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "Invalid arguments to SelectBestModelCandidate."); + } + + // Default implementation: all candidates are unsupported, sets `selected_index` to SIZE_MAX. + *selected_index = SIZE_MAX; + return nullptr; + } + // Function ORT calls to release an EP instance. void ReleaseEp(OrtEp* ep); diff --git a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h index d4d93152c4f80..9dbf25a8251a4 100644 --- a/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h +++ b/onnxruntime/core/session/plugin_ep/forward_to_factory_impl.h @@ -106,6 +106,17 @@ struct ForwardToFactoryImpl { return static_cast(this_ptr)->DeinitGraphicsInterop(ep_device); } + static OrtStatus* ORT_API_CALL SelectBestModelCandidate( + OrtEpFactory* this_ptr, + _In_ const OrtHardwareDevice* device, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, + size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, + size_t* selected_index) noexcept { + return static_cast(this_ptr)->SelectBestModelCandidate( + device, candidates, num_candidates, session_options, selected_index); + } + static void ORT_API_CALL ReleaseEp(OrtEpFactory* this_ptr, OrtEp* ep) noexcept { static_cast(this_ptr)->ReleaseEp(ep); } diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc index 6137b23111bf9..77fe29d52e6e9 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc @@ -4,6 +4,7 @@ #include "ep_factory.h" #include +#include #include "ep.h" #include "ep_allocator.h" @@ -13,6 +14,22 @@ #include "core/session/onnxruntime_session_options_config_keys.h" +namespace { +int CompatibilityRank(OrtCompiledModelCompatibility c) { + switch (c) { + case OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL: + return 3; + case OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION: + return 2; + case OrtCompiledModelCompatibility_EP_NOT_APPLICABLE: + return 1; + case OrtCompiledModelCompatibility_EP_UNSUPPORTED: + default: + return 0; + } +} +} // namespace + ExampleEpFactory::ExampleEpFactory(const char* ep_name, ApiPtrs apis, const OrtLogger& default_logger) : OrtEpFactory{}, ApiPtrs(apis), @@ -45,6 +62,7 @@ ExampleEpFactory::ExampleEpFactory(const char* ep_name, ApiPtrs apis, const OrtL GetNumCustomOpDomains = GetNumCustomOpDomainsImpl; GetCustomOpDomains = GetCustomOpDomainsImpl; ValidateCompiledModelCompatibilityInfo = ValidateCompiledModelCompatibilityInfoImpl; + SelectBestModelCandidate = SelectBestModelCandidateImpl; // setup the OrtMemoryInfo instances required by the EP. // We pretend the device the EP is running on is GPU. @@ -514,3 +532,59 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::ValidateCompiledModelCompatibilityInfo *model_compatibility = OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL; return nullptr; } + +OrtStatus* ORT_API_CALL ExampleEpFactory::SelectBestModelCandidateImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* device, + const OrtKeyValuePairs* const* candidates, + size_t num_candidates, + const OrtSessionOptions* /*session_options*/, + size_t* selected_index) noexcept { + auto& factory = *static_cast(this_ptr); + + if (selected_index == nullptr) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, "selected_index cannot be nullptr"); + } + + *selected_index = std::numeric_limits::max(); + + if (candidates == nullptr || num_candidates == 0) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, "candidates cannot be nullptr or empty"); + } + + const OrtHardwareDevice* devices[] = {device}; + + int best_rank = -1; + size_t best_idx = std::numeric_limits::max(); + + for (size_t i = 0; i < num_candidates; ++i) { + const char* compatibility_info = + factory.ort_api.GetKeyValue(candidates[i], "ep_compatibility_info"); + + if (compatibility_info == nullptr) { + return factory.ort_api.CreateStatus(ORT_INVALID_ARGUMENT, + "candidate metadata is missing required key: ep_compatibility_info"); + } + + OrtCompiledModelCompatibility compatibility = OrtCompiledModelCompatibility_EP_UNSUPPORTED; + OrtStatus* status = ValidateCompiledModelCompatibilityInfoImpl( + this_ptr, devices, 1, compatibility_info, &compatibility); + if (status != nullptr) { + return status; + } + + const int rank = CompatibilityRank(compatibility); + if (rank > best_rank) { + best_rank = rank; + best_idx = i; + } + } + + if (best_rank <= CompatibilityRank(OrtCompiledModelCompatibility_EP_UNSUPPORTED)) { + *selected_index = std::numeric_limits::max(); + } else { + *selected_index = best_idx; + } + + return nullptr; +} diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h index 91478047afb0a..31bc4abeff887 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.h @@ -106,6 +106,14 @@ class ExampleEpFactory : public OrtEpFactory, public ApiPtrs { const char* compatibility_info, OrtCompiledModelCompatibility* model_compatibility) noexcept; + static OrtStatus* ORT_API_CALL SelectBestModelCandidateImpl( + OrtEpFactory* this_ptr, + const OrtHardwareDevice* device, + const OrtKeyValuePairs* const* candidates, + size_t num_candidates, + const OrtSessionOptions* session_options, + size_t* selected_index) noexcept; + const std::string ep_name_; // EP name const std::string vendor_{"Contoso"}; // EP vendor name const uint32_t vendor_id_{0xB357}; // EP vendor ID diff --git a/onnxruntime/test/autoep/test_ep_compatibility.cc b/onnxruntime/test/autoep/test_ep_compatibility.cc new file mode 100644 index 0000000000000..437ebc70ef08f --- /dev/null +++ b/onnxruntime/test/autoep/test_ep_compatibility.cc @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include "gtest/gtest.h" + +#include "core/session/onnxruntime_c_api.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/session/abi_devices.h" +#include "test/autoep/test_autoep_utils.h" + +using namespace onnxruntime; + +TEST(EpCompatibilitySelectBestTest, SelectBestModelCandidate_UsesHardwareDevices) { + Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "EpCompatSelectBestExampleEp"}; + + onnxruntime::test::RegisteredEpDeviceUniquePtr example_ep; + onnxruntime::test::Utils::RegisterAndGetExampleEp( + env, onnxruntime::test::Utils::example_ep_info, example_ep); + + ASSERT_NE(example_ep.get(), nullptr); + + const OrtEpDevice* ep_device = example_ep.get(); + OrtEpFactory* factory = ep_device->GetMutableFactory(); + ASSERT_NE(factory, nullptr); + ASSERT_NE(factory->SelectBestModelCandidate, nullptr); + + const std::string ep_name = factory->GetName(factory); + + const std::string optimal = + ep_name + ";version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + + const std::string prefer_recompile = + ep_name + ";version=9.9.9;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + + const std::string unsupported = + "SomeOtherEp;version=0.1.0;ort_api_version=" + std::to_string(ORT_API_VERSION) + ";hardware_architecture=arch1"; + + const OrtApi* api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + ASSERT_NE(api, nullptr); + + OrtKeyValuePairs* kvp0 = nullptr; + OrtKeyValuePairs* kvp1 = nullptr; + OrtKeyValuePairs* kvp2 = nullptr; + api->CreateKeyValuePairs(&kvp0); + api->CreateKeyValuePairs(&kvp1); + api->CreateKeyValuePairs(&kvp2); + api->AddKeyValuePair(kvp0, "ep_compatibility_info", unsupported.c_str()); + api->AddKeyValuePair(kvp1, "ep_compatibility_info", prefer_recompile.c_str()); + api->AddKeyValuePair(kvp2, "ep_compatibility_info", optimal.c_str()); + + const OrtKeyValuePairs* candidates[] = {kvp0, kvp1, kvp2}; + + size_t selected_index = std::numeric_limits::max(); + OrtStatus* st = factory->SelectBestModelCandidate( + factory, ep_device->device, candidates, 3, nullptr, &selected_index); + + ASSERT_EQ(st, nullptr) << (st ? api->GetErrorMessage(st) : ""); + + EXPECT_EQ(selected_index, 2u); + + api->ReleaseKeyValuePairs(kvp0); + api->ReleaseKeyValuePairs(kvp1); + api->ReleaseKeyValuePairs(kvp2); +} \ No newline at end of file