diff --git a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h index 351eafc2a4675..071a9d264a355 100644 --- a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h +++ b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h @@ -41,10 +41,17 @@ enum COREMLFlags { COREML_FLAG_LAST = COREML_FLAG_USE_CPU_AND_GPU, }; -// MLComputeUnits can be one of the following values: -// 'MLComputeUnitsCPUAndNeuralEngine|MLComputeUnitsCPUAndGPU|MLComputeUnitsCPUOnly|MLComputeUnitsAll' -// these values are intended to be used with Ort::SessionOptions::AppendExecutionProvider (C++ API) -// and SessionOptionsAppendExecutionProvider (C API). For the old API, use COREMLFlags instead. +// Set the MLComputeUnits provider option to one of the kCoremlProviderOption_MLComputeUnits_* values below. +// Each value maps to the corresponding Core ML compute-unit mode +// (https://developer.apple.com/documentation/coreml/mlcomputeunits). +// Use these values with SessionOptionsAppendExecutionProvider or SessionOptionsAppendExecutionProvider_V2. +// The legacy CoreML-specific OrtSessionOptionsAppendExecutionProvider_CoreML API uses a COREMLFlags bitmask +// instead. +// MLComputeUnitsCPUAndNeuralEngine requires Core ML 6 (iOS 16 / macOS 13) and is unavailable on earlier systems. +static const char* const kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine = "CPUAndNeuralEngine"; +static const char* const kCoremlProviderOption_MLComputeUnits_CPUAndGPU = "CPUAndGPU"; +static const char* const kCoremlProviderOption_MLComputeUnits_CPUOnly = "CPUOnly"; +static const char* const kCoremlProviderOption_MLComputeUnits_ALL = "ALL"; static const char* const kCoremlProviderOption_MLComputeUnits = "MLComputeUnits"; static const char* const kCoremlProviderOption_ModelFormat = "ModelFormat"; // same as COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 9dd3dfaf6c75a..b78275df0a228 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -37,6 +37,10 @@ CoreMLExecutionProvider::CoreMLExecutionProvider(const CoreMLOptions& options) CoreMLExecutionProvider::~CoreMLExecutionProvider() {} +ProviderOptions CoreMLExecutionProvider::GetProviderOptions() const { + return coreml_options_.ToProviderOptions(); +} + std::vector> CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer, const IKernelLookup& /*kernel_lookup*/, diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index 574ae1fc0106b..b91aeb2d8fd77 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -28,6 +28,8 @@ class CoreMLExecutionProvider : public IExecutionProvider { std::vector& node_compute_funcs) override; #endif + ProviderOptions GetProviderOptions() const override; + private: // The bit flags which define bool options for COREML EP, bits are defined as // COREMLFlags in include/onnxruntime/core/providers/coreml/coreml_provider_factory.h diff --git a/onnxruntime/core/providers/coreml/coreml_options.cc b/onnxruntime/core/providers/coreml/coreml_options.cc index c441a2eff56e0..7f89b7bd86e8a 100644 --- a/onnxruntime/core/providers/coreml/coreml_options.cc +++ b/onnxruntime/core/providers/coreml/coreml_options.cc @@ -43,10 +43,10 @@ CoreMLOptions::CoreMLOptions(uint32_t coreml_flags) { void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& options) { const std::unordered_map available_computeunits_options = { - {"CPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, - {"CPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU}, - {"CPUOnly", COREML_FLAG_USE_CPU_ONLY}, - {"ALL", COREML_FLAG_USE_NONE}, + {kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine, COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE}, + {kCoremlProviderOption_MLComputeUnits_CPUAndGPU, COREML_FLAG_USE_CPU_AND_GPU}, + {kCoremlProviderOption_MLComputeUnits_CPUOnly, COREML_FLAG_USE_CPU_ONLY}, + {kCoremlProviderOption_MLComputeUnits_ALL, COREML_FLAG_USE_NONE}, }; const std::unordered_map available_modelformat_options = { {"MLProgram", COREML_FLAG_CREATE_MLPROGRAM}, @@ -98,4 +98,34 @@ void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& option } } } + +ProviderOptions CoreMLOptions::ToProviderOptions() const { + // MLComputeUnits uses constants defined in coreml_provider_factory.h. The other strings match + // those accepted by ValidateAndParseProviderOption. + const char* compute_units = kCoremlProviderOption_MLComputeUnits_ALL; + if (ComputeUnits(COREML_FLAG_USE_CPU_ONLY)) { + compute_units = kCoremlProviderOption_MLComputeUnits_CPUOnly; + } else if (ComputeUnits(COREML_FLAG_USE_CPU_AND_GPU)) { + compute_units = kCoremlProviderOption_MLComputeUnits_CPUAndGPU; + } else if (ComputeUnits(COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE)) { + compute_units = kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine; + } + + ProviderOptions options{ + {kCoremlProviderOption_MLComputeUnits, compute_units}, + {kCoremlProviderOption_ModelFormat, create_mlprogram_ ? "MLProgram" : "NeuralNetwork"}, + {kCoremlProviderOption_RequireStaticInputShapes, require_static_shape_ ? "1" : "0"}, + {kCoremlProviderOption_EnableOnSubgraphs, enable_on_subgraph_ ? "1" : "0"}, + {kCoremlProviderOption_SpecializationStrategy, strategy_.empty() ? std::string{"Default"} : strategy_}, + // The effective value: profiling applies to MLProgram models only. + {kCoremlProviderOption_ProfileComputePlan, ProfileComputePlan() ? "1" : "0"}, + {kCoremlProviderOption_AllowLowPrecisionAccumulationOnGPU, allow_low_precision_accumulation_on_gpu_ ? "1" : "0"}, + }; + + if (!model_cache_directory_.empty()) { + options[kCoremlProviderOption_ModelCacheDirectory] = model_cache_directory_; + } + + return options; +} } // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/coreml_options.h b/onnxruntime/core/providers/coreml/coreml_options.h index d7ee04b3f8a79..fe4f65a9ab13c 100644 --- a/onnxruntime/core/providers/coreml/coreml_options.h +++ b/onnxruntime/core/providers/coreml/coreml_options.h @@ -5,6 +5,7 @@ #include "core/common/inlined_containers.h" #include "core/framework/execution_provider.h" +#include "core/framework/provider_options.h" namespace onnxruntime { @@ -36,6 +37,10 @@ class CoreMLOptions { bool ProfileComputePlan() const { return profile_compute_plan_ && create_mlprogram_; } std::string_view ModelCacheDirectory() const { return model_cache_directory_; } + + // Returns the effective options for IExecutionProvider::GetProviderOptions using the provider option names + // defined in coreml_provider_factory.h. Options supplied through legacy flags are reported in the same format. + ProviderOptions ToProviderOptions() const; // The options specified by the user are const, but if there's an error setting up caching we disable it // so that the EP can still be used. The error is logged for the user to investigate. void DisableModelCache() const { model_cache_directory_.clear(); } diff --git a/onnxruntime/core/providers/coreml/model/host_utils.h b/onnxruntime/core/providers/coreml/model/host_utils.h index 92eda24e5c46d..824d2d13ac7cc 100644 --- a/onnxruntime/core/providers/coreml/model/host_utils.h +++ b/onnxruntime/core/providers/coreml/model/host_utils.h @@ -62,6 +62,11 @@ #define MINIMUM_COREML_VERSION 5 // first version we support +// Core ML 6 (iOS 16 / macOS 13) introduced MLComputeUnitsCPUAndNeuralEngine, the only compute-units mode that +// enables the Neural Engine without also enabling the GPU. Earlier versions cannot honor a Neural-Engine-only +// request, so device-based EP selection does not offer the Neural Engine below this version. +#define MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION 6 + namespace onnxruntime { namespace coreml { namespace util { diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_coreml.cc b/onnxruntime/core/session/plugin_ep/ep_factory_coreml.cc new file mode 100644 index 0000000000000..1c6f678067578 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_factory_coreml.cc @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// CoreML can be built on non-Apple platforms to test model conversion. Such builds report Core ML 7 but cannot +// execute models, even when device discovery finds GPUs. Require __APPLE__ so the factory cannot advertise those +// GPUs. On Apple platforms, the factory accepts any discovered GPU regardless of vendor. GPU discovery currently +// supports only Apple Silicon Macs. Support for iOS and Intel Macs remains TODO in +// core/platform/apple/device_discovery.cc. +#if defined(USE_COREML) && defined(__APPLE__) + +#include "core/session/plugin_ep/ep_factory_coreml.h" + +#include +#include + +#include + +#include "core/common/common.h" +#include "core/framework/error_code_helper.h" +#include "core/framework/provider_options.h" +#include "core/providers/coreml/coreml_provider_factory.h" +#include "core/providers/coreml/coreml_provider_factory_creator.h" +#include "core/providers/coreml/model/host_utils.h" +#include "core/session/abi_devices.h" +#include "core/session/abi_logger.h" +#include "core/session/abi_session_options_impl.h" +#include "core/session/plugin_ep/ep_api.h" +#include "core/session/plugin_ep/ep_factory_internal.h" +#include "core/session/ort_apis.h" + +namespace onnxruntime { +namespace { + +// Returns whether the factory advertises an OrtEpDevice for the given hardware device type and Core ML version. +// Nothing is advertised below Core ML 5 (MINIMUM_COREML_VERSION), because CoreMLExecutionProvider rejects earlier +// versions during construction. Advertising a device there would make provider creation fail after selection +// instead of allowing the policy to select another EP. NPU devices additionally require +// MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION. CPU devices are never advertised. +bool CanClaimDeviceType(OrtHardwareDeviceType device_type, int32_t coreml_version) { + if (coreml_version < MINIMUM_COREML_VERSION) { + return false; + } + + switch (device_type) { + case OrtHardwareDeviceType::OrtHardwareDeviceType_GPU: + return true; + + case OrtHardwareDeviceType::OrtHardwareDeviceType_NPU: + // The NPU is claimed only on Core ML 6 or later (iOS 16 / macOS 13): earlier Core ML has no MLComputeUnits + // mode that enables the Neural Engine without also enabling the GPU (CPUAndNeuralEngine requires Core ML 6), + // so an NPU selection could not be honored. Core ML can still use the ANE through ALL on those systems. + // The factory therefore does not advertise the NPU on Core ML 5 systems: iOS 15 devices with an ANE and + // Apple Silicon Macs running macOS 12. + return coreml_version >= MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION; + + default: + // The CPU device is deliberately not claimed. DEFAULT and PREFER_CPU select a single preferred EP for the CPU + // device. The NPU- and GPU-preferring policies first select an EP for the accelerator and then use the same CPU + // selection as a fallback. + // If CoreML claimed the CPU, it would compete with the ORT CPU EP and other CPU-based EPs to be selected as the + // primary CPU EP. + // CoreML-on-CPU remains available through an explicit MLComputeUnits=CPUOnly option. Core ML may also use its + // internal CPU path for operations unsupported by the selected compute units. + return false; + } +} + +// The factory advertises at most one NPU and one GPU (see SelectDevicesToClaim), so pointers to both the selected +// hardware devices and the newly created OrtEpDevice instances fit in fixed-size arrays. GetSupportedDevices +// therefore needs no dynamic allocation for local storage. CreateEpDevice reports failures through OrtStatus, +// so GetSupportedDevices satisfies the base interface's noexcept requirement. +constexpr size_t kMaxClaimedDevices = 2; + +struct ClaimedDevices { + std::array devices{}; + size_t count = 0; +}; + +// Selects the hardware devices to advertise. The result contains only devices accepted by CanClaimDeviceType, +// at most one device of each type, and no more than max_ep_devices devices in total. +// MLComputeUnits selects device classes rather than specific physical devices, so CoreML cannot target a particular +// GPU or NPU when multiple devices of that type exist. Apple device discovery currently reports at most one device +// of each type. Additional same type devices are intentionally ignored. +ClaimedDevices SelectDevicesToClaim(gsl::span devices, int32_t coreml_version, + size_t max_ep_devices) { + ClaimedDevices selected; + bool npu_claimed = false; + bool gpu_claimed = false; + + for (size_t i = 0; i < devices.size() && selected.count < max_ep_devices; ++i) { + const OrtHardwareDevice* device = devices[i]; + + // CanClaimDeviceType checks the Core ML version and excludes CPU devices. + if (!CanClaimDeviceType(device->type, coreml_version)) { + continue; + } + + // CanClaimDeviceType accepts only NPU and GPU devices, so every accepted non-NPU device is a GPU. + // Only the first device of each type is selected. If CanClaimDeviceType is extended to accept another device type, + // update this selection logic as well. + const bool is_npu = device->type == OrtHardwareDeviceType::OrtHardwareDeviceType_NPU; + bool& claimed = is_npu ? npu_claimed : gpu_claimed; + + if (claimed) { + continue; + } + + claimed = true; + + selected.devices[selected.count++] = device; + } + + return selected; +} + +// Validates that 'devices' contains exactly one NPU, one GPU, or one NPU plus one GPU. Otherwise, returns +// ORT_INVALID_ARGUMENT. The CPU device is not valid because the factory does not advertise it. +// If 'options' does not contain MLComputeUnits, derives a default from the selection: +// CPUAndNeuralEngine for NPU, CPUAndGPU for GPU, and ALL for NPU plus GPU. +// A caller-provided value may disable selected accelerators, as CPUOnly does, but it must not enable an +// accelerator that was not selected. Unrecognized values are left for CoreMLOptions to validate during +// provider creation. +OrtStatus* ValidateDeviceSelectionAndResolveMLComputeUnits(const OrtHardwareDevice* const* devices, + size_t num_devices, ProviderOptions& options) { + if (num_devices == 0) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "CoreML EP factory requires at least one device to be selected."); + } + + if (devices == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "CoreML device list is null."); + } + + size_t num_npu = 0; + size_t num_gpu = 0; + for (size_t i = 0; i < num_devices; ++i) { + if (devices[i] == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "CoreML device is null."); + } + + switch (devices[i]->type) { + case OrtHardwareDeviceType::OrtHardwareDeviceType_NPU: + ++num_npu; + break; + case OrtHardwareDeviceType::OrtHardwareDeviceType_GPU: + ++num_gpu; + break; + default: + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "CoreML EP factory only supports NPU and GPU devices."); + } + } + + // Reject multiple devices of the same type. Core ML's MLComputeUnits selects a device class, not a specific + // physical device, so the CoreML EP cannot honor a selection containing multiple NPUs or GPUs. + // With current device discovery, such entries can only be duplicates. Future discovery could expose multiple + // physical devices of one type. + if (num_npu > 1 || num_gpu > 1) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "CoreML EP factory supports one NPU, one GPU, or one NPU plus one GPU. " + "At most one device of each type can be selected."); + } + + // This helper does not recheck the Core ML version. CreateIExecutionProvider receives only devices previously + // advertised by GetSupportedDevices, which uses CanClaimDeviceType to enforce the version requirements. + // Therefore, a selected NPU implies Core ML 6 or later, so CPUAndNeuralEngine is available. + // MLComputeUnits is the only option derived from the selection. ModelFormat is not device-specific and changes + // only when the caller explicitly sets it. + const auto compute_units_it = options.find(kCoremlProviderOption_MLComputeUnits); + if (compute_units_it == options.end()) { + options[kCoremlProviderOption_MLComputeUnits] = + (num_npu > 0 && num_gpu > 0) ? kCoremlProviderOption_MLComputeUnits_ALL + : num_npu > 0 ? kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine + : kCoremlProviderOption_MLComputeUnits_CPUAndGPU; + return nullptr; + } + + // A user-provided MLComputeUnits value may disable selected accelerators (e.g. CPUOnly), but it must not enable + // an unselected accelerator. Otherwise, PREFER_NPU could be silently redirected to the GPU. This check only + // enforces compatibility with the selected devices. CoreMLOptions::ValidateAndParseProviderOption later validates + // the option value itself. If that method is extended to accept a new value that enables an accelerator, update + // the classification below accordingly. + const std::string& compute_units = compute_units_it->second; + const bool enables_npu = + compute_units == kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine || + compute_units == kCoremlProviderOption_MLComputeUnits_ALL; + const bool enables_gpu = + compute_units == kCoremlProviderOption_MLComputeUnits_CPUAndGPU || + compute_units == kCoremlProviderOption_MLComputeUnits_ALL; + + if ((enables_npu && num_npu == 0) || (enables_gpu && num_gpu == 0)) { + const std::string message = MakeString("MLComputeUnits value '", compute_units, + "' enables a device type that was not selected for the CoreML EP."); + + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, message.c_str()); + } + + return nullptr; +} + +} // namespace + +OrtStatus* CoreMLEpFactory::GetSupportedDevices(EpFactoryInternal& ep_factory, + const OrtHardwareDevice* const* devices, + size_t num_devices, + OrtEpDevice** ep_devices, + size_t max_ep_devices, + size_t* p_num_ep_devices) noexcept { + const ClaimedDevices selected = + SelectDevicesToClaim(gsl::make_span(devices, num_devices), coreml::util::CoreMLVersion(), max_ep_devices); + + // Do not attach MLComputeUnits to individual OrtEpDevice instances. Per-device defaults would assign different + // values to the NPU and GPU, creating a conflict when both are selected. CreateIExecutionProvider instead computes + // a single value from all selected devices. Any MLComputeUnits value already present in the session options was + // provided by the caller. + // + // Publish the OrtEpDevice instances only after all creations succeed. If a creation fails, release the devices + // created by earlier calls and leave ep_devices and p_num_ep_devices unchanged, following ORT's C API convention + // for failed calls. 'selected' contains no more than max_ep_devices entries, so publishing cannot exceed the + // capacity of ep_devices. + std::array created{}; + size_t num_created = 0; + + for (size_t i = 0; i < selected.count; ++i) { + OrtEpDevice* ep_device = nullptr; + OrtStatus* status = + OrtExecutionProviderApi::CreateEpDevice(&ep_factory, selected.devices[i], nullptr, nullptr, &ep_device); + if (status != nullptr) { + for (size_t j = 0; j < num_created; ++j) { + OrtExecutionProviderApi::ReleaseEpDevice(created[j]); + } + + return status; + } + + created[num_created++] = ep_device; + } + + std::copy_n(created.begin(), num_created, ep_devices); + *p_num_ep_devices = num_created; + + return nullptr; +} + +OrtStatus* CoreMLEpFactory::CreateIExecutionProvider(const OrtHardwareDevice* const* devices, + const OrtKeyValuePairs* const* /*ep_metadata_pairs*/, + size_t num_devices, + const OrtSessionOptions* session_options, + const OrtLogger* session_logger, + std::unique_ptr* ep) { + *ep = nullptr; + + ProviderOptions options = GetOptionsFromSessionOptions(session_options->value); + + // OrtEpDevice instances carry no MLComputeUnits default, so any existing session option can only come from + // the caller. Validate the device selection, derive MLComputeUnits when absent, and reject recognized values + // that enable an unselected accelerator. + // CoreMLOptions performs the remaining option validation during provider creation. + ORT_API_RETURN_IF_ERROR(ValidateDeviceSelectionAndResolveMLComputeUnits(devices, num_devices, options)); + + const auto provider_factory = CoreMLProviderFactoryCreator::Create(options); + *ep = provider_factory->CreateProvider(); + (*ep)->SetLogger(session_logger->ToInternal()); + + return nullptr; +} + +} // namespace onnxruntime + +#endif // defined(USE_COREML) && defined(__APPLE__) diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_coreml.h b/onnxruntime/core/session/plugin_ep/ep_factory_coreml.h new file mode 100644 index 0000000000000..74271427ff0b9 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_factory_coreml.h @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +// CoreML can be built on non-Apple platforms to test model conversion. Such builds report Core ML 7 but cannot +// execute models, even when device discovery finds GPUs. Require __APPLE__ so the factory cannot advertise those +// GPUs. On Apple platforms, the factory accepts any discovered GPU regardless of vendor. GPU discovery currently +// supports only Apple Silicon Macs. Support for iOS and Intel Macs remains TODO in +// core/platform/apple/device_discovery.cc. +#if defined(USE_COREML) && defined(__APPLE__) + +#include "core/session/plugin_ep/ep_factory_internal_impl.h" + +#include "core/common/common.h" +#include "core/framework/ortdevice.h" +#include "core/graph/constants.h" + +namespace onnxruntime { + +class CoreMLEpFactory : public EpFactoryInternalImpl { + public: + // Report "Microsoft" and VendorIds::MICROSOFT as the factory vendor identifiers, like the other internal + // factories (CPU, DML, and WebGPU): the factory vendor identifies who provides the EP implementation, not the + // hardware vendor. Discovered Apple hardware devices keep their own Apple vendor identifiers. + // Because the factory vendor does not match the Apple hardware vendor, device ordering gives CoreML no vendor + // affinity. When multiple EPs target the same NPU or GPU with equal vendor affinity, they are ordered by + // EP name. + CoreMLEpFactory() + : EpFactoryInternalImpl(kCoreMLExecutionProvider, "Microsoft", OrtDevice::VendorIds::MICROSOFT) {} + + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CoreMLEpFactory); + + private: + OrtStatus* GetSupportedDevices(EpFactoryInternal& ep_factory, + const OrtHardwareDevice* const* devices, + size_t num_devices, + OrtEpDevice** ep_devices, + size_t max_ep_devices, + size_t* p_num_ep_devices) noexcept override; + + OrtStatus* CreateIExecutionProvider(const OrtHardwareDevice* const* devices, + const OrtKeyValuePairs* const* ep_metadata_pairs, + size_t num_devices, + const OrtSessionOptions* session_options, + const OrtLogger* session_logger, + std::unique_ptr* ep) override; +}; + +} // namespace onnxruntime + +#endif // defined(USE_COREML) && defined(__APPLE__) diff --git a/onnxruntime/core/session/plugin_ep/ep_library_internal.cc b/onnxruntime/core/session/plugin_ep/ep_library_internal.cc index 0f8913a01188a..ca852b367adcf 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_internal.cc +++ b/onnxruntime/core/session/plugin_ep/ep_library_internal.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "core/session/plugin_ep/ep_library_internal.h" +#include "core/session/plugin_ep/ep_factory_coreml.h" #include "core/session/plugin_ep/ep_factory_cpu.h" #include "core/session/plugin_ep/ep_factory_dml.h" #include "core/session/plugin_ep/ep_factory_webgpu.h" @@ -14,6 +15,14 @@ std::unique_ptr EpLibraryInternal::CreateCpuEp() { return std::make_unique(std::move(internal_factory)); } +#if defined(USE_COREML) && defined(__APPLE__) +std::unique_ptr EpLibraryInternal::CreateCoreMLEp() { + auto coreml_factory_impl = std::make_unique(); + auto internal_factory = std::make_unique(std::move(coreml_factory_impl)); + return std::make_unique(std::move(internal_factory)); +} +#endif + #if defined(USE_DML) std::unique_ptr EpLibraryInternal::CreateDmlEp() { @@ -48,6 +57,12 @@ std::vector> EpLibraryInternal::CreateInterna internal_eps.push_back(CreateDmlEp()); #endif +// Unlike WebGPU, CoreML has no plugin-EP adapter, so it does not need an ORT_USE_EP_API_ADAPTERS exclusion. +// This matches DML. +#if defined(USE_COREML) && defined(__APPLE__) + internal_eps.push_back(CreateCoreMLEp()); +#endif + return internal_eps; } diff --git a/onnxruntime/core/session/plugin_ep/ep_library_internal.h b/onnxruntime/core/session/plugin_ep/ep_library_internal.h index 352dba9928021..8d246e5539cbb 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_library_internal.h @@ -45,6 +45,9 @@ class EpLibraryInternal : public EpLibrary { private: static std::unique_ptr CreateCpuEp(); +#if defined(USE_COREML) && defined(__APPLE__) + static std::unique_ptr CreateCoreMLEp(); +#endif #if defined(USE_DML) static std::unique_ptr CreateDmlEp(); #endif diff --git a/onnxruntime/test/autoep/test_autoep_utils.h b/onnxruntime/test/autoep/test_autoep_utils.h index 42f330a0124b9..c57f68a1404cf 100644 --- a/onnxruntime/test/autoep/test_autoep_utils.h +++ b/onnxruntime/test/autoep/test_autoep_utils.h @@ -4,7 +4,10 @@ #pragma once #include +#include +#include #include +#include #include "core/session/onnxruntime_cxx_api.h" @@ -54,5 +57,20 @@ struct Utils { LoadExampleEpHooksPtr& example_ep_hooks); }; +// Runs the mul_1.onnx inference test with ep_name selected in two ways: +// - automatically through "test.ep_to_select", when test_auto_select is true. +// - explicitly through SessionOptionsAppendExecutionProvider_V2. +// Both paths pass provider_options to the EP: as prefixed session options on the auto-selection path and as +// ep_options on the V2 path. The V2 path uses the devices chosen by select_devices, or the first OrtEpDevice +// advertised for ep_name when no selector is provided. If library_path is provided, the helper registers the +// plugin EP library before creating the session. +// disable_cpu_ep_fallback applies to both paths. It requires the selected EP to handle the entire graph, preventing +// a false pass caused by unsupported nodes falling back to the ORT CPU EP. +void RunBasicTest(const std::string& ep_name, std::optional library_path, + const Ort::KeyValuePairs& provider_options = Ort::KeyValuePairs{}, + const std::function&)>& select_devices = nullptr, + bool test_auto_select = true, + bool disable_cpu_ep_fallback = false); + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/autoep/test_selection.cc b/onnxruntime/test/autoep/test_selection.cc index ebbbc40d46fa9..8b84559019fbe 100644 --- a/onnxruntime/test/autoep/test_selection.cc +++ b/onnxruntime/test/autoep/test_selection.cc @@ -11,6 +11,7 @@ #include "core/session/abi_session_options_impl.h" #include "core/session/onnxruntime_cxx_api.h" #include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" +#include "core/session/onnxruntime_session_options_config_keys.h" #include "test_allocator.h" #include "test/autoep/test_autoep_utils.h" @@ -74,9 +75,18 @@ static void TestInference(Ort::Env& env, const std::basic_string& mod // auto select using policy std::optional policy = std::nullopt, std::optional delegate = std::nullopt, - bool test_session_creation_only = false) { + bool test_session_creation_only = false, + // If true, disables fallback of unsupported graph nodes to the ORT CPU + // EP. Session creation fails unless the selected non-CPU EP supports the + // entire graph. Setting this to true while explicitly selecting the ORT + // CPU EP is invalid and causes session creation to fail. + bool disable_cpu_ep_fallback = false) { Ort::SessionOptions session_options; + if (disable_cpu_ep_fallback) { + session_options.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1"); + } + if (library_path && IsRegistered(ep_to_select) == false) { ASSERT_ORTSTATUS_OK(Ort::GetApi().RegisterExecutionProviderLibrary(env, ep_to_select.c_str(), library_path->c_str())); @@ -138,10 +148,11 @@ static void TestInference(Ort::Env& env, const std::basic_string& mod } } -namespace { void RunBasicTest(const std::string& ep_name, std::optional library_path, - const Ort::KeyValuePairs& provider_options = Ort::KeyValuePairs{}, - const std::function&)>& select_devices = nullptr) { + const Ort::KeyValuePairs& provider_options, + const std::function&)>& select_devices, + bool test_auto_select, + bool disable_cpu_ep_fallback) { const auto run_test = [&](bool auto_select) { std::vector> inputs(1); auto& input = inputs.back(); @@ -160,13 +171,19 @@ void RunBasicTest(const std::string& ep_name, std::optional +#include +#include +#include +#include + +#include +#include +#include + +#include "core/common/inlined_containers.h" +#include "core/graph/constants.h" +#include "core/session/onnxruntime_cxx_api.h" + +#if defined(__APPLE__) +#include "core/common/logging/logging.h" +#include "core/common/pci_vendor_ids.h" +#include "core/framework/execution_provider.h" +#include "core/framework/ortdevice.h" +#include "core/providers/coreml/coreml_provider_factory.h" +#include "core/providers/coreml/model/host_utils.h" +#include "core/session/abi_devices.h" +#include "core/session/abi_session_options_impl.h" +#include "core/session/plugin_ep/ep_factory_coreml.h" +#include "core/session/plugin_ep/ep_factory_internal.h" +#endif + +#include "test_allocator.h" +#include "test/autoep/test_autoep_utils.h" +#include "test/shared_lib/utils.h" +#include "test/util/include/api_asserts.h" +#include "test/util/include/asserts.h" + +extern std::unique_ptr ort_env; + +namespace onnxruntime { +namespace test { + +#if defined(__APPLE__) +namespace { +// The CoreML EP factory creates OrtEpDevice instances only for claimable NPU and GPU hardware. The tests use +// GetHardwareDevices and the runtime Core ML version to determine which devices the factory should advertise, +// rather than relying on GetEpDevices itself. A device-dependent test skips only if the required hardware is +// unavailable or excluded by the Core ML version check. Otherwise, the test fails if the factory is not registered +// or does not return the expected OrtEpDevice. The tests also verify that no devices are advertised below Core ML 5 +// and that no NPU is advertised below Core ML 6. +struct CoreMLDevices { + const OrtEpDevice* npu = nullptr; + const OrtEpDevice* gpu = nullptr; +}; + +// Hardware device types reported by device discovery, independent of EP support. +struct HardwareDeviceTypes { + bool has_npu = false; + bool has_gpu = false; +}; + +HardwareDeviceTypes GetHardwareDeviceTypes() { + HardwareDeviceTypes found; + const OrtApi* c_api = &Ort::GetApi(); + size_t num_devices = 0; + + Ort::ThrowOnError(c_api->GetNumHardwareDevices(*ort_env, &num_devices)); + if (num_devices == 0) { + return found; + } + + InlinedVector devices(num_devices); + Ort::ThrowOnError(c_api->GetHardwareDevices(*ort_env, devices.data(), num_devices)); + + for (const OrtHardwareDevice* device : devices) { + const OrtHardwareDeviceType type = c_api->HardwareDevice_Type(device); + found.has_npu |= type == OrtHardwareDeviceType_NPU; + found.has_gpu |= type == OrtHardwareDeviceType_GPU; + } + + return found; +} + +CoreMLDevices GetCoreMLEpDevices() { + CoreMLDevices found; + size_t num_npu = 0; + size_t num_gpu = 0; + const OrtApi* c_api = &Ort::GetApi(); + const OrtEpDevice* const* ep_devices = nullptr; + size_t num_devices = 0; + + Ort::ThrowOnError(c_api->GetEpDevices(*ort_env, &ep_devices, &num_devices)); + + for (size_t i = 0; i < num_devices; ++i) { + const OrtEpDevice* ep_device = ep_devices[i]; + if (strcmp(c_api->EpDevice_EpName(ep_device), kCoreMLExecutionProvider) != 0) { + continue; + } + + const OrtHardwareDeviceType type = c_api->HardwareDevice_Type(c_api->EpDevice_Device(ep_device)); + if (type == OrtHardwareDeviceType_NPU) { + ++num_npu; + if (found.npu == nullptr) { + found.npu = ep_device; + } + } else if (type == OrtHardwareDeviceType_GPU) { + ++num_gpu; + if (found.gpu == nullptr) { + found.gpu = ep_device; + } + } else { + ADD_FAILURE() << "CoreML EP advertised an OrtEpDevice with unexpected hardware device type " << type; + } + } + + // The factory may advertise at most one NPU and one GPU. This helper retains only the first OrtEpDevice of each + // type, so the separate counters ensure that any additional OrtEpDevice of the same type is detected. + EXPECT_LE(num_npu, 1u) << "CoreML EP advertised more than one NPU OrtEpDevice."; + EXPECT_LE(num_gpu, 1u) << "CoreML EP advertised more than one GPU OrtEpDevice."; + + return found; +} + +// Returns whether this machine has an NPU that the CoreML EP factory may advertise at the runtime Core ML version. +bool CoreMLCanClaimNpu(const HardwareDeviceTypes& hardware) { + return hardware.has_npu && coreml::util::CoreMLVersion() >= MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION; +} + +// Returns whether this machine has a GPU that the CoreML EP factory may advertise at the runtime Core ML version. +bool CoreMLCanClaimGpu(const HardwareDeviceTypes& hardware) { + return hardware.has_gpu && coreml::util::CoreMLVersion() >= MINIMUM_COREML_VERSION; +} + +// Verifies that a session for mul_1.onnx assigns its input to the CoreML EP. +void AssertMul1InputAssignedToCoreML(Ort::Session& session) { + const OrtApi* c_api = &Ort::GetApi(); + const OrtEpDevice* input_ep_device = nullptr; + ASSERT_ORTSTATUS_OK(c_api->SessionGetEpDeviceForInputs(session, &input_ep_device, 1)); + ASSERT_NE(input_ep_device, nullptr); + ASSERT_STREQ(c_api->EpDevice_EpName(input_ep_device), kCoreMLExecutionProvider); +} + +// Verifies the input assignment and runs the inference. Used by the tests that create the session themselves. +void AssertMul1SessionRunsOnCoreML(Ort::Session& session) { + ASSERT_NO_FATAL_FAILURE(AssertMul1InputAssignedToCoreML(session)); + + std::vector> inputs(1); + auto& input = inputs.back(); + input.name = "X"; + input.dims = {3, 2}; + input.values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + auto allocator = std::make_unique(); + RunSession(allocator.get(), session, inputs, "Y", + {3, 2}, {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}, nullptr); +} +} // namespace + +TEST(AutoEpSelection, CoreMLEP) { + const HardwareDeviceTypes hardware = GetHardwareDeviceTypes(); + const bool can_claim_npu = CoreMLCanClaimNpu(hardware); + const bool can_claim_gpu = CoreMLCanClaimGpu(hardware); + const CoreMLDevices coreml_devices = GetCoreMLEpDevices(); + + // Verify that the factory does not advertise devices rejected by its version requirements. + if (hardware.has_npu && !can_claim_npu) { + EXPECT_EQ(coreml_devices.npu, nullptr) << "CoreML EP must not advertise the NPU below Core ML 6."; + } + if (hardware.has_gpu && !can_claim_gpu) { + EXPECT_EQ(coreml_devices.gpu, nullptr) << "CoreML EP must not advertise devices below Core ML 5."; + } + + // Skip when the factory cannot advertise any discovered accelerator. For example, on iOS versions earlier + // than 16, discovery may report an NPU that cannot be advertised because NPU selection requires Core ML 6, + // while GPU discovery does not currently support iOS. + if (!can_claim_npu && !can_claim_gpu) { + GTEST_SKIP() << "No hardware device that the CoreML EP factory can advertise on this machine."; + } + + if (can_claim_npu) { + ASSERT_NE(coreml_devices.npu, nullptr) << "CoreML EP did not claim the NPU hardware device."; + } + if (can_claim_gpu) { + ASSERT_NE(coreml_devices.gpu, nullptr) << "CoreML EP did not claim the GPU hardware device."; + } + + // Verify the CoreML factory's vendor name and vendor ID, along with the hardware vendor ID. The factory reports + // "Microsoft", like the other internal factories: the factory vendor identifies who provides the EP + // implementation, not the hardware vendor. + // Apple device discovery assigns pci_vendor_ids::kApple to every NPU and GPU it reports. The + // CoreML factory intentionally accepts any discovered GPU regardless of vendor, so this hardware vendor ID + // assertion must be updated if discovery later reports a non-Apple GPU (see the multi-GPU TODO in + // core/platform/apple/device_discovery.cc). + const OrtApi* c_api = &Ort::GetApi(); + size_t num_ep_devices = 0; + const OrtEpDevice* const* all_ep_devices = nullptr; + ASSERT_ORTSTATUS_OK(c_api->GetEpDevices(*ort_env, &all_ep_devices, &num_ep_devices)); + + for (size_t i = 0; i < num_ep_devices; ++i) { + const OrtEpDevice* ep_device = all_ep_devices[i]; + if (strcmp(c_api->EpDevice_EpName(ep_device), kCoreMLExecutionProvider) != 0) { + continue; + } + + EXPECT_STREQ(c_api->EpDevice_EpVendor(ep_device), "Microsoft"); + EXPECT_EQ(c_api->HardwareDevice_VendorId(c_api->EpDevice_Device(ep_device)), pci_vendor_ids::kApple); + EXPECT_EQ(ep_device->ep_factory->GetVendorId(ep_device->ep_factory), OrtDevice::VendorIds::MICROSOFT); + } + + // Both the "test.ep_to_select" auto-selection path and the AppendExecutionProvider_V2 path select CoreML as + // the only non-CPU EP. Disabling CPU fallback makes session creation fail unless CoreML takes the entire graph. + // CoreMLEPPreferNpu and CoreMLEPPreferGpu check the input assignment for policy-based selection. + RunBasicTest(kCoreMLExecutionProvider, std::nullopt, Ort::KeyValuePairs{}, /*select_devices*/ nullptr, + /*test_auto_select*/ true, /*disable_cpu_ep_fallback*/ true); +} + +// Tests explicit device selection through AppendExecutionProvider_V2. Each test case selects one or more CoreML +// devices, optionally provides MLComputeUnits in ep_options, and verifies that the session assigns the graph to +// CoreML and runs successfully. Test cases are skipped individually only when a required device is unavailable. +// Test cases with MLComputeUnits verify that the caller may narrow accelerator use within the selected devices: +// for an NPU-and-GPU selection, CPUAndGPU enables only the GPU accelerator and CPUAndNeuralEngine enables only +// the NPU. CPUOnly disables all selected accelerators while the graph remains assigned to CoreML. +TEST(AutoEpSelection, CoreMLEPExplicitDeviceSelection) { + enum class Selection { kNpuAndGpu, + kGpuOnly, + kNpuOnly, + kEveryAdvertisedDevice }; + + struct TestCase { + const char* description; + Selection selection; + const char* compute_units; // nullptr: MLComputeUnits omitted + }; + + const TestCase test_cases[] = { + {"NPU and GPU, MLComputeUnits omitted", Selection::kNpuAndGpu, nullptr}, + {"NPU and GPU, CPUAndGPU", Selection::kNpuAndGpu, "CPUAndGPU"}, + {"NPU and GPU, CPUAndNeuralEngine", Selection::kNpuAndGpu, "CPUAndNeuralEngine"}, + {"GPU only, MLComputeUnits omitted", Selection::kGpuOnly, nullptr}, + {"NPU only, MLComputeUnits omitted", Selection::kNpuOnly, nullptr}, + {"every advertised device, CPUOnly", Selection::kEveryAdvertisedDevice, "CPUOnly"}, + }; + + const HardwareDeviceTypes hardware = GetHardwareDeviceTypes(); + const bool can_claim_npu = CoreMLCanClaimNpu(hardware); + const bool can_claim_gpu = CoreMLCanClaimGpu(hardware); + if (!can_claim_npu && !can_claim_gpu) { + GTEST_SKIP() << "No hardware device that the CoreML EP factory can advertise on this machine."; + } + + const CoreMLDevices coreml_devices = GetCoreMLEpDevices(); + if (can_claim_npu) { + ASSERT_NE(coreml_devices.npu, nullptr) << "CoreML EP did not claim the NPU hardware device."; + } + if (can_claim_gpu) { + ASSERT_NE(coreml_devices.gpu, nullptr) << "CoreML EP did not claim the GPU hardware device."; + } + + for (const TestCase& test_case : test_cases) { + SCOPED_TRACE(test_case.description); + + const bool wants_npu = test_case.selection == Selection::kNpuAndGpu || + test_case.selection == Selection::kNpuOnly || + (test_case.selection == Selection::kEveryAdvertisedDevice && can_claim_npu); + const bool wants_gpu = test_case.selection == Selection::kNpuAndGpu || + test_case.selection == Selection::kGpuOnly || + (test_case.selection == Selection::kEveryAdvertisedDevice && can_claim_gpu); + if ((wants_npu && !can_claim_npu) || (wants_gpu && !can_claim_gpu)) { + continue; + } + + std::vector devices; + if (wants_npu) { + devices.push_back(coreml_devices.npu); + } + if (wants_gpu) { + devices.push_back(coreml_devices.gpu); + } + + Ort::KeyValuePairs ep_options; + if (test_case.compute_units != nullptr) { + ep_options.Add(kCoremlProviderOption_MLComputeUnits, test_case.compute_units); + } + + // Run only the AppendExecutionProvider_V2 path because it passes the test case's full device selection to the + // factory. The "test.ep_to_select" path would pass only the first CoreML device. Disable ORT CPU fallback so + // CoreML must execute the entire graph. + RunBasicTest( + kCoreMLExecutionProvider, std::nullopt, ep_options, + [&devices](std::vector& selected) { selected = devices; }, + /*test_auto_select*/ false, /*disable_cpu_ep_fallback*/ true); + } +} + +// PREFER_NPU must select the CoreML EP for the Apple Neural Engine instead of falling back to the ORT CPU EP. +// MAX_EFFICIENCY and MIN_OVERALL_POWER use the same selector as PREFER_NPU, so the test verifies the same result +// for both aliases. +TEST(AutoEpSelection, CoreMLEPPreferNpu) { + if (!CoreMLCanClaimNpu(GetHardwareDeviceTypes())) { + GTEST_SKIP() << "No NPU hardware device, or this Core ML version cannot advertise it."; + } + ASSERT_NE(GetCoreMLEpDevices().npu, nullptr) << "CoreML EP did not claim the NPU hardware device."; + + for (const OrtExecutionProviderDevicePolicy policy : {OrtExecutionProviderDevicePolicy_PREFER_NPU, + OrtExecutionProviderDevicePolicy_MAX_EFFICIENCY, + OrtExecutionProviderDevicePolicy_MIN_OVERALL_POWER}) { + SCOPED_TRACE(testing::Message() << "policy " << policy); + Ort::SessionOptions session_options; + session_options.SetEpSelectionPolicy(policy); + Ort::Session session(*ort_env, ORT_TSTR("testdata/mul_1.onnx"), session_options); + + ASSERT_NO_FATAL_FAILURE(AssertMul1SessionRunsOnCoreML(session)); + } +} + +// PREFER_GPU must select the CoreML EP. When the internal WebGPU EP is also registered, both EPs advertise an +// OrtEpDevice for the same Apple GPU, and the test verifies that CoreML is selected over WebGPU. +// Without the internal WebGPU EP, the test simply verifies that PREFER_GPU selects CoreML. +// MAX_PERFORMANCE uses the same selector and is covered by the same assertions. +TEST(AutoEpSelection, CoreMLEPPreferGpu) { + if (!CoreMLCanClaimGpu(GetHardwareDeviceTypes())) { + GTEST_SKIP() << "No GPU hardware device that the CoreML EP factory can advertise on this machine."; + } + + const CoreMLDevices coreml_devices = GetCoreMLEpDevices(); + ASSERT_NE(coreml_devices.gpu, nullptr) << "CoreML EP did not claim the GPU hardware device."; + +#if defined(USE_WEBGPU) && !defined(ORT_USE_EP_API_ADAPTERS) + // Verify that WebGPU also advertises an OrtEpDevice for the same GPU. Otherwise, selecting CoreML would not show + // that it is preferred over WebGPU for the same hardware device. + bool matching_webgpu_device_present = false; + { + const OrtApi* c_api = &Ort::GetApi(); + const OrtHardwareDevice* coreml_gpu_hw = c_api->EpDevice_Device(coreml_devices.gpu); + const OrtEpDevice* const* ep_devices = nullptr; + size_t num_devices = 0; + + ASSERT_ORTSTATUS_OK(c_api->GetEpDevices(*ort_env, &ep_devices, &num_devices)); + for (size_t i = 0; i < num_devices; ++i) { + if (strcmp(c_api->EpDevice_EpName(ep_devices[i]), kWebGpuExecutionProvider) == 0 && + c_api->EpDevice_Device(ep_devices[i]) == coreml_gpu_hw) { + matching_webgpu_device_present = true; + break; + } + } + } + + ASSERT_TRUE(matching_webgpu_device_present) + << "Expected WebGPU to also advertise an OrtEpDevice for the CoreML GPU's hardware device in this build."; +#endif + + for (const OrtExecutionProviderDevicePolicy policy : {OrtExecutionProviderDevicePolicy_PREFER_GPU, + OrtExecutionProviderDevicePolicy_MAX_PERFORMANCE}) { + SCOPED_TRACE(testing::Message() << "policy " << policy); + Ort::SessionOptions session_options; + session_options.SetEpSelectionPolicy(policy); + Ort::Session session(*ort_env, ORT_TSTR("testdata/mul_1.onnx"), session_options); + + ASSERT_NO_FATAL_FAILURE(AssertMul1SessionRunsOnCoreML(session)); + } +} + +// MLComputeUnits must not enable an accelerator outside the selected devices. The policy path uses PREFER_NPU to +// select only the NPU, then tests CPUAndGPU and ALL. The AppendExecutionProvider_V2 path selects only the GPU, then +// tests CPUAndNeuralEngine and ALL. Each value must be rejected because it enables an unselected accelerator. +TEST(AutoEpSelection, CoreMLEPConflictingComputeUnitsRejected) { + const HardwareDeviceTypes hardware = GetHardwareDeviceTypes(); + const bool can_claim_npu = CoreMLCanClaimNpu(hardware); + const bool can_claim_gpu = CoreMLCanClaimGpu(hardware); + if (!can_claim_npu && !can_claim_gpu) { + GTEST_SKIP() << "No hardware device that the CoreML EP factory can advertise on this machine."; + } + + const CoreMLDevices coreml_devices = GetCoreMLEpDevices(); + + const auto expect_rejected = [](Ort::SessionOptions& session_options, const char* explanation, + bool check_error_code) { + try { + Ort::Session session(*ort_env, ORT_TSTR("testdata/mul_1.onnx"), session_options); + FAIL() << "Expected session creation to fail: " << explanation; + } catch (const Ort::Exception& ex) { + if (check_error_code) { + EXPECT_EQ(ex.GetOrtErrorCode(), ORT_INVALID_ARGUMENT); + } + EXPECT_THAT(ex.what(), ::testing::HasSubstr("was not selected")); + } + }; + + if (can_claim_npu) { + ASSERT_NE(coreml_devices.npu, nullptr) << "CoreML EP did not claim the NPU hardware device."; + + for (const char* compute_units : {"CPUAndGPU", "ALL"}) { + SCOPED_TRACE(compute_units); + Ort::SessionOptions session_options; + session_options.SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy_PREFER_NPU); + const std::string option_key = + OrtSessionOptions::GetProviderOptionPrefix(kCoreMLExecutionProvider) + kCoremlProviderOption_MLComputeUnits; + session_options.AddConfigEntry(option_key.c_str(), compute_units); + expect_rejected(session_options, "the value enables the GPU, which PREFER_NPU did not select.", + /*check_error_code*/ true); + } + } + + if (can_claim_gpu) { + ASSERT_NE(coreml_devices.gpu, nullptr) << "CoreML EP did not claim the GPU hardware device."; + + for (const char* compute_units : {"CPUAndNeuralEngine", "ALL"}) { + SCOPED_TRACE(compute_units); + Ort::KeyValuePairs ep_options; + ep_options.Add(kCoremlProviderOption_MLComputeUnits, compute_units); + std::vector ep_devices{Ort::ConstEpDevice{coreml_devices.gpu}}; + Ort::SessionOptions session_options; + session_options.AppendExecutionProvider_V2(*ort_env, ep_devices, ep_options); + // The V2 registration path currently reports the factory's ORT_INVALID_ARGUMENT as ORT_FAIL, so only the + // diagnostic message is verified here. The policy path above verifies ORT_INVALID_ARGUMENT. + expect_rejected(session_options, "the value enables the NPU, but only the GPU was selected.", + /*check_error_code*/ false); + } + } +} + +// The factory must reject selecting the same device type twice. AppendExecutionProvider_V2 validation +// checks only that all selected devices use the same EP and factory, so the duplicate reaches the factory-specific +// validation. +TEST(AutoEpSelection, CoreMLEPDuplicateDeviceSelectionRejected) { + const HardwareDeviceTypes hardware = GetHardwareDeviceTypes(); + if (!CoreMLCanClaimNpu(hardware) && !CoreMLCanClaimGpu(hardware)) { + GTEST_SKIP() << "No hardware device that the CoreML EP factory can advertise on this machine."; + } + + const CoreMLDevices coreml_devices = GetCoreMLEpDevices(); + ASSERT_TRUE(coreml_devices.npu != nullptr || coreml_devices.gpu != nullptr) + << "CoreML EP did not advertise any supported NPU or GPU device."; + + // The factory checks for duplicate NPU and GPU selections separately, so test each available device type. + for (const OrtEpDevice* device : {coreml_devices.npu, coreml_devices.gpu}) { + if (device == nullptr) { + continue; + } + SCOPED_TRACE(testing::Message() << "device type " + << Ort::GetApi().HardwareDevice_Type(Ort::GetApi().EpDevice_Device(device))); + + std::vector ep_devices{Ort::ConstEpDevice{device}, Ort::ConstEpDevice{device}}; + Ort::SessionOptions session_options; + session_options.AppendExecutionProvider_V2(*ort_env, ep_devices, Ort::KeyValuePairs{}); + + try { + Ort::Session session(*ort_env, ORT_TSTR("testdata/mul_1.onnx"), session_options); + FAIL() << "Expected session creation to fail: the same CoreML device was selected twice."; + } catch (const Ort::Exception& ex) { + // The V2 registration path currently reports the factory's ORT_INVALID_ARGUMENT as ORT_FAIL, so this test + // verifies only the diagnostic message. + EXPECT_THAT(ex.what(), ::testing::HasSubstr("At most one device of each type can be selected")); + } + } +} + +// Verifies that an unknown MLComputeUnits value is not rejected by the factory but reaches CoreMLOptions, which +// rejects it during provider creation, so session creation fails with the CoreMLOptions diagnostic. +TEST(AutoEpSelection, CoreMLEPUnknownComputeUnitsRejectedByProvider) { + const CoreMLDevices coreml_devices = GetCoreMLEpDevices(); + const OrtEpDevice* device = coreml_devices.npu != nullptr ? coreml_devices.npu : coreml_devices.gpu; + if (device == nullptr) { + GTEST_SKIP() << "No CoreML OrtEpDevice on this machine."; + } + + Ort::KeyValuePairs ep_options; + ep_options.Add(kCoremlProviderOption_MLComputeUnits, "NotAComputeUnitsValue"); + std::vector ep_devices{Ort::ConstEpDevice{device}}; + + Ort::SessionOptions session_options; + session_options.AppendExecutionProvider_V2(*ort_env, ep_devices, ep_options); + + try { + Ort::Session session(*ort_env, ORT_TSTR("testdata/mul_1.onnx"), session_options); + FAIL() << "Expected session creation to fail: MLComputeUnits=NotAComputeUnitsValue is not a valid value."; + } catch (const Ort::Exception& ex) { + EXPECT_THAT(ex.what(), ::testing::HasSubstr("Invalid value for option")); + EXPECT_THAT(ex.what(), ::testing::HasSubstr("NotAComputeUnitsValue")); + } +} + +// The tests below call CoreMLEpFactory through its EpFactoryInternal wrapper using synthetic hardware devices. +// Device advertising depends only on the device type and runtime Core ML version, so the tests require no physical +// accelerator and can run on Intel Macs that meet the version requirement. Any published OrtEpDevice instances are +// released through the EP API. +namespace { + +// Creates a synthetic hardware device of the given type. The device's vendor does not affect whether the factory +// advertises the device. The Apple values below match those reported by device discovery. +OrtHardwareDevice MakeSyntheticHardwareDevice(OrtHardwareDeviceType type) { + OrtHardwareDevice device{}; + device.type = type; + device.vendor_id = pci_vendor_ids::kApple; + device.vendor = "Apple"; + return device; +} + +// Creates the CoreML EP for the given devices through CoreMLEpFactory::CreateIExecutionProvider and writes +// its reported MLComputeUnits value to compute_units. If requested_compute_units is non-null, it is passed through +// the session options so the factory validates it against the selection instead of deriving a default. +void GetMLComputeUnitsForSelection(gsl::span devices, + const char* requested_compute_units, std::string& compute_units) { + EpFactoryInternal factory{std::make_unique()}; + + OrtSessionOptions session_options{}; + if (requested_compute_units != nullptr) { + const std::string key = OrtSessionOptions::GetProviderOptionPrefix(kCoreMLExecutionProvider) + + kCoremlProviderOption_MLComputeUnits; + ASSERT_STATUS_OK(session_options.value.config_options.AddConfigEntry(key.c_str(), requested_compute_units)); + } + + std::unique_ptr ep; + ASSERT_ORTSTATUS_OK(factory.CreateIExecutionProvider(devices.data(), nullptr, devices.size(), &session_options, + logging::LoggingManager::DefaultLogger().ToExternal(), &ep)); + ASSERT_NE(ep, nullptr); + + compute_units = ep->GetProviderOptions().at(kCoremlProviderOption_MLComputeUnits); +} + +// Number of calls to the injected GetVersion callback. +int g_injected_version_calls = 0; + +} // namespace + +// Verifies that GetSupportedDevices preserves input order, advertises at most one NPU and one GPU, ignores +// duplicate devices of either type, and excludes CPU devices. +// It publishes no more than max_ep_devices devices and leaves the remaining output entries unchanged. +// Every successful call writes the output count, including zero when no devices are advertised. +TEST(AutoEpSelection, CoreMLEPGetSupportedDevicesEnumeration) { + if (coreml::util::CoreMLVersion() < MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION) { + GTEST_SKIP() << "This test requires Core ML 6 or later because it expects the factory to advertise an NPU."; + } + + EpFactoryInternal factory{std::make_unique()}; + OrtEpFactory* c_factory = &factory; + ASSERT_NE(c_factory->GetSupportedDevices, nullptr); + + const OrtHardwareDevice gpu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_GPU); + const OrtHardwareDevice npu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_NPU); + const OrtHardwareDevice second_gpu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_GPU); + const OrtHardwareDevice second_npu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_NPU); + const OrtHardwareDevice cpu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_CPU); + + constexpr size_t kOutputCapacity = 5; + OrtEpDevice untouched_marker{}; + + // Calls GetSupportedDevices on the given hardware devices with max_ep_devices as the output capacity. Returns the + // hardware device behind each published OrtEpDevice in publication order and releases the instances. Output + // entries beyond the published count must keep their marker value. + const auto published_hardware = [&](gsl::span hardware, size_t max_ep_devices) { + std::array ep_devices{}; + ep_devices.fill(&untouched_marker); + size_t num_ep_devices = 42; // a successful call must overwrite this + Ort::Status status{c_factory->GetSupportedDevices(c_factory, hardware.data(), hardware.size(), + ep_devices.data(), max_ep_devices, &num_ep_devices)}; + + InlinedVector result; + EXPECT_TRUE(status.IsOK()) << status.GetErrorMessage(); + EXPECT_LE(num_ep_devices, max_ep_devices); + if (!status.IsOK() || num_ep_devices > max_ep_devices) { + return result; + } + + for (size_t i = 0; i < ep_devices.size(); ++i) { + if (i >= num_ep_devices) { + EXPECT_EQ(ep_devices[i], &untouched_marker) << "Entry " << i << " lies beyond the published count."; + continue; + } + + EXPECT_NE(ep_devices[i], &untouched_marker) << "Published entry " << i << " was not written."; + EXPECT_NE(ep_devices[i], nullptr) << "Published entry " << i << " is null."; + if (ep_devices[i] == nullptr || ep_devices[i] == &untouched_marker) { + continue; + } + + result.push_back(Ort::GetApi().EpDevice_Device(ep_devices[i])); + Ort::GetEpApi().ReleaseEpDevice(ep_devices[i]); + } + + return result; + }; + + // Only the first GPU and NPU are advertised, in that order. + const std::array gpu_first{&gpu, &npu, &second_gpu, &second_npu, &cpu}; + EXPECT_THAT(published_hardware(gpu_first, kOutputCapacity), ::testing::ElementsAre(&gpu, &npu)); + + // Skip the leading CPU device and preserve the NPU-before-GPU order. + const std::array cpu_first{&cpu, &npu, &gpu}; + EXPECT_THAT(published_hardware(cpu_first, kOutputCapacity), ::testing::ElementsAre(&npu, &gpu)); + + // The output capacity limits the total number of published devices. A capacity of 1 keeps only the first + // supported device. A capacity of 0 produces no devices. + EXPECT_THAT(published_hardware(gpu_first, 1), ::testing::ElementsAre(&gpu)); + EXPECT_THAT(published_hardware(gpu_first, 0), ::testing::IsEmpty()); + + // When the input contains only a CPU device, no devices are published and the output count is set to zero. + const std::array cpu_only{&cpu}; + EXPECT_THAT(published_hardware(cpu_only, kOutputCapacity), ::testing::IsEmpty()); +} + +// If the second OrtEpDevice creation fails, GetSupportedDevices must return an error, release the first device, +// and leave the output array and count unchanged, following ORT's C API convention for failed calls. +// The test replaces the GetVersion callback on EpFactoryInternal with one that returns a valid version on the +// first call and an invalid version on the second. CreateEpDevice validates each version, causing the second +// creation to fail. +TEST(AutoEpSelection, CoreMLEPGetSupportedDevicesRollsBackOnCreateFailure) { + if (coreml::util::CoreMLVersion() < MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION) { + GTEST_SKIP() << "This test requires Core ML 6 or later so the factory can advertise both an NPU and a GPU."; + } + + EpFactoryInternal factory{std::make_unique()}; + OrtEpFactory* c_factory = &factory; + g_injected_version_calls = 0; + c_factory->GetVersion = [](const OrtEpFactory*) noexcept -> const char* { + // A valid version for the first device and an invalid one for the second, so the second CreateEpDevice fails. + return ++g_injected_version_calls == 1 ? "1.0.0" : "not a version"; + }; + + const OrtHardwareDevice npu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_NPU); + const OrtHardwareDevice gpu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_GPU); + const std::array hardware{&npu, &gpu}; + + OrtEpDevice untouched_marker{}; + std::array ep_devices{&untouched_marker, &untouched_marker}; + size_t num_ep_devices = 42; + Ort::Status status{c_factory->GetSupportedDevices(c_factory, hardware.data(), hardware.size(), ep_devices.data(), + ep_devices.size(), &num_ep_devices)}; + + ASSERT_FALSE(status.IsOK()) + << "GetSupportedDevices succeeded although the second CreateEpDevice was made to fail."; + EXPECT_EQ(g_injected_version_calls, 2) + << "GetVersion must be called for both device creation attempts."; + EXPECT_THAT(ep_devices, ::testing::Each(&untouched_marker)); + EXPECT_EQ(num_ep_devices, size_t{42}); +} + +// Verifies that CreateIExecutionProvider derives MLComputeUnits from the selected devices and preserves a +// caller-provided value compatible with the selection. Synthetic devices let these cases run without physical +// accelerators. +TEST(AutoEpSelection, CoreMLEPDerivedMLComputeUnits) { + const OrtHardwareDevice npu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_NPU); + const OrtHardwareDevice gpu = MakeSyntheticHardwareDevice(OrtHardwareDeviceType_GPU); + const std::array npu_only{&npu}; + const std::array gpu_only{&gpu}; + const std::array npu_and_gpu{&npu, &gpu}; + std::string compute_units; + + ASSERT_NO_FATAL_FAILURE(GetMLComputeUnitsForSelection(npu_only, nullptr, compute_units)); + EXPECT_EQ(compute_units, kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine); + + ASSERT_NO_FATAL_FAILURE(GetMLComputeUnitsForSelection(gpu_only, nullptr, compute_units)); + EXPECT_EQ(compute_units, kCoremlProviderOption_MLComputeUnits_CPUAndGPU); + + ASSERT_NO_FATAL_FAILURE(GetMLComputeUnitsForSelection(npu_and_gpu, nullptr, compute_units)); + EXPECT_EQ(compute_units, kCoremlProviderOption_MLComputeUnits_ALL); + + // The caller can disable some or all selected accelerators. The factory keeps the provided MLComputeUnits value. + ASSERT_NO_FATAL_FAILURE( + GetMLComputeUnitsForSelection(gpu_only, kCoremlProviderOption_MLComputeUnits_CPUOnly, compute_units)); + EXPECT_EQ(compute_units, kCoremlProviderOption_MLComputeUnits_CPUOnly); + + ASSERT_NO_FATAL_FAILURE( + GetMLComputeUnitsForSelection(npu_and_gpu, kCoremlProviderOption_MLComputeUnits_CPUAndGPU, compute_units)); + EXPECT_EQ(compute_units, kCoremlProviderOption_MLComputeUnits_CPUAndGPU); + + ASSERT_NO_FATAL_FAILURE(GetMLComputeUnitsForSelection( + npu_and_gpu, kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine, compute_units)); + EXPECT_EQ(compute_units, kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine); +} + +#endif // defined(__APPLE__) + +#if !defined(__APPLE__) +// CoreML can be built on non-Apple platforms to test model conversion, but it cannot execute models there. +// The factory checks only the device type and Core ML version. Because the non-Apple CoreMLVersion stub reports +// Core ML 7, the factory would accept accelerators discovered on a non-Apple system. Verify that the platform guards +// prevent the CoreML factory from being registered on non-Apple builds. +// Registration is checked with GetHardwareDeviceEpIncompatibilityDetails rather than through the published device +// list. The function looks up the factory by EP name before checking hardware compatibility, so any discovered +// device, including the CPU, is sufficient and the test does not depend on accelerator discovery. +TEST(AutoEpSelection, CoreMLEPIsNotRegisteredOnNonApplePlatforms) { + const OrtApi* c_api = &Ort::GetApi(); + + size_t num_devices = 0; + ASSERT_ORTSTATUS_OK(c_api->GetNumHardwareDevices(*ort_env, &num_devices)); + ASSERT_GT(num_devices, 0u) << "Expected device discovery to report at least the CPU device."; + + InlinedVector devices(num_devices); + ASSERT_ORTSTATUS_OK(c_api->GetHardwareDevices(*ort_env, devices.data(), num_devices)); + + OrtDeviceEpIncompatibilityDetails* details = nullptr; + Ort::Status status{c_api->GetHardwareDeviceEpIncompatibilityDetails(*ort_env, kCoreMLExecutionProvider, + devices.front(), &details)}; + // If the CoreML factory is unexpectedly registered, the call succeeds and returns an object that must be released + // before the test reports the failure. + if (details != nullptr) { + c_api->ReleaseDeviceEpIncompatibilityDetails(details); + } + + ASSERT_FALSE(status.IsOK()) << "A factory named " << kCoreMLExecutionProvider << " is registered on a non-Apple " + << "platform, where CoreML cannot execute models."; + EXPECT_EQ(status.GetErrorCode(), ORT_INVALID_ARGUMENT); + // ORT_INVALID_ARGUMENT alone does not prove that the factory is absent. If the EP name matches a registered + // factory, GetHardwareDeviceEpIncompatibilityDetails invokes the factory's compatibility callback, + // which may return the same error code. Verify the diagnostic to confirm that no factory matched the CoreML EP name. + EXPECT_THAT(status.GetErrorMessage(), ::testing::HasSubstr("No valid factory found for execution provider")); +} +#endif // !defined(__APPLE__) + +} // namespace test +} // namespace onnxruntime + +#endif // defined(USE_COREML) diff --git a/onnxruntime/test/providers/coreml/coreml_basic_test.cc b/onnxruntime/test/providers/coreml/coreml_basic_test.cc index 2fb8d1a09d873..81fd46dee2365 100644 --- a/onnxruntime/test/providers/coreml/coreml_basic_test.cc +++ b/onnxruntime/test/providers/coreml/coreml_basic_test.cc @@ -3736,5 +3736,75 @@ TEST(CoreMLExecutionProviderTest, GatherScalarIndicesRank5DataNotSupported) { } #endif // !(ORT_MINIMAL_BUILD) + +// Verifies that GetProviderOptions reports effective CoreML options using public option names, +// whether the EP is configured through ProviderOptions or legacy flags. +TEST(CoreMLExecutionProviderTest, GetProviderOptionsReportsEffectiveOptions) { + const auto reported_options = [](const ProviderOptions& options) { + return CoreMLProviderFactoryCreator::Create(options)->CreateProvider()->GetProviderOptions(); + }; + + // Default values are reported even when no options are provided. An unset cache directory is omitted. + { + const ProviderOptions reported = reported_options({}); + EXPECT_EQ(reported.at(kCoremlProviderOption_MLComputeUnits), kCoremlProviderOption_MLComputeUnits_ALL); + EXPECT_EQ(reported.at(kCoremlProviderOption_ModelFormat), "NeuralNetwork"); + EXPECT_EQ(reported.at(kCoremlProviderOption_RequireStaticInputShapes), "0"); + EXPECT_EQ(reported.at(kCoremlProviderOption_EnableOnSubgraphs), "0"); + EXPECT_EQ(reported.at(kCoremlProviderOption_SpecializationStrategy), "Default"); + EXPECT_EQ(reported.at(kCoremlProviderOption_ProfileComputePlan), "0"); + EXPECT_EQ(reported.at(kCoremlProviderOption_AllowLowPrecisionAccumulationOnGPU), "0"); + EXPECT_EQ(reported.count(kCoremlProviderOption_ModelCacheDirectory), size_t{0}); + } + + // Explicitly provided option values are reported unchanged. + { + const std::string cache_directory = + std::filesystem::temp_directory_path().append("coreml_provider_options_test").string(); + const ProviderOptions reported = reported_options({ + {kCoremlProviderOption_MLComputeUnits, kCoremlProviderOption_MLComputeUnits_CPUOnly}, + {kCoremlProviderOption_ModelFormat, "MLProgram"}, + {kCoremlProviderOption_RequireStaticInputShapes, "1"}, + {kCoremlProviderOption_EnableOnSubgraphs, "1"}, + {kCoremlProviderOption_SpecializationStrategy, "FastPrediction"}, + {kCoremlProviderOption_ProfileComputePlan, "1"}, + {kCoremlProviderOption_AllowLowPrecisionAccumulationOnGPU, "1"}, + {kCoremlProviderOption_ModelCacheDirectory, cache_directory}, + }); + EXPECT_EQ(reported.at(kCoremlProviderOption_MLComputeUnits), kCoremlProviderOption_MLComputeUnits_CPUOnly); + EXPECT_EQ(reported.at(kCoremlProviderOption_ModelFormat), "MLProgram"); + EXPECT_EQ(reported.at(kCoremlProviderOption_RequireStaticInputShapes), "1"); + EXPECT_EQ(reported.at(kCoremlProviderOption_EnableOnSubgraphs), "1"); + EXPECT_EQ(reported.at(kCoremlProviderOption_SpecializationStrategy), "FastPrediction"); + EXPECT_EQ(reported.at(kCoremlProviderOption_ProfileComputePlan), "1"); + EXPECT_EQ(reported.at(kCoremlProviderOption_AllowLowPrecisionAccumulationOnGPU), "1"); + EXPECT_EQ(reported.at(kCoremlProviderOption_ModelCacheDirectory), cache_directory); + } + + // ProfileComputePlan applies only to MLProgram, so it is reported as "0" for NeuralNetwork even when set to "1". + EXPECT_EQ(reported_options({{kCoremlProviderOption_ProfileComputePlan, "1"}}) + .at(kCoremlProviderOption_ProfileComputePlan), + "0"); + + EXPECT_EQ(reported_options({{kCoremlProviderOption_MLComputeUnits, kCoremlProviderOption_MLComputeUnits_CPUAndGPU}}) + .at(kCoremlProviderOption_MLComputeUnits), + kCoremlProviderOption_MLComputeUnits_CPUAndGPU); + + EXPECT_EQ(reported_options({{kCoremlProviderOption_MLComputeUnits, + kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine}}) + .at(kCoremlProviderOption_MLComputeUnits), + kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine); + + // The legacy flags constructor reports the same names and values. + { + const ProviderOptions reported = + CoreMLProviderFactoryCreator::Create(COREML_FLAG_USE_CPU_ONLY | COREML_FLAG_CREATE_MLPROGRAM) + ->CreateProvider() + ->GetProviderOptions(); + EXPECT_EQ(reported.at(kCoremlProviderOption_MLComputeUnits), kCoremlProviderOption_MLComputeUnits_CPUOnly); + EXPECT_EQ(reported.at(kCoremlProviderOption_ModelFormat), "MLProgram"); + EXPECT_EQ(reported.count(kCoremlProviderOption_ModelCacheDirectory), size_t{0}); + } +} } // namespace test } // namespace onnxruntime