Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,17 @@ enum COREMLFlags {
COREML_FLAG_LAST = COREML_FLAG_USE_CPU_AND_GPU,
};

// MLComputeUnits can be one of the following values:
// 'MLComputeUnitsCPUAndNeuralEngine|MLComputeUnitsCPUAndGPU|MLComputeUnitsCPUOnly|MLComputeUnitsAll'
// these values are intended to be used with Ort::SessionOptions::AppendExecutionProvider (C++ API)
// and SessionOptionsAppendExecutionProvider (C API). For the old API, use COREMLFlags instead.
// Set the MLComputeUnits provider option to one of the kCoremlProviderOption_MLComputeUnits_* values below.
// Each value maps to the corresponding Core ML compute-unit mode
// (https://developer.apple.com/documentation/coreml/mlcomputeunits).
// Use these values with SessionOptionsAppendExecutionProvider or SessionOptionsAppendExecutionProvider_V2.
// The legacy CoreML-specific OrtSessionOptionsAppendExecutionProvider_CoreML API uses a COREMLFlags bitmask
// instead.
// MLComputeUnitsCPUAndNeuralEngine requires Core ML 6 (iOS 16 / macOS 13) and is unavailable on earlier systems.
static const char* const kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine = "CPUAndNeuralEngine";
static const char* const kCoremlProviderOption_MLComputeUnits_CPUAndGPU = "CPUAndGPU";
static const char* const kCoremlProviderOption_MLComputeUnits_CPUOnly = "CPUOnly";
static const char* const kCoremlProviderOption_MLComputeUnits_ALL = "ALL";
static const char* const kCoremlProviderOption_MLComputeUnits = "MLComputeUnits";
static const char* const kCoremlProviderOption_ModelFormat = "ModelFormat";
// same as COREML_FLAG_ONLY_ALLOW_STATIC_INPUT_SHAPES
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ CoreMLExecutionProvider::CoreMLExecutionProvider(const CoreMLOptions& options)

CoreMLExecutionProvider::~CoreMLExecutionProvider() {}

ProviderOptions CoreMLExecutionProvider::GetProviderOptions() const {
return coreml_options_.ToProviderOptions();
}

std::vector<std::unique_ptr<ComputeCapability>>
CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer,
const IKernelLookup& /*kernel_lookup*/,
Expand Down
2 changes: 2 additions & 0 deletions onnxruntime/core/providers/coreml/coreml_execution_provider.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ class CoreMLExecutionProvider : public IExecutionProvider {
std::vector<NodeComputeInfo>& node_compute_funcs) override;
#endif

ProviderOptions GetProviderOptions() const override;

private:
// The bit flags which define bool options for COREML EP, bits are defined as
// COREMLFlags in include/onnxruntime/core/providers/coreml/coreml_provider_factory.h
Expand Down
38 changes: 34 additions & 4 deletions onnxruntime/core/providers/coreml/coreml_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@

void CoreMLOptions::ValidateAndParseProviderOption(const ProviderOptions& options) {
const std::unordered_map<std::string, COREMLFlags> available_computeunits_options = {
{"CPUAndNeuralEngine", COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE},
{"CPUAndGPU", COREML_FLAG_USE_CPU_AND_GPU},
{"CPUOnly", COREML_FLAG_USE_CPU_ONLY},
{"ALL", COREML_FLAG_USE_NONE},
{kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine, COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE},
{kCoremlProviderOption_MLComputeUnits_CPUAndGPU, COREML_FLAG_USE_CPU_AND_GPU},
{kCoremlProviderOption_MLComputeUnits_CPUOnly, COREML_FLAG_USE_CPU_ONLY},
{kCoremlProviderOption_MLComputeUnits_ALL, COREML_FLAG_USE_NONE},
};
const std::unordered_map<std::string, COREMLFlags> available_modelformat_options = {
{"MLProgram", COREML_FLAG_CREATE_MLPROGRAM},
Expand Down Expand Up @@ -98,4 +98,34 @@
}
}
}

ProviderOptions CoreMLOptions::ToProviderOptions() const {
// MLComputeUnits uses constants defined in coreml_provider_factory.h. The other strings match
// those accepted by ValidateAndParseProviderOption.
const char* compute_units = kCoremlProviderOption_MLComputeUnits_ALL;
if (ComputeUnits(COREML_FLAG_USE_CPU_ONLY)) {
compute_units = kCoremlProviderOption_MLComputeUnits_CPUOnly;
} else if (ComputeUnits(COREML_FLAG_USE_CPU_AND_GPU)) {
compute_units = kCoremlProviderOption_MLComputeUnits_CPUAndGPU;
} else if (ComputeUnits(COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE)) {
compute_units = kCoremlProviderOption_MLComputeUnits_CPUAndNeuralEngine;
}

ProviderOptions options{
{kCoremlProviderOption_MLComputeUnits, compute_units},
{kCoremlProviderOption_ModelFormat, create_mlprogram_ ? "MLProgram" : "NeuralNetwork"},
{kCoremlProviderOption_RequireStaticInputShapes, require_static_shape_ ? "1" : "0"},
{kCoremlProviderOption_EnableOnSubgraphs, enable_on_subgraph_ ? "1" : "0"},
{kCoremlProviderOption_SpecializationStrategy, strategy_.empty() ? std::string{"Default"} : strategy_},

Check warning on line 119 in onnxruntime/core/providers/coreml/coreml_options.cc

View workflow job for this annotation

GitHub Actions / Optional Lint C++

[cpplint] reported by reviewdog 🐶 Add #include <string> for string [build/include_what_you_use] [4] Raw Output: onnxruntime/core/providers/coreml/coreml_options.cc:119: Add #include <string> for string [build/include_what_you_use] [4]
// The effective value: profiling applies to MLProgram models only.
{kCoremlProviderOption_ProfileComputePlan, ProfileComputePlan() ? "1" : "0"},
{kCoremlProviderOption_AllowLowPrecisionAccumulationOnGPU, allow_low_precision_accumulation_on_gpu_ ? "1" : "0"},
};

if (!model_cache_directory_.empty()) {
options[kCoremlProviderOption_ModelCacheDirectory] = model_cache_directory_;
}

return options;
}
} // namespace onnxruntime
5 changes: 5 additions & 0 deletions onnxruntime/core/providers/coreml/coreml_options.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#include "core/common/inlined_containers.h"
#include "core/framework/execution_provider.h"
#include "core/framework/provider_options.h"

namespace onnxruntime {

Expand Down Expand Up @@ -36,6 +37,10 @@ class CoreMLOptions {
bool ProfileComputePlan() const { return profile_compute_plan_ && create_mlprogram_; }

std::string_view ModelCacheDirectory() const { return model_cache_directory_; }

// Returns the effective options for IExecutionProvider::GetProviderOptions using the provider option names
// defined in coreml_provider_factory.h. Options supplied through legacy flags are reported in the same format.
ProviderOptions ToProviderOptions() const;
// The options specified by the user are const, but if there's an error setting up caching we disable it
// so that the EP can still be used. The error is logged for the user to investigate.
void DisableModelCache() const { model_cache_directory_.clear(); }
Expand Down
5 changes: 5 additions & 0 deletions onnxruntime/core/providers/coreml/model/host_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@

#define MINIMUM_COREML_VERSION 5 // first version we support

// Core ML 6 (iOS 16 / macOS 13) introduced MLComputeUnitsCPUAndNeuralEngine, the only compute-units mode that
// enables the Neural Engine without also enabling the GPU. Earlier versions cannot honor a Neural-Engine-only
// request, so device-based EP selection does not offer the Neural Engine below this version.
#define MINIMUM_COREML_VERSION_FOR_NEURAL_ENGINE_SELECTION 6

namespace onnxruntime {
namespace coreml {
namespace util {
Expand Down
Loading
Loading