Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,10 @@ public struct OrtApi
public IntPtr Graph_GetModelMetadata;
public IntPtr GetModelCompatibilityForEpDevices;
public IntPtr CreateExternalInitializerInfo;

public IntPtr TensorTypeAndShape_HasShape;
public IntPtr KernelInfo_GetConfigEntries;
public IntPtr RegisterExecutionProviderLibraryWithOptions;
}

internal static class NativeMethods
Expand Down Expand Up @@ -847,7 +851,7 @@ static NativeMethods()
api_.CreateSyncStreamForEpDevice,
typeof(DOrtCreateSyncStreamForEpDevice));

OrtSyncStream_GetHandle =
OrtSyncStream_GetHandle =
(DOrtSyncStream_GetHandle)Marshal.GetDelegateForFunctionPointer(
api_.SyncStream_GetHandle,
typeof(DOrtSyncStream_GetHandle));
Expand All @@ -861,6 +865,11 @@ static NativeMethods()
(DOrtCopyTensors)Marshal.GetDelegateForFunctionPointer(
api_.CopyTensors,
typeof(DOrtCopyTensors));

OrtRegisterExecutionProviderLibraryWithOptions =
(DOrtRegisterExecutionProviderLibraryWithOptions)Marshal.GetDelegateForFunctionPointer(
api_.RegisterExecutionProviderLibraryWithOptions,
typeof(DOrtRegisterExecutionProviderLibraryWithOptions));
}

internal class NativeLib
Expand Down Expand Up @@ -2780,6 +2789,22 @@ out IntPtr /* OrtSyncStream** */ stream
byte[] /* const char* */ registration_name,
byte[] /* const ORTCHAR_T* */ path);

/// <summary>
/// Register an execution provider library. The provided options are passed to EP factories after creation.
/// The library must implement CreateEpFactories and ReleaseEpFactory.
/// </summary>
/// <param name="env">Environment to add the EP library to.</param>
/// <param name="registration_name">Name to register the library under.</param>
/// <param name="path">Absolute path to the library.</param>
/// <param name="options">Options passed to OrtEpFactory::SetEnvironmentOptions after creation.</param>
/// <returns>OrtStatus*</returns>
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
public delegate IntPtr /* OrtStatus* */ DOrtRegisterExecutionProviderLibraryWithOptions(
IntPtr /* OrtEnv* */ env,
byte[] /* const char* */ registration_name,
byte[] /* const ORTCHAR_T* */ path,
IntPtr /* const OrtKeyValuePairs* */ options);

/// <summary>
/// Unregister an execution provider library.
/// </summary>
Expand All @@ -2792,6 +2817,7 @@ out IntPtr /* OrtSyncStream** */ stream
byte[] /* const char* */ registration_name);

public static DOrtRegisterExecutionProviderLibrary OrtRegisterExecutionProviderLibrary;
public static DOrtRegisterExecutionProviderLibraryWithOptions OrtRegisterExecutionProviderLibraryWithOptions;
public static DOrtUnregisterExecutionProviderLibrary OrtUnregisterExecutionProviderLibrary;

/// <summary>
Expand Down
45 changes: 42 additions & 3 deletions csharp/src/Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs
Original file line number Diff line number Diff line change
Expand Up @@ -523,9 +523,6 @@ public OrtLoggingLevel EnvLogLevel
/// A registered execution provider library can be used by all sessions created with the OrtEnv instance.
/// Devices the execution provider can utilize are added to the values returned by GetEpDevices() and can
/// be used in SessionOptions.AppendExecutionProvider to select an execution provider for a device.
///
/// Coming: A selection policy can be specified and ORT will automatically select the best execution providers
/// and devices for the model.
/// </summary>
/// <param name="registrationName">The name to register the library under.</param>
/// <param name="libraryPath">The path to the library to register.</param>
Expand All @@ -540,6 +537,48 @@ public void RegisterExecutionProviderLibrary(string registrationName, string lib
NativeMethods.OrtRegisterExecutionProviderLibrary(handle, registrationNameUtf8, pathUtf8));
}

/// <summary>
/// Register an execution provider library with the OrtEnv instance. The provided options are passed to
/// EP factory instances after creation.
///
/// A registered execution provider library can be used by all sessions created with the OrtEnv instance.
/// Devices the execution provider can utilize are added to the values returned by GetEpDevices() and can
/// be used in SessionOptions.AppendExecutionProvider to select an execution provider for a device.
/// </summary>
/// <param name="registrationName">The name to register the library under.</param>
/// <param name="libraryPath">The path to the library to register.</param>
/// <param name="options">Optional options to pass to each EP factory after creation. May be null.</param>
/// <see cref="GetEpDevices"/>
/// <see cref="SessionOptions.AppendExecutionProvider(OrtEnv, IReadOnlyList{OrtEpDevice}, IReadOnlyDictionary{string, string})"/>
public void RegisterExecutionProviderLibrary(string registrationName, string libraryPath,
IReadOnlyDictionary<string, string> options)
{
var registrationNameUtf8 = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(registrationName);
var pathUtf8 = NativeOnnxValueHelper.GetPlatformSerializedString(libraryPath);

if (options != null && options.Count > 0)
{
// this creates an OrtKeyValuePairs instance with a backing native instance
using var optionsKvps = new OrtKeyValuePairs(options);

NativeApiStatus.VerifySuccess(
NativeMethods.OrtRegisterExecutionProviderLibraryWithOptions(
handle,
registrationNameUtf8,
pathUtf8,
optionsKvps.Handle));
}
else
{
NativeApiStatus.VerifySuccess(
NativeMethods.OrtRegisterExecutionProviderLibraryWithOptions(
handle,
registrationNameUtf8,
pathUtf8,
IntPtr.Zero)); // Options OrtKeyValuePairs
}
}

/// <summary>
/// Unregister an execution provider library from the OrtEnv instance.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,37 @@ public void RegisterUnregisterLibrary()
}
}

[Fact]
public void RegisterLibraryWithOptions()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
const string registrationName = "example_plugin_ep_kernel_registry";
const string epName = "ExampleKernelEp";

string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), "example_plugin_ep_kernel_registry.dll");
Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist.");

Dictionary<string, string> options = new Dictionary<string, string> { { "some_env_config", "2" } };
ortEnvInstance.RegisterExecutionProviderLibrary(registrationName, libFullPath, options);
try
{
// check OrtEpDevice was found
var epDevices = ortEnvInstance.GetEpDevices();
var epDevice = epDevices.FirstOrDefault(d => string.Equals(epName, d.EpName, StringComparison.OrdinalIgnoreCase));
Assert.NotNull(epDevice);

// The example EP stores the env config in the OrtEpDevice metadata for testing convenience.
var epMetadata = epDevice.EpMetadata.Entries;
Assert.Equal("2", epMetadata["some_env_config"]);
}
finally
{ // unregister
ortEnvInstance.UnregisterExecutionProviderLibrary(registrationName);
}
}
}

[Fact]
public void AppendToSessionOptionsV2()
{
Expand Down Expand Up @@ -194,7 +225,7 @@ public void SetEpSelectionPolicyDelegate()

// doesn't matter what the value is. should fallback to ORT CPU EP
sessionOptions.SetEpSelectionPolicyDelegate(SelectionPolicyDelegate);

var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx");

// session should load successfully
Expand Down
9 changes: 6 additions & 3 deletions include/onnxruntime/core/session/environment.h
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ class Environment {
const OrtArenaCfg* arena_cfg = nullptr);

#if !defined(ORT_MINIMAL_BUILD)
Status RegisterExecutionProviderLibrary(const std::string& registration_name, const ORTCHAR_T* lib_path);
Status RegisterExecutionProviderLibrary(const std::string& registration_name, const ORTCHAR_T* lib_path,
const OrtKeyValuePairs* options);
Status UnregisterExecutionProviderLibrary(const std::string& registration_name);

// convert an OrtEpFactory* to EpFactoryInternal* if possible.
Expand Down Expand Up @@ -206,14 +207,16 @@ class Environment {

Status RegisterExecutionProviderLibrary(const std::string& registration_name,
std::unique_ptr<EpLibrary> ep_library,
const std::vector<EpFactoryInternal*>& internal_factories = {});
const std::vector<EpFactoryInternal*>& internal_factories = {},
const OrtKeyValuePairs* options = nullptr);

struct EpInfo {
// calls EpLibrary::Load
// for each factory gets the OrtEpDevice instances and adds to execution_devices
// internal_factory is set if this is an internal EP
static Status Create(std::unique_ptr<EpLibrary> library_in, std::unique_ptr<EpInfo>& out,
const std::vector<EpFactoryInternal*>& internal_factories = {});
const std::vector<EpFactoryInternal*>& internal_factories = {},
const OrtKeyValuePairs* options = nullptr);

// removes entries for this library from execution_devices
// calls EpLibrary::Unload
Expand Down
18 changes: 18 additions & 0 deletions include/onnxruntime/core/session/onnxruntime_c_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -6608,6 +6608,24 @@ struct OrtApi {
* \since Version 1.24
*/
ORT_API2_STATUS(KernelInfo_GetConfigEntries, _In_ const OrtKernelInfo* info, _Outptr_ OrtKeyValuePairs** out);

/** \brief Register an execution provider library with ORT. The provided options are passed to
* OrtEpFactory::SetEnvironmentOptions after factory creation.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's good to document how the options are passed through. maybe also mention when it would be useful to specify such options.

*
* The library must export 'CreateEpFactories' and 'ReleaseEpFactory' functions.
* See OrtEpApi for more details.
*
* \param[in] env The OrtEnv instance to register the library in.
* \param[in] registration_name The name to register the execution provider library under.
* \param[in] path The path to the execution provider library.
* \param[in] options Map of options to pass to each OrtEpFactory via OrtEpFactory::SetEnvironmentOptions.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe mention that this parameter is optional.

*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
* \since Version 1.24.
*/
ORT_API2_STATUS(RegisterExecutionProviderLibraryWithOptions, _In_ OrtEnv* env, _In_ const char* registration_name,
_In_ const ORTCHAR_T* path, _In_opt_ const OrtKeyValuePairs* options);
};

/*
Expand Down
10 changes: 8 additions & 2 deletions include/onnxruntime/core/session/onnxruntime_cxx_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -1199,8 +1199,14 @@ struct Env : detail::Base<OrtEnv> {
void ReleaseSharedAllocator(const OrtEpDevice* ep_device,
OrtDeviceMemoryType mem_type); ///< Wraps OrtApi::ReleaseSharedAllocator

Env& RegisterExecutionProviderLibrary(const char* registration_name, const std::basic_string<ORTCHAR_T>& path); ///< Wraps OrtApi::RegisterExecutionProviderLibrary
Env& UnregisterExecutionProviderLibrary(const char* registration_name); ///< Wraps OrtApi::UnregisterExecutionProviderLibrary
///< Wraps OrtApi::RegisterExecutionProviderLibrary
Env& RegisterExecutionProviderLibrary(const char* registration_name, const std::basic_string<ORTCHAR_T>& path);

///< Wraps OrtApi::RegisterExecutionProviderLibraryWithOptions
Env& RegisterExecutionProviderLibraryWithOptions(const char* registration_name, const std::basic_string<ORTCHAR_T>& path,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

one thing that bothered me before is how Env::RegisterExecutionProviderLibrary() took both a char* and a std::basic_string parameter.

would it be possible to make them consistent? I guess an argument could be made for making it consistently inconsistent too though...

const OrtKeyValuePairs* options);

Env& UnregisterExecutionProviderLibrary(const char* registration_name); ///< Wraps OrtApi::UnregisterExecutionProviderLibrary

std::vector<ConstEpDevice> GetEpDevices() const;

Expand Down
7 changes: 7 additions & 0 deletions include/onnxruntime/core/session/onnxruntime_cxx_inline.h
Original file line number Diff line number Diff line change
Expand Up @@ -835,6 +835,13 @@ inline Env& Env::RegisterExecutionProviderLibrary(const char* registration_name,
return *this;
}

inline Env& Env::RegisterExecutionProviderLibraryWithOptions(const char* registration_name,
const std::basic_string<ORTCHAR_T>& path,
const OrtKeyValuePairs* options) {
ThrowOnError(GetApi().RegisterExecutionProviderLibraryWithOptions(p_, registration_name, path.c_str(), options));
return *this;
}

inline Env& Env::UnregisterExecutionProviderLibrary(const char* registration_name) {
ThrowOnError(GetApi().UnregisterExecutionProviderLibrary(p_, registration_name));
return *this;
Expand Down
14 changes: 8 additions & 6 deletions include/onnxruntime/core/session/onnxruntime_ep_c_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -1510,12 +1510,13 @@ struct OrtEpFactory {
_In_opt_ const OrtKeyValuePairs* stream_options,
_Outptr_ OrtSyncStreamImpl** stream);

/** \brief Set environment options on this EP factory.
/** \brief Sets environment options that are provided by the application during EP library registration.
*
* Environment options can be set by ORT after calling the library's 'CreateEpFactories' function to
* create EP factories.
* If defined, ORT calls this function during EP library registration directly after creating the factory instance.
* Valid option keys and values are defined by the EP library. However, some common EP-agnostic options are listed
* below.
*
* Supported options:
* Common EP options:
* "allow_virtual_devices": Allows EP factory to specify OrtEpDevice instances that use custom
* virtual OrtHardwareDevices, which can be created via OrtEpApi::CreateHardwareDevice().
*
Expand All @@ -1528,10 +1529,11 @@ struct OrtEpFactory {
* -# "1": Creation of virtual devices is allowed.
*
* \param[in] this_ptr The OrtEpFactory instance.
* \param[in] options The configuration options.
* \param[in] options The configuration options. Do not cache pointers to the OrtKeyValuePairs instance or its
* keys and values. Key and value strings should be copied if necessary.
*
* \note Implementation of this function is optional.
* An EP factory should only implement this if it needs to handle any environment options.
* An EP factory should implement this if it needs to handle any environment options.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
Expand Down
4 changes: 4 additions & 0 deletions onnxruntime/core/session/abi_key_value_pairs.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ struct OrtKeyValuePairs {
Sync();
}

bool HasKey(const std::string& key) const {
return entries_.find(key) != entries_.end();
}

void Add(const char* key, const char* value) {
// ignore if either are nullptr.
if (key && value) {
Expand Down
Loading
Loading