Enable CUDA provider option configuration for C#

csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs

@@ -192,6 +192,22 @@ public struct OrtApi
         public IntPtr ModelMetadataGetGraphDescription;
     }
 
+    #region ORT Provider options
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct OrtCUDAProviderOptionsNative
+    {
+        public int device_id;                                  // cuda device with id=0 as default device.
+        public OrtCudnnConvAlgoSearch cudnn_conv_algo_search;  // cudnn conv algo search option
+        public UIntPtr gpu_mem_limit;                          // default cuda memory limitation to maximum finite value of size_t.
+        public int arena_extend_strategy;                      // default area extend strategy to KNextPowerOfTwo.
+        public int do_copy_in_default_stream;                  // Whether to do copies in the default stream or use separate streams.
+        public int has_user_compute_stream;                    // indicator of user specified CUDA compute stream.
+        public IntPtr user_compute_stream;                     // user specified CUDA compute stream.
+    }
+
+    #endregion
+
     internal static class NativeMethods
     {
         private const string nativeLib = "onnxruntime";
@@ -256,6 +272,8 @@ static NativeMethods()
             OrtRegisterCustomOpsLibrary = (DOrtRegisterCustomOpsLibrary)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary, typeof(DOrtRegisterCustomOpsLibrary));
             OrtAddSessionConfigEntry = (DOrtAddSessionConfigEntry)Marshal.GetDelegateForFunctionPointer(api_.AddSessionConfigEntry, typeof(DOrtAddSessionConfigEntry));
             OrtAddInitializer = (DOrtAddInitializer)Marshal.GetDelegateForFunctionPointer(api_.AddInitializer, typeof(DOrtAddInitializer));
+            SessionOptionsAppendExecutionProvider_CUDA = (DSessionOptionsAppendExecutionProvider_CUDA)Marshal.GetDelegateForFunctionPointer(
+                                                             api_.SessionOptionsAppendExecutionProvider_CUDA, typeof(DSessionOptionsAppendExecutionProvider_CUDA));
 
             OrtCreateRunOptions = (DOrtCreateRunOptions)Marshal.GetDelegateForFunctionPointer(api_.CreateRunOptions, typeof(DOrtCreateRunOptions));
             OrtReleaseRunOptions = (DOrtReleaseRunOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseRunOptions, typeof(DOrtReleaseRunOptions));
@@ -561,6 +579,17 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_CUDA(IntPtr /*(OrtSessionOptions*) */ options, int device_id);
 
+        /// <summary>
+        /// Append a CUDA EP instance (configured based on given provider options) to the native OrtSessionOptions instance
+        /// </summary>
+        /// <param name="options">Native OrtSessionOptions instance</param>
+        /// <param name="cudaProviderOptions">Native OrtCUDAProviderOptions instance</param>
+        public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_CUDA(
+                                               IntPtr /*(OrtSessionOptions*)*/ options,
+                                               ref OrtCUDAProviderOptionsNative cudaProviderOptions);
+
+        public static DSessionOptionsAppendExecutionProvider_CUDA SessionOptionsAppendExecutionProvider_CUDA;
+
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_ROCM(IntPtr /*(OrtSessionOptions*) */ options, int device_id);
 

csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.cs

@@ -0,0 +1,108 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntime
+{
+    /// <summary>
+    /// Search Algorithm for Cudnn Conv.
+    /// </summary>
+    public enum OrtCudnnConvAlgoSearch
+    {
+        EXHAUSTIVE,  //!< expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx
+        HEURISTIC,   //!< lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7
+        DEFAULT,     //!< default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM
+    }
+
+    /// <summary>
+    /// Provider options for CUDA EP.
+    /// </summary>
+    public struct OrtCUDAProviderOptions
+    {
+        public int device_id;                                   //!< cuda device with id=0 as default device.
+        public OrtCudnnConvAlgoSearch cudnn_conv_algo_search;   //!< cudnn conv algo search option
+        public UIntPtr gpu_mem_limit;                           //!< default cuda memory limitation to maximum finite value of size_t.
+        public int arena_extend_strategy;                       //!< default area extend strategy to KNextPowerOfTwo.
+        public int do_copy_in_default_stream;                   //!< Whether to do copies in the default stream or use separate streams.
+    }
+
+    /// <summary>
+    /// Holds provider options configuration for creating an InferenceSession.
+    /// </summary>
+    public class ProviderOptions : SafeHandle
+    {
+        internal IntPtr Handle
+        {
+            get
+            {
+                return handle;
+            }
+        }
+
+        #region Constructor and Factory methods
+
+        /// <summary>
+        /// Constructs an empty ProviderOptions
+        /// </summary>
+        public ProviderOptions()
+            : base(IntPtr.Zero, true)
+        {
+        }
+
+        #endregion
+
+        #region Public Methods
+
+        /// <summary>
+        /// Get CUDA provider options with default setting.
+        /// </summary>
+        /// <returns> CUDA provider options instance.  </returns>
+        public static OrtCUDAProviderOptions GetDefaultCUDAProviderOptions()
+        {
+            OrtCUDAProviderOptions cuda_options;
+            cuda_options.device_id = 0;
+            cuda_options.cudnn_conv_algo_search = OrtCudnnConvAlgoSearch.EXHAUSTIVE;
+            if (IntPtr.Size == 8)
+            {
+                cuda_options.gpu_mem_limit = (UIntPtr)UInt64.MaxValue;
+            }
+            else
+            {
+                cuda_options.gpu_mem_limit = (UIntPtr)UInt32.MaxValue;
+            }
+            cuda_options.arena_extend_strategy = 0;
+            cuda_options.do_copy_in_default_stream = 1;
+
+            return cuda_options;
+        }
+        #endregion
+
+        #region Public Properties
+
+        /// <summary>
+        /// Overrides SafeHandle.IsInvalid
+        /// </summary>
+        /// <value>returns true if handle is equal to Zero</value>
+        public override bool IsInvalid { get { return handle == IntPtr.Zero; } }
+
+        #endregion
+
+        #region SafeHandle
+        /// <summary>
+        /// Overrides SafeHandle.ReleaseHandle() to properly dispose of
+        /// the native instance of SessionOptions
+        /// </summary>
+        /// <returns>always returns true</returns>
+        protected override bool ReleaseHandle()
+        {
+            handle = IntPtr.Zero;
+            return true;
+        }
+
+        #endregion
+    }
+}

csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs

@@ -75,6 +75,30 @@ public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId = 0)
             return options;
         }
 
+        /// <summary>
+        /// A helper method to construct a SessionOptions object for CUDA execution.
+        /// Use only if CUDA is installed and you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="cuda_options">CUDA EP provider options to configure the CUDA EP instance</param>>
+        /// <returns>A SessionsOptions() object configured for execution with CUDA provider options.</returns>
+        public static SessionOptions MakeSessionOptionWithCudaProvider(OrtCUDAProviderOptions cuda_options)
+        {
+            CheckCudaExecutionProviderDLLs();
+            SessionOptions options = new SessionOptions();
+
+            OrtCUDAProviderOptionsNative cuda_options_native;
+            cuda_options_native.device_id = cuda_options.device_id;
+            cuda_options_native.cudnn_conv_algo_search = cuda_options.cudnn_conv_algo_search;
+            cuda_options_native.gpu_mem_limit = cuda_options.gpu_mem_limit;
+            cuda_options_native.arena_extend_strategy = cuda_options.arena_extend_strategy;
+            cuda_options_native.do_copy_in_default_stream = cuda_options.do_copy_in_default_stream;
+            cuda_options_native.has_user_compute_stream = 0;
+            cuda_options_native.user_compute_stream = IntPtr.Zero;
+            NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_CUDA(options.Handle, ref cuda_options_native));
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options.Handle, 1));
+            return options;
+        }
+
         /// <summary>
         /// A helper method to construct a SessionOptions object for Nuphar execution.
         /// Use only if you have the onnxruntime package specific to this Execution Provider.
@@ -148,6 +172,24 @@ public void AppendExecutionProvider_CUDA(int deviceId)
             NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(handle, deviceId));
         }
 
+        /// <summary>
+        /// Append a CUDA EP instance (based on specified configuration) to the SessionOptions instance
+        /// Use only if you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="cuda_options">CUDA EP provider options to configure the CUDA EP instance</param>
+        public void AppendExecutionProvider_CUDA(OrtCUDAProviderOptions cuda_options)
+        {
+            OrtCUDAProviderOptionsNative cuda_options_native;
+            cuda_options_native.device_id = cuda_options.device_id;
+            cuda_options_native.cudnn_conv_algo_search = cuda_options.cudnn_conv_algo_search;
+            cuda_options_native.gpu_mem_limit = cuda_options.gpu_mem_limit;
+            cuda_options_native.arena_extend_strategy = cuda_options.arena_extend_strategy;
+            cuda_options_native.do_copy_in_default_stream = cuda_options.do_copy_in_default_stream;
+            cuda_options_native.has_user_compute_stream = 0;
+            cuda_options_native.user_compute_stream = IntPtr.Zero;
+            NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_CUDA(handle, ref cuda_options_native));
+        }
+
         /// <summary>
         /// Use only if you have the onnxruntime package specific to this Execution Provider.
         /// </summary>
@@ -325,6 +367,7 @@ public void AddFreeDimensionOverrideByName(string dimName, long dimValue)
                 NativeApiStatus.VerifySuccess(NativeMethods.OrtAddFreeDimensionOverrideByName(handle, pinnedDimName.Pointer, dimValue));
             }
         }
+
         #endregion
 
         internal IntPtr Handle

csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs

@@ -226,6 +226,18 @@ public void CanCreateAndDisposeSessionWithModelPath()
                 }
             }
         }
+#if USE_CUDA
+        [Fact]
+        private void TestCUDAProviderOptions()
+        {
+
+            OrtCUDAProviderOptions cuda_options = ProviderOptions.GetDefaultCUDAProviderOptions();
+            using (var sessionOptions = new SessionOptions())
+            {
+                sessionOptions.AppendExecutionProvider_CUDA(cuda_options);
+            }
+        }
+#endif
 
         [Theory]
         [InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, true)]