diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index c8d6864e9a478..6ab76110cc469 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -201,6 +201,11 @@ public struct OrtApi public IntPtr ReleasePrepackedWeightsContainer; public IntPtr CreateSessionWithPrepackedWeightsContainer; public IntPtr CreateSessionFromArrayWithPrepackedWeightsContainer; + public IntPtr SessionOptionsAppendExecutionProvider_TensorRT_V2; + public IntPtr CreateTensorRTProviderOptions; + public IntPtr UpdateTensorRTProviderOptions; + public IntPtr GetTensorRTProviderOptionsAsString; + public IntPtr ReleaseTensorRTProviderOptions; } internal static class NativeMethods @@ -271,6 +276,8 @@ static NativeMethods() OrtRegisterCustomOpsLibrary = (DOrtRegisterCustomOpsLibrary)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary, typeof(DOrtRegisterCustomOpsLibrary)); OrtAddSessionConfigEntry = (DOrtAddSessionConfigEntry)Marshal.GetDelegateForFunctionPointer(api_.AddSessionConfigEntry, typeof(DOrtAddSessionConfigEntry)); OrtAddInitializer = (DOrtAddInitializer)Marshal.GetDelegateForFunctionPointer(api_.AddInitializer, typeof(DOrtAddInitializer)); + SessionOptionsAppendExecutionProvider_TensorRT = (DSessionOptionsAppendExecutionProvider_TensorRT)Marshal.GetDelegateForFunctionPointer( + api_.SessionOptionsAppendExecutionProvider_TensorRT, typeof(DSessionOptionsAppendExecutionProvider_TensorRT)); OrtCreateRunOptions = (DOrtCreateRunOptions)Marshal.GetDelegateForFunctionPointer(api_.CreateRunOptions, typeof(DOrtCreateRunOptions)); OrtReleaseRunOptions = (DOrtReleaseRunOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseRunOptions, typeof(DOrtReleaseRunOptions)); @@ -354,6 +361,12 @@ static NativeMethods() OrtCreatePrepackedWeightsContainer = (DOrtCreatePrepackedWeightsContainer)Marshal.GetDelegateForFunctionPointer(api_.CreatePrepackedWeightsContainer, typeof(DOrtCreatePrepackedWeightsContainer)); OrtReleasePrepackedWeightsContainer = (DOrtReleasePrepackedWeightsContainer)Marshal.GetDelegateForFunctionPointer(api_.ReleasePrepackedWeightsContainer, typeof(DOrtReleasePrepackedWeightsContainer)); + SessionOptionsAppendExecutionProvider_TensorRT_V2 = (DSessionOptionsAppendExecutionProvider_TensorRT_V2)Marshal.GetDelegateForFunctionPointer( + api_.SessionOptionsAppendExecutionProvider_TensorRT_V2, typeof(DSessionOptionsAppendExecutionProvider_TensorRT_V2)); + OrtCreateTensorRTProviderOptions = (DOrtCreateTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.CreateTensorRTProviderOptions, typeof(DOrtCreateTensorRTProviderOptions)); + OrtUpdateTensorRTProviderOptions = (DOrtUpdateTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.UpdateTensorRTProviderOptions, typeof(DOrtUpdateTensorRTProviderOptions)); + OrtGetTensorRTProviderOptionsAsString = (DOrtGetTensorRTProviderOptionsAsString)Marshal.GetDelegateForFunctionPointer(api_.GetTensorRTProviderOptionsAsString, typeof(DOrtGetTensorRTProviderOptionsAsString)); + OrtReleaseTensorRTProviderOptions = (DOrtReleaseTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseTensorRTProviderOptions, typeof(DOrtReleaseTensorRTProviderOptions)); } [DllImport(nativeLib, CharSet = charSet)] @@ -376,6 +389,50 @@ static NativeMethods() #endregion Runtime/Environment API + #region Provider Options API + + /// + /// Creates native OrtTensorRTProviderOptions instance + /// + /// (output) native instance of OrtTensorRTProviderOptions + public delegate IntPtr /* OrtStatus* */DOrtCreateTensorRTProviderOptions( + out IntPtr /*(OrtTensorRTProviderOptions**)*/ trtProviderOptionsInstance); + public static DOrtCreateTensorRTProviderOptions OrtCreateTensorRTProviderOptions; + + /// + /// Updates native OrtTensorRTProviderOptions instance using given key/value pairs + /// + /// native instance of OrtTensorRTProviderOptions + /// configuration keys of OrtTensorRTProviderOptions + /// configuration values of OrtTensorRTProviderOptions + /// number of configuration keys + public delegate IntPtr /* OrtStatus* */DOrtUpdateTensorRTProviderOptions( + IntPtr /*(OrtTensorRTProviderOptions*)*/ trtProviderOptionsInstance, + IntPtr[] /*(const char* const *)*/ providerOptionsKeys, + IntPtr[] /*(const char* const *)*/ providerOptionsValues, + UIntPtr /*(size_t)*/ numKeys); + public static DOrtUpdateTensorRTProviderOptions OrtUpdateTensorRTProviderOptions; + + /// + /// Get native OrtTensorRTProviderOptionsV2 in serialized string + /// + /// instance of OrtAllocator + /// is a UTF-8 null terminated string allocated using 'allocator' + public delegate IntPtr /* OrtStatus* */DOrtGetTensorRTProviderOptionsAsString( + IntPtr /*(OrtTensorRTProviderOptionsV2**)*/ trtProviderOptionsInstance, + IntPtr /*(OrtAllocator*)*/ allocator, + out IntPtr /*(char**)*/ptr); + public static DOrtGetTensorRTProviderOptionsAsString OrtGetTensorRTProviderOptionsAsString; + + /// + /// Releases native OrtTensorRTProviderOptions instance + /// + /// native instance of OrtTensorRTProviderOptions to be released + public delegate void DOrtReleaseTensorRTProviderOptions(IntPtr /*(OrtTensorRTProviderOptions*)*/ trtProviderOptionsInstance); + public static DOrtReleaseTensorRTProviderOptions OrtReleaseTensorRTProviderOptions; + + #endregion + #region Status API public delegate ErrorCode DOrtGetErrorCode(IntPtr /*(OrtStatus*)*/status); public static DOrtGetErrorCode OrtGetErrorCode; @@ -640,6 +697,26 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca //[DllImport(nativeLib, CharSet = charSet)] //public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path); + // + /// + /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance + /// + /// Native OrtSessionOptions instance + /// Native OrtTensorRTProviderOptions instance + public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_TensorRT( + IntPtr /*(OrtSessionOptions*)*/ options, + IntPtr /*(const OrtTensorRTProviderOptions*)*/ trtProviderOptions); + public static DSessionOptionsAppendExecutionProvider_TensorRT SessionOptionsAppendExecutionProvider_TensorRT; + + /// + /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance + /// + /// Native OrtSessionOptions instance + /// Native OrtTensorRTProviderOptionsV2 instance + public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_TensorRT_V2( + IntPtr /*(OrtSessionOptions*)*/ options, + IntPtr /*(const OrtTensorRTProviderOptionsV2*)*/ trtProviderOptions); + public static DSessionOptionsAppendExecutionProvider_TensorRT_V2 SessionOptionsAppendExecutionProvider_TensorRT_V2; /// /// Free Dimension override (by denotation) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.cs index efb619f345d4f..ee57a02b8120b 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.cs @@ -5,6 +5,8 @@ using System; using System.Runtime.InteropServices; using System.Text; +using System.Collections.Generic; +using System.Linq; namespace Microsoft.ML.OnnxRuntime { @@ -77,6 +79,31 @@ internal static string StringFromNativeUtf8(IntPtr nativeUtf8) Marshal.Copy(nativeUtf8, buffer, 0, len); return Encoding.UTF8.GetString(buffer, 0, buffer.Length); } + + /// + /// Run helper + /// + /// names to convert to zero terminated utf8 and pin + /// delegate for string extraction from inputs + /// list to add pinned memory to for later disposal + /// + internal static IntPtr[] ConvertNamesToUtf8(IReadOnlyCollection names, NameExtractor extractor, + DisposableList cleanupList) + { + var result = new IntPtr[names.Count]; + for (int i = 0; i < names.Count; ++i) + { + var name = extractor(names.ElementAt(i)); + var utf8Name = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(name); + var pinnedHandle = new PinnedGCHandle(GCHandle.Alloc(utf8Name, GCHandleType.Pinned)); + result[i] = pinnedHandle.Pointer; + cleanupList.Add(pinnedHandle); + } + return result; + } + + // Delegate for string extraction from an arbitrary input/output object + internal delegate string NameExtractor(TInput input); } internal static class TensorElementTypeConverter diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.cs new file mode 100644 index 0000000000000..e288ac367c5b2 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + /// + /// Holds the options for configuring a TensorRT Execution Provider instance + /// + public class OrtTensorRTProviderOptions : SafeHandle + { + internal IntPtr Handle + { + get + { + return handle; + } + } + + private int _deviceId = 0; + private string _deviceIdStr = "device_id"; + + #region Constructor + + /// + /// Constructs an empty OrtTensorRTProviderOptions instance + /// + public OrtTensorRTProviderOptions() : base(IntPtr.Zero, true) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorRTProviderOptions(out handle)); + } + + #endregion + + #region Public Methods + + /// + /// Get TensorRT EP provider options + /// + /// return C# UTF-16 encoded string + public string GetOptions() + { + var allocator = OrtAllocator.DefaultInstance; + + // Process provider options string + IntPtr providerOptions = IntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorRTProviderOptionsAsString(handle, allocator.Pointer, out providerOptions)); + using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0)) + { + return NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions); + } + } + + /// + /// Updates the configuration knobs of OrtTensorRTProviderOptions that will eventually be used to configure a TensorRT EP + /// Please refer to the following on different key/value pairs to configure a TensorRT EP and their meaning: + /// https://www.onnxruntime.ai/docs/reference/execution-providers/TensorRT-ExecutionProvider.html + /// + /// key/value pairs used to configure a TensorRT Execution Provider + public void UpdateOptions(Dictionary providerOptions) + { + + using (var cleanupList = new DisposableList()) + { + var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Keys.ToArray(), n => n, cleanupList); + var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Values.ToArray(), n => n, cleanupList); + + NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateTensorRTProviderOptions(handle, keysArray, valuesArray, (UIntPtr)providerOptions.Count)); + + if (providerOptions.ContainsKey(_deviceIdStr)) + { + _deviceId = Int32.Parse(providerOptions[_deviceIdStr]); + } + } + } + + /// + /// Get device id of TensorRT EP. + /// + /// device id + public int GetDeviceId() + { + return _deviceId; + } + + #endregion + + #region Public Properties + + /// + /// Overrides SafeHandle.IsInvalid + /// + /// returns true if handle is equal to Zero + public override bool IsInvalid { get { return handle == IntPtr.Zero; } } + + #endregion + + #region Private Methods + + + #endregion + + #region SafeHandle + /// + /// Overrides SafeHandle.ReleaseHandle() to properly dispose of + /// the native instance of OrtTensorRTProviderOptions + /// + /// always returns true + protected override bool ReleaseHandle() + { + NativeMethods.OrtReleaseTensorRTProviderOptions(handle); + handle = IntPtr.Zero; + return true; + } + + #endregion + } + + /// + /// This helper class contains methods to handle values of provider options + /// + public class ProviderOptionsValueHelper + { + /// + /// Parse from string and save to dictionary + /// + /// C# string + /// Dictionary instance to store the parsing result of s + public static void StringToDict(string s, Dictionary dict) + { + string[] paris = s.Split(';'); + + foreach (var p in paris) + { + string[] keyValue = p.Split('='); + if (keyValue.Length != 2) + { + throw new ArgumentException("Make sure input string contains key-value paris, e.g. key1=value1;key2=value2...", "s"); + } + dict.Add(keyValue[0], keyValue[1]); + } + } + } + +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index 11a24d93acc07..b989e8a777382 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -4,6 +4,7 @@ using System; using System.Runtime.InteropServices; using System.Text; +using System.Collections.Generic; namespace Microsoft.ML.OnnxRuntime { @@ -100,6 +101,33 @@ public static SessionOptions MakeSessionOptionWithTensorrtProvider(int deviceId } } + /// + /// A helper method to construct a SessionOptions object for TensorRT execution provider. + /// Use only if CUDA/TensorRT are installed and you have the onnxruntime package specific to this Execution Provider. + /// + /// TensorRT EP provider options + /// A SessionsOptions() object configured for execution on provider options + public static SessionOptions MakeSessionOptionWithTensorrtProvider(OrtTensorRTProviderOptions trtProviderOptions) + { + CheckTensorrtExecutionProviderDLLs(); + SessionOptions options = new SessionOptions(); + try + { + // Make sure that CUDA EP uses the same device id as TensorRT EP. + int deviceId = trtProviderOptions.GetDeviceId() ; + + NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT(options.Handle, trtProviderOptions.Handle)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options.Handle, deviceId)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options.Handle, 1)); + return options; + } + catch (Exception e) + { + options.Dispose(); + throw e; + } + } + /// /// A helper method to construct a SessionOptions object for Nuphar execution. /// Use only if you have the onnxruntime package specific to this Execution Provider. @@ -205,6 +233,16 @@ public void AppendExecutionProvider_Tensorrt(int deviceId) NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Tensorrt(handle, deviceId)); } + /// + /// Append a TensorRT EP instance (based on specified configuration) to the SessionOptions instance. + /// Use only if you have the onnxruntime package specific to this Execution Provider. + /// + /// TensorRT EP provider options + public void AppendExecutionProvider_Tensorrt(OrtTensorRTProviderOptions trtProviderOptions) + { + NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT(handle, trtProviderOptions.Handle)); + } + /// /// Use only if you have the onnxruntime package specific to this Execution Provider. /// diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index ef5f88aa01903..e28a1d52aa5ac 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -259,6 +259,74 @@ private void CanRunInferenceOnAModelWithTensorRT() } } } + + [Fact] + private void TestTensorRTProviderOptions() + { + string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); + string calTablePath = "squeezenet_calibration.flatbuffers"; + string enginePath = "./"; + string engineDecrptLibPath = "engine_decryp"; + + using (var cleanUp = new DisposableListTest()) + { + var trtProviderOptions = new OrtTensorRTProviderOptions(); + cleanUp.Add(trtProviderOptions); + + var providerOptionsDict = new Dictionary(); + providerOptionsDict["device_id"] = "0"; + providerOptionsDict["trt_fp16_enable"] = "1"; + providerOptionsDict["trt_int8_enable"] = "1"; + providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath; + providerOptionsDict["trt_engine_cache_enable"] = "1"; + providerOptionsDict["trt_engine_cache_path"] = enginePath; + providerOptionsDict["trt_engine_decryption_enable"] = "0"; + providerOptionsDict["trt_engine_decryption_lib_path"] = engineDecrptLibPath; + trtProviderOptions.UpdateOptions(providerOptionsDict); + + var resultProviderOptionsDict = new Dictionary(); + ProviderOptionsValueHelper.StringToDict(trtProviderOptions.GetOptions(), resultProviderOptionsDict); + + // test provider options configuration + string value; + value = resultProviderOptionsDict["device_id"]; + Assert.Equal("0", value); + value = resultProviderOptionsDict["trt_fp16_enable"]; + Assert.Equal("1", value); + value = resultProviderOptionsDict["trt_int8_enable"]; + Assert.Equal("1", value); + value = resultProviderOptionsDict["trt_int8_calibration_table_name"]; + Assert.Equal(calTablePath, value); + value = resultProviderOptionsDict["trt_engine_cache_enable"]; + Assert.Equal("1", value); + value = resultProviderOptionsDict["trt_engine_cache_path"]; + Assert.Equal(enginePath, value); + value = resultProviderOptionsDict["trt_engine_decryption_enable"]; + Assert.Equal("0", value); + value = resultProviderOptionsDict["trt_engine_decryption_lib_path"]; + Assert.Equal(engineDecrptLibPath, value); + + // test correctness of provider options + SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(trtProviderOptions); + cleanUp.Add(options); + + var session = new InferenceSession(modelPath, options); + cleanUp.Add(session); + + var inputMeta = session.InputMetadata; + var container = new List(); + float[] inputData = LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model + foreach (var name in inputMeta.Keys) + { + Assert.Equal(typeof(float), inputMeta[name].ElementType); + Assert.True(inputMeta[name].IsTensor); + var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions); + container.Add(NamedOnnxValue.CreateFromTensor(name, tensor)); + } + + session.Run(container); + } + } #endif [Theory] diff --git a/csharp/testdata/squeezenet_calibration.flatbuffers b/csharp/testdata/squeezenet_calibration.flatbuffers new file mode 100644 index 0000000000000..e5cad768f4fe1 Binary files /dev/null and b/csharp/testdata/squeezenet_calibration.flatbuffers differ diff --git a/include/onnxruntime/core/common/string_helper.h b/include/onnxruntime/core/common/string_helper.h new file mode 100644 index 0000000000000..09f153ff1e5e8 --- /dev/null +++ b/include/onnxruntime/core/common/string_helper.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/session/onnxruntime_c_api.h" + +namespace onnxruntime { +#ifdef USE_TENSORRT +static char* StrDup(const std::string& str, _Inout_ OrtAllocator* allocator) { + char* output_string = reinterpret_cast(allocator->Alloc(allocator, str.size() + 1)); + memcpy(output_string, str.c_str(), str.size()); + output_string[str.size()] = '\0'; + return output_string; +} +#endif +} // namespace onnxruntime diff --git a/include/onnxruntime/core/platform/tensorrt_provider_options.h b/include/onnxruntime/core/platform/tensorrt_provider_options.h new file mode 100644 index 0000000000000..9f5f0ba546547 --- /dev/null +++ b/include/onnxruntime/core/platform/tensorrt_provider_options.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +/// +/// Options for the TensorRT provider that are passed to SessionOptionsAppendExecutionProvider_TensorRT_V2. +/// Please note that this sturct is identical to OrtTensorRTProviderOptions but only to be used internally. +// User can only get the instance of OrtTensorRTProviderOptionsV2 via CreateTensorRTProviderOptions. +/// +struct OrtTensorRTProviderOptionsV2 { + int device_id; // cuda device id. + int has_user_compute_stream; // indicator of user specified CUDA compute stream. + void* user_compute_stream; // user specified CUDA compute stream. + int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability + int trt_min_subgraph_size; // minimum size of TensorRT subgraphs + size_t trt_max_workspace_size; // maximum workspace size for TensorRT. + int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true + int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true + const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. + int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true + int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true + int trt_dla_core; // DLA core number. Default 0 + int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true + int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true + const char* trt_engine_cache_path; // specify engine cache path + int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true + const char* trt_engine_decryption_lib_path; // specify engine decryption library path + int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true +}; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 271a2864c0dd0..cf26fb9492d15 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -176,6 +176,7 @@ ORT_RUNTIME_CLASS(ThreadPoolParams); ORT_RUNTIME_CLASS(ThreadingOptions); ORT_RUNTIME_CLASS(ArenaCfg); ORT_RUNTIME_CLASS(PrepackedWeightsContainer); +ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); #ifdef _WIN32 typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; @@ -198,6 +199,7 @@ typedef OrtStatus* OrtStatusPtr; _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION #define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) +#define ORT_CLASS_RELEASE2(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X##V2 * input) // When passing in an allocator to any ORT function, be sure that the allocator object // is not destroyed until the last allocated object using it is freed. @@ -1241,7 +1243,7 @@ struct OrtApi { ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); /** - * Append TensorRT execution provider to the session options + * Append TensorRT execution provider to the session options with TensorRT provider options. * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT, @@ -1387,6 +1389,62 @@ struct OrtApi { _In_ const void* model_data, size_t model_data_length, _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, _Outptr_ OrtSession** out); + + /** + * Append TensorRT execution provider to the session options with TensorRT provider options. + * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. + * Note: this API is slightly different than SessionOptionsAppendExecutionProvider_TensorRT. + * SessionOptionsAppendExecutionProvider_TensorRT takes struct OrtTensorRTProviderOptions which is open to user as argument, + * but this API takes opaque struct OrtTensorRTProviderOptionsV2 which must be created by CreateTensorRTProviderOptions. + * User needs to instantiate OrtTensorRTProviderOptions as well as allocate/release buffers for some members of OrtTensorRTProviderOptions. + * However, for using OrtTensorRTProviderOptionsV2, CreateTensorRTProviderOptions and ReleaseTensorRTProviderOptions will do the memory allocation and release for you. + * + * \param options - OrtSessionOptions instance + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT_V2, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); + + /** + * Use this API to create the configuration of a TensorRT Execution Provider which is an instance of OrtTensorRTProviderOptionsV2. + * + * \param out - pointer to the pointer of TensorRT EP provider options instance. + */ + ORT_API2_STATUS(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); + + /** + * Use this API to set appropriate configuration knobs of a TensorRT Execution Provider. + * + * Please reference to https://www.onnxruntime.ai/docs/reference/execution-providers/TensorRT-ExecutionProvider.html#c-api-example + * to know the available keys and values. Key should be in string format of the member of OrtTensorRTProviderOptions and value should be its related range. + * For example, key="trt_max_workspace_size" and value="2147483648" + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param provider_options_keys - array of UTF-8 null-terminated string for provider options keys + * \param provider_options_values - array of UTF-8 null-terminated string for provider options values + * \param num_keys - number of keys + */ + ORT_API2_STATUS(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized TensorRT provider options string. + * + * For example, "trt_max_workspace_size=2147483648;trt_max_partition_iterations=10;trt_int8_enable=1;......" + * + * \param tensorrt_options - OrTensorRTProviderOptionsV2 instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + */ + ORT_API2_STATUS(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** + * Use this API to release the instance of OrtTensorRTProviderV2. + */ + ORT_CLASS_RELEASE2(TensorRTProviderOptions); }; /* diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index d29e4b10b6336..ac045673183f1 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -21,6 +21,9 @@ #include "core/util/math.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/TensorSeq.h" +#include "core/framework/provider_options.h" +#include "core/common/string_helper.h" + #include "core/framework/fallback_cpu_capability.h" #include "core/framework/random_generator.h" @@ -98,6 +101,7 @@ using IndexedSubGraph_MetaDef = IndexedSubGraph::MetaDef; #include "core/providers/dnnl/dnnl_provider_factory.h" #include "core/providers/tensorrt/tensorrt_provider_factory.h" #include "core/providers/openvino/openvino_provider_factory.h" +#include "core/platform/tensorrt_provider_options.h" // The filename extension for a shared library is different per platform #ifdef _WIN32 @@ -1127,6 +1131,20 @@ INcclService& INcclService::GetInstance() { } // namespace cuda #endif +void UpdateProviderInfo_Tensorrt(OrtTensorRTProviderOptions* provider_options, const ProviderOptions& options) { + if (auto provider = s_library_tensorrt.Get()) { + provider->UpdateProviderOptions(reinterpret_cast(provider_options), options); + } +} + +ProviderOptions GetProviderInfo_Tensorrt(const OrtTensorRTProviderOptions* provider_options) { + if (auto provider = s_library_tensorrt.Get()) { + return provider->GetProviderOptions(reinterpret_cast(provider_options)); + } + + return {}; +} + } // namespace onnxruntime ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena) { @@ -1217,3 +1235,122 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ Or return nullptr; API_IMPL_END } + +ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2, _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options) { + return OrtApis::SessionOptionsAppendExecutionProvider_TensorRT(options, reinterpret_cast(tensorrt_options)); +} + +ORT_API_STATUS_IMPL(OrtApis::CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out) { + API_IMPL_BEGIN +#ifdef USE_TENSORRT + *out = new OrtTensorRTProviderOptionsV2(); + (*out)->device_id = 0; + (*out)->has_user_compute_stream = 0; + (*out)->user_compute_stream = nullptr; + (*out)->trt_max_partition_iterations = 1000; + (*out)->trt_min_subgraph_size = 1; + (*out)->trt_max_workspace_size = 1 << 30; + (*out)->trt_fp16_enable = false; + (*out)->trt_int8_enable = false; + (*out)->trt_int8_calibration_table_name = nullptr; + (*out)->trt_int8_use_native_calibration_table = false; + (*out)->trt_dla_enable = false; + (*out)->trt_dla_core = false; + (*out)->trt_dump_subgraphs = false; + (*out)->trt_engine_cache_enable= false; + (*out)->trt_engine_cache_path = nullptr; + (*out)->trt_engine_decryption_enable = false; + (*out)->trt_engine_decryption_lib_path = nullptr; + (*out)->trt_force_sequential_engine_build = false; + return nullptr; +#else + ORT_UNUSED_PARAMETER(out); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::UpdateTensorRTProviderOptions, + _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + size_t num_keys) { + API_IMPL_BEGIN +#ifdef USE_TENSORRT + onnxruntime::ProviderOptions provider_options_map; + for (size_t i = 0; i != num_keys; ++i) { + if (provider_options_keys[i] == nullptr || provider_options_keys[i][0] == '\0' || + provider_options_values[i] == nullptr || provider_options_values[i][0] == '\0') { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "key/value cannot be empty"); + } + + provider_options_map[provider_options_keys[i]] = provider_options_values[i]; + } + + onnxruntime::UpdateProviderInfo_Tensorrt(reinterpret_cast(tensorrt_options), + reinterpret_cast(provider_options_map)); + return nullptr; +#else + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(provider_options_keys); + ORT_UNUSED_PARAMETER(provider_options_values); + ORT_UNUSED_PARAMETER(num_keys); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, + _Outptr_ char** ptr) { + API_IMPL_BEGIN +#ifdef USE_TENSORRT + onnxruntime::ProviderOptions options = onnxruntime::GetProviderInfo_Tensorrt(reinterpret_cast(tensorrt_options)); + onnxruntime::ProviderOptions::iterator it = options.begin(); + std::string options_str = ""; + + while (it != options.end()) { + if (options_str == "") { + options_str += it->first; + options_str += "="; + options_str += it->second; + } else { + options_str += ";"; + options_str += it->first; + options_str += "="; + options_str += it->second; + } + it++; + } + + *ptr = onnxruntime::StrDup(options_str, allocator); + return nullptr; +#else + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(allocator); + ORT_UNUSED_PARAMETER(ptr); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +#endif + API_IMPL_END +} + +ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* ptr) { +#ifdef USE_TENSORRT + if (ptr != nullptr) { + if (ptr->trt_int8_calibration_table_name != nullptr) { + delete ptr->trt_int8_calibration_table_name; + } + + if (ptr->trt_engine_cache_path != nullptr) { + delete ptr->trt_engine_cache_path; + } + + if (ptr->trt_engine_decryption_lib_path != nullptr) { + delete ptr->trt_engine_decryption_lib_path; + } + } + + delete ptr; +#else + ORT_UNUSED_PARAMETER(ptr); +#endif +} diff --git a/onnxruntime/core/providers/shared_library/provider_host_api.h b/onnxruntime/core/providers/shared_library/provider_host_api.h index ab8d4e1da7958..44bbdc9a8fb84 100644 --- a/onnxruntime/core/providers/shared_library/provider_host_api.h +++ b/onnxruntime/core/providers/shared_library/provider_host_api.h @@ -1,5 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +// +#include "core/framework/provider_options.h" namespace onnxruntime { @@ -11,6 +13,13 @@ struct Provider { virtual std::shared_ptr CreateExecutionProviderFactory(int /*device_id*/) { return nullptr; } virtual void* GetInfo() { return nullptr; } // Returns a provider specific information interface if it exists + + // Convert provider options struct to ProviderOptions which is a map + virtual ProviderOptions GetProviderOptions(const void* /*provider options struct*/) { return {}; } + + // Update provider options from key-value string configuration + virtual void UpdateProviderOptions(void* /*provider options to be configured*/, const ProviderOptions& /*key-value string provider options*/){}; + virtual void Shutdown() = 0; }; diff --git a/onnxruntime/core/providers/tensorrt/symbols.txt b/onnxruntime/core/providers/tensorrt/symbols.txt index 47950c476c5e8..b7d03850ffee2 100644 --- a/onnxruntime/core/providers/tensorrt/symbols.txt +++ b/onnxruntime/core/providers/tensorrt/symbols.txt @@ -1 +1 @@ -OrtSessionOptionsAppendExecutionProvider_Tensorrt +OrtSessionOptionsAppendExecutionProvider_Tensorrt \ No newline at end of file diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index 3eb91671d4292..cfc43350a210e 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -69,6 +69,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions } ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtExecutionProviderInfo& info) { + const ProviderOptions options{ {tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, {tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.max_partition_iterations)}, @@ -87,7 +88,34 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE {tensorrt::provider_option_names::kDecryptionLibPath, MakeStringWithClassicLocale(info.engine_decryption_lib_path)}, {tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)}, }; + return options; +} + +ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const OrtTensorRTProviderOptions& info) { + auto empty_if_null = [](const char* s) { return s != nullptr ? std::string{s} : std::string{}; }; + const std::string kInt8CalibTable_ = empty_if_null(info.trt_int8_calibration_table_name); + const std::string kCachePath_ = empty_if_null(info.trt_engine_cache_path); + const std::string kDecryptionLibPath_ = empty_if_null(info.trt_engine_decryption_lib_path); + + const ProviderOptions options{ + {tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, + {tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.trt_max_partition_iterations)}, + {tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.trt_min_subgraph_size)}, + {tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.trt_max_workspace_size)}, + {tensorrt::provider_option_names::kFp16Enable, MakeStringWithClassicLocale(info.trt_fp16_enable)}, + {tensorrt::provider_option_names::kInt8Enable, MakeStringWithClassicLocale(info.trt_int8_enable)}, + {tensorrt::provider_option_names::kInt8CalibTable, kInt8CalibTable_}, + {tensorrt::provider_option_names::kInt8UseNativeCalibTable, MakeStringWithClassicLocale(info.trt_int8_use_native_calibration_table)}, + {tensorrt::provider_option_names::kDLAEnable, MakeStringWithClassicLocale(info.trt_dla_enable)}, + {tensorrt::provider_option_names::kDLACore, MakeStringWithClassicLocale(info.trt_dla_core)}, + {tensorrt::provider_option_names::kDumpSubgraphs, MakeStringWithClassicLocale(info.trt_dump_subgraphs)}, + {tensorrt::provider_option_names::kEngineCacheEnable, MakeStringWithClassicLocale(info.trt_engine_cache_enable)}, + {tensorrt::provider_option_names::kCachePath, kCachePath_}, + {tensorrt::provider_option_names::kDecryptionEnable, MakeStringWithClassicLocale(info.trt_engine_decryption_enable)}, + {tensorrt::provider_option_names::kDecryptionLibPath, kDecryptionLibPath_}, + {tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.trt_force_sequential_engine_build)}, + }; return options; } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h index 2b82bcb1fd8f3..b6d879bf72558 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h @@ -34,5 +34,6 @@ struct TensorrtExecutionProviderInfo { static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); static ProviderOptions ToProviderOptions(const TensorrtExecutionProviderInfo& info); + static ProviderOptions ToProviderOptions(const OrtTensorRTProviderOptions& info); }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc index 86fb41d8cd034..f2bb266356416 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc @@ -5,6 +5,8 @@ #include "core/providers/tensorrt/tensorrt_provider_factory.h" #include #include "tensorrt_execution_provider.h" +#include "core/framework/provider_options.h" +#include using namespace onnxruntime; @@ -70,6 +72,63 @@ struct Tensorrt_Provider : Provider { return std::make_shared(info); } + void UpdateProviderOptions(void* provider_options, const ProviderOptions& options) override { + auto internal_options = onnxruntime::TensorrtExecutionProviderInfo::FromProviderOptions(options); + auto& trt_options = *reinterpret_cast(provider_options); + trt_options.device_id = internal_options.device_id; + trt_options.trt_max_partition_iterations = internal_options.max_partition_iterations; + trt_options.trt_min_subgraph_size = internal_options.min_subgraph_size; + trt_options.trt_max_workspace_size = internal_options.max_workspace_size; + trt_options.trt_fp16_enable = internal_options.fp16_enable; + trt_options.trt_int8_enable = internal_options.int8_enable; + + char* dest = nullptr; + auto str_size = internal_options.int8_calibration_table_name.size(); + if (str_size == 0) { + trt_options.trt_int8_calibration_table_name = nullptr; + } else { + dest = new char[str_size + 1]; + strncpy(dest, internal_options.int8_calibration_table_name.c_str(), str_size); + dest[str_size] = '\0'; + trt_options.trt_int8_calibration_table_name = (const char*)dest; + } + + trt_options.trt_int8_use_native_calibration_table = internal_options.int8_use_native_calibration_table; + trt_options.trt_dla_enable = internal_options.dla_enable; + trt_options.trt_dla_core = internal_options.dla_core; + trt_options.trt_dump_subgraphs = internal_options.dump_subgraphs; + trt_options.trt_engine_cache_enable = internal_options.engine_cache_enable; + + str_size = internal_options.engine_cache_path.size(); + if (str_size == 0) { + trt_options.trt_engine_cache_path = nullptr; + } else { + dest = new char[str_size + 1]; + strncpy(dest, internal_options.engine_cache_path.c_str(), str_size); + dest[str_size] = '\0'; + trt_options.trt_engine_cache_path = (const char*)dest; + } + + trt_options.trt_engine_decryption_enable = internal_options.engine_decryption_enable; + + str_size = internal_options.engine_decryption_lib_path.size(); + if (str_size == 0) { + trt_options.trt_engine_decryption_lib_path = nullptr; + } else { + dest = new char[str_size + 1]; + strncpy(dest, internal_options.engine_decryption_lib_path.c_str(), str_size); + dest[str_size] = '\0'; + trt_options.trt_engine_decryption_lib_path = (const char*)dest; + } + + trt_options.trt_force_sequential_engine_build = internal_options.force_sequential_engine_build; + } + + ProviderOptions GetProviderOptions(const void* provider_options) override { + auto& options = *reinterpret_cast(provider_options); + return onnxruntime::TensorrtExecutionProviderInfo::ToProviderOptions(options); + } + void Shutdown() override { Shutdown_DeleteRegistry(); } diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index e73bdbe18a2fc..9e819cb8079d1 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -1976,6 +1976,42 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT, ORT_UNUSED_PARAMETER(tensorrt_options); return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled."); } + +ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options) { + ORT_UNUSED_PARAMETER(options); + ORT_UNUSED_PARAMETER(tensorrt_options); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled."); +} + +ORT_API_STATUS_IMPL(OrtApis::CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out) { + ORT_UNUSED_PARAMETER(out); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +} + +ORT_API_STATUS_IMPL(OrtApis::UpdateTensorRTProviderOptions, + _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + size_t num_keys) { + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(provider_options_keys); + ORT_UNUSED_PARAMETER(provider_options_values); + ORT_UNUSED_PARAMETER(num_keys); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +} + +ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, + _Outptr_ char** ptr) { + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(allocator); + ORT_UNUSED_PARAMETER(ptr); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +} + +ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* ptr) { + ORT_UNUSED_PARAMETER(ptr); +} #endif static constexpr OrtApiBase ort_api_base = { @@ -2022,7 +2058,7 @@ Second example, if we wanted to add and remove some members, we'd do this: In GetApi we now make it return ort_api_3 for version 3. */ -static constexpr OrtApi ort_api_1_to_8 = { +static constexpr OrtApi ort_api_1_to_9 = { // NOTE: The ordering of these fields MUST not change after that version has shipped since existing binaries depend on this ordering. // Shipped as version 1 - DO NOT MODIFY (see above text for more information) @@ -2228,6 +2264,11 @@ static constexpr OrtApi ort_api_1_to_8 = { // End of Version 8 - DO NOT MODIFY ABOVE (see above text for more information) // Version 9 - In development, feel free to add/remove/rearrange here + &OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2, + &OrtApis::CreateTensorRTProviderOptions, + &OrtApis::UpdateTensorRTProviderOptions, + &OrtApis::GetTensorRTProviderOptionsAsString, + &OrtApis::ReleaseTensorRTProviderOptions, }; // Assert to do a limited check to ensure Version 1 of OrtApi never changes (will detect an addition or deletion but not if they cancel out each other) @@ -2236,7 +2277,7 @@ static_assert(offsetof(OrtApi, ReleaseCustomOpDomain) / sizeof(void*) == 101, "S ORT_API(const OrtApi*, OrtApis::GetApi, uint32_t version) { if (version >= 1 && version <= ORT_API_VERSION) - return &ort_api_1_to_8; + return &ort_api_1_to_9; fprintf(stderr, "The given version [%u] is not supported, only version 1 to %u is supported in this build.\n", version, ORT_API_VERSION); diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index 2d99582724562..f18682385f4a8 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -276,5 +276,13 @@ ORT_API_STATUS_IMPL(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ co _In_ const void* model_data, size_t model_data_length, _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, _Outptr_ OrtSession** out); - +ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_TensorRT_V2, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); +ORT_API_STATUS_IMPL(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); +ORT_API_STATUS_IMPL(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + size_t num_keys); +ORT_API_STATUS_IMPL(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); +ORT_API(void, ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2*); } // namespace OrtApis diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index c356d10288730..ffc56e678f3f3 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -1497,3 +1497,62 @@ TEST(CApiTest, ConfigureCudaArenaAndDemonstrateMemoryArenaShrinkage) { // run_option.AddConfigEntry(kOrtRunOptionsConfigEnableMemoryArenaShrinkage, "cpu:0;gpu:0"); } #endif + +#ifdef USE_TENSORRT + +// This test uses CreateTensorRTProviderOptions/UpdateTensorRTProviderOptions APIs to configure and create a TensorRT Execution Provider +TEST(CApiTest, TestConfigureTensorRTProviderOptions) { + const auto& api = Ort::GetApi(); + OrtTensorRTProviderOptionsV2* trt_options; + OrtAllocator* allocator; + char* trt_options_str; + ASSERT_TRUE(api.CreateTensorRTProviderOptions(&trt_options) == nullptr); + std::unique_ptr rel_trt_options(trt_options, api.ReleaseTensorRTProviderOptions); + + const char* engine_cache_path = "./trt_engine_folder"; + + std::vector keys{"device_id", "trt_fp16_enable", "trt_int8_enable", "trt_engine_cache_enable", "trt_engine_cache_path"}; + + std::vector values{"0", "1", "0", "1", engine_cache_path}; + + ASSERT_TRUE(api.UpdateTensorRTProviderOptions(rel_trt_options.get(), keys.data(), values.data(), 5) == nullptr); + + ASSERT_TRUE(api.GetAllocatorWithDefaultOptions(&allocator) == nullptr); + ASSERT_TRUE(api.GetTensorRTProviderOptionsAsString(rel_trt_options.get(), allocator, &trt_options_str) == nullptr); + std::string s(trt_options_str); + ASSERT_TRUE(s.find(engine_cache_path) != std::string::npos); + ASSERT_TRUE(api.AllocatorFree(allocator, (void*)trt_options_str) == nullptr); + + Ort::SessionOptions session_options; + ASSERT_TRUE(api.SessionOptionsAppendExecutionProvider_TensorRT_V2(static_cast(session_options), rel_trt_options.get()) == nullptr); + + // simple inference test + // prepare inputs + std::vector inputs(1); + Input& input = inputs.back(); + input.name = "X"; + input.dims = {3, 2}; + input.values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + + // prepare expected inputs and outputs + std::vector expected_dims_y = {3, 2}; + std::vector expected_values_y = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; + std::basic_string model_uri = MODEL_URI; + + // if session creation passes, model loads fine + Ort::Session session(*ort_env, model_uri.c_str(), session_options); + auto default_allocator = std::make_unique(); + + //without preallocated output tensor + RunSession(default_allocator.get(), + session, + inputs, + "Y", + expected_dims_y, + expected_values_y, + nullptr); + + struct stat buffer; + ASSERT_TRUE(stat(engine_cache_path, &buffer) == 0); +} +#endif \ No newline at end of file