From 5e7405119371c642dc2475793856c99f6c270f3f Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Mon, 29 Jun 2026 00:38:54 -0700 Subject: [PATCH 1/2] remove skip_layer_norm_strict_mode --- .../providers/cuda/cuda_provider_options.h | 4 +- .../contrib_ops/cuda/bert/skip_layer_norm.cc | 88 ++++++------------- .../contrib_ops/cuda/bert/skip_layer_norm.h | 1 - .../providers/cuda/cuda_execution_provider.h | 1 - .../cuda/plugin/cuda_kernel_adapter.h | 5 -- .../test/contrib_ops/skiplayernorm_op_test.cc | 28 +----- 6 files changed, 33 insertions(+), 94 deletions(-) diff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_options.h b/include/onnxruntime/core/providers/cuda/cuda_provider_options.h index 3b7a1e99346d7..f4f4dac5fc5f2 100644 --- a/include/onnxruntime/core/providers/cuda/cuda_provider_options.h +++ b/include/onnxruntime/core/providers/cuda/cuda_provider_options.h @@ -33,8 +33,8 @@ struct OrtCUDAProviderOptionsV2 { int tunable_op_enable = 0; // flag specifying if TunableOp is enabled. int tunable_op_tuning_enable = 0; // flag specifying if TunableOp is enabled for tuning, this relies on TunableOp is enabled. int tunable_op_max_tuning_duration_ms = 0; // Max tuning duration time limit for TunableOp. - int enable_skip_layer_norm_strict_mode = 0; // flag specifying if SkipLayerNorm is in strict mode. If true, use LayerNormalization kernel. - // The strict mode has better accuracy but lower performance. + int enable_skip_layer_norm_strict_mode = 0; // [Deprecated] No longer has any effect. SkipLayerNorm always accumulates in fp32. + // Kept for backward compatibility; the value is ignored. int prefer_nhwc = 0; // make the CUDA EP NHWC preferred int use_ep_level_unified_stream = 0; // flag specifying if ep level stream is used or not int use_tf32 = 1; // use TF32 diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc index aefd86a6ebd10..8557e326e5b15 100644 --- a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc +++ b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/cuda/cuda_common.h" -#include "core/providers/cuda/nn/layer_norm_impl.h" #include "core/common/narrow.h" #include "skip_layer_norm.h" #include "skip_layer_norm_impl.h" @@ -42,26 +41,14 @@ template SkipLayerNorm::SkipLayerNorm(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info) { ORT_ENFORCE(op_kernel_info.GetAttr("epsilon", &epsilon_).IsOK()); ORT_ENFORCE(epsilon_ >= 0); - -#ifdef BUILD_CUDA_EP_AS_PLUGIN - // Plugin adapter cannot static_cast to CUDAExecutionProvider directly. - // Use the adapter shim that reads the config from the per-EP runtime map. - strict_ = onnxruntime::cuda::GetCudaKernelAdapterSkipLayerNormStrictMode(op_kernel_info.GetExecutionProvider()); -#else - const CUDAExecutionProvider* cuda_ep = static_cast(op_kernel_info.GetExecutionProvider()); - strict_ = cuda_ep->IsSkipLayerNormInStrictMode(); -#endif + // Note: the enable_skip_layer_norm_strict_mode provider option is deprecated and ignored. + // The kernel always accumulates in fp32, so the previous strict-mode path is no longer needed. } template Status SkipLayerNorm::ComputeInternal(OpKernelContext* ctx) const { const Tensor* input = ctx->Input(0); const Tensor* skip = ctx->Input(1); - if (strict_ && skip->Shape() != input->Shape()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "'input' and 'skip' shall have same shape when enable_skip_layer_norm_strict_mode is True"); - } - const Tensor* gamma = ctx->Input(2); const Tensor* beta = Simplified ? nullptr : ctx->Input(3); @@ -94,53 +81,34 @@ Status SkipLayerNorm::ComputeInternal(OpKernelContext* ctx) const const int skip_size = onnxruntime::narrow(skip->Shape().Size()); - if (strict_) { - HostApplyLayerNorm( - GetDeviceProp(), + if constexpr (std::is_same_v) { + LaunchSkipLayerNormKernel( Stream(ctx), - reinterpret_cast(output->MutableData()), // Y_data - nullptr, // mean_data - nullptr, // inv_var_data - reinterpret_cast(input->Data()), // X_data - row_count, // n1 - hidden_size, // n2 - (double)epsilon_, // epsilon - reinterpret_cast(gamma->Data()), // gamma - (beta != nullptr) ? reinterpret_cast(beta->Data()) : nullptr, // beta - 0, // no broadcast for gamma/beta - reinterpret_cast(skip->Data()), // skip or residual to add - (bias != nullptr) ? reinterpret_cast(bias->Data()) : nullptr, // bias to add - sum_output != nullptr ? reinterpret_cast(sum_output->MutableData()) : nullptr); + reinterpret_cast(output->MutableData()), + sum_output != nullptr ? reinterpret_cast(sum_output->MutableData()) : nullptr, + reinterpret_cast(input->Data()), + reinterpret_cast(skip->Data()), + (bias != nullptr) ? reinterpret_cast(bias->Data()) : nullptr, + reinterpret_cast(gamma->Data()), + (beta != nullptr) ? reinterpret_cast(beta->Data()) : nullptr, + epsilon_, + hidden_size, + row_count, + skip_size); } else { - if constexpr (std::is_same_v) { - LaunchSkipLayerNormKernel( - Stream(ctx), - reinterpret_cast(output->MutableData()), - sum_output != nullptr ? reinterpret_cast(sum_output->MutableData()) : nullptr, - reinterpret_cast(input->Data()), - reinterpret_cast(skip->Data()), - (bias != nullptr) ? reinterpret_cast(bias->Data()) : nullptr, - reinterpret_cast(gamma->Data()), - (beta != nullptr) ? reinterpret_cast(beta->Data()) : nullptr, - epsilon_, - hidden_size, - row_count, - skip_size); - } else { - LaunchSkipLayerNormKernel( - Stream(ctx), - reinterpret_cast(output->MutableData()), - sum_output != nullptr ? reinterpret_cast(sum_output->MutableData()) : nullptr, - reinterpret_cast(input->Data()), - reinterpret_cast(skip->Data()), - (bias != nullptr) ? reinterpret_cast(bias->Data()) : nullptr, - reinterpret_cast(gamma->Data()), - (beta != nullptr) ? reinterpret_cast(beta->Data()) : nullptr, - epsilon_, - hidden_size, - row_count, - skip_size); - } + LaunchSkipLayerNormKernel( + Stream(ctx), + reinterpret_cast(output->MutableData()), + sum_output != nullptr ? reinterpret_cast(sum_output->MutableData()) : nullptr, + reinterpret_cast(input->Data()), + reinterpret_cast(skip->Data()), + (bias != nullptr) ? reinterpret_cast(bias->Data()) : nullptr, + reinterpret_cast(gamma->Data()), + (beta != nullptr) ? reinterpret_cast(beta->Data()) : nullptr, + epsilon_, + hidden_size, + row_count, + skip_size); } CUDA_RETURN_IF_ERROR(cudaGetLastError()); diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h index 06b3945427833..9d47373fba4f0 100644 --- a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h +++ b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h @@ -19,7 +19,6 @@ class SkipLayerNorm final : public CudaKernel { private: float epsilon_; - bool strict_; }; } // namespace cuda diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h index a3a7c2dd13c52..7b78457c6a120 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -83,7 +83,6 @@ class CUDAExecutionProvider : public IExecutionProvider { bool DoCopyOnDefaultStream() const { return info_.do_copy_in_default_stream; } bool GetCudnnConvUseMaxWorkspace() const { return info_.cudnn_conv_use_max_workspace; } bool GetCudnnConv1dPadToNc1d() const { return info_.cudnn_conv1d_pad_to_nc1d; } - bool IsSkipLayerNormInStrictMode() const { return info_.enable_skip_layer_norm_strict_mode; } bool IsNHWCPreferred() const { return info_.prefer_nhwc; } bool IsFuseConvBias() const { return info_.fuse_conv_bias; } bool UseTF32() const { return info_.use_tf32; } diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h index 06fe635e35716..9e7e44e4a0a53 100644 --- a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h +++ b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h @@ -738,11 +738,6 @@ inline void SetCudaKernelAdapterRuntimeConfigForProvider( PL_CUDA_CALL_THROW(cudaGetDeviceProperties(&config->device_prop, config->device_id)); } -inline bool GetCudaKernelAdapterSkipLayerNormStrictMode(const void* provider) { - const auto config = detail::GetCudaKernelAdapterRuntimeConfigForProvider(provider); - return config->skip_layer_norm_strict_mode; -} - // Global aliases and shims using Status = onnxruntime::common::Status; using MLFloat16 = onnxruntime::MLFloat16; diff --git a/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc b/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc index b28fec5111f2b..fdeb0f3b4f0ba 100644 --- a/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc +++ b/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc @@ -12,7 +12,6 @@ namespace test { constexpr float epsilon_ = 1e-12f; static void RunOneTest( - bool strict, const std::vector& input_data, const std::vector& skip_data, const std::vector& gamma_data, @@ -136,14 +135,7 @@ static void RunOneTest( ToBFloat16(sum_output_data)); } - if (strict) { - Ort::CUDAProviderOptions cuda_options; - std::unordered_map options = {{"enable_skip_layer_norm_strict_mode", "1"}}; - cuda_options.Update(options); - execution_providers.push_back(CudaExecutionProviderWithOptions(std::move(cuda_options))); - } else { - execution_providers.push_back(DefaultCudaExecutionProvider()); - } + execution_providers.push_back(DefaultCudaExecutionProvider()); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } else if (HasCudaEnvironment(530 /*min_cuda_architecture*/) || @@ -187,14 +179,7 @@ static void RunOneTest( } else if (dml_ep != nullptr) { execution_providers.push_back(DefaultDmlExecutionProvider()); } else { - if (strict) { - Ort::CUDAProviderOptions cuda_options; - std::unordered_map options = {{"enable_skip_layer_norm_strict_mode", "1"}}; - cuda_options.Update(options); - execution_providers.push_back(CudaExecutionProviderWithOptions(std::move(cuda_options))); - } else { - execution_providers.push_back(DefaultCudaExecutionProvider()); - } + execution_providers.push_back(DefaultCudaExecutionProvider()); } test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); @@ -220,16 +205,9 @@ static void RunTest( bool use_token_count = false, bool broadcast_skip = false, bool no_batch_size = false) { - RunOneTest(false, input_data, skip_data, gamma_data, beta_data, bias_data, output_data, sum_output_data, + RunOneTest(input_data, skip_data, gamma_data, beta_data, bias_data, output_data, sum_output_data, epsilon, batch_size, sequence_length, hidden_size, use_float16, use_bfloat16, no_beta, simplified, use_token_count, broadcast_skip, no_batch_size); - - // strict mode does not support skip broadcasting. - if (!broadcast_skip) { - RunOneTest(true, input_data, skip_data, gamma_data, beta_data, bias_data, output_data, sum_output_data, - epsilon, batch_size, sequence_length, hidden_size, use_float16, use_bfloat16, no_beta, simplified, - use_token_count, broadcast_skip, no_batch_size); - } } TEST(SkipLayerNormTest, SkipLayerNormPrePack) { From cf7453329e69781741fa2a78269b881b05caad60 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Mon, 29 Jun 2026 21:48:49 +0000 Subject: [PATCH 2/2] address feedbacks --- .../providers/cuda/cuda_provider_options.h | 4 +-- .../cuda/cuda_execution_provider_info.cc | 12 +++++-- .../cuda/cuda_execution_provider_info.h | 2 -- .../providers/cuda/cuda_provider_factory.cc | 2 -- .../core/providers/cuda/cuda_stream_handle.cc | 4 ++- .../core/providers/cuda/plugin/cuda_ep.cc | 1 - .../core/providers/cuda/plugin/cuda_ep.h | 35 +++++++++---------- .../providers/cuda/plugin/cuda_ep_factory.cc | 4 --- .../cuda/plugin/cuda_kernel_adapter.h | 2 -- 9 files changed, 32 insertions(+), 34 deletions(-) diff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_options.h b/include/onnxruntime/core/providers/cuda/cuda_provider_options.h index f4f4dac5fc5f2..7a44cb8e76daf 100644 --- a/include/onnxruntime/core/providers/cuda/cuda_provider_options.h +++ b/include/onnxruntime/core/providers/cuda/cuda_provider_options.h @@ -33,8 +33,8 @@ struct OrtCUDAProviderOptionsV2 { int tunable_op_enable = 0; // flag specifying if TunableOp is enabled. int tunable_op_tuning_enable = 0; // flag specifying if TunableOp is enabled for tuning, this relies on TunableOp is enabled. int tunable_op_max_tuning_duration_ms = 0; // Max tuning duration time limit for TunableOp. - int enable_skip_layer_norm_strict_mode = 0; // [Deprecated] No longer has any effect. SkipLayerNorm always accumulates in fp32. - // Kept for backward compatibility; the value is ignored. + int enable_skip_layer_norm_strict_mode = 0; // [Deprecated] Accepted for ABI/back-compat but not stored in EP info. SkipLayerNorm always accumulates in fp32. + // Setting it has no effect on computation or output. int prefer_nhwc = 0; // make the CUDA EP NHWC preferred int use_ep_level_unified_stream = 0; // flag specifying if ep level stream is used or not int use_tf32 = 1; // use TF32 diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc index 14195703d5963..25b77a268830b 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc @@ -30,6 +30,7 @@ constexpr const char* kCudnnConv1dPadToNc1d = "cudnn_conv1d_pad_to_nc1d"; constexpr const char* kTunableOpEnable = "tunable_op_enable"; constexpr const char* kTunableOpTuningEnable = "tunable_op_tuning_enable"; constexpr const char* kTunableOpMaxTuningDurationMs = "tunable_op_max_tuning_duration_ms"; +// [Deprecated] Accepted but ignored: SkipLayerNorm always accumulates in fp32. constexpr const char* kEnableSkipLayerNormStrictMode = "enable_skip_layer_norm_strict_mode"; constexpr const char* kPreferNHWCMode = "prefer_nhwc"; constexpr const char* kUseEPLevelUnifiedStream = "use_ep_level_unified_stream"; @@ -115,7 +116,15 @@ CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const P .AddAssignmentToReference(cuda::provider_option_names::kCudnnConvUseMaxWorkspace, info.cudnn_conv_use_max_workspace) .AddAssignmentToReference(cuda::provider_option_names::kEnableCudaGraph, info.enable_cuda_graph) .AddAssignmentToReference(cuda::provider_option_names::kCudnnConv1dPadToNc1d, info.cudnn_conv1d_pad_to_nc1d) - .AddAssignmentToReference(cuda::provider_option_names::kEnableSkipLayerNormStrictMode, info.enable_skip_layer_norm_strict_mode) + .AddValueParser( + cuda::provider_option_names::kEnableSkipLayerNormStrictMode, + [](const std::string& value_str) -> Status { + // [Deprecated] Accept the option for backward compatibility, but do not store it: + // SkipLayerNorm always accumulates in fp32, so strict mode has no effect. + bool ignored = false; + ORT_RETURN_IF_ERROR(ParseStringWithClassicLocale(value_str, ignored)); + return Status::OK(); + }) .AddAssignmentToReference(cuda::provider_option_names::kPreferNHWCMode, info.prefer_nhwc) .AddAssignmentToReference(cuda::provider_option_names::kUseEPLevelUnifiedStream, info.use_ep_level_unified_stream) .AddAssignmentToReference(cuda::provider_option_names::kUseTF32, info.use_tf32) @@ -170,7 +179,6 @@ ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecution {cuda::provider_option_names::kTunableOpEnable, MakeStringWithClassicLocale(info.tunable_op.enable)}, {cuda::provider_option_names::kTunableOpTuningEnable, MakeStringWithClassicLocale(info.tunable_op.tuning_enable)}, {cuda::provider_option_names::kTunableOpMaxTuningDurationMs, MakeStringWithClassicLocale(info.tunable_op.max_tuning_duration_ms)}, - {cuda::provider_option_names::kEnableSkipLayerNormStrictMode, MakeStringWithClassicLocale(info.enable_skip_layer_norm_strict_mode)}, {cuda::provider_option_names::kPreferNHWCMode, MakeStringWithClassicLocale(info.prefer_nhwc)}, {cuda::provider_option_names::kUseEPLevelUnifiedStream, MakeStringWithClassicLocale(info.use_ep_level_unified_stream)}, {cuda::provider_option_names::kUseTF32, MakeStringWithClassicLocale(info.use_tf32)}, diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h index bfd50ca8d40a1..2d7f5d2ad041c 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h @@ -71,7 +71,6 @@ struct CUDAExecutionProviderInfo { cuda::TunableOpInfo tunable_op{}; - bool enable_skip_layer_norm_strict_mode{false}; bool prefer_nhwc{false}; bool use_ep_level_unified_stream{false}; @@ -105,7 +104,6 @@ struct std::hash<::onnxruntime::CUDAExecutionProviderInfo> { (static_cast(info.tunable_op.enable) << 24) ^ (static_cast(info.tunable_op.tuning_enable) << 25) ^ (static_cast(info.cudnn_conv1d_pad_to_nc1d) << 26) ^ - (static_cast(info.enable_skip_layer_norm_strict_mode) << 27) ^ (static_cast(info.prefer_nhwc) << 28) ^ (static_cast(info.use_ep_level_unified_stream) << 29) ^ (static_cast(info.use_tf32) << 30) ^ diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index d6a5dc41e1d04..66d3617c8da9a 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -242,7 +242,6 @@ struct CUDA_Provider : Provider { info.tunable_op.enable = params->tunable_op_enable; info.tunable_op.tuning_enable = params->tunable_op_tuning_enable; info.tunable_op.max_tuning_duration_ms = params->tunable_op_max_tuning_duration_ms; - info.enable_skip_layer_norm_strict_mode = params->enable_skip_layer_norm_strict_mode != 0; info.use_ep_level_unified_stream = params->use_ep_level_unified_stream != 0; info.use_tf32 = params->use_tf32 != 0; info.sdpa_kernel = params->sdpa_kernel; @@ -276,7 +275,6 @@ struct CUDA_Provider : Provider { cuda_options.cudnn_conv_use_max_workspace = internal_options.cudnn_conv_use_max_workspace; cuda_options.enable_cuda_graph = internal_options.enable_cuda_graph; cuda_options.cudnn_conv1d_pad_to_nc1d = internal_options.cudnn_conv1d_pad_to_nc1d; - cuda_options.enable_skip_layer_norm_strict_mode = internal_options.enable_skip_layer_norm_strict_mode; cuda_options.prefer_nhwc = internal_options.prefer_nhwc; cuda_options.use_ep_level_unified_stream = internal_options.use_ep_level_unified_stream; cuda_options.use_tf32 = internal_options.use_tf32; diff --git a/onnxruntime/core/providers/cuda/cuda_stream_handle.cc b/onnxruntime/core/providers/cuda/cuda_stream_handle.cc index c4e3bd7e63e5c..39bc2ea35ed85 100644 --- a/onnxruntime/core/providers/cuda/cuda_stream_handle.cc +++ b/onnxruntime/core/providers/cuda/cuda_stream_handle.cc @@ -214,7 +214,9 @@ void* CudaStream::GetResource(int version, int id) const { return reinterpret_cast(ep_info_.cudnn_conv1d_pad_to_nc1d); break; case CudaResource::enable_skip_layer_norm_strict_mode_t: - return reinterpret_cast(ep_info_.enable_skip_layer_norm_strict_mode); + // [Deprecated] SkipLayerNorm always accumulates in fp32; the strict-mode option no longer + // affects computation. Kept for backward compatibility and always reported as false. + return reinterpret_cast(false); break; case CudaResource::prefer_nhwc_t: return reinterpret_cast(ep_info_.prefer_nhwc); diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_ep.cc b/onnxruntime/core/providers/cuda/plugin/cuda_ep.cc index 73fa92d19cd1f..ac2ac04b26cfe 100644 --- a/onnxruntime/core/providers/cuda/plugin/cuda_ep.cc +++ b/onnxruntime/core/providers/cuda/plugin/cuda_ep.cc @@ -202,7 +202,6 @@ CudaEp::CudaEp(CudaEpFactory& factory, const Config& config, const OrtLogger& lo // below — no function-signature change. onnxruntime::cuda::detail::CudaKernelAdapterRuntimeConfig adapter_config; adapter_config.use_tf32 = config_.use_tf32; - adapter_config.skip_layer_norm_strict_mode = config_.enable_skip_layer_norm_strict_mode; adapter_config.cudnn_conv_algo = config_.cudnn_conv_algo; adapter_config.cudnn_conv_use_max_workspace = config_.cudnn_conv_use_max_workspace; adapter_config.cudnn_conv1d_pad_to_nc1d = config_.cudnn_conv1d_pad_to_nc1d; diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_ep.h b/onnxruntime/core/providers/cuda/plugin/cuda_ep.h index 346f73a4cfa0b..b05d4a2d3146d 100644 --- a/onnxruntime/core/providers/cuda/plugin/cuda_ep.h +++ b/onnxruntime/core/providers/cuda/plugin/cuda_ep.h @@ -24,24 +24,23 @@ class CudaEp : public onnxruntime::ep::adapter::Ep { public: /// Configuration parameters for the CUDA EP, parsed from session options. struct Config { - bool prefer_nhwc = false; ///< Use NHWC data layout when available. - bool use_tf32 = true; ///< Enable TF32 math on Ampere+ GPUs. - bool enable_skip_layer_norm_strict_mode = false; ///< Strict mode for SkipLayerNorm kernel. - int device_id = 0; ///< CUDA device ordinal. - int cudnn_conv_algo = 0; ///< cuDNN convolution algorithm selection. - bool cudnn_conv_use_max_workspace = true; ///< Use maximum workspace for cuDNN conv algo search. - bool cudnn_conv1d_pad_to_nc1d = false; ///< Pad 1D convolutions to NC1D format. - bool fuse_conv_bias = false; ///< Enable cuDNN frontend conv+bias fusion. - int sdpa_kernel = 0; ///< Attention backend bitmask override. - bool enable_cuda_graph = false; ///< Enable CUDA graph capture and replay. - int min_num_runs_before_cuda_graph_capture = 2; ///< Warm-up runs before graph capture begins. - bool has_user_compute_stream = false; ///< Whether user provided an external CUDA stream. - void* user_compute_stream = nullptr; ///< User-provided CUDA stream (cudaStream_t cast to void*). - bool do_copy_in_default_stream = true; ///< Use default stream for H2D/D2H copies. - bool use_ep_level_unified_stream = false; ///< Force all ops to share one stream (no concurrency). - void* external_alloc = nullptr; ///< External GPU memory allocation function pointer. - void* external_free = nullptr; ///< External GPU memory deallocation function pointer. - void* external_empty_cache = nullptr; ///< External GPU memory cache-clear function pointer. + bool prefer_nhwc = false; ///< Use NHWC data layout when available. + bool use_tf32 = true; ///< Enable TF32 math on Ampere+ GPUs. + int device_id = 0; ///< CUDA device ordinal. + int cudnn_conv_algo = 0; ///< cuDNN convolution algorithm selection. + bool cudnn_conv_use_max_workspace = true; ///< Use maximum workspace for cuDNN conv algo search. + bool cudnn_conv1d_pad_to_nc1d = false; ///< Pad 1D convolutions to NC1D format. + bool fuse_conv_bias = false; ///< Enable cuDNN frontend conv+bias fusion. + int sdpa_kernel = 0; ///< Attention backend bitmask override. + bool enable_cuda_graph = false; ///< Enable CUDA graph capture and replay. + int min_num_runs_before_cuda_graph_capture = 2; ///< Warm-up runs before graph capture begins. + bool has_user_compute_stream = false; ///< Whether user provided an external CUDA stream. + void* user_compute_stream = nullptr; ///< User-provided CUDA stream (cudaStream_t cast to void*). + bool do_copy_in_default_stream = true; ///< Use default stream for H2D/D2H copies. + bool use_ep_level_unified_stream = false; ///< Force all ops to share one stream (no concurrency). + void* external_alloc = nullptr; ///< External GPU memory allocation function pointer. + void* external_free = nullptr; ///< External GPU memory deallocation function pointer. + void* external_empty_cache = nullptr; ///< External GPU memory cache-clear function pointer. }; CudaEp(CudaEpFactory& factory, const Config& config, const OrtLogger& logger); diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc b/onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc index d445d8bab033c..8d4f7c7c06244 100644 --- a/onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc +++ b/onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc @@ -482,7 +482,6 @@ OrtStatus* ORT_API_CALL CudaEpFactory::CreateEpImpl( const std::string prefer_nhwc_key = ep_options_prefix + "prefer_nhwc"; const std::string prefer_nhwc_layout_key = ep_options_prefix + "prefer_nhwc_layout"; const std::string use_tf32_key = ep_options_prefix + "use_tf32"; - const std::string skip_layer_norm_key = ep_options_prefix + "enable_skip_layer_norm_strict_mode"; const std::string cudnn_use_max_workspace_key = ep_options_prefix + "cudnn_conv_use_max_workspace"; const std::string cudnn_conv1d_pad_key = ep_options_prefix + "cudnn_conv1d_pad_to_nc1d"; const std::string cudnn_conv_algo_key = ep_options_prefix + "cudnn_conv_algo"; @@ -505,9 +504,6 @@ OrtStatus* ORT_API_CALL CudaEpFactory::CreateEpImpl( {prefer_nhwc_key, prefer_nhwc_layout_key, "ep.cuda.prefer_nhwc_layout", "prefer_nhwc", "prefer_nhwc_layout"}, config.prefer_nhwc); read_session_config_bool({use_tf32_key, "ep.cuda.use_tf32", "use_tf32"}, config.use_tf32); - read_session_config_bool( - {skip_layer_norm_key, "ep.cuda.enable_skip_layer_norm_strict_mode", "enable_skip_layer_norm_strict_mode"}, - config.enable_skip_layer_norm_strict_mode); read_session_config_bool( {cudnn_use_max_workspace_key, "ep.cuda.cudnn_conv_use_max_workspace", "cudnn_conv_use_max_workspace"}, config.cudnn_conv_use_max_workspace); diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h index 9e7e44e4a0a53..b6b563475b00b 100644 --- a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h +++ b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h @@ -517,7 +517,6 @@ namespace cuda { namespace detail { struct CudaKernelAdapterRuntimeConfig { bool use_tf32 = true; - bool skip_layer_norm_strict_mode = false; int cudnn_conv_algo = 0; bool cudnn_conv_use_max_workspace = true; bool cudnn_conv1d_pad_to_nc1d = false; @@ -727,7 +726,6 @@ inline void SetCudaKernelAdapterRuntimeConfigForProvider( // AttentionKernelOptions contains std::once_flag (not copyable), so assign // the plain-data fields individually rather than relying on operator=. config->use_tf32 = init_config.use_tf32; - config->skip_layer_norm_strict_mode = init_config.skip_layer_norm_strict_mode; config->cudnn_conv_algo = init_config.cudnn_conv_algo; config->cudnn_conv_use_max_workspace = init_config.cudnn_conv_use_max_workspace; config->cudnn_conv1d_pad_to_nc1d = init_config.cudnn_conv1d_pad_to_nc1d;