Skip to content

[CUDA] Deprecate SkipLayerNorm strict mode #29388

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kunal-vaishnavi merged 2 commits into from
Jun 30, 2026
Merged

[CUDA] Deprecate SkipLayerNorm strict mode #29388

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e74051
remove skip_layer_norm_strict_mode
tianleiwu Jun 29, 2026
cf74533
address feedbacks
tianleiwu Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions include/onnxruntime/core/providers/cuda/cuda_provider_options.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ struct OrtCUDAProviderOptionsV2 {
int tunable_op_enable = 0; // flag specifying if TunableOp is enabled.
int tunable_op_tuning_enable = 0; // flag specifying if TunableOp is enabled for tuning, this relies on TunableOp is enabled.
int tunable_op_max_tuning_duration_ms = 0; // Max tuning duration time limit for TunableOp.
int enable_skip_layer_norm_strict_mode = 0; // flag specifying if SkipLayerNorm is in strict mode. If true, use LayerNormalization kernel.
// The strict mode has better accuracy but lower performance.
int enable_skip_layer_norm_strict_mode = 0; // [Deprecated] Accepted for ABI/back-compat but not stored in EP info. SkipLayerNorm always accumulates in fp32.
// Setting it has no effect on computation or output.
int prefer_nhwc = 0; // make the CUDA EP NHWC preferred
int use_ep_level_unified_stream = 0; // flag specifying if ep level stream is used or not
int use_tf32 = 1; // use TF32
Expand Down
88 changes: 28 additions & 60 deletions onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// Licensed under the MIT License.

#include "core/providers/cuda/cuda_common.h"
#include "core/providers/cuda/nn/layer_norm_impl.h"
#include "core/common/narrow.h"
#include "skip_layer_norm.h"
#include "skip_layer_norm_impl.h"
Expand Down Expand Up @@ -42,26 +41,14 @@ template <typename T, bool Simplified>
SkipLayerNorm<T, Simplified>::SkipLayerNorm(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info) {
ORT_ENFORCE(op_kernel_info.GetAttr<float>("epsilon", &epsilon_).IsOK());
ORT_ENFORCE(epsilon_ >= 0);

#ifdef BUILD_CUDA_EP_AS_PLUGIN
// Plugin adapter cannot static_cast to CUDAExecutionProvider directly.
// Use the adapter shim that reads the config from the per-EP runtime map.
strict_ = onnxruntime::cuda::GetCudaKernelAdapterSkipLayerNormStrictMode(op_kernel_info.GetExecutionProvider());
#else
const CUDAExecutionProvider* cuda_ep = static_cast<const CUDAExecutionProvider*>(op_kernel_info.GetExecutionProvider());
strict_ = cuda_ep->IsSkipLayerNormInStrictMode();
#endif
// Note: the enable_skip_layer_norm_strict_mode provider option is deprecated and ignored.
// The kernel always accumulates in fp32, so the previous strict-mode path is no longer needed.
}

template <typename T, bool Simplified>
Status SkipLayerNorm<T, Simplified>::ComputeInternal(OpKernelContext* ctx) const {
const Tensor* input = ctx->Input<Tensor>(0);
const Tensor* skip = ctx->Input<Tensor>(1);
if (strict_ && skip->Shape() != input->Shape()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"'input' and 'skip' shall have same shape when enable_skip_layer_norm_strict_mode is True");
}

const Tensor* gamma = ctx->Input<Tensor>(2);

const Tensor* beta = Simplified ? nullptr : ctx->Input<Tensor>(3);
Expand Down Expand Up @@ -94,53 +81,34 @@ Status SkipLayerNorm<T, Simplified>::ComputeInternal(OpKernelContext* ctx) const

const int skip_size = onnxruntime::narrow<int>(skip->Shape().Size());

if (strict_) {
HostApplyLayerNorm<CudaT, float, CudaT, Simplified>(
GetDeviceProp(),
if constexpr (std::is_same_v<T, BFloat16>) {
LaunchSkipLayerNormKernel<nv_bfloat16, Simplified>(
Stream(ctx),
reinterpret_cast<CudaT*>(output->MutableData<T>()), // Y_data
nullptr, // mean_data
nullptr, // inv_var_data
reinterpret_cast<const CudaT*>(input->Data<T>()), // X_data
row_count, // n1
hidden_size, // n2
(double)epsilon_, // epsilon
reinterpret_cast<const CudaT*>(gamma->Data<T>()), // gamma
(beta != nullptr) ? reinterpret_cast<const CudaT*>(beta->Data<T>()) : nullptr, // beta
0, // no broadcast for gamma/beta
reinterpret_cast<const CudaT*>(skip->Data<T>()), // skip or residual to add
(bias != nullptr) ? reinterpret_cast<const CudaT*>(bias->Data<T>()) : nullptr, // bias to add
sum_output != nullptr ? reinterpret_cast<CudaT*>(sum_output->MutableData<T>()) : nullptr);
reinterpret_cast<nv_bfloat16*>(output->MutableData<T>()),
sum_output != nullptr ? reinterpret_cast<nv_bfloat16*>(sum_output->MutableData<T>()) : nullptr,
reinterpret_cast<const nv_bfloat16*>(input->Data<T>()),
reinterpret_cast<const nv_bfloat16*>(skip->Data<T>()),
(bias != nullptr) ? reinterpret_cast<const nv_bfloat16*>(bias->Data<T>()) : nullptr,
reinterpret_cast<const nv_bfloat16*>(gamma->Data<T>()),
(beta != nullptr) ? reinterpret_cast<const nv_bfloat16*>(beta->Data<T>()) : nullptr,
epsilon_,
hidden_size,
row_count,
skip_size);
} else {
if constexpr (std::is_same_v<T, BFloat16>) {
LaunchSkipLayerNormKernel<nv_bfloat16, Simplified>(
Stream(ctx),
reinterpret_cast<nv_bfloat16*>(output->MutableData<T>()),
sum_output != nullptr ? reinterpret_cast<nv_bfloat16*>(sum_output->MutableData<T>()) : nullptr,
reinterpret_cast<const nv_bfloat16*>(input->Data<T>()),
reinterpret_cast<const nv_bfloat16*>(skip->Data<T>()),
(bias != nullptr) ? reinterpret_cast<const nv_bfloat16*>(bias->Data<T>()) : nullptr,
reinterpret_cast<const nv_bfloat16*>(gamma->Data<T>()),
(beta != nullptr) ? reinterpret_cast<const nv_bfloat16*>(beta->Data<T>()) : nullptr,
epsilon_,
hidden_size,
row_count,
skip_size);
} else {
LaunchSkipLayerNormKernel<CudaT, Simplified>(
Stream(ctx),
reinterpret_cast<CudaT*>(output->MutableData<T>()),
sum_output != nullptr ? reinterpret_cast<CudaT*>(sum_output->MutableData<T>()) : nullptr,
reinterpret_cast<const CudaT*>(input->Data<T>()),
reinterpret_cast<const CudaT*>(skip->Data<T>()),
(bias != nullptr) ? reinterpret_cast<const CudaT*>(bias->Data<T>()) : nullptr,
reinterpret_cast<const CudaT*>(gamma->Data<T>()),
(beta != nullptr) ? reinterpret_cast<const CudaT*>(beta->Data<T>()) : nullptr,
epsilon_,
hidden_size,
row_count,
skip_size);
}
LaunchSkipLayerNormKernel<CudaT, Simplified>(
Stream(ctx),
reinterpret_cast<CudaT*>(output->MutableData<T>()),
sum_output != nullptr ? reinterpret_cast<CudaT*>(sum_output->MutableData<T>()) : nullptr,
reinterpret_cast<const CudaT*>(input->Data<T>()),
reinterpret_cast<const CudaT*>(skip->Data<T>()),
(bias != nullptr) ? reinterpret_cast<const CudaT*>(bias->Data<T>()) : nullptr,
reinterpret_cast<const CudaT*>(gamma->Data<T>()),
(beta != nullptr) ? reinterpret_cast<const CudaT*>(beta->Data<T>()) : nullptr,
epsilon_,
hidden_size,
row_count,
skip_size);
}

CUDA_RETURN_IF_ERROR(cudaGetLastError());
Expand Down
1 change: 0 additions & 1 deletion onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ class SkipLayerNorm final : public CudaKernel {

private:
float epsilon_;
bool strict_;
};

} // namespace cuda
Expand Down
1 change: 0 additions & 1 deletion onnxruntime/core/providers/cuda/cuda_execution_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ class CUDAExecutionProvider : public IExecutionProvider {
bool DoCopyOnDefaultStream() const { return info_.do_copy_in_default_stream; }
bool GetCudnnConvUseMaxWorkspace() const { return info_.cudnn_conv_use_max_workspace; }
bool GetCudnnConv1dPadToNc1d() const { return info_.cudnn_conv1d_pad_to_nc1d; }
bool IsSkipLayerNormInStrictMode() const { return info_.enable_skip_layer_norm_strict_mode; }
bool IsNHWCPreferred() const { return info_.prefer_nhwc; }
bool IsFuseConvBias() const { return info_.fuse_conv_bias; }
bool UseTF32() const { return info_.use_tf32; }
Expand Down
12 changes: 10 additions & 2 deletions onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ constexpr const char* kCudnnConv1dPadToNc1d = "cudnn_conv1d_pad_to_nc1d";
constexpr const char* kTunableOpEnable = "tunable_op_enable";
constexpr const char* kTunableOpTuningEnable = "tunable_op_tuning_enable";
constexpr const char* kTunableOpMaxTuningDurationMs = "tunable_op_max_tuning_duration_ms";
// [Deprecated] Accepted but ignored: SkipLayerNorm always accumulates in fp32.
constexpr const char* kEnableSkipLayerNormStrictMode = "enable_skip_layer_norm_strict_mode";
constexpr const char* kPreferNHWCMode = "prefer_nhwc";
constexpr const char* kUseEPLevelUnifiedStream = "use_ep_level_unified_stream";
Expand Down Expand Up @@ -115,7 +116,15 @@ CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const P
.AddAssignmentToReference(cuda::provider_option_names::kCudnnConvUseMaxWorkspace, info.cudnn_conv_use_max_workspace)
.AddAssignmentToReference(cuda::provider_option_names::kEnableCudaGraph, info.enable_cuda_graph)
.AddAssignmentToReference(cuda::provider_option_names::kCudnnConv1dPadToNc1d, info.cudnn_conv1d_pad_to_nc1d)
.AddAssignmentToReference(cuda::provider_option_names::kEnableSkipLayerNormStrictMode, info.enable_skip_layer_norm_strict_mode)
.AddValueParser(
cuda::provider_option_names::kEnableSkipLayerNormStrictMode,
[](const std::string& value_str) -> Status {
// [Deprecated] Accept the option for backward compatibility, but do not store it:
// SkipLayerNorm always accumulates in fp32, so strict mode has no effect.
bool ignored = false;
ORT_RETURN_IF_ERROR(ParseStringWithClassicLocale(value_str, ignored));
return Status::OK();
})
.AddAssignmentToReference(cuda::provider_option_names::kPreferNHWCMode, info.prefer_nhwc)
.AddAssignmentToReference(cuda::provider_option_names::kUseEPLevelUnifiedStream, info.use_ep_level_unified_stream)
.AddAssignmentToReference(cuda::provider_option_names::kUseTF32, info.use_tf32)
Expand Down Expand Up @@ -170,7 +179,6 @@ ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecution
{cuda::provider_option_names::kTunableOpEnable, MakeStringWithClassicLocale(info.tunable_op.enable)},
{cuda::provider_option_names::kTunableOpTuningEnable, MakeStringWithClassicLocale(info.tunable_op.tuning_enable)},
{cuda::provider_option_names::kTunableOpMaxTuningDurationMs, MakeStringWithClassicLocale(info.tunable_op.max_tuning_duration_ms)},
{cuda::provider_option_names::kEnableSkipLayerNormStrictMode, MakeStringWithClassicLocale(info.enable_skip_layer_norm_strict_mode)},
{cuda::provider_option_names::kPreferNHWCMode, MakeStringWithClassicLocale(info.prefer_nhwc)},
{cuda::provider_option_names::kUseEPLevelUnifiedStream, MakeStringWithClassicLocale(info.use_ep_level_unified_stream)},
{cuda::provider_option_names::kUseTF32, MakeStringWithClassicLocale(info.use_tf32)},
Expand Down
2 changes: 0 additions & 2 deletions onnxruntime/core/providers/cuda/cuda_execution_provider_info.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ struct CUDAExecutionProviderInfo {

cuda::TunableOpInfo tunable_op{};

bool enable_skip_layer_norm_strict_mode{false};
bool prefer_nhwc{false};

bool use_ep_level_unified_stream{false};
Expand Down Expand Up @@ -105,7 +104,6 @@ struct std::hash<::onnxruntime::CUDAExecutionProviderInfo> {
(static_cast<size_t>(info.tunable_op.enable) << 24) ^
(static_cast<size_t>(info.tunable_op.tuning_enable) << 25) ^
(static_cast<size_t>(info.cudnn_conv1d_pad_to_nc1d) << 26) ^
(static_cast<size_t>(info.enable_skip_layer_norm_strict_mode) << 27) ^
(static_cast<size_t>(info.prefer_nhwc) << 28) ^
(static_cast<size_t>(info.use_ep_level_unified_stream) << 29) ^
(static_cast<size_t>(info.use_tf32) << 30) ^
Expand Down
2 changes: 0 additions & 2 deletions onnxruntime/core/providers/cuda/cuda_provider_factory.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,6 @@ struct CUDA_Provider : Provider {
info.tunable_op.enable = params->tunable_op_enable;
info.tunable_op.tuning_enable = params->tunable_op_tuning_enable;
info.tunable_op.max_tuning_duration_ms = params->tunable_op_max_tuning_duration_ms;
info.enable_skip_layer_norm_strict_mode = params->enable_skip_layer_norm_strict_mode != 0;
info.use_ep_level_unified_stream = params->use_ep_level_unified_stream != 0;
info.use_tf32 = params->use_tf32 != 0;
info.sdpa_kernel = params->sdpa_kernel;
Expand Down Expand Up @@ -276,7 +275,6 @@ struct CUDA_Provider : Provider {
cuda_options.cudnn_conv_use_max_workspace = internal_options.cudnn_conv_use_max_workspace;
cuda_options.enable_cuda_graph = internal_options.enable_cuda_graph;
cuda_options.cudnn_conv1d_pad_to_nc1d = internal_options.cudnn_conv1d_pad_to_nc1d;
cuda_options.enable_skip_layer_norm_strict_mode = internal_options.enable_skip_layer_norm_strict_mode;
cuda_options.prefer_nhwc = internal_options.prefer_nhwc;
cuda_options.use_ep_level_unified_stream = internal_options.use_ep_level_unified_stream;
cuda_options.use_tf32 = internal_options.use_tf32;
Expand Down
4 changes: 3 additions & 1 deletion onnxruntime/core/providers/cuda/cuda_stream_handle.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,9 @@ void* CudaStream::GetResource(int version, int id) const {
return reinterpret_cast<void*>(ep_info_.cudnn_conv1d_pad_to_nc1d);
break;
case CudaResource::enable_skip_layer_norm_strict_mode_t:
return reinterpret_cast<void*>(ep_info_.enable_skip_layer_norm_strict_mode);
// [Deprecated] SkipLayerNorm always accumulates in fp32; the strict-mode option no longer
// affects computation. Kept for backward compatibility and always reported as false.
return reinterpret_cast<void*>(false);
break;
case CudaResource::prefer_nhwc_t:
return reinterpret_cast<void*>(ep_info_.prefer_nhwc);
Expand Down
1 change: 0 additions & 1 deletion onnxruntime/core/providers/cuda/plugin/cuda_ep.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,6 @@ CudaEp::CudaEp(CudaEpFactory& factory, const Config& config, const OrtLogger& lo
// below — no function-signature change.
onnxruntime::cuda::detail::CudaKernelAdapterRuntimeConfig adapter_config;
adapter_config.use_tf32 = config_.use_tf32;
adapter_config.skip_layer_norm_strict_mode = config_.enable_skip_layer_norm_strict_mode;
adapter_config.cudnn_conv_algo = config_.cudnn_conv_algo;
adapter_config.cudnn_conv_use_max_workspace = config_.cudnn_conv_use_max_workspace;
adapter_config.cudnn_conv1d_pad_to_nc1d = config_.cudnn_conv1d_pad_to_nc1d;
Expand Down
35 changes: 17 additions & 18 deletions onnxruntime/core/providers/cuda/plugin/cuda_ep.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,23 @@ class CudaEp : public onnxruntime::ep::adapter::Ep {
public:
/// Configuration parameters for the CUDA EP, parsed from session options.
struct Config {
bool prefer_nhwc = false; ///< Use NHWC data layout when available.
bool use_tf32 = true; ///< Enable TF32 math on Ampere+ GPUs.
bool enable_skip_layer_norm_strict_mode = false; ///< Strict mode for SkipLayerNorm kernel.
int device_id = 0; ///< CUDA device ordinal.
int cudnn_conv_algo = 0; ///< cuDNN convolution algorithm selection.
bool cudnn_conv_use_max_workspace = true; ///< Use maximum workspace for cuDNN conv algo search.
bool cudnn_conv1d_pad_to_nc1d = false; ///< Pad 1D convolutions to NC1D format.
bool fuse_conv_bias = false; ///< Enable cuDNN frontend conv+bias fusion.
int sdpa_kernel = 0; ///< Attention backend bitmask override.
bool enable_cuda_graph = false; ///< Enable CUDA graph capture and replay.
int min_num_runs_before_cuda_graph_capture = 2; ///< Warm-up runs before graph capture begins.
bool has_user_compute_stream = false; ///< Whether user provided an external CUDA stream.
void* user_compute_stream = nullptr; ///< User-provided CUDA stream (cudaStream_t cast to void*).
bool do_copy_in_default_stream = true; ///< Use default stream for H2D/D2H copies.
bool use_ep_level_unified_stream = false; ///< Force all ops to share one stream (no concurrency).
void* external_alloc = nullptr; ///< External GPU memory allocation function pointer.
void* external_free = nullptr; ///< External GPU memory deallocation function pointer.
void* external_empty_cache = nullptr; ///< External GPU memory cache-clear function pointer.
bool prefer_nhwc = false; ///< Use NHWC data layout when available.
bool use_tf32 = true; ///< Enable TF32 math on Ampere+ GPUs.
int device_id = 0; ///< CUDA device ordinal.
int cudnn_conv_algo = 0; ///< cuDNN convolution algorithm selection.
bool cudnn_conv_use_max_workspace = true; ///< Use maximum workspace for cuDNN conv algo search.
bool cudnn_conv1d_pad_to_nc1d = false; ///< Pad 1D convolutions to NC1D format.
bool fuse_conv_bias = false; ///< Enable cuDNN frontend conv+bias fusion.
int sdpa_kernel = 0; ///< Attention backend bitmask override.
bool enable_cuda_graph = false; ///< Enable CUDA graph capture and replay.
int min_num_runs_before_cuda_graph_capture = 2; ///< Warm-up runs before graph capture begins.
bool has_user_compute_stream = false; ///< Whether user provided an external CUDA stream.
void* user_compute_stream = nullptr; ///< User-provided CUDA stream (cudaStream_t cast to void*).
bool do_copy_in_default_stream = true; ///< Use default stream for H2D/D2H copies.
bool use_ep_level_unified_stream = false; ///< Force all ops to share one stream (no concurrency).
void* external_alloc = nullptr; ///< External GPU memory allocation function pointer.
void* external_free = nullptr; ///< External GPU memory deallocation function pointer.
void* external_empty_cache = nullptr; ///< External GPU memory cache-clear function pointer.
};

CudaEp(CudaEpFactory& factory, const Config& config, const OrtLogger& logger);
Expand Down
4 changes: 0 additions & 4 deletions onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,6 @@ OrtStatus* ORT_API_CALL CudaEpFactory::CreateEpImpl(
const std::string prefer_nhwc_key = ep_options_prefix + "prefer_nhwc";
const std::string prefer_nhwc_layout_key = ep_options_prefix + "prefer_nhwc_layout";
const std::string use_tf32_key = ep_options_prefix + "use_tf32";
const std::string skip_layer_norm_key = ep_options_prefix + "enable_skip_layer_norm_strict_mode";
const std::string cudnn_use_max_workspace_key = ep_options_prefix + "cudnn_conv_use_max_workspace";
const std::string cudnn_conv1d_pad_key = ep_options_prefix + "cudnn_conv1d_pad_to_nc1d";
const std::string cudnn_conv_algo_key = ep_options_prefix + "cudnn_conv_algo";
Expand All @@ -505,9 +504,6 @@ OrtStatus* ORT_API_CALL CudaEpFactory::CreateEpImpl(
{prefer_nhwc_key, prefer_nhwc_layout_key, "ep.cuda.prefer_nhwc_layout", "prefer_nhwc", "prefer_nhwc_layout"},
config.prefer_nhwc);
read_session_config_bool({use_tf32_key, "ep.cuda.use_tf32", "use_tf32"}, config.use_tf32);
read_session_config_bool(
{skip_layer_norm_key, "ep.cuda.enable_skip_layer_norm_strict_mode", "enable_skip_layer_norm_strict_mode"},
config.enable_skip_layer_norm_strict_mode);
read_session_config_bool(
{cudnn_use_max_workspace_key, "ep.cuda.cudnn_conv_use_max_workspace", "cudnn_conv_use_max_workspace"},
config.cudnn_conv_use_max_workspace);
Expand Down
Loading
Loading