From d1bdf0f38f50b8c499f22ae1d50770b819f27678 Mon Sep 17 00:00:00 2001 From: Lin Yuan Date: Sat, 3 Nov 2018 16:33:25 -0700 Subject: [PATCH] use cudnnGet instead of cudnnFind when determinism required --- src/operator/nn/cudnn/cudnn_convolution-inl.h | 8 ++++---- src/operator/nn/cudnn/cudnn_deconvolution-inl.h | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/operator/nn/cudnn/cudnn_convolution-inl.h b/src/operator/nn/cudnn/cudnn_convolution-inl.h index 3bd6c5a3826b..a2dce33e91fb 100644 --- a/src/operator/nn/cudnn/cudnn_convolution-inl.h +++ b/src/operator/nn/cudnn/cudnn_convolution-inl.h @@ -635,8 +635,8 @@ class CuDNNConvolutionOp { std::vector fwd_results(MaxForwardAlgos(s->dnn_handle_)); int actual_fwd_algos = 0; auto fwd_algo_discoverer = - param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionForwardAlgorithm_v7 - : cudnnFindConvolutionForwardAlgorithm; + (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ? + cudnnGetConvolutionForwardAlgorithm_v7 : cudnnFindConvolutionForwardAlgorithm; CUDNN_CALL((*fwd_algo_discoverer)(s->dnn_handle_, in_desc_, filter_desc_, @@ -657,8 +657,8 @@ class CuDNNConvolutionOp { // In cudnn v7.1.4, find() returned wgrad algos that could fail for large c if we // were summing into the output (i.e. beta != 0). Get() returned OK algos though. auto bwd_filter_algo_discoverer = - param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionBackwardFilterAlgorithm_v7 - : cudnnFindConvolutionBackwardFilterAlgorithm; + (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ? + cudnnGetConvolutionBackwardFilterAlgorithm_v7 : cudnnFindConvolutionBackwardFilterAlgorithm; CUDNN_CALL((*bwd_filter_algo_discoverer)(s->dnn_handle_, in_desc_, out_desc_, diff --git a/src/operator/nn/cudnn/cudnn_deconvolution-inl.h b/src/operator/nn/cudnn/cudnn_deconvolution-inl.h index 72ba2c95fc6a..7616acd6f687 100644 --- a/src/operator/nn/cudnn/cudnn_deconvolution-inl.h +++ b/src/operator/nn/cudnn/cudnn_deconvolution-inl.h @@ -562,8 +562,8 @@ class CuDNNDeconvolutionOp { std::vector fwd_results(MaxForwardAlgos(s->dnn_handle_)); int actual_fwd_algos = 0; auto fwd_algo_discoverer = - param_.cudnn_tune.value() == deconv::kOff ? cudnnGetConvolutionForwardAlgorithm_v7 - : cudnnFindConvolutionForwardAlgorithm; + (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ? + cudnnGetConvolutionForwardAlgorithm_v7 : cudnnFindConvolutionForwardAlgorithm; CUDNN_CALL((*fwd_algo_discoverer)(s->dnn_handle_, out_desc_, filter_desc_, @@ -584,8 +584,8 @@ class CuDNNDeconvolutionOp { // In cudnn v7.1.4, find() returned wgrad algos that could fail for large c if we // were summing into the output (i.e. beta != 0). Get() returned OK algos though. auto bwd_filter_algo_discoverer = - param_.cudnn_tune.value() == deconv::kOff ? cudnnGetConvolutionBackwardFilterAlgorithm_v7 - : cudnnFindConvolutionBackwardFilterAlgorithm; + (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ? + cudnnGetConvolutionBackwardFilterAlgorithm_v7 : cudnnFindConvolutionBackwardFilterAlgorithm; CUDNN_CALL((*bwd_filter_algo_discoverer)(s->dnn_handle_, out_desc_, in_desc_, @@ -603,8 +603,8 @@ class CuDNNDeconvolutionOp { std::vector bwd_data_results(max_bwd_data_algos); int actual_bwd_data_algos = 0; auto bwd_data_algo_discoverer = - param_.cudnn_tune.value() == deconv::kOff ? cudnnGetConvolutionBackwardDataAlgorithm_v7 - : cudnnFindConvolutionBackwardDataAlgorithm; + (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ? + cudnnGetConvolutionBackwardDataAlgorithm_v7 : cudnnFindConvolutionBackwardDataAlgorithm; CUDNN_CALL((*bwd_data_algo_discoverer)(s->dnn_handle_, filter_desc_, in_desc_,