use cudnnGet instead of cudnnFind when determinism required

jlcontreras · Nov 3, 2018 · d1bdf0f · d1bdf0f
1 parent 3d6ef7b
commit d1bdf0f
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 10 deletions.
diff --git a/src/operator/nn/cudnn/cudnn_convolution-inl.h b/src/operator/nn/cudnn/cudnn_convolution-inl.h
@@ -635,8 +635,8 @@ class CuDNNConvolutionOp {
     std::vector<cudnnConvolutionFwdAlgoPerf_t> fwd_results(MaxForwardAlgos(s->dnn_handle_));
     int actual_fwd_algos = 0;
     auto fwd_algo_discoverer =
-      param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionForwardAlgorithm_v7
-                                              : cudnnFindConvolutionForwardAlgorithm;
+      (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ?
+        cudnnGetConvolutionForwardAlgorithm_v7 : cudnnFindConvolutionForwardAlgorithm;
     CUDNN_CALL((*fwd_algo_discoverer)(s->dnn_handle_,
                                       in_desc_,
                                       filter_desc_,
@@ -657,8 +657,8 @@ class CuDNNConvolutionOp {
     // In cudnn v7.1.4, find() returned wgrad algos that could fail for large c if we
     // were summing into the output (i.e. beta != 0).  Get() returned OK algos though.
     auto bwd_filter_algo_discoverer =
-      param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionBackwardFilterAlgorithm_v7
-                                              : cudnnFindConvolutionBackwardFilterAlgorithm;
+      (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ?
+        cudnnGetConvolutionBackwardFilterAlgorithm_v7 : cudnnFindConvolutionBackwardFilterAlgorithm;
     CUDNN_CALL((*bwd_filter_algo_discoverer)(s->dnn_handle_,
                                              in_desc_,
                                              out_desc_,

diff --git a/src/operator/nn/cudnn/cudnn_deconvolution-inl.h b/src/operator/nn/cudnn/cudnn_deconvolution-inl.h
@@ -562,8 +562,8 @@ class CuDNNDeconvolutionOp {
     std::vector<cudnnConvolutionFwdAlgoPerf_t> fwd_results(MaxForwardAlgos(s->dnn_handle_));
     int actual_fwd_algos = 0;
     auto fwd_algo_discoverer =
-        param_.cudnn_tune.value() == deconv::kOff ? cudnnGetConvolutionForwardAlgorithm_v7
-                                                : cudnnFindConvolutionForwardAlgorithm;
+        (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ?
+          cudnnGetConvolutionForwardAlgorithm_v7 : cudnnFindConvolutionForwardAlgorithm;
     CUDNN_CALL((*fwd_algo_discoverer)(s->dnn_handle_,
                                       out_desc_,
                                       filter_desc_,
@@ -584,8 +584,8 @@ class CuDNNDeconvolutionOp {
     // In cudnn v7.1.4, find() returned wgrad algos that could fail for large c if we
     // were summing into the output (i.e. beta != 0).  Get() returned OK algos though.
     auto bwd_filter_algo_discoverer =
-        param_.cudnn_tune.value() == deconv::kOff ? cudnnGetConvolutionBackwardFilterAlgorithm_v7
-                                                  : cudnnFindConvolutionBackwardFilterAlgorithm;
+      (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ?
+        cudnnGetConvolutionBackwardFilterAlgorithm_v7 : cudnnFindConvolutionBackwardFilterAlgorithm;
     CUDNN_CALL((*bwd_filter_algo_discoverer)(s->dnn_handle_,
                                              out_desc_,
                                              in_desc_,
@@ -603,8 +603,8 @@ class CuDNNDeconvolutionOp {
     std::vector<cudnnConvolutionBwdDataAlgoPerf_t> bwd_data_results(max_bwd_data_algos);
     int actual_bwd_data_algos = 0;
     auto bwd_data_algo_discoverer =
-        param_.cudnn_tune.value() == deconv::kOff ? cudnnGetConvolutionBackwardDataAlgorithm_v7
-                                                : cudnnFindConvolutionBackwardDataAlgorithm;
+      (param_.cudnn_tune.value() == conv::kOff || dmlc::GetEnv("MXNET_ENFORCE_DETERMINISM", 0)) ?
+        cudnnGetConvolutionBackwardDataAlgorithm_v7 : cudnnFindConvolutionBackwardDataAlgorithm;
     CUDNN_CALL((*bwd_data_algo_discoverer)(s->dnn_handle_,
                                            filter_desc_,
                                            in_desc_,