Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[Flaky Test] Fix test_gluon_model_zoo.test_models when MXNET_MKLDNN_DEBUG=1 #12069

Merged
merged 9 commits into from
Aug 14, 2018
Merged

[Flaky Test] Fix test_gluon_model_zoo.test_models when MXNET_MKLDNN_DEBUG=1 #12069

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d43e9c4
reorder inputs
azai91 Aug 7, 2018
9cf9fec
use function flatten vs build in method
azai91 Aug 7, 2018
8c797b3
update similar array atoi to 0.01
azai91 Aug 7, 2018
92c268b
fix reorder
azai91 Aug 7, 2018
db6f1cd
enable MXNET_MKLDNN_DEBUG in CI
azai91 Aug 8, 2018
d0a68c6
add exclude debug flag
azai91 Aug 9, 2018
eb04161
fix lint
azai91 Aug 9, 2018
4aaf82f
add warning log for excluded op
azai91 Aug 12, 2018
3e635d7
retrigger
azai91 Aug 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 7 additions & 21 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,9 +527,7 @@ sanity_check() {
unittest_ubuntu_python2_cpu() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train
Expand All @@ -539,9 +537,7 @@ unittest_ubuntu_python2_cpu() {
unittest_ubuntu_python3_cpu() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
Expand All @@ -550,9 +546,7 @@ unittest_ubuntu_python3_cpu() {
unittest_ubuntu_python3_cpu_mkldnn() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl
Expand All @@ -561,9 +555,7 @@ unittest_ubuntu_python3_cpu_mkldnn() {
unittest_ubuntu_python2_gpu() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
}
Expand Down Expand Up @@ -595,9 +587,7 @@ tutorialtest_ubuntu_python2_gpu() {
unittest_ubuntu_python3_gpu() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
}
Expand All @@ -615,9 +605,7 @@ unittest_ubuntu_python3_gpu_nocudnn() {
unittest_ubuntu_python2_quantization_gpu() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu
}
Expand All @@ -627,9 +615,7 @@ unittest_ubuntu_python2_quantization_gpu() {
unittest_ubuntu_python3_quantization_gpu() {
set -ex
export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu
}
Expand Down
2 changes: 1 addition & 1 deletion python/mxnet/gluon/nn/basic_layers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ def __init__(self, **kwargs):
super(Flatten, self).__init__(**kwargs)

def hybrid_forward(self, F, x):
return x.reshape((0, -1))
return F.Flatten(x)

def __repr__(self):
return self.__class__.__name__
Expand Down
2 changes: 2 additions & 0 deletions src/operator/nn/lrn.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,8 @@ NNVM_REGISTER_OP(_backward_LRN)
.set_attr<nnvm::TIsBackward>("TIsBackward", true)
#if MXNET_USE_MKLDNN == 1
.set_attr<FComputeEx>("FComputeEx<cpu>", LRNGradComputeExCPU)
// Native compute requires norm while MKLDNN does not so cannot be compared in debug mode
.set_attr<bool>("TExcludeMKLDNNDebug", true)
#endif
.set_attr<FCompute>("FCompute<cpu>", LRNGradCompute<cpu>);

Expand Down
12 changes: 8 additions & 4 deletions src/operator/nn/mkldnn/mkldnn_base.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,9 +473,11 @@ void OpCheck::Init(const std::vector<mxnet::NDArray> &inputs_,
auto ctx = inputs_[0].ctx();
CHECK(!MKLDNNStream::Get()->HasOps());
for (size_t i = 0; i < inputs_.size(); i++) {
inputs.emplace_back(inputs_[i].shape(), ctx,
false, inputs_[i].dtype());
auto mem = inputs_[i].GetMKLDNNData();
NDArray data = inputs_[i];
inputs.emplace_back(data.shape(), ctx, false, data.dtype());
if (data.IsMKLDNNData() && data.IsView())
data = data.Reorder2Default();
auto mem = data.GetMKLDNNData();
inputs[i].CopyFrom(*mem);
}
for (size_t i = 0; i < outputs_.size(); i++) {
Expand All @@ -494,6 +496,8 @@ void OpCheck::Run(mxnet::FCompute fn, const nnvm::NodeAttrs &attrs,
const std::vector<mxnet::NDArray> &inputs_,
const std::vector<mxnet::OpReqType> &req,
const std::vector<mxnet::NDArray> &outputs_) {
static auto& is_excluded = Op::GetAttr<bool>("TExcludeMKLDNNDebug");
if (is_excluded.get(attrs.op, false)) return;
std::vector<mxnet::TBlob> in_blobs(inputs.size());
for (size_t i = 0; i < in_blobs.size(); i++) in_blobs[i] = inputs[i].data();
std::vector<mxnet::TBlob> out_blobs(outputs.size());
Expand All @@ -509,7 +513,7 @@ void OpCheck::Run(mxnet::FCompute fn, const nnvm::NodeAttrs &attrs,
if (req[i] == kNullOp)
continue;
MSHADOW_TYPE_SWITCH(outputs[i].dtype(), DType, {
bool similar = SimilarArray<DType>(outputs[i], outputs_[i], 1e-3, 1e-3);
bool similar = SimilarArray<DType>(outputs[i], outputs_[i], 1e-2, 1e-2);
if (!similar) {
LOG(ERROR) << attrs.op->name << " fails";
}
Expand Down