From b759cc39786cbb680eb604899c53861e5158a117 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Mon, 17 Feb 2020 13:38:55 -0800 Subject: [PATCH 1/8] Added CPU fix --- src/operator/spatial_transformer.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index 6c413f884df9..be21ef45068f 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -41,8 +41,8 @@ inline void BilinearSamplingForward(const Tensor &output, DType *out = output.dptr_; const DType *data = input.dptr_; const DType *grid = grid_src.dptr_; - const int o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); - const int i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); + const index_t o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); + const index_t i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); for (index_t n = 0; n < static_cast(o_n); ++n) { for (index_t c = 0; c < static_cast(o_c); ++c) { for (index_t h = 0; h < static_cast(o_h); ++h) { @@ -51,23 +51,23 @@ inline void BilinearSamplingForward(const Tensor &output, const index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; const DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; const DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; - const auto top_left_y = static_cast(std::floor(y_real)); - const auto top_left_x = static_cast(std::floor(x_real)); + const auto top_left_y = static_cast(std::floor(y_real)); + const auto top_left_x = static_cast(std::floor(x_real)); const DType top_left_y_w = 1.0 - (y_real - top_left_y); const DType top_left_x_w = 1.0 - (x_real - top_left_x); - const int data_index = n * i_c * i_h * i_w + c * i_h * i_w + + const index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; - DType top_left_v = 0; - DType top_right_v = 0; - DType bottom_left_v = 0; - DType bottom_right_v = 0; - if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + index_t top_left_v = 0; + index_t top_right_v = 0; + index_t bottom_left_v = 0; + index_t bottom_right_v = 0; + if (between(top_left_x, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) top_left_v = *(data + data_index); - if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + if (between(top_left_x + 1, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) top_right_v = *(data + data_index + 1); - if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + if (between(top_left_x, 0L, i_w-1) && between(top_left_y + 1, 0L, i_h-1)) bottom_left_v = *(data + data_index + i_w); - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + if (between(top_left_x+1, 0L, i_w-1) && between(top_left_y + 1, 0L, i_h-1)) bottom_right_v = *(data + data_index + i_w + 1); *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + From 1f795ac726bb62c89d7e76d088815353b87988d0 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Mon, 17 Feb 2020 14:26:45 -0800 Subject: [PATCH 2/8] Added fix for backward on CPU --- src/operator/spatial_transformer.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index be21ef45068f..ebcb050e584d 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -88,9 +88,9 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, DType *grid_src = grid_src_data.dptr_; const DType *grad = output_grad.dptr_; const DType *data = input_data.dptr_; - const int o_n = output_grad.size(0), o_c = output_grad.size(1), + const index_t o_n = output_grad.size(0), o_c = output_grad.size(1), o_h = output_grad.size(2), o_w = output_grad.size(3); - const int i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); + const index_t i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); for (index_t n = 0; n < static_cast(o_n); ++n) { for (index_t h = 0; h < static_cast(o_h); ++h) { for (index_t w = 0; w < static_cast(o_w); ++w) { @@ -99,34 +99,34 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, const index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; const DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; const DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; - const auto top_left_y = static_cast(std::floor(y_real)); - const auto top_left_x = static_cast(std::floor(x_real)); + const auto top_left_y = static_cast(std::floor(y_real)); + const auto top_left_x = static_cast(std::floor(x_real)); const DType top_left_y_w = 1.0 - (y_real - top_left_y); const DType top_left_x_w = 1.0 - (x_real - top_left_x); for (index_t c = 0; c < static_cast(o_c); ++c) { index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; - const int data_index = n * i_c * i_h * i_w + c * i_h * i_w + + const index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; // calc 4 vertex value in input data - DType top_left_v = 0; - DType top_right_v = 0; - DType bottom_left_v = 0; - DType bottom_right_v = 0; - if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + index_t top_left_v = 0; + index_t top_right_v = 0; + index_t bottom_left_v = 0; + index_t bottom_right_v = 0; + if (between(top_left_x, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) { *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; top_left_v = *(data + data_index); } - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + if (between(top_left_x+1, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) { *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w * (1.0 - top_left_x_w); top_right_v = *(data + data_index + 1); } - if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + if (between(top_left_x, 0L, i_w-1) && between(top_left_y+1, 0L, i_h-1)) { *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) * top_left_x_w; bottom_left_v = *(data + data_index + i_w); } - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + if (between(top_left_x+1, 0L, i_w-1) && between(top_left_y+1, 0L, i_h-1)) { *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); bottom_right_v = *(data + data_index + i_w + 1); From 195c801b487e3858544392723645e168d728c943 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Mon, 17 Feb 2020 15:49:19 -0800 Subject: [PATCH 3/8] Fixed lint error --- src/operator/spatial_transformer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index ebcb050e584d..e379e86af600 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -41,7 +41,8 @@ inline void BilinearSamplingForward(const Tensor &output, DType *out = output.dptr_; const DType *data = input.dptr_; const DType *grid = grid_src.dptr_; - const index_t o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); + const index_t o_n = output.size(0), o_c = output.size(1), + o_h = output.size(2), o_w = output.size(3); const index_t i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); for (index_t n = 0; n < static_cast(o_n); ++n) { for (index_t c = 0; c < static_cast(o_c); ++c) { From f8f73381029b078ee2383f231e24a9b38fe4fe06 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Mon, 17 Feb 2020 16:10:17 -0800 Subject: [PATCH 4/8] index_t for lower bound instead of hardcoded long int --- src/operator/spatial_transformer.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index e379e86af600..979a89f6cba7 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -62,13 +62,14 @@ inline void BilinearSamplingForward(const Tensor &output, index_t top_right_v = 0; index_t bottom_left_v = 0; index_t bottom_right_v = 0; - if (between(top_left_x, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) + index_t lower_bound = 0; + if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) top_left_v = *(data + data_index); - if (between(top_left_x + 1, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) + if (between(top_left_x + 1, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) top_right_v = *(data + data_index + 1); - if (between(top_left_x, 0L, i_w-1) && between(top_left_y + 1, 0L, i_h-1)) + if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y + 1, lower_bound, i_h-1)) bottom_left_v = *(data + data_index + i_w); - if (between(top_left_x+1, 0L, i_w-1) && between(top_left_y + 1, 0L, i_h-1)) + if (between(top_left_x+1, lower_bound, i_w-1) && between(top_left_y + 1, lower_bound, i_h-1)) bottom_right_v = *(data + data_index + i_w + 1); *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + @@ -113,21 +114,22 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, index_t top_right_v = 0; index_t bottom_left_v = 0; index_t bottom_right_v = 0; - if (between(top_left_x, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) { + index_t lower_bound = 0; + if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) { *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; top_left_v = *(data + data_index); } - if (between(top_left_x+1, 0L, i_w-1) && between(top_left_y, 0L, i_h-1)) { + if (between(top_left_x+1, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) { *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w * (1.0 - top_left_x_w); top_right_v = *(data + data_index + 1); } - if (between(top_left_x, 0L, i_w-1) && between(top_left_y+1, 0L, i_h-1)) { + if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y+1, lower_bound, i_h-1)) { *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) * top_left_x_w; bottom_left_v = *(data + data_index + i_w); } - if (between(top_left_x+1, 0L, i_w-1) && between(top_left_y+1, 0L, i_h-1)) { + if (between(top_left_x+1, lower_bound, i_w-1) && between(top_left_y+1, lower_bound, i_h-1)) { *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); bottom_right_v = *(data + data_index + i_w + 1); From 3be467d75b9a4c9ab8173eaa2ccf2f094ad168aa Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Mon, 17 Feb 2020 16:16:51 -0800 Subject: [PATCH 5/8] Fixed remaining lint errors --- src/operator/spatial_transformer.cc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index 979a89f6cba7..cd47f2e58eb4 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -63,13 +63,17 @@ inline void BilinearSamplingForward(const Tensor &output, index_t bottom_left_v = 0; index_t bottom_right_v = 0; index_t lower_bound = 0; - if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) + if (between(top_left_x, lower_bound, i_w-1) && + between(top_left_y, lower_bound, i_h-1)) top_left_v = *(data + data_index); - if (between(top_left_x + 1, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) + if (between(top_left_x + 1, lower_bound, i_w-1) && + between(top_left_y, lower_bound, i_h-1)) top_right_v = *(data + data_index + 1); - if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y + 1, lower_bound, i_h-1)) + if (between(top_left_x, lower_bound, i_w-1) && + between(top_left_y + 1, lower_bound, i_h-1)) bottom_left_v = *(data + data_index + i_w); - if (between(top_left_x+1, lower_bound, i_w-1) && between(top_left_y + 1, lower_bound, i_h-1)) + if (between(top_left_x+1, lower_bound, i_w-1) && + between(top_left_y + 1, lower_bound, i_h-1)) bottom_right_v = *(data + data_index + i_w + 1); *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + @@ -115,21 +119,25 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, index_t bottom_left_v = 0; index_t bottom_right_v = 0; index_t lower_bound = 0; - if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) { + if (between(top_left_x, lower_bound, i_w-1) && + between(top_left_y, lower_bound, i_h-1)) { *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; top_left_v = *(data + data_index); } - if (between(top_left_x+1, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) { + if (between(top_left_x+1, lower_bound, i_w-1) && + between(top_left_y, lower_bound, i_h-1)) { *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w * (1.0 - top_left_x_w); top_right_v = *(data + data_index + 1); } - if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y+1, lower_bound, i_h-1)) { + if (between(top_left_x, lower_bound, i_w-1) && + between(top_left_y+1, lower_bound, i_h-1)) { *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) * top_left_x_w; bottom_left_v = *(data + data_index + i_w); } - if (between(top_left_x+1, lower_bound, i_w-1) && between(top_left_y+1, lower_bound, i_h-1)) { + if (between(top_left_x+1, lower_bound, i_w-1) && + between(top_left_y+1, lower_bound, i_h-1)) { *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); bottom_right_v = *(data + data_index + i_w + 1); From 1fc5f7f15b1fe8973b8b1215a57d3db8980de5ab Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Mon, 17 Feb 2020 16:26:39 -0800 Subject: [PATCH 6/8] Removed trailing whitespace --- src/operator/spatial_transformer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index cd47f2e58eb4..ba3e349bbedc 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -66,7 +66,7 @@ inline void BilinearSamplingForward(const Tensor &output, if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) top_left_v = *(data + data_index); - if (between(top_left_x + 1, lower_bound, i_w-1) && + if (between(top_left_x + 1, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) top_right_v = *(data + data_index + 1); if (between(top_left_x, lower_bound, i_w-1) && From 3fcfd6af4b74861a48161546cad64d6c6a9572bc Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Tue, 18 Feb 2020 14:32:29 -0800 Subject: [PATCH 7/8] Reverting to DType for vertices --- src/operator/spatial_transformer.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index ba3e349bbedc..8dde5268a3b2 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -58,10 +58,10 @@ inline void BilinearSamplingForward(const Tensor &output, const DType top_left_x_w = 1.0 - (x_real - top_left_x); const index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; - index_t top_left_v = 0; - index_t top_right_v = 0; - index_t bottom_left_v = 0; - index_t bottom_right_v = 0; + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; index_t lower_bound = 0; if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) @@ -114,10 +114,10 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, const index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; // calc 4 vertex value in input data - index_t top_left_v = 0; - index_t top_right_v = 0; - index_t bottom_left_v = 0; - index_t bottom_right_v = 0; + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; index_t lower_bound = 0; if (between(top_left_x, lower_bound, i_w-1) && between(top_left_y, lower_bound, i_h-1)) { From 5ac85a6e90da4bc0f10505aa6a89b2b83473e174 Mon Sep 17 00:00:00 2001 From: Connor Goggins Date: Thu, 20 Feb 2020 13:06:43 -0800 Subject: [PATCH 8/8] Added nightly test for SpatialTransformer --- tests/nightly/test_large_array.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/nightly/test_large_array.py b/tests/nightly/test_large_array.py index 8b36d09cbaf8..0dfeda47385f 100644 --- a/tests/nightly/test_large_array.py +++ b/tests/nightly/test_large_array.py @@ -468,6 +468,7 @@ def check_col2im(): assert res.shape[2] == 2 assert res.shape[3] == 2 assert res.shape[4] == 1 + def check_embedding(): data = nd.random_normal(shape=(LARGE_TENSOR_SHAPE, 1)) weight = nd.random_normal(shape=(LARGE_TENSOR_SHAPE, 1)) @@ -479,6 +480,21 @@ def check_embedding(): assert out.shape[0] == LARGE_TENSOR_SHAPE assert out.shape[1] == 1 assert out.shape[2] == 1 + + def check_spatial_transformer(): + data = nd.random_normal(shape=(2, 2**29, 1, 6)) + loc = nd.random_normal(shape=(2, 6)) + transform_type = 'affine' + sampler_type = 'bilinear' + target_shape = (2, 6) + + res = nd.SpatialTransformer(data=data, loc=loc, transform_type=transform_type, + sampler_type=sampler_type, target_shape=target_shape) + + assert res.shape[0] == 2 + assert res.shape[1] == 536870912 + assert res.shape[2] == 2 + assert res.shape[3] == 6 check_gluon_embedding() check_fully_connected() @@ -501,6 +517,7 @@ def check_embedding(): check_instance_norm() check_col2im() check_embedding() + check_spatial_transformer() def test_tensor():