From bd2b5bf235b3751a96e7a00ca1b7693ae923bfb2 Mon Sep 17 00:00:00 2001 From: Przemek Tredak Date: Tue, 7 Apr 2020 15:58:01 -0700 Subject: [PATCH 1/2] Fix ElemwiseSum for more than 4 inputs --- src/operator/tensor/elemwise_sum.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/tensor/elemwise_sum.h b/src/operator/tensor/elemwise_sum.h index e89e9d799903..259c80ddddac 100644 --- a/src/operator/tensor/elemwise_sum.h +++ b/src/operator/tensor/elemwise_sum.h @@ -94,7 +94,7 @@ void ElementWiseSumCompute_(const nnvm::NodeAttrs& attrs, Kernel::Launch(s, out_size, out_dptr, req[0], in_0_dptr); for (size_t i = 1; i < size; ++i) { DType* in_dptr = in_data[i].dptr(); - Kernel::Launch(s, out_size, out_dptr, req[0], out_dptr, in_dptr); + Kernel::Launch(s, out_size, out_dptr, kWriteTo, out_dptr, in_dptr); } break; } From 942b0cc5d05d6acc99b825e948107eb91aecd833 Mon Sep 17 00:00:00 2001 From: Przemek Tredak Date: Thu, 9 Apr 2020 13:22:56 -0700 Subject: [PATCH 2/2] Added test --- tests/python/unittest/test_operator.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 6cbbc5dd0509..df4a77f66ea4 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -9875,6 +9875,25 @@ def test_grad(input_shape, kernel, stride=1, dilate=1, pad=0): pad = 1 ) +def test_elemwise_sum_for_gradient_accumulation(): + for nrepeat in range(1, 10): + stored_grad = dict() + for grad_req in ['write', 'add']: + a = mx.nd.array([1]) + b = mx.nd.array([2]) + if grad_req == 'write': + a.attach_grad(grad_req='write') + elif grad_req == 'add': + a.attach_grad(grad_req='add') + a.grad[:] = 0 + with mx.autograd.record(): + for _ in range(nrepeat): + b = b * a + b.backward() + stored_grad[grad_req] = a.grad.asscalar() + assert stored_grad['write'] == stored_grad['add'] + assert stored_grad['write'] == 2 * nrepeat + if __name__ == '__main__': import nose