From 1b5d96ab09634c58964c82e79850432990a8267e Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Fri, 19 Jul 2019 23:53:24 +0530 Subject: [PATCH 1/7] n-th order grad test support --- .../python/unittest/test_higher_order_grad.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 0f07d014d435..1b0cfe09c022 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -30,10 +30,16 @@ def sin(x): def grad_grad_op(x): return -nd.sin(x) + def grad_grad_grad_op(x): + return -nd.cos(x) + for dim in range(1, 5): shape = rand_shape_nd(dim) array = random_arrays(shape) check_second_order_unary(array, sin, grad_grad_op) + # TODO(kshitij12345): Remove + check_nth_order_unary(array, sin, + [grad_grad_op, grad_grad_grad_op], [2, 3]) @with_seed() @@ -44,10 +50,16 @@ def cos(x): def grad_grad_op(x): return -nd.cos(x) + def grad_grad_grad_op(x): + return nd.sin(x) + for dim in range(1, 5): shape = rand_shape_nd(dim) array = random_arrays(shape) check_second_order_unary(array, cos, grad_grad_op) + # TODO(kshitij12345): Remove + check_nth_order_unary(array, cos, + [grad_grad_op, grad_grad_grad_op], [2, 3]) @with_seed() @@ -69,6 +81,9 @@ def test_log(): def log(x): return nd.log(x) + def grad_op(x): + return 1/x + def grad_grad_op(x): return -1/(x**2) @@ -76,6 +91,8 @@ def grad_grad_op(x): shape = rand_shape_nd(dim) array = random_arrays(shape) check_second_order_unary(array, log, grad_grad_op) + # TODO(kshitij12345): Remove + check_nth_order_unary(array, log, [grad_op, grad_grad_op], [1, 2]) @with_seed() @@ -148,6 +165,9 @@ def grad_grad_op(x): shape = rand_shape_nd(dim) array = random_arrays(shape) check_second_order_unary(array, sigmoid, grad_grad_op) + # TODO(kshitij12345): Remove + check_nth_order_unary(array, sigmoid, [grad_op, grad_grad_op], [1, 2]) + check_nth_order_unary(array, sigmoid, grad_grad_op, 2) def check_second_order_unary(x, op, grad_grad_op): @@ -174,6 +194,41 @@ def check_second_order_unary(x, op, grad_grad_op): assert_almost_equal(expected_grad_grad, x.grad.asnumpy()) +def check_nth_order_unary(x, op, grad_ops, orders): + if isinstance(orders, int): + orders = [orders] + grad_ops = [grad_ops] + + x = nd.array(x) + x.attach_grad() + + order = max(orders) + expected_grads = [grad_op(x) for grad_op in grad_ops] + computed_grads = [] + head_grads = [] + + # Perform compute. + with autograd.record(): + y = op(x) + for current_order in range(1, order+1): + head_grad = nd.random.normal(shape=x.shape) + y = autograd.grad(heads=y, variables=x, head_grads=head_grad, + create_graph=True, retain_graph=True)[0] + if current_order in orders: + computed_grads.append(y) + head_grads.append(head_grad) + + # Validate all the gradients. + for order, grad, computed_grad in \ + zip(orders, expected_grads, computed_grads): + # Compute expected values. + expected_grad = grad.asnumpy() + for head_grad in head_grads[:order]: + expected_grad *= head_grad.asnumpy() + + assert_almost_equal(expected_grad, computed_grad.asnumpy()) + + if __name__ == '__main__': import nose nose.runmodule() From 1f74614391e182e299e2fdcce1036515c4e5fb4f Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Sat, 20 Jul 2019 00:24:42 +0530 Subject: [PATCH 2/7] use check_nth_order_unary for second order check --- .../python/unittest/test_higher_order_grad.py | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 1b0cfe09c022..43c5317707a1 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -171,27 +171,7 @@ def grad_grad_op(x): def check_second_order_unary(x, op, grad_grad_op): - x = nd.array(x) - grad_grad_x = grad_grad_op(x) - x.attach_grad() - - # Manual head_grads. - y_grad = nd.random.normal(shape=x.shape) - head_grad_grads = nd.random.normal(shape=x.shape) - - # Perform compute. - with autograd.record(): - y = op(x) - x_grad = autograd.grad(heads=y, variables=x, head_grads=y_grad, - create_graph=True, retain_graph=True)[0] - x_grad.backward(head_grad_grads) - - # Compute expected values. - expected_grad_grad = grad_grad_x.asnumpy() * head_grad_grads.asnumpy() * \ - y_grad.asnumpy() - - # Validate the gradients. - assert_almost_equal(expected_grad_grad, x.grad.asnumpy()) + check_nth_order_unary(x, op, grad_grad_op, 2) def check_nth_order_unary(x, op, grad_ops, orders): From 4e5c4ccc5d68ef7147fd80be4e8014a24f034329 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Wed, 24 Jul 2019 00:16:48 +0530 Subject: [PATCH 3/7] add docstring to check_nth_order_unary --- .../python/unittest/test_higher_order_grad.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 43c5317707a1..bbe2d3958001 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -175,6 +175,35 @@ def check_second_order_unary(x, op, grad_grad_op): def check_nth_order_unary(x, op, grad_ops, orders): + """Assert n-th order autograd gradient against expected gradient. + + Multiple order of gradients can be checked by passing list of + function computing the particular order gradient and passing the + corresponding list of order. + + Note + ---- + 1. Orders should always be monotonically increasing. + 2. Elements of grads_ops should correspond to elements of orders + i.e. grads_op = [grad_op, grad_grad_grad_op] should be passed with + orders = [1, 3] + + Parameters + ---------- + x : mxnet.NDArray + Input Array. + op : Callable + Operation to perform on Input Array. + grad_ops : Callable or List of Callable + Function to compute and assert gradient of given order. + orders : int or List of int + Order/s to assert expected and computed gradients. + + Returns + ------- + None + + """ if isinstance(orders, int): orders = [orders] grad_ops = [grad_ops] From f7cd88533c783af2d7de2bd97f699a3a1d8361e6 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Wed, 11 Sep 2019 22:51:02 +0530 Subject: [PATCH 4/7] retrigger CI From dccc2e89fb84ce9f3fe8363cb172cb9335a044df Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Sat, 14 Sep 2019 20:30:35 +0530 Subject: [PATCH 5/7] add assertions for requirements of orders --- tests/python/unittest/test_higher_order_grad.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 1b3d40c7b320..2936f6b5bb3f 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -359,10 +359,14 @@ def check_nth_order_unary(x, op, grad_ops, orders, rtol=None, atol=None): orders = [orders] grad_ops = [grad_ops] + assert all(i < j for i, j in zip(orders[0:-1], orders[1:])), \ + "orders should be monotonically increasing" + assert set(orders) == len(orders), "orders should have unique elements" + highest_order = max(orders) + x = nd.array(x) x.attach_grad() - order = max(orders) expected_grads = [grad_op(x) for grad_op in grad_ops] computed_grads = [] head_grads = [] @@ -370,7 +374,7 @@ def check_nth_order_unary(x, op, grad_ops, orders, rtol=None, atol=None): # Perform compute. with autograd.record(): y = op(x) - for current_order in range(1, order+1): + for current_order in range(1, highest_order+1): head_grad = nd.random.normal(shape=x.shape) y = autograd.grad(heads=y, variables=x, head_grads=head_grad, create_graph=True, retain_graph=True)[0] From c1c14d2ddec6f9133f08e5335d1df6f9e2ad6d49 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Sat, 14 Sep 2019 21:12:35 +0530 Subject: [PATCH 6/7] fix assert condition --- tests/python/unittest/test_higher_order_grad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/unittest/test_higher_order_grad.py b/tests/python/unittest/test_higher_order_grad.py index 2936f6b5bb3f..9c758c8467e3 100644 --- a/tests/python/unittest/test_higher_order_grad.py +++ b/tests/python/unittest/test_higher_order_grad.py @@ -361,7 +361,8 @@ def check_nth_order_unary(x, op, grad_ops, orders, rtol=None, atol=None): assert all(i < j for i, j in zip(orders[0:-1], orders[1:])), \ "orders should be monotonically increasing" - assert set(orders) == len(orders), "orders should have unique elements" + assert len(set(orders)) == len(orders), \ + "orders should have unique elements" highest_order = max(orders) x = nd.array(x) From 82174a4f4f67a71651dfcd09779766bbf2f5c2d4 Mon Sep 17 00:00:00 2001 From: kshitij12345 Date: Sun, 15 Sep 2019 00:00:12 +0530 Subject: [PATCH 7/7] retrigger CI