diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py
index 5d15b27fa7ea..3a56dc887647 100644
--- a/tests/python/unittest/test_gluon.py
+++ b/tests/python/unittest/test_gluon.py
@@ -1672,7 +1672,7 @@ def test(net, x):
 
     assert_almost_equal(y1.asnumpy(), y2.asnumpy(), rtol=1e-3, atol=1e-5)
     for key in grads1:
-        assert_almost_equal(grads1[key].asnumpy(), grads2[key].asnumpy(), rtol=1e-3, atol=1e-5)
+        assert_almost_equal(grads1[key].asnumpy(), grads2[key].asnumpy(), rtol=1e-3, atol=1e-4)
 
 @with_seed()
 def test_hybrid_static_memory():