From 94179ad1f6d8f34677d7db3bd8d69f27789ffdc1 Mon Sep 17 00:00:00 2001 From: Josh Fromm Date: Wed, 17 Mar 2021 13:47:52 -0700 Subject: [PATCH 1/2] Relax simulated qnn tests to prevent flakiness. --- .../relay/test_op_qnn_simulated_dequantize.py | 10 +++++----- .../relay/test_op_qnn_simulated_quantize.py | 17 ++++++++++++----- tests/python/topi/python/test_topi_qnn.py | 8 ++++++-- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/python/relay/test_op_qnn_simulated_dequantize.py b/tests/python/relay/test_op_qnn_simulated_dequantize.py index 0cc04e4998eb..a9333c916561 100644 --- a/tests/python/relay/test_op_qnn_simulated_dequantize.py +++ b/tests/python/relay/test_op_qnn_simulated_dequantize.py @@ -81,7 +81,7 @@ def verify_simulated_dequantize_simple(dtype): dtype = relay.var("dtype", shape=[]) vm = build_simulated_dequantize(input_data, scale, zp, dtype) sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out) + np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5) def test_simulated_dequantize(): @@ -112,7 +112,7 @@ def test_dynamic_channels(): dtype = relay.var("dtype", shape=[]) vm = build_simulated_dequantize(input_data, scale, zp, dtype, axis=0) sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out) + np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5) # Now get the perchannel quantize output and compare without recompiling. scale_np = np.array([0.5, 0.25]).astype("float32") @@ -128,7 +128,7 @@ def test_dynamic_channels(): ) # Run the simulated quantize without recompiling and confirm results match. sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out) + np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5) def test_dynamic_dtype(): @@ -153,7 +153,7 @@ def test_dynamic_dtype(): dtype = relay.var("dtype", shape=[]) vm = build_simulated_dequantize(input_data, scale, zp, dtype) sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out) + np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5) # Now test int8 to float32 compilation. data = np.random.uniform(low=0, high=255, size=[2, 5]).astype("int8") @@ -168,7 +168,7 @@ def test_dynamic_dtype(): # Run the simulated quantize without recompiling and confirm results match. dtype_np = np.int32(SQNN_DTYPE_TO_CODE["int8"]) sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out) + np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5) if __name__ == "__main__": diff --git a/tests/python/relay/test_op_qnn_simulated_quantize.py b/tests/python/relay/test_op_qnn_simulated_quantize.py index ee4ba209dcb8..a50438c1bab7 100644 --- a/tests/python/relay/test_op_qnn_simulated_quantize.py +++ b/tests/python/relay/test_op_qnn_simulated_quantize.py @@ -24,6 +24,13 @@ from tvm.topi.nn.qnn import SQNN_DTYPE_TO_CODE +def test_allclose_with_rounding(a, b): + # Find number of mismatches in inputs. + mismatch = a != b + # Allow some rounding errors due to GPU fp32 arithmetic. + assert np.sum(mismatch) <= 3 + + def quantize_test_driver(in_dtype, quant_args, axis, out_dtype, in_data): shape = in_data.shape input_data = relay.var("input_data", shape=shape, dtype=in_dtype) @@ -82,7 +89,7 @@ def verify_simulated_quantize_simple(dtype): dtype = relay.var("dtype", shape=[]) vm = build_simulated_quantize(input_data, scale, zp, dtype) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_q_out.asnumpy(), q_out) + test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) def test_simulated_quantize(): @@ -113,7 +120,7 @@ def test_dynamic_channels(): dtype = relay.var("dtype", shape=[]) vm = build_simulated_quantize(input_data, scale, zp, dtype, axis=0) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_q_out.asnumpy(), q_out) + test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) # Now get the perchannel quantize output and compare without recompiling. scale_np = np.array([0.5, 0.25]).astype("float32") @@ -130,7 +137,7 @@ def test_dynamic_channels(): ) # Run the simulated quantize without recompiling and confirm results match. sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_q_out.asnumpy(), q_out) + test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) def test_dynamic_dtype(): @@ -155,7 +162,7 @@ def test_dynamic_dtype(): dtype = relay.var("dtype", shape=[]) vm = build_simulated_quantize(input_data, scale, zp, dtype) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_q_out.asnumpy(), q_out) + test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) # Now test float32 to int32 compilation. # Get the reference quantize output. @@ -169,7 +176,7 @@ def test_dynamic_dtype(): # Run the simulated quantize without recompiling and confirm results match. dtype_np = np.int32(SQNN_DTYPE_TO_CODE["int32"]) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - np.testing.assert_equal(sim_q_out.asnumpy(), q_out) + test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) if __name__ == "__main__": diff --git a/tests/python/topi/python/test_topi_qnn.py b/tests/python/topi/python/test_topi_qnn.py index a63f34fe08d0..386f77335f1a 100644 --- a/tests/python/topi/python/test_topi_qnn.py +++ b/tests/python/topi/python/test_topi_qnn.py @@ -72,7 +72,9 @@ def check_device(device, ctx): func(a, d, s, z, q) # Check correctness against the true qnn output. - tvm.testing.assert_allclose(q.asnumpy(), real_q_out.asnumpy().astype("float32")) + mismatch = q.asnumpy() != real_q_out.asnumpy().astype("float32") + # Allow some rounding errors due to GPU fp32 arithmetic. + assert np.sum(mismatch) <= 3 for target, ctx in tvm.testing.enabled_targets(): check_device(target, ctx) @@ -137,7 +139,9 @@ def check_device(device, ctx): func(a, d, s, z, dq) # Check correctness against the true qnn output. - tvm.testing.assert_allclose(dq.asnumpy(), real_dq_out.asnumpy().astype("float32")) + tvm.testing.assert_allclose( + dq.asnumpy(), real_dq_out.asnumpy().astype("float32"), rtol=1e-5 + ) for target, ctx in tvm.testing.enabled_targets(): check_device(target, ctx) From de0b9ce5673b24389547d766e614de2fa2aa8267 Mon Sep 17 00:00:00 2001 From: Josh Fromm Date: Wed, 17 Mar 2021 17:20:14 -0700 Subject: [PATCH 2/2] Change name of helper to make pytest happy. --- tests/python/relay/test_op_qnn_simulated_quantize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python/relay/test_op_qnn_simulated_quantize.py b/tests/python/relay/test_op_qnn_simulated_quantize.py index a50438c1bab7..c0fa0648d879 100644 --- a/tests/python/relay/test_op_qnn_simulated_quantize.py +++ b/tests/python/relay/test_op_qnn_simulated_quantize.py @@ -24,7 +24,7 @@ from tvm.topi.nn.qnn import SQNN_DTYPE_TO_CODE -def test_allclose_with_rounding(a, b): +def allclose_with_rounding(a, b): # Find number of mismatches in inputs. mismatch = a != b # Allow some rounding errors due to GPU fp32 arithmetic. @@ -89,7 +89,7 @@ def verify_simulated_quantize_simple(dtype): dtype = relay.var("dtype", shape=[]) vm = build_simulated_quantize(input_data, scale, zp, dtype) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) + allclose_with_rounding(sim_q_out.asnumpy(), q_out) def test_simulated_quantize(): @@ -120,7 +120,7 @@ def test_dynamic_channels(): dtype = relay.var("dtype", shape=[]) vm = build_simulated_quantize(input_data, scale, zp, dtype, axis=0) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) + allclose_with_rounding(sim_q_out.asnumpy(), q_out) # Now get the perchannel quantize output and compare without recompiling. scale_np = np.array([0.5, 0.25]).astype("float32") @@ -137,7 +137,7 @@ def test_dynamic_channels(): ) # Run the simulated quantize without recompiling and confirm results match. sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) + allclose_with_rounding(sim_q_out.asnumpy(), q_out) def test_dynamic_dtype(): @@ -162,7 +162,7 @@ def test_dynamic_dtype(): dtype = relay.var("dtype", shape=[]) vm = build_simulated_quantize(input_data, scale, zp, dtype) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) + allclose_with_rounding(sim_q_out.asnumpy(), q_out) # Now test float32 to int32 compilation. # Get the reference quantize output. @@ -176,7 +176,7 @@ def test_dynamic_dtype(): # Run the simulated quantize without recompiling and confirm results match. dtype_np = np.int32(SQNN_DTYPE_TO_CODE["int32"]) sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np) - test_allclose_with_rounding(sim_q_out.asnumpy(), q_out) + allclose_with_rounding(sim_q_out.asnumpy(), q_out) if __name__ == "__main__":