Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Relay][QNN] Relax simulated qnn tests to prevent flakiness. #7684

Merged
merged 2 commits into from
Mar 18, 2021
Merged

[Relay][QNN] Relax simulated qnn tests to prevent flakiness. #7684

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
94179ad
Relax simulated qnn tests to prevent flakiness.
jwfromm Mar 17, 2021
de0b9ce
Change name of helper to make pytest happy.
jwfromm Mar 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions tests/python/relay/test_op_qnn_simulated_dequantize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def verify_simulated_dequantize_simple(dtype):
dtype = relay.var("dtype", shape=[])
vm = build_simulated_dequantize(input_data, scale, zp, dtype)
sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out)
np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5)


def test_simulated_dequantize():
Expand Down Expand Up @@ -112,7 +112,7 @@ def test_dynamic_channels():
dtype = relay.var("dtype", shape=[])
vm = build_simulated_dequantize(input_data, scale, zp, dtype, axis=0)
sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out)
np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5)

# Now get the perchannel quantize output and compare without recompiling.
scale_np = np.array([0.5, 0.25]).astype("float32")
Expand All @@ -128,7 +128,7 @@ def test_dynamic_channels():
)
# Run the simulated quantize without recompiling and confirm results match.
sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out)
np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5)


def test_dynamic_dtype():
Expand All @@ -153,7 +153,7 @@ def test_dynamic_dtype():
dtype = relay.var("dtype", shape=[])
vm = build_simulated_dequantize(input_data, scale, zp, dtype)
sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out)
np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5)

# Now test int8 to float32 compilation.
data = np.random.uniform(low=0, high=255, size=[2, 5]).astype("int8")
Expand All @@ -168,7 +168,7 @@ def test_dynamic_dtype():
# Run the simulated quantize without recompiling and confirm results match.
dtype_np = np.int32(SQNN_DTYPE_TO_CODE["int8"])
sim_dq_out = vm.invoke("main", input_data=data_fp, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_dq_out.asnumpy(), dq_out)
np.testing.assert_allclose(sim_dq_out.asnumpy(), dq_out, rtol=1e-5)


if __name__ == "__main__":
Expand Down
17 changes: 12 additions & 5 deletions tests/python/relay/test_op_qnn_simulated_quantize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@
from tvm.topi.nn.qnn import SQNN_DTYPE_TO_CODE


def allclose_with_rounding(a, b):
# Find number of mismatches in inputs.
mismatch = a != b
# Allow some rounding errors due to GPU fp32 arithmetic.
assert np.sum(mismatch) <= 3


def quantize_test_driver(in_dtype, quant_args, axis, out_dtype, in_data):
shape = in_data.shape
input_data = relay.var("input_data", shape=shape, dtype=in_dtype)
Expand Down Expand Up @@ -82,7 +89,7 @@ def verify_simulated_quantize_simple(dtype):
dtype = relay.var("dtype", shape=[])
vm = build_simulated_quantize(input_data, scale, zp, dtype)
sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_q_out.asnumpy(), q_out)
allclose_with_rounding(sim_q_out.asnumpy(), q_out)


def test_simulated_quantize():
Expand Down Expand Up @@ -113,7 +120,7 @@ def test_dynamic_channels():
dtype = relay.var("dtype", shape=[])
vm = build_simulated_quantize(input_data, scale, zp, dtype, axis=0)
sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_q_out.asnumpy(), q_out)
allclose_with_rounding(sim_q_out.asnumpy(), q_out)

# Now get the perchannel quantize output and compare without recompiling.
scale_np = np.array([0.5, 0.25]).astype("float32")
Expand All @@ -130,7 +137,7 @@ def test_dynamic_channels():
)
# Run the simulated quantize without recompiling and confirm results match.
sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_q_out.asnumpy(), q_out)
allclose_with_rounding(sim_q_out.asnumpy(), q_out)


def test_dynamic_dtype():
Expand All @@ -155,7 +162,7 @@ def test_dynamic_dtype():
dtype = relay.var("dtype", shape=[])
vm = build_simulated_quantize(input_data, scale, zp, dtype)
sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_q_out.asnumpy(), q_out)
allclose_with_rounding(sim_q_out.asnumpy(), q_out)

# Now test float32 to int32 compilation.
# Get the reference quantize output.
Expand All @@ -169,7 +176,7 @@ def test_dynamic_dtype():
# Run the simulated quantize without recompiling and confirm results match.
dtype_np = np.int32(SQNN_DTYPE_TO_CODE["int32"])
sim_q_out = vm.invoke("main", input_data=data, scale=scale_np, zp=zp_np, dtype=dtype_np)
np.testing.assert_equal(sim_q_out.asnumpy(), q_out)
allclose_with_rounding(sim_q_out.asnumpy(), q_out)


if __name__ == "__main__":
Expand Down
8 changes: 6 additions & 2 deletions tests/python/topi/python/test_topi_qnn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ def check_device(device, ctx):
func(a, d, s, z, q)

# Check correctness against the true qnn output.
tvm.testing.assert_allclose(q.asnumpy(), real_q_out.asnumpy().astype("float32"))
mismatch = q.asnumpy() != real_q_out.asnumpy().astype("float32")
# Allow some rounding errors due to GPU fp32 arithmetic.
assert np.sum(mismatch) <= 3

for target, ctx in tvm.testing.enabled_targets():
check_device(target, ctx)
Expand Down Expand Up @@ -137,7 +139,9 @@ def check_device(device, ctx):
func(a, d, s, z, dq)

# Check correctness against the true qnn output.
tvm.testing.assert_allclose(dq.asnumpy(), real_dq_out.asnumpy().astype("float32"))
tvm.testing.assert_allclose(
dq.asnumpy(), real_dq_out.asnumpy().astype("float32"), rtol=1e-5
)

for target, ctx in tvm.testing.enabled_targets():
check_device(target, ctx)
Expand Down