Arm backend: Fix sigmoid int16 and int32 flakyness

MLBEDSW-10642 has now been resolved. Step vela pin and remove all
related pytest.mark_flaky in sigmoid int16 and int32 tests. Also updates
dw_conv2d tests as new constraints were introduced in vela compilation.

Signed-off-by: Oscar Andersson <[email protected]>
Change-Id: I8c1aff31fb173fa6fdebc1d5b757623f9c2e1321

Author: oscarandersson8218

backends/arm/test/ops/test_depthwise_conv.py

@@ -202,8 +202,8 @@ def test_convolution_2d_tosa_BI_depth_wise(test_module: torch.nn.Module):
 
 
 x_fails = {
-    "3x3_2x8x198x198_gp8_st3": "MLETORCH-516: AssertionError: Output 0 does not match reference output.",
-    "two_dw_conv2d": "MLETORCH-516: AssertionError: Output 0 does not match reference output.",
+    "3x3_2x8x198x198_gp8_st3": "MLBEDSW-10752: Vela issues with batches > 1 for dw_conv",
+    "two_dw_conv2d": "MLBEDSW-10752: Vela issues with batches > 1 for dw_conv",
 }
 
 

backends/arm/test/ops/test_sigmoid_16bit.py

@@ -4,7 +4,6 @@
 # LICENSE file in the root directory of this source tree.
 
 import pytest
-
 import torch
 from executorch.backends.arm.quantizer import (
     get_symmetric_quantization_config,
@@ -91,10 +90,13 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+        Sigmoid(),
+        (test_data(),),
+        Sigmoid.aten_op,
+        Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer())
     pipeline.run()
@@ -108,13 +110,13 @@ def test_sigmoid_tosa_BI(test_data):
     },
     strict=False,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
-def test_sigmoid_tosa_BI_add_sigmoid(test_data):
+def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.run()
 
@@ -131,8 +133,7 @@ def test_sigmoid_tosa_BI_add_sigmoid(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
-def test_sigmoid_u55_BI(test_data):
+def test_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(),
         (test_data(),),
@@ -148,8 +149,7 @@ def test_sigmoid_u55_BI(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
-def test_sigmoid_u55_BI_add_sigmoid(test_data):
+def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
         (test_data(),),
@@ -163,7 +163,6 @@ def test_sigmoid_u55_BI_add_sigmoid(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_u85_BI(test_data):
     pipeline = EthosU85PipelineBI(
@@ -181,10 +180,10 @@ def test_sigmoid_u85_BI(test_data):
     "test_data",
     test_data_suite,
     xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
+        "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
+@pytest.mark.flaky(reruns=5)  # MLETORCH-787: Investigate int16-int8 rescaling precision
 @common.XfailIfNoCorstone320
 def test_sigmoid_u85_BI_add_sigmoid(test_data):
     pipeline = EthosU85PipelineBI(

backends/arm/test/ops/test_sigmoid_32bit.py

@@ -3,7 +3,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import pytest
 import torch
 from executorch.backends.arm.quantizer import TOSAQuantizer
 from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -107,34 +106,33 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         Sigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer())
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
-def test_sigmoid_tosa_BI_add_sigmoid(test_data):
+def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer())
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
-def test_sigmoid_u55_BI(test_data):
+def test_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(),
         (test_data(),),
@@ -147,8 +145,7 @@ def test_sigmoid_u55_BI(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
-def test_sigmoid_u55_BI_add_sigmoid(test_data):
+def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
         (test_data(),),
@@ -162,9 +159,7 @@ def test_sigmoid_u55_BI_add_sigmoid(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
-@pytest.mark.flaky(reruns=5)
 def test_sigmoid_u85_BI(test_data):
     pipeline = EthosU85PipelineBI(
         Sigmoid(),
@@ -180,15 +175,8 @@ def test_sigmoid_u85_BI(test_data):
 @common.parametrize(
     "test_data",
     test_data_suite,
-    xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
-        "rand": "AssertionError: Output 0 does not match reference output.",
-        "rand_4d": "AssertionError: Output 0 does not match reference output.",
-    },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
-@pytest.mark.flaky(reruns=5)
 def test_sigmoid_u85_BI_add_sigmoid(test_data):
     pipeline = EthosU85PipelineBI(
         SigmoidAddSigmoid(),

examples/arm/setup.sh

@@ -60,7 +60,7 @@ fi
 
 # vela
 vela_repo_url="https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela"
-vela_rev="859cc066178a87ff28230c1ce9bd370f1e98aa5a"
+vela_rev="677651add50745478367da009936e4505d627399"
 
 ########
 ### Functions