Refine autoscan pass (#37363)

python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt

@@ -72,6 +72,7 @@ set_tests_properties(test_trt_conv3d_op PROPERTIES TIMEOUT 60)
 set_tests_properties(test_trt_conv3d_transpose_op PROPERTIES TIMEOUT 60)
 set_tests_properties(test_trt_nearest_interp_v2_op PROPERTIES TIMEOUT 30)
 set_tests_properties(test_emb_eltwise_layernorm_fuse_pass PROPERTIES TIMEOUT 120)
+set_tests_properties(test_fc_fuse_pass PROPERTIES TIMEOUT 120)
 
 if (WITH_MKLDNN)
   set_tests_properties(test_mkldnn_prelu_op PROPERTIES TIMEOUT 300)

python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py

@@ -31,7 +31,8 @@
 from program_config import TensorConfig, OpConfig, ProgramConfig, create_fake_model, create_quant_model
 
 import hypothesis
-from hypothesis import given, settings, seed, example, assume
+from hypothesis import given, settings, seed, reproduce_failure
+import hypothesis.strategies as st
 
 logging.basicConfig(level=logging.INFO, format="%(message)s")
 
@@ -78,6 +79,11 @@ def __init__(self, *args, **kwargs):
         abs_dir = os.path.abspath(os.path.dirname(__file__))
         self.cache_dir = os.path.join(abs_dir,
                                       str(self.__module__) + '_cache_dir')
+        self.available_passes_in_framework = set()
+        self.num_ran_programs = 0
+        self.num_invalid_programs = 0
+        self.num_skipped_tests = 0
+        self.num_predictor_kinds = 0
 
     @abc.abstractmethod
     def sample_program_configs(self):
@@ -99,9 +105,8 @@ def add_skip_case(
             note: str):
         self.skip_cases.append((teller, reason, note))
 
-    @abc.abstractmethod
     def is_program_valid(self, program_config: ProgramConfig) -> bool:
-        raise NotImplementedError
+        return True
 
     def run_test_config(self, model, params, prog_config, pred_config,
                         feed_data) -> Dict[str, np.ndarray]:
@@ -110,6 +115,8 @@ def run_test_config(self, model, params, prog_config, pred_config,
         '''
         pred_config.set_model_buffer(model, len(model), params, len(params))
         predictor = paddle_infer.create_predictor(pred_config)
+        self.available_passes_in_framework = self.available_passes_in_framework | set(
+            pred_config.pass_builder().all_passes())
 
         for name, _ in prog_config.inputs.items():
             input_tensor = predictor.get_input_handle(name)
@@ -277,39 +284,118 @@ def __init__(self, *args, **kwargs):
     def check_op_version(self):
         status = True
         for pass_name in self.passes:
+            if pass_name not in self.available_passes_in_framework:
+                continue
             if not PassVersionChecker.IsCompatible(pass_name):
                 self.fail_log('{} version check failed.'.format(pass_name))
                 status = False
         return status
 
-    def assert_op_size(self, fusion_before_num, fusion_after_num, origin_model):
+    def add_skip_pass_case(self):
+        return
+
+    def assert_op_list(self, op_list_after_fusion):
         if not self.passes:
             raise ValueError(
-                'In PassAutoScan you should give a valid pass name.')
+                "In PassAutoScan you should give a valid pass name.")
         last_passed_program = os.path.join(self.cache_dir,
-                                           self.passes[-1] + '.pdmodel')
+                                           self.passes[-1] + ".pdmodel")
+        if not os.path.exists(last_passed_program):
+            raise ValueError(
+                "Cannot find file {}, please make sure that your pass name is correct".
+                format(last_passed_program))
         model_bytes = paddle.static.load_from_file(last_passed_program)
         pg = paddle.static.deserialize_program(model_bytes)
         main_block = pg.desc.block(0)
-        after_op_size = main_block.op_size()
-        pg = paddle.static.deserialize_program(origin_model)
-        main_block = pg.desc.block(0)
-        before_op_size = main_block.op_size()
-        self.assertTrue(before_op_size == fusion_before_num,
-                        'before fusion op size is {}, but got {}!'.format(
-                            before_op_size, fusion_before_num))
-        self.assertTrue(after_op_size == fusion_after_num,
-                        'after fusion op size is {}, but got {}!'.format(
-                            after_op_size, fusion_after_num))
+        after_op_list = list()
+        for i in range(main_block.op_size()):
+            if main_block.op(i).type() in ["feed", "fetch"]:
+                continue
+            after_op_list.append(main_block.op(i).type())
+        self.assertTrue(
+            op_list_after_fusion == after_op_list,
+            "Expected operator list after fusion is {}, but now it's {}".format(
+                op_list_after_fusion, after_op_list), )
 
-    def run_test(self, quant=False, *args, **kwargs):
+    def run_and_statis(
+            self,
+            quant=False,
+            max_examples=100,
+            reproduce=None,
+            min_success_num=25,
+            max_duration=180,
+            passes=None, ):
+        if os.getenv('HYPOTHESIS_TEST_PROFILE', 'ci') == "dev":
+            max_examples *= 10
+            min_success_num *= 10
+            # while at ce phase, there's no limit on time
+            max_duration = -1
+        start_time = time.time()
+        settings.register_profile(
+            "ci",
+            max_examples=max_examples,
+            suppress_health_check=hypothesis.HealthCheck.all(),
+            deadline=None,
+            print_blob=True,
+            derandomize=True,
+            report_multiple_bugs=False, )
+        settings.load_profile("ci")
+        assert passes is not None, "Parameter of passes must be defined in function run_and_statis."
+        self.passes = passes
+
+        self.add_skip_pass_case()
+
+        def program_generator(draw):
+            return self.sample_program_config(draw)
+
+        def run_test(prog_config):
+            return self.run_test(quant=quant, prog_configs=[prog_config])
+
+        generator = st.composite(program_generator)
+        loop_func = given(generator())(run_test)
+        if reproduce is not None:
+            loop_func = reproduce(loop_func)
+        logging.info("Start to running test of {}".format(type(self)))
+        loop_func()
+        logging.info(
+            "===================Statistical Information===================")
+        logging.info("Number of Generated Programs: {}".format(
+            self.num_ran_programs + self.num_invalid_programs))
+        logging.info("Number of Invalid Programs: {}".format(
+            self.num_invalid_programs))
+        logging.info("Number of Ran Programs: {}".format(self.num_ran_programs))
+        logging.info("Number of Skipped Tests: {}".format(
+            self.num_skipped_tests))
+        successful_ran_programs = int(self.num_ran_programs -
+                                      self.num_skipped_tests /
+                                      self.num_predictor_kinds)
+        logging.info(
+            "Number of successfully ran programs approximately equal to {}".
+            format(successful_ran_programs))
+        if successful_ran_programs < min_success_num:
+            logging.warning(
+                "satisfied_programs = ran_programs - num_skipped_tests / num_predictor_kinds"
+            )
+            logging.error(
+                "At least {} programs need to ran successfully, but now only about {} programs satisfied.".
+                format(min_success_num, successful_ran_programs))
+            assert False
+        used_time = time.time() - start_time
+        if max_duration > 0 and used_time > max_duration:
+            logging.error(
+                "The duration exceeds {} seconds, if this is neccessary, try to set a larger number for parameter `max_duration`.".
+                format(max_duration))
+            assert False
+
+    def run_test(self, quant=False, prog_configs=None):
         status = True
 
-        for prog_config in self.sample_program_configs(*args, **kwargs):
+        for prog_config in prog_configs:
             # if program is invalid, we should skip that cases.
             if not self.is_program_valid(prog_config):
+                self.num_invalid_programs += 1
                 continue
-
+            self.num_ran_programs += 1
             model, params = create_fake_model(prog_config)
             if quant:
                 model, params = create_quant_model(model, params)
@@ -330,13 +416,16 @@ def run_test(self, quant=False, *args, **kwargs):
                                      feed_data))
             self.success_log('RUN_CPU_BASELINE done')
 
-            for pred_config, nodes_num, (
+            self.num_predictor_kinds = 0
+            for pred_config, op_list, (
                     atol, rtol) in self.sample_predictor_configs(prog_config):
+                self.num_predictor_kinds += 1
                 # skip info
                 skip_flag = False
                 for skip_info in self.skip_cases:
                     if skip_info[0](prog_config, pred_config):
                         skip_flag = True
+                        self.num_skipped_tests += 1
                         if skip_info[1] == SkipReasons.PASS_ACCURACY_ERROR:
                             self.skip_log("[PASS_ACCURACY_ERROR] " + skip_info[
                                 2] + ' ' + ' vs ' + self.inference_config_str(
@@ -357,7 +446,7 @@ def run_test(self, quant=False, *args, **kwargs):
                     self.assert_tensors_near(atol, rtol, results[-1],
                                              results[0])
                     if not skip_flag:
-                        self.assert_op_size(nodes_num[0], nodes_num[1], model)
+                        self.assert_op_list(op_list)
 
                 except Exception as e:
                     self.fail_log(

python/paddle/fluid/tests/unittests/ir/inference/program_config.py

@@ -34,17 +34,24 @@ class TensorConfig:
 
     def __init__(self,
                  lod: Optional[List[List[int]]]=None,
-                 data_gen: Optional[Callable[..., np.array]]=None):
+                 data_gen: Optional[Callable[..., np.array]]=None,
+                 shape: Optional[List[List[int]]]=None):
         '''
         shape: The shape of the tensor.
         dtype: The data type of the tensor.
         data: The value of WeightVar. for input, it should be None 
         '''
         self.lod = lod
-        self.data_gen = data_gen
-        self.data = data_gen()
-        self.dtype = data_gen().dtype
-        self.shape = data_gen().shape
+        if data_gen is not None:
+            self.data_gen = data_gen
+            self.data = data_gen()
+            self.dtype = data_gen().dtype
+            self.shape = data_gen().shape
+        else:
+            assert shape is not None, "While data_gen is not defined, shape must not be None"
+            self.data = np.random.normal(0.0, 1.0, shape).astype(np.float32)
+            self.shape = shape
+            self.dtype = self.data.dtype
 
     def __repr__(self):
         return str({'shape': self.shape, 'lod': self.lod, 'dtype': self.dtype})
@@ -57,11 +64,15 @@ def __init__(self,
                  type: str,
                  inputs: Dict[str, List[str]],
                  outputs: Dict[str, List[str]],
-                 attrs: Dict[str, Any]):
+                 attrs: Dict[str, Any]=None,
+                 **kwargs):
         self.type = type
         self.inputs = inputs
         self.outputs = outputs
         self.attrs = attrs
+        if self.attrs is None:
+            self.attrs = dict()
+        self.attrs.update(kwargs)
 
     def __repr__(self):
         log_str = self.type

python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py

@@ -71,7 +71,19 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool:
 
         return True
 
-    def sample_program_configs(self, *args, **kwargs):
+    def sample_program_config(self, draw):
+        is_sparse = draw(st.booleans())
+        is_distributed = draw(st.booleans())
+        padding_idx = draw(st.integers())
+        axis = draw(st.integers(min_value=-4, max_value=4))
+        op_type = draw(st.sampled_from(['lookup_table', 'lookup_table_v2']))
+        epsilon = draw(st.floats(min_value=0, max_value=0.001))
+        # begin_norm_axis has to be 2
+        begin_norm_axis = 2
+        batch_size = draw(st.integers(min_value=1, max_value=4))
+        input_dim = draw(st.sampled_from([32, 64]))
+        weight_size = draw(st.sampled_from([[64, 64], [64, 32]]))
+
         def generate_input(attrs):
             if attrs[0]['op_type'] == 'lookup_table':
                 return np.random.randint(
@@ -101,19 +113,19 @@ def generate_weight2(attrs):
                     np.float32)
 
         attrs = [{
-            'is_sparse': kwargs['is_sparse'],
-            'is_distributed': kwargs['is_distributed'],
-            'padding_idx': kwargs['padding_idx'],
-            'op_type': kwargs['op_type']
+            'is_sparse': is_sparse,
+            'is_distributed': is_distributed,
+            'padding_idx': padding_idx,
+            'op_type': op_type
         }, {
-            'axis': kwargs['axis']
+            'axis': axis
         }, {
-            'begin_norm_axis': kwargs['begin_norm_axis'],
-            'epsilon': kwargs['epsilon']
+            'begin_norm_axis': begin_norm_axis,
+            'epsilon': epsilon
         }, {
-            'batch_size': kwargs['batch_size'],
-            'input_dim': kwargs['input_dim'],
-            'weight_size': kwargs['weight_size']
+            'batch_size': batch_size,
+            'input_dim': input_dim,
+            'weight_size': weight_size
         }]
 
         emb_op1 = OpConfig(
@@ -203,13 +215,12 @@ def generate_weight2(attrs):
             },
             outputs=["layer_norm_output1"])
 
-        yield program_config
+        return program_config
 
     def sample_predictor_configs(self, program_config):
         # only used in gpu passes and trt passes.
-        config = self.create_inference_config(
-            passes=['embedding_eltwise_layernorm_fuse_pass'], use_gpu=True)
-        yield config, (10, 5), (1e-5, 1e-5)
+        config = self.create_inference_config(use_gpu=True)
+        yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5)
         # trt static_shape
         config = self.create_trt_inference_config()
         config.enable_tensorrt_engine(
@@ -219,7 +230,7 @@ def sample_predictor_configs(self, program_config):
             precision_mode=paddle_infer.PrecisionType.Float32,
             use_static=False,
             use_calib_mode=False)
-        yield config, (10, 5), (1e-5, 1e-5)
+        yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5)
         # trt dynamic_shape
         config = self.create_trt_inference_config()
         config.enable_tensorrt_engine(
@@ -257,7 +268,7 @@ def sample_predictor_configs(self, program_config):
                 "input_data2": [2, 128],
                 "input_data3": [2, 128]
             })
-        yield config, (10, 5), (1e-5, 1e-5)
+        yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5)
 
     def add_skip_pass_case(self):
         def teller1(program_config, predictor_config):
@@ -272,26 +283,13 @@ def teller1(program_config, predictor_config):
         self.add_skip_case(teller1, SkipReasons.PASS_ACCURACY_ERROR,
                            "The pass output has diff in a specific case.")
 
-    @given(
-        is_sparse=st.booleans(),
-        is_distributed=st.booleans(),
-        padding_idx=st.integers(),
-        axis=st.integers(
-            min_value=-4, max_value=4),
-        op_type=st.sampled_from(['lookup_table', 'lookup_table_v2']),
-        epsilon=st.floats(
-            min_value=0, max_value=0.001),
-        begin_norm_axis=st.integers(
-            min_value=-4, max_value=4),
-        batch_size=st.integers(
-            min_value=1, max_value=4),
-        input_dim=st.sampled_from([32, 64]),
-        weight_size=st.sampled_from([[64, 64], [64, 32]]))
-    def test(self, *args, **kwargs):
-        assume(kwargs['begin_norm_axis'] == 2)
-
-        self.add_skip_pass_case()
-        self.run_test(quant=False, *args, **kwargs)
+    def test(self):
+        # this fuse need to fix, now there's no program can ran successfully
+        self.run_and_statis(
+            quant=False,
+            max_examples=50,
+            passes=["embedding_eltwise_layernorm_fuse_pass"],
+            min_success_num=0)
 
 
 if __name__ == "__main__":