Implement GHA performance tests in nightly validation to compare read_model and ovc.convert

.github/workflows/linux.yml

@@ -1427,7 +1427,7 @@ jobs:
           export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
           python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_performance.html --self-contained-html -v
         env:
-          TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}}
+          TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'nightly'}}
           TEST_DEVICE: CPU
 
       - name: Upload Test Results

tests/model_hub_tests/models_hub_common/constants.py

@@ -10,11 +10,13 @@
 loading and heating and includes measurement only one of 2 models - got through convert and read_model.
 Both "converted" and "read_model" modes will be 2 * runtime_measure_duration 
 '''
-runtime_measure_duration = os.environ.get('RUNTIME_MEASURE_DURATION', '60')
+precommit_runtime_measure_duration = os.environ.get('PRECOMMIT_RUNTIME_MEASURE_DURATION', '60')
+nightly_runtime_measure_duration = os.environ.get('NIGHTLY_RUNTIME_MEASURE_DURATION', '15')
 '''
 @brief Time in seconds of heating before measurement
 '''
-runtime_heat_duration = os.environ.get('RUNTIME_HEAT_DURATION', '5')
+precommit_runtime_heat_duration = os.environ.get('PRECOMMIT_RUNTIME_HEAT_DURATION', '5')
+nigtly_runtime_heat_duration = os.environ.get('NIGHTLY_RUNTIME_HEAT_DURATION', '5')
 
 
 tf_hub_cache_dir = os.environ.get('TFHUB_CACHE_DIR',

tests/model_hub_tests/models_hub_common/test_performance_model.py

@@ -120,14 +120,12 @@ def get_read_model(self, model_path: str):
         core = ov.Core()
         return core.read_model(model=model_path)
 
-    def heat_hardware(self, ov_model, inputs) -> None:
-        _, heat_n_repeats, _ = utils.measure(utils.nano_secs(const.runtime_heat_duration), ov_model, (inputs,))
+    def heat_hardware(self, ov_model, inputs, conf) -> None:
+        _, heat_n_repeats, _ = utils.measure(conf.runtime_heat_duration, ov_model, (inputs,))
         print('heat done in {} repeats'.format(heat_n_repeats))
 
-    def measure_inference(self, ov_model, inputs) -> ModelResults:
-        time_slices, infer_n_repeats, real_runtime = utils.measure(utils.nano_secs(const.runtime_measure_duration),
-                                                                   ov_model,
-                                                                   (inputs,))
+    def measure_inference(self, ov_model, inputs, conf) -> ModelResults:
+        time_slices, infer_n_repeats, real_runtime = utils.measure(conf.runtime_measure_duration, ov_model, (inputs,))
         print('measurement done in {} repeats'.format(infer_n_repeats))
         infer_throughput = float(infer_n_repeats * (10 ** 9)) / real_runtime
         infer_mean_time_ns = np.mean(time_slices)
@@ -141,15 +139,15 @@ def measure_inference(self, ov_model, inputs) -> ModelResults:
         results.infer_variance = infer_variance
         return results
 
-    def infer_model(self, ov_model, inputs) -> ModelResults:
-        self.heat_hardware(ov_model, inputs)
-        return self.measure_inference(ov_model, inputs)
+    def infer_model(self, ov_model, inputs, conf) -> ModelResults:
+        self.heat_hardware(ov_model, inputs, conf)
+        return self.measure_inference(ov_model, inputs, conf)
 
     def compile_model(self, model, ie_device):
         core = ov.Core()
         return core.compile_model(model, ie_device)
 
-    def __run(self, model_name, model_link, ie_device):
+    def __run(self, model_name, model_link, ie_device, conf):
         results = Results()
         results.status = None
         try:
@@ -168,11 +166,11 @@ def __run(self, model_name, model_link, ie_device):
             results.status = Status.INFER_CONVERTED_MODEL
             results.converted_model_results = utils.call_with_timer('Infer converted model',
                                                                     self.infer_model,
-                                                                    (converted_model, inputs))
+                                                                    (converted_model, inputs, conf))
             results.status = Status.INFER_READ_MODEL
             results.read_model_results = utils.call_with_timer('Infer read model',
                                                                self.infer_model,
-                                                               (read_model, inputs))
+                                                               (read_model, inputs, conf))
 
             infer_time_ratio = (results.converted_model_results.infer_mean_time /
                                 results.read_model_results.infer_mean_time)
@@ -196,10 +194,10 @@ def __run(self, model_name, model_link, ie_device):
                                                                          ex_type=ex_type.__name__, ex_value=ex_value)
         return results
 
-    def run(self, model_name, model_link, ie_device):
+    def run(self, model_name, model_link, ie_device, conf):
         self.result = Results()
         t0 = time.time()
-        self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device], model_name,
+        self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device, conf], model_name,
                                           self.infer_timeout)
         t1 = time.time()
         utils.print_stat('test run time {} secs', (t1 - t0))

tests/model_hub_tests/models_hub_common/utils.py

@@ -32,6 +32,28 @@ def get_models_list(file_name: str):
     return models
 
 
+def get_skipped_model_links(filename: str):
+    links = set()
+    if not os.path.exists(filename):
+        return links
+    with open(filename) as f:
+        for model_info in f:
+            model_info = model_info.strip()
+            model_name, model_link = model_info.split(',')
+            links.add(model_link)
+    return links
+
+
+def get_models_list_not_skipped(model_list_file: str, skip_list_file: str):
+    skipped_links = get_skipped_model_links(skip_list_file)
+    not_skipped_models = []
+    for model_name, model_link, mark, reason in get_models_list(model_list_file):
+        if model_link in skipped_links:
+            continue
+        not_skipped_models.append((model_name, model_link, mark, reason))
+    return not_skipped_models
+
+
 def compare_two_tensors(ov_res, fw_res, eps):
     is_ok = True
     if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True):