Opencv

flybirds/core/dsl/globalization/i18n.py

@@ -89,6 +89,8 @@
         "screenshot": ["全屏截图"],
         "ocr": ["全屏扫描"],
         "change ocr lang [{param}]": ["切换OCR语言[{param}]"],
+        "exist image [{param}]": ["存在图像[{param}]"],
+        "not exist image [{param}]": ["不存在图像[{param}]"],
         "information association of failed operation, run the {param1} time"
         " :[{param2}]": ["失败运行的信息关联,运行第{param1}次:[{param2}]"],
         "text[{selector}]property[{param2}]is {param3}": [
@@ -100,6 +102,7 @@
         "click[{selector}]": ["点击[{selector}]"],
         "click text[{selector}]": ["点击文案[{selector}]"],
         "click ocr text[{selector}]": ["点击扫描文案[{selector}]"],
+        "click image[{selector}]": ["点击图像[{selector}]"],
         "click position[{x},{y}]": ["点击屏幕位置[{x},{y}]"],
         "in[{selector}]input[{param2}]": ["在[{selector}]中输入[{param2}]"],
         "in ocr[{selector}]input[{param2}]": ["在扫描文字[{selector}]中输入[{param2}]"],

flybirds/core/dsl/step/element.py

@@ -71,6 +71,17 @@ def click_ocr_text(context, selector=None):
     g_Context.step.click_ocr_text(context, selector)
 
 
+@step("click image[{selector}]")
+@ele_wrap
+def click_image(context, selector=None):
+    """
+    Click on the image
+    :param context: step context
+    :param selector: locator string for text element (or None).
+    """
+    g_Context.step.click_image(context, selector)
+
+
 @step("click position[{x},{y}]")
 @ele_wrap
 def click_coordinates(context, x=None, y=None):
@@ -383,3 +394,13 @@ def find_text_from_parent(context, p_selector=None, c_selector=None,
     """
     g_Context.step.find_text_from_parent(context, p_selector, c_selector,
                                          param3)
+
+@step("exist image [{param}]")
+def img_exist(context, param):
+    g_Context.step.img_exist(context, param)
+
+
+@step("not exist image [{param}]")
+def img_not_exist(context, param):
+    g_Context.step.img_not_exist(context, param)
+

flybirds/core/plugin/plugins/default/__init__.py

@@ -0,0 +1,3 @@
+#! usr/bin/python
+# -*- coding:utf-8 -*-
+from .ui_driver import SIFT

flybirds/core/plugin/plugins/default/app_base_step.py

@@ -31,6 +31,12 @@ def init_device(self, context, param=None):
     def change_ocr_lang(self, context, param=None):
         step_common.change_ocr_lang(context, lang=param)
 
+    def img_exist(self, context, param):
+        step_verify.img_exist(context, param)
+
+    def img_not_exist(self, context, param):
+        step_verify.img_not_exist(context, param)
+
     def connect_device(self, context, param):
         step_app.connect_device(context, param)
 
@@ -61,6 +67,9 @@ def click_coordinates(self, context, x, y):
     def click_ocr_text(self, context, selector):
         step_click.click_ocr_text(context, selector)
 
+    def click_image(self, context, selector):
+        step_click.click_image(context, selector)
+
     def sleep(self, context, param):
         step_common.sleep(context, param)
 

flybirds/core/plugin/plugins/default/screen.py

@@ -5,8 +5,9 @@
 import os
 import time
 import traceback
+from baseImage import Image
 from base64 import b64decode
-from PIL import Image
+from .ui_driver import SIFT
 
 import flybirds.core.global_resource as gr
 import flybirds.utils.file_helper as file_helper
@@ -46,7 +47,7 @@ def screen_shot(path):
         log.info("[screen_shot] screen shot end!")
 
     @staticmethod
-    def screen_link_to_behave(scenario, step_index, tag=None):
+    def screen_link_to_behave(scenario, step_index, tag=None, link=True):
         """
         screenshot address and linked to the <scr> tag
         The label information is placed in the description of the scene,
@@ -90,17 +91,21 @@ def screen_link_to_behave(scenario, step_index, tag=None):
                 'embeddingsTags, stepIndex={}, <image class ="screenshot"'
                 ' width="375" src="{}" />'.format(step_index, src_path)
             )
-            scenario.description.append(data)
+            if link is True:
+                scenario.description.append(data)
             screen_path = os.path.join(current_screen_dir, file_name)
             g_Context.screen.screen_shot(screen_path)
             return screen_path
 
     @staticmethod
     def image_ocr(img_path):
+        """
+        Take a screenshot and ocr
+        """
         log.info(f"[image ocr path] image path is:{img_path}")
         ocr = g_Context.ocr_driver_instance
         g_Context.ocr_result = ocr.ocr(img_path, cls=True)
-        g_Context.image_size = Image.open(img_path).size
+        g_Context.image_size = Image(img_path).size
         log.info(f"[image ocr path] image size is:{g_Context.image_size}")
         for line in g_Context.ocr_result:
             log.info(f"[image ocr result] scan line info is:{line}")
@@ -113,3 +118,19 @@ def image_ocr(img_path):
             # log.info(f"[image ocr result] scan txt info is:{txt}")
             # score = line[1][1]
             # log.info(f"[image ocr result] scan score info is:{score}")
+
+
+    @staticmethod
+    def image_verify(img_source_path, img_search_path):
+        """
+        Take a screenshot and verify image
+        """
+        match = SIFT()
+        img_source = Image(img_source_path)
+        img_search = Image(img_search_path)
+
+        result = match.find_all_results(img_source, img_search)
+        return result
+
+
+

flybirds/core/plugin/plugins/default/screen_record.py

@@ -168,7 +168,7 @@ def stop_record(self):
                 return "No recording service"
 
             message = ""
-            proc.wait()
+            proc.wait(timeout=30)
             proc_code = proc.poll()
             if proc_code is None:
                 proc.terminate()

flybirds/core/plugin/plugins/default/step/click.py

@@ -8,6 +8,7 @@
 import flybirds.utils.dsl_helper as dsl_helper
 from flybirds.core.global_context import GlobalContext as g_Context
 from flybirds.core.plugin.plugins.default.step.verify import ocr_txt_exist
+from flybirds.core.plugin.plugins.default.step.common import img_verify
 
 
 def click_ele(context, param):
@@ -115,7 +116,17 @@ def click_ocr_text(context, param):
             x = (box[0][0] + box[1][0]) / 2
             y = (box[0][1] + box[2][1]) / 2
             poco_instance = gr.get_value("pocoInstance")
-            x_coordinate = float(x) / g_Context.image_size[0]
-            y_coordinate = float(y) / g_Context.image_size[1]
+            x_coordinate = float(x) / g_Context.image_size[1]
+            y_coordinate = float(y) / g_Context.image_size[0]
             poco_instance.click([x_coordinate, y_coordinate])
 
+
+def click_image(context, param):
+    result = img_verify(context, param)
+    x = result[0].get('rect').x + result[0].get('rect').width / 2
+    y = result[0].get('rect').y + result[0].get('rect').height / 2
+    poco_instance = gr.get_value("pocoInstance")
+    x_coordinate = float(x) / g_Context.image_size[1]
+    y_coordinate = float(y) / g_Context.image_size[0]
+    poco_instance.click([x_coordinate, y_coordinate])
+

flybirds/core/plugin/plugins/default/step/common.py

@@ -25,13 +25,13 @@ def screenshot(context):
 
 def ocr(context):
     step_index = context.cur_step_index - 1
-    image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_")
+    image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_", False)
     BaseScreen.image_ocr(image_path)
 
 
 def change_ocr_lang(context,lang=None):
     """
-    init ocr
+    change ocr language
     """
     ocr_instance = ui_driver.init_ocr(lang)
     gr.set_value("ocrInstance", ocr_instance)
@@ -84,3 +84,14 @@ def prev_fail_scenario_relevance(context, param1, param2):
     except Exception:
         log.warn("rerun failed senario error")
         log.warn(traceback.format_exc())
+
+
+def img_verify(context, search_image_path):
+    """
+    verify image exist or not
+    """
+    step_index = context.cur_step_index - 1
+    source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_", False)
+    result = BaseScreen.image_verify(source_image_path, search_image_path)
+    return result
+

flybirds/core/plugin/plugins/default/step/verify.py

@@ -6,6 +6,7 @@
 import time
 
 import flybirds.core.global_resource as gr
+import flybirds.utils.flybirds_log as log
 from flybirds.core.global_context import GlobalContext as g_Context
 import flybirds.core.plugin.plugins.default.ui_driver.poco.poco_ele \
     as poco_ele
@@ -16,7 +17,7 @@
 import flybirds.utils.dsl_helper as dsl_helper
 import flybirds.utils.verify_helper as verify
 from flybirds.core.exceptions import FlybirdVerifyException
-from flybirds.core.plugin.plugins.default.step.common import ocr
+from flybirds.core.plugin.plugins.default.step.common import ocr,img_verify
 
 
 def wait_text_exist(context, param):
@@ -333,3 +334,39 @@ def paddle_fix_txt(txt):
     return txt
 
 
+def img_exist(context, param):
+    start = time.time()
+    step_index = context.cur_step_index - 1
+    result = img_verify(context, param)
+    if len(result) == 0:
+        src_path = "../../../{}".format(param)
+        data = (
+            'embeddingsTags, stepIndex={}, <image class ="screenshot"'
+            ' width="375" src="{}" />'.format(step_index, src_path)
+        )
+        context.scenario.description.append(data)
+        # context.cur_step_index += 1
+        raise Exception("[image exist verify] image not found !")
+    else:
+        log.info(f"[image exist verify] cost time:{time.time() - start}")
+        log.info(f"[image exist verify] result:{result}")
+
+
+def img_not_exist(context, param):
+    start = time.time()
+    step_index = context.cur_step_index - 1
+    result = img_verify(context, param)
+    if len(result) == 0:
+        log.info(f"[image not exist verify] cost time:{time.time() - start}")
+        log.info(f"[image not exist verify] result:{result}")
+    else:
+        src_path = "../../../{}".format(param)
+        data = (
+            'embeddingsTags, stepIndex={}, <image class ="screenshot"'
+            ' width="375" src="{}" />'.format(step_index, src_path)
+        )
+        context.scenario.description.append(data)
+        # context.cur_step_index += 1
+        raise Exception("[image not exist verify] image found !")
+
+

flybirds/core/plugin/plugins/default/ui_driver/__init__.py

@@ -0,0 +1,3 @@
+#! usr/bin/python
+# -*- coding:utf-8 -*-
+from .opencv import SIFT

flybirds/core/plugin/plugins/default/ui_driver/opencv/__init__.py

@@ -0,0 +1,7 @@
+#! usr/bin/python
+# -*- coding:utf-8 -*-
+from .matchTemplate import MatchTemplate
+from .sift import SIFT
+from .utils import generate_result, get_keypoint_from_matches, keypoint_distance, rectangle_transform
+from .exceptions import NoEnoughPointsError, PerspectiveTransformError, HomographyError, MatchResultError, InputImageError
+from .base import BaseKeypoint