From 1b1200b40d0085641fc1ebfa35040f82231b9268 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Thu, 28 Jul 2022 17:22:47 +0800 Subject: [PATCH 01/10] add opencv image --- flybirds/core/dsl/globalization/i18n.py | 1 + flybirds/core/dsl/step/common.py | 5 + .../plugin/plugins/default/app_base_step.py | 3 + .../core/plugin/plugins/default/screen.py | 15 + .../plugin/plugins/default/step/common.py | 7 + .../ui_driver/image_registration/__init__.py | 1 + .../image_registration/exceptions.py | 46 ++ .../image_registration/matching/__init__.py | 4 + .../matching/keypoint/__init__.py | 6 + .../matching/keypoint/akaze/__init__.py | 1 + .../matching/keypoint/akaze/akaze.py | 37 ++ .../matching/keypoint/akaze/akaze.pyi | 21 + .../matching/keypoint/base.py | 580 ++++++++++++++++++ .../matching/keypoint/base.pyi | 80 +++ .../matching/keypoint/orb/__init__.py | 1 + .../matching/keypoint/orb/orb.py | 129 ++++ .../matching/keypoint/orb/orb.pyi | 38 ++ .../matching/keypoint/sift/__init__.py | 1 + .../matching/keypoint/sift/sift.py | 38 ++ .../matching/keypoint/sift/sift.pyi | 23 + .../matching/keypoint/surf/__init__.py | 1 + .../matching/keypoint/surf/surf.py | 38 ++ .../matching/keypoint/surf/surf.pyi | 25 + .../matching/template/__init__.py | 3 + .../matching/template/matchTemplate.py | 249 ++++++++ .../matching/template/matchTemplate.pyi | 56 ++ .../image_registration/utils/__init__.py | 159 +++++ .../image_registration/utils/__init__.pyi | 27 + requirements.txt | 3 +- 29 files changed, 1597 insertions(+), 1 deletion(-) create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/exceptions.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi diff --git a/flybirds/core/dsl/globalization/i18n.py b/flybirds/core/dsl/globalization/i18n.py index f64961ef..a466c11f 100644 --- a/flybirds/core/dsl/globalization/i18n.py +++ b/flybirds/core/dsl/globalization/i18n.py @@ -89,6 +89,7 @@ "screenshot": ["全屏截图"], "ocr": ["全屏扫描"], "change ocr lang [{param}]": ["切换OCR语言[{param}]"], + "image search [{param}]": ["查找图像[{param}]"], "information association of failed operation, run the {param1} time" " :[{param2}]": ["失败运行的信息关联,运行第{param1}次:[{param2}]"], "text[{selector}]property[{param2}]is {param3}": [ diff --git a/flybirds/core/dsl/step/common.py b/flybirds/core/dsl/step/common.py index 992c7c89..ceb49696 100644 --- a/flybirds/core/dsl/step/common.py +++ b/flybirds/core/dsl/step/common.py @@ -29,6 +29,11 @@ def scan(context): g_Context.step.ocr(context) +@step("image search [{param}]") +def img_verify(context, param): + g_Context.step.img_verify(context, param) + + @step("information association of failed operation," " run the {param1} time :[{param2}]" ) diff --git a/flybirds/core/plugin/plugins/default/app_base_step.py b/flybirds/core/plugin/plugins/default/app_base_step.py index 96039d6d..46edfee3 100644 --- a/flybirds/core/plugin/plugins/default/app_base_step.py +++ b/flybirds/core/plugin/plugins/default/app_base_step.py @@ -31,6 +31,9 @@ def init_device(self, context, param=None): def change_ocr_lang(self, context, param=None): step_common.change_ocr_lang(context, lang=param) + def img_verify(self, context, param): + step_common.img_verify(context, param) + def connect_device(self, context, param): step_app.connect_device(context, param) diff --git a/flybirds/core/plugin/plugins/default/screen.py b/flybirds/core/plugin/plugins/default/screen.py index 34bb48a7..180becf2 100644 --- a/flybirds/core/plugin/plugins/default/screen.py +++ b/flybirds/core/plugin/plugins/default/screen.py @@ -5,8 +5,10 @@ import os import time import traceback +import baseImage from base64 import b64decode from PIL import Image +from flybirds.core.plugin.plugins.default.ui_driver.image_registration.matching.keypoint.sift import SIFT import flybirds.core.global_resource as gr import flybirds.utils.file_helper as file_helper @@ -113,3 +115,16 @@ def image_ocr(img_path): # log.info(f"[image ocr result] scan txt info is:{txt}") # score = line[1][1] # log.info(f"[image ocr result] scan score info is:{score}") + + + @staticmethod + def image_verify(img_source_path, img_search_path): + match = SIFT() + img_source = baseImage.Image(img_source_path) + img_search = baseImage.Image(img_search_path) + + start = time.time() + + result = match.find_all_results(img_source, img_search) + print(time.time() - start) + print(result) diff --git a/flybirds/core/plugin/plugins/default/step/common.py b/flybirds/core/plugin/plugins/default/step/common.py index 8ed2612b..4cbd82ab 100644 --- a/flybirds/core/plugin/plugins/default/step/common.py +++ b/flybirds/core/plugin/plugins/default/step/common.py @@ -84,3 +84,10 @@ def prev_fail_scenario_relevance(context, param1, param2): except Exception: log.warn("rerun failed senario error") log.warn(traceback.format_exc()) + + +def img_verify(context, search_image_path): + step_index = context.cur_step_index - 1 + source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_") + BaseScreen.image_verify(source_image_path, search_image_path) + diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py @@ -0,0 +1 @@ + diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/exceptions.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/exceptions.py new file mode 100644 index 00000000..dcf67cc5 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/exceptions.py @@ -0,0 +1,46 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +class BaseError(Exception): + def __init__(self, message="", *args, **kwargs): + self.message = message + + def __repr__(self): + return repr(self.message) + + +class NoModuleError(BaseError): + """ Missing dependent module """ + + +class CreateExtractorError(BaseError): + """ An error occurred while create Extractor """ + + +class NoEnoughPointsError(BaseError): + """ detect not enough feature points in input images""" + + +class CudaSurfInputImageError(BaseError): + """ The image size does not conform to CUDA standard """ + # https://stackoverflow.com/questions/42492060/surf-cuda-error-while-computing-descriptors-and-keypoints + # https://github.com/opencv/opencv_contrib/blob/master/modules/xfeatures2d/src/surf.cuda.cpp#L151 + + +class CudaOrbDetectorError(BaseError): + """ An CvError when orb detector error occurred """ + + +class HomographyError(BaseError): + """ An error occurred while findHomography """ + + +class MatchResultError(BaseError): + """ An error occurred while result out of screen""" + + +class PerspectiveTransformError(BaseError): + """ An error occurred while perspectiveTransform """ + + +class InputImageError(BaseError): + """ An error occurred while input image place/dtype/channels error""" diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py new file mode 100644 index 00000000..3f77c355 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py @@ -0,0 +1,4 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +from .template import MatchTemplate, CudaMatchTemplate +from .keypoint import SIFT, ORB, AKAZE, SURF, CUDA_ORB diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py new file mode 100644 index 00000000..1ee1c973 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py @@ -0,0 +1,6 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +from .akaze import AKAZE +from .sift import SIFT +from .orb import ORB, CUDA_ORB +from .surf import SURF diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py new file mode 100644 index 00000000..62a878ee --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py @@ -0,0 +1 @@ +from .akaze import AKAZE diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py new file mode 100644 index 00000000..64be81be --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py @@ -0,0 +1,37 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from baseImage.constant import Place + +from image_registration.matching.keypoint.base import BaseKeypoint + + +class AKAZE(BaseKeypoint): + METHOD_NAME = 'AKAZE' + Dtype = np.uint8 + Place = (Place.UMat, Place.Ndarray) + + def create_matcher(self, **kwargs) -> cv2.DescriptorMatcher: + """ + 创建特征点匹配器 + + Returns: + cv2.FlannBasedMatcher + """ + matcher = cv2.BFMatcher_create(cv2.NORM_L1) + return matcher + + def create_detector(self, **kwargs): + descriptor_type = kwargs.get('descriptor_type', cv2.AKAZE_DESCRIPTOR_MLDB) + descriptor_size = kwargs.get('descriptor_size', 0) + descriptor_channels = kwargs.get('descriptor_channels', 3) + threshold = kwargs.get('_threshold', 0.001) + diffusivity = kwargs.get('diffusivity', cv2.KAZE_DIFF_PM_G2) + nOctaveLayers = kwargs.get('nOctaveLayers', 4) + nOctaves = kwargs.get('nOctaves', 4) + + detector = cv2.AKAZE_create(descriptor_type=descriptor_type, descriptor_size=descriptor_size, + descriptor_channels=descriptor_channels, threshold=threshold, + diffusivity=diffusivity, nOctaves=nOctaves, nOctaveLayers=nOctaveLayers,) + return detector diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi new file mode 100644 index 00000000..f5c3f7dd --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi @@ -0,0 +1,21 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from typing import Union, Tuple, List +from baseImage import Image + +from image_registration.matching.keypoint.base import BaseKeypoint + + +image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] +keypoint_type = Tuple[cv2.KeyPoint, ...] +matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] +good_match_type = List[cv2.DMatch] + + +class AKAZE(BaseKeypoint): + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, + descriptor_type: int = cv2.AKAZE_DESCRIPTOR_MLDB, descriptor_size: int = 0, + descriptor_channels: int = 3, _threshold: float = 0.001, diffusivity: int = cv2.KAZE_DIFF_PM_G2, + nOctaveLayers: int = 4, nOctaves: int = 4): ... diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py new file mode 100644 index 00000000..43db9f62 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py @@ -0,0 +1,580 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from baseImage import Image, Rect + +from image_registration.matching import MatchTemplate +from image_registration.utils import (generate_result, get_keypoint_from_matches, keypoint_distance, rectangle_transform) +from image_registration.exceptions import (NoEnoughPointsError, PerspectiveTransformError, HomographyError, MatchResultError, + InputImageError) +from typing import List + + +class BaseKeypoint(object): + FILTER_RATIO = 1 + METHOD_NAME = None + Dtype = None + Place = None + template = MatchTemplate() + + def __init__(self, threshold=0.8, rgb=True, **kwargs): + """ + 初始化 + + Args: + threshold: 识别阈值(0~1) + rgb: 是否使用rgb通道进行校验 + """ + self.threshold = threshold + self.rgb = rgb + self.detector = self.create_detector(**kwargs) + self.matcher = self.create_matcher(**kwargs) + + def create_matcher(self, **kwargs): + raise NotImplementedError + + def create_detector(self, **kwargs): + raise NotImplementedError + + def find_best_result(self, im_source, im_search, threshold=None, rgb=None, **kwargs): + """ + 通过特征点匹配,在im_source中找到最符合im_search的范围 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + threshold: 识别阈值(0~1) + rgb: 是否使用rgb通道进行校验 + + Returns: + + """ + max_count = 1 + ret = self.find_all_results(im_source=im_source, im_search=im_search, threshold=threshold, rgb=rgb, + max_count=max_count, **kwargs) + if ret: + return ret[0] + return None + + def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_count=10, max_iter_counts=20, distance_threshold=150): + """ + 通过特征点匹配,在im_source中找到全部符合im_search的范围 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + threshold: 识别阈值(0~1) + rgb: 是否使用rgb通道进行校验 + max_count: 最多可以返回的匹配数量 + max_iter_counts: 最大的搜索次数,需要大于max_count + distance_threshold: 距离阈值,特征点(first_point)大于该阈值后,不做后续筛选 + + Returns: + + """ + threshold = self.threshold if threshold is None else threshold + rgb = self.rgb if rgb is None else rgb + + im_source, im_search = self.input_image_check(im_source, im_search) + result = [] + if im_source.channels == 1: + rgb = False + + kp_src, des_src = self.get_keypoint_and_descriptor(image=im_source) + kp_sch, des_sch = self.get_keypoint_and_descriptor(image=im_search) + + kp_src, kp_sch = list(kp_src), list(kp_sch) + # 在特征点集中,匹配最接近的特征点 + matches = np.array(self.match_keypoint(des_sch=des_sch, des_src=des_src)) + kp_sch_point = np.array([(kp.pt[0], kp.pt[1], kp.angle) for kp in kp_sch]) + kp_src_matches_point = np.array([[(*kp_src[dMatch.trainIdx].pt, kp_src[dMatch.trainIdx].angle) + if dMatch else np.nan for dMatch in match] for match in matches]) + _max_iter_counts = 0 + src_pop_list = [] + while True: + # 这里没有用matches判断nan, 是因为类型不对 + if (np.count_nonzero(~np.isnan(kp_src_matches_point)) == 0) or (len(result) == max_count) or (_max_iter_counts >= max_iter_counts): + break + _max_iter_counts += 1 + filtered_good_point, angle, first_point = self.filter_good_point(matches=matches, kp_src=kp_src, + kp_sch=kp_sch, + kp_sch_point=kp_sch_point, + kp_src_matches_point=kp_src_matches_point) + if first_point.distance > distance_threshold: + break + + rect, confidence = None, 0 + try: + rect, confidence = self.extract_good_points(im_source=im_source, im_search=im_search, kp_src=kp_src, + kp_sch=kp_sch, good=filtered_good_point, angle=angle, rgb=rgb) + # print(f'good:{len(filtered_good_point)}, rect={rect}, confidence={confidence}') + except PerspectiveTransformError: + pass + finally: + + if rect and confidence >= threshold: + br, tl = rect.br, rect.tl + # 移除改范围内的所有特征点 ??有可能因为透视变换的原因，删除了多余的特征点 + for index, match in enumerate(kp_src_matches_point): + x, y = match[:, 0], match[:, 1] + flag = np.argwhere((x < br.x) & (x > tl.x) & (y < br.y) & (y > tl.y)) + for _index in flag: + src_pop_list.append(matches[index, _index][0].trainIdx) + kp_src_matches_point[index, _index, :] = np.nan + matches[index, _index] = np.nan + result.append(generate_result(rect, confidence)) + else: + for match in filtered_good_point: + flags = np.argwhere(matches[match.queryIdx, :] == match) + for _index in flags: + kp_src_matches_point[match.queryIdx, _index, :] = np.nan + matches[match.queryIdx, _index] = np.nan + + # 一下代码用于删除目标图片中的特征点,以后会用 + # src_pop_list = list(set(src_pop_list)) + # src_pop_list.sort(reverse=True) + # mask = np.ones(len(des_src), dtype=bool) + # for v in src_pop_list: + # kp_src.pop(v) + # mask[v] = False + # + # des_src = des_src[mask, ...] + return result + + def get_keypoint_and_descriptor(self, image): + """ + 获取图像关键点(keypoint)与描述符(descriptor) + + Args: + image: 待检测的灰度图像 + + Returns: + + """ + if image.channels == 3: + image = image.cvtColor(cv2.COLOR_BGR2GRAY).data + else: + image = image.data + keypoint, descriptor = self.detector.detectAndCompute(image, None) + + if len(keypoint) < 2: + raise NoEnoughPointsError('{} detect not enough feature points in input images'.format(self.METHOD_NAME)) + return keypoint, descriptor + + @staticmethod + def filter_good_point(matches, kp_src, kp_sch, kp_sch_point, kp_src_matches_point): + """ 筛选最佳点 """ + # 假设第一个点,及distance最小的点,为基准点 + sort_list = [sorted(match, key=lambda x: x is np.nan and float('inf') or x.distance)[0] + for match in matches] + sort_list = [v for v in sort_list if v is not np.nan] + + first_good_point: cv2.DMatch = sorted(sort_list, key=lambda x: x.distance)[0] + first_good_point_train: cv2.KeyPoint = kp_src[first_good_point.trainIdx] + first_good_point_query: cv2.KeyPoint = kp_sch[first_good_point.queryIdx] + first_good_point_angle = first_good_point_train.angle - first_good_point_query.angle + + def get_points_origin_angle(point_x, point_y, offset): + points_origin_angle = np.arctan2( + (point_y - offset.pt[1]), + (point_x - offset.pt[0]) + ) * 180 / np.pi + + points_origin_angle = np.where( + points_origin_angle == 0, + points_origin_angle, points_origin_angle - offset.angle + ) + points_origin_angle = np.where( + points_origin_angle >= 0, + points_origin_angle, points_origin_angle + 360 + ) + return points_origin_angle + + # 计算模板图像上,该点与其他特征点的旋转角 + first_good_point_sch_origin_angle = get_points_origin_angle(kp_sch_point[:, 0], kp_sch_point[:, 1], + first_good_point_query) + + # 计算目标图像中,该点与其他特征点的夹角 + kp_sch_rotate_angle = kp_sch_point[:, 2] + first_good_point_angle + kp_sch_rotate_angle = np.where(kp_sch_rotate_angle >= 360, kp_sch_rotate_angle - 360, kp_sch_rotate_angle) + kp_sch_rotate_angle = kp_sch_rotate_angle.reshape(kp_sch_rotate_angle.shape + (1,)) + + kp_src_angle = kp_src_matches_point[:, :, 2] + good_point = np.array([matches[index][array[0]] for index, array in + enumerate(np.argsort(np.abs(kp_src_angle - kp_sch_rotate_angle)))]) + + # 计算各点以first_good_point为原点的旋转角 + good_point_nan = (np.nan, np.nan) + good_point_pt = np.array([good_point_nan if dMatch is np.nan else (*kp_src[dMatch.trainIdx].pt, ) + for dMatch in good_point]) + good_point_origin_angle = get_points_origin_angle(good_point_pt[:, 0], good_point_pt[:, 1], + first_good_point_train) + threshold = round(5 / 360, 2) * 100 + point_bool = (np.abs(good_point_origin_angle - first_good_point_sch_origin_angle) / 360) * 100 < threshold + _, index = np.unique(good_point_pt[point_bool], return_index=True, axis=0) + good = good_point[point_bool] + good = good[index] + return good, int(first_good_point_angle), first_good_point + + def match_keypoint(self, des_sch, des_src, k=10): + """ + 特征点匹配 + + Args: + des_src: 待匹配图像的描述符集 + des_sch: 图片模板的描述符集 + k(int): 获取多少匹配点 + + Returns: + List[List[cv2.DMatch]]: 包含最匹配的描述符 + """ + # k=2表示每个特征点取出2个最匹配的对应点 + matches = self.matcher.knnMatch(des_sch, des_src, k) + return matches + + def get_good_in_matches(self, matches): + """ + 特征点过滤 + + Args: + matches: 特征点集 + + Returns: + List[cv2.DMatch]: 过滤后的描述符集 + """ + if not matches: + return None + good = [] + for match_index in range(len(matches)): + match = matches[match_index] + for DMatch_index in range(len(match)): + if match[DMatch_index].distance <= self.FILTER_RATIO * match[-1].distance: + good.append(match[DMatch_index]) + return good + + def extract_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle, rgb): + """ + 根据匹配点(good)数量,提取识别区域 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + kp_src: 关键点集 + kp_sch: 关键点集 + good: 描述符集 + angle: 旋转角度 + rgb: 是否使用rgb通道进行校验 + + Returns: + 范围,和置信度 + """ + len_good = len(good) + confidence, rect, target_img = None, None, None + + if len_good == 0: + pass + elif len_good == 1: + target_img, rect = self._handle_one_good_points(im_source=im_source, im_search=im_search, + kp_sch=kp_sch, kp_src=kp_src, good=good, angle=angle) + elif len_good == 2: + target_img, rect = self._handle_two_good_points(im_source=im_source, im_search=im_search, + kp_sch=kp_sch, kp_src=kp_src, good=good, angle=angle) + elif len_good == 3: + target_img, rect = self._handle_three_good_points(im_source=im_source, im_search=im_search, + kp_sch=kp_sch, kp_src=kp_src, good=good, angle=angle) + else: # len > 4 + target_img, rect = self._handle_many_good_points(im_source=im_source, im_search=im_search, + kp_sch=kp_sch, kp_src=kp_src, good=good) + + if target_img: + confidence = self._cal_confidence(im_source=im_search, im_search=target_img, rgb=rgb) + + return rect, confidence + + def _handle_one_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle): + """ + 特征点匹配数量等于1时,根据特征点的大小,对矩形进行缩放,并根据旋转角度,获取识别的目标图片 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + kp_sch: 关键点集 + kp_src: 关键点集 + good: 描述符集 + angle: 旋转角度 + + Returns: + 待验证的图片 + """ + sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query')[0] + src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train')[0] + + scale = src_point.size / sch_point.size + h, w = im_search.size + _h, _w = h * scale, w * scale + src = np.float32(rectangle_transform(point=sch_point.pt, size=(h, w), mapping_point=src_point.pt, + mapping_size=(_h, _w), angle=angle)) + dst = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) + output = self._perspective_transform(im_source=im_source, im_search=im_search, src=src, dst=dst) + rect = self._get_perspective_area_rect(im_source=im_source, src=src) + return output, rect + + def _handle_two_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle): + """ + 特征点匹配数量等于2时,根据两点距离差,对矩形进行缩放,并根据旋转角度,获取识别的目标图片 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + kp_sch: 关键点集 + kp_src: 关键点集 + good: 描述符集 + angle: 旋转角度 + + Returns: + 待验证的图片 + """ + sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query') + src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train') + + sch_distance = keypoint_distance(sch_point[0], sch_point[1]) + src_distance = keypoint_distance(src_point[0], src_point[1]) + + try: + scale = src_distance / sch_distance # 计算缩放大小 + except ZeroDivisionError: + if src_distance == sch_distance: + scale = 1 + else: + return None, None + + h, w = im_search.size + _h, _w = h * scale, w * scale + src = np.float32(rectangle_transform(point=sch_point[0].pt, size=(h, w), mapping_point=src_point[0].pt, + mapping_size=(_h, _w), angle=angle)) + dst = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) + output = self._perspective_transform(im_source=im_source, im_search=im_search, src=src, dst=dst) + rect = self._get_perspective_area_rect(im_source=im_source, src=src) + return output, rect + + def _handle_three_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle): + """ + 特征点匹配数量等于3时,根据三个点组成的三角面积差,对矩形进行缩放,并根据旋转角度,获取识别的目标图片 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + kp_sch: 关键点集 + kp_src: 关键点集 + good: 描述符集 + angle: 旋转角度 + + Returns: + 待验证的图片 + """ + sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query') + src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train') + + def _area(point_list): + p1_2 = keypoint_distance(point_list[0], point_list[1]) + p1_3 = keypoint_distance(point_list[0], point_list[2]) + p2_3 = keypoint_distance(point_list[1], point_list[2]) + + s = (p1_2 + p1_3 + p2_3) / 2 + area = (s * (s - p1_2) * (s - p1_3) * (s - p2_3)) ** 0.5 + return area + + sch_area = _area(sch_point) + src_area = _area(src_point) + + try: + scale = src_area / sch_area # 计算缩放大小 + except ZeroDivisionError: + if sch_area == src_area: + scale = 1 + else: + return None, None + + h, w = im_search.size + _h, _w = h * scale, w * scale + src = np.float32(rectangle_transform(point=sch_point[0].pt, size=(h, w), mapping_point=src_point[0].pt, + mapping_size=(_h, _w), angle=angle)) + dst = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) + output = self._perspective_transform(im_source=im_source, im_search=im_search, src=src, dst=dst) + rect = self._get_perspective_area_rect(im_source=im_source, src=src) + return output, rect + + def _handle_many_good_points(self, im_source, im_search, kp_src, kp_sch, good): + """ + 特征点匹配数量>=4时,使用单矩阵映射,获取识别的目标图片 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + kp_sch: 关键点集 + kp_src: 关键点集 + good: 描述符集 + + Returns: + 透视变换后的图片 + """ + + sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape( + -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) + # M是转化矩阵 + M, mask = self._find_homography(sch_pts, img_pts) + # 计算四个角矩阵变换后的坐标，也就是在大图中的目标区域的顶点坐标: + h, w = im_search.size + h_s, w_s = im_source.size + pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2) + try: + dst: np.ndarray = cv2.perspectiveTransform(pts, M) + # img = im_source.clone().data + # img2 = cv2.polylines(img, [np.int32(dst)], True, 255, 3, cv2.LINE_AA) + # Image(img).imshow('dst') + pypts = [tuple(npt[0]) for npt in dst.tolist()] + src = np.array([pypts[0], pypts[3], pypts[1], pypts[2]], dtype=np.float32) + dst = np.float32([[0, 0], [w, 0], [0, h], [w, h]]) + output = self._perspective_transform(im_source=im_source, im_search=im_search, src=src, dst=dst) + except cv2.error as err: + raise PerspectiveTransformError(err) + + # img = im_source.clone().data + # cv2.polylines(img, [np.int32(dst)], True, 255, 3, cv2.LINE_AA) + # Image(img).imshow() + # cv2.waitKey(0) + + rect = self._get_perspective_area_rect(im_source=im_source, src=src) + return output, rect + + @staticmethod + def _target_image_crop(img, rect): + """ + 截取目标图像 + + Args: + img: 图像 + rect: 图像范围 + + Returns: + 裁剪后的图像 + """ + try: + target_img = img.crop(rect) + except OverflowError: + raise MatchResultError(f"Target area({rect}) out of screen{img.size}") + return target_img + + def _cal_confidence(self, im_source, im_search, rgb): + """ + 将截图和识别结果缩放到大小一致,并计算可信度 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + rgb:是否使用rgb通道进行校验 + + Returns: + + """ + h, w = im_source.size + im_search = im_search.resize(w, h) + if rgb: + confidence = self.template.cal_rgb_confidence(im_source=im_source, im_search=im_search) + else: + confidence = self.template.cal_ccoeff_confidence(im_source=im_source, im_search=im_search) + + confidence = (1 + confidence) / 2 + return confidence + + def input_image_check(self, im_source, im_search): + im_source = self._image_check(im_source) + im_search = self._image_check(im_search) + + if im_source.place != im_search.place: + raise InputImageError('输入图片类型必须相同, source={}, search={}'.format(im_source.place, im_search.place)) + elif im_source.dtype != im_search.dtype: + raise InputImageError('输入图片数据类型必须相同, source={}, search={}'.format(im_source.dtype, im_search.dtype)) + elif im_source.channels != im_search.channels: + raise InputImageError('输入图片通道必须相同, source={}, search={}'.format(im_source.channels, im_search.channels)) + + return im_source, im_search + + def _image_check(self, data): + if not isinstance(data, Image): + data = Image(data, dtype=self.Dtype) + + if data.place not in self.Place: + raise TypeError(f'{self.METHOD_NAME}方法,Image类型必须为(Place.UMat, Place.Ndarray)') + return data + + @staticmethod + def _find_homography(sch_pts, src_pts): + """ + 多组特征点对时，求取单向性矩阵 + """ + try: + # M, mask = cv2.findHomography(sch_pts, src_pts, cv2.RANSAC) + M, mask = cv2.findHomography(sch_pts, src_pts, cv2.USAC_MAGSAC, 4.0, None, 2000, 0.99) + except cv2.error: + import traceback + traceback.print_exc() + raise HomographyError("OpenCV error in _find_homography()...") + else: + if mask is None: + raise HomographyError("In _find_homography(), find no mask...") + else: + return M, mask + + @staticmethod + def _perspective_transform(im_source, im_search, src, dst): + """ + 根据四对对应点计算透视变换, 并裁剪相应图片 + + Args: + im_source: 待匹配图像 + im_search: 待匹配模板 + src: 目标图像中相应四边形顶点的坐标 (左上,右上,左下,右下) + dst: 源图像中四边形顶点的坐标 (左上,右上,左下,右下) + + Returns: + + """ + h, w = im_search.size + matrix = cv2.getPerspectiveTransform(src=src, dst=dst) + # warpPerspective https://github.com/opencv/opencv/issues/11784 + output = im_source.warpPerspective(matrix, size=(w, h), flags=cv2.INTER_CUBIC) + + return output + + @staticmethod + def _get_perspective_area_rect(im_source, src): + """ + 根据矩形四个顶点坐标,获取在原图中的最大外接矩形 + + Args: + im_source: 待匹配图像 + src: 目标图像中相应四边形顶点的坐标 + + Returns: + 最大外接矩形 + """ + h, w = im_source.size + + x = [int(i[0]) for i in src] + y = [int(i[1]) for i in src] + x_min, x_max = min(x), max(x) + y_min, y_max = min(y), max(y) + # 挑选出目标矩形区域可能会有越界情况，越界时直接将其置为边界： + # 超出左边界取0，超出右边界取w_s-1，超出下边界取0，超出上边界取h_s-1 + # 当x_min小于0时，取0。 x_max小于0时，取0。 + x_min, x_max = int(max(x_min, 0)), int(max(x_max, 0)) + # 当x_min大于w_s时，取值w_s-1。 x_max大于w_s-1时，取w_s-1。 + x_min, x_max = int(min(x_min, w - 1)), int(min(x_max, w - 1)) + # 当y_min小于0时，取0。 y_max小于0时，取0。 + y_min, y_max = int(max(y_min, 0)), int(max(y_max, 0)) + # 当y_min大于h_s时，取值h_s-1。 y_max大于h_s-1时，取h_s-1。 + y_min, y_max = int(min(y_min, h - 1)), int(min(y_max, h - 1)) + rect = Rect(x=x_min, y=y_min, width=(x_max - x_min), height=(y_max - y_min)) + return rect diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi new file mode 100644 index 00000000..4903930f --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi @@ -0,0 +1,80 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from typing import Union, Optional, Type, Any, Tuple, List +from baseImage.constant import Place +from baseImage import Image, Rect + +from image_registration.matching import MatchTemplate, CudaMatchTemplate + + +image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] +keypoint_type = List[cv2.KeyPoint, ...] +matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] +good_match_type = List[cv2.DMatch] +Detector = Union[cv2.Feature2D, cv2.cuda.Feature2DAsync] +Matcher = Union[cv2.DescriptorMatcher, cv2.cuda.DescriptorMatcher] + + +class BaseKeypoint(object): + FILTER_RATIO: int = 0.59 + METHOD_NAME: str + Dtype: Union[Type[np.uint8], Type[np.int8], Type[np.uint16], Type[np.int16], Type[np.int32], Type[np.float32], Type[np.float64]] + Place: Place + template: Union[MatchTemplate, CudaMatchTemplate] + + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, **kwargs): + self.detector: Detector = None + self.matcher: Matcher = None + self.threshold: Union[int, float] = 0.8 + self.rgb: bool = True + ... + + def create_matcher(self, **kwargs) -> Matcher: ... + + def create_detector(self, **kwargs) -> Detector: ... + + def find_best_result(self, im_source: image_type, im_search: image_type, threshold: Union[int, float] = None, rgb: bool = None) -> Optional[Rect]: ... + + def find_all_results(self, im_source: image_type, im_search: image_type, threshold: Union[int, float] = None, rgb: bool = None, max_count: int = 10, max_iter_counts: int = 20, distance_threshold: int = 150): ... + + def get_keypoint_and_descriptor(self, image: Image) -> Tuple[keypoint_type, np.ndarray]: ... + + def filter_good_point(self, matches: matches_type, kp_src: keypoint_type, kp_sch: keypoint_type, + kp_sch_point: np.ndarray, kp_src_matches_point: np.ndarray) -> Tuple[good_match_type, float, cv2.DMatch]: ... + + def get_rect_from_good_matches(self, im_source: Image, im_search: Image, kp_src: keypoint_type, + des_src: np.ndarray, kp_sch:keypoint_type, des_sch: np.ndarray) -> \ + Tuple[Optional[Rect], matches_type, good_match_type]: ... + + def match_keypoint(self, des_sch: np.ndarray, des_src: np.ndarray, k: int = 2) -> matches_type: ... + + def get_good_in_matches(self, matches: matches_type) -> good_match_type: ... + + def extract_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: float, rgb: bool) -> Union[Optional[Rect], Union[None, int, float]]: ... + + def _handle_one_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: Union[float, int]) -> Tuple[Image, Rect]: ... + + def _handle_two_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: Union[float, int]) -> Tuple[Image, Rect]: ... + + def _handle_three_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: Union[float, int]) -> Tuple[Image, Rect]: ... + + def _handle_many_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type) -> Tuple[Image, Rect]: ... + + def _target_image_crop(self, img: Image, rect: Rect) -> Image: ... + + def _cal_confidence(self, im_source: Image, im_search: Image, rgb: bool) -> Union[int, float]: ... + + def input_image_check(self, im_source: image_type, im_search: image_type) -> Tuple[Image, Image]: ... + + def _image_check(self, data: Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image]) -> Image: ... + + @staticmethod + def _find_homography(sch_pts: np.ndarray, src_pts: np.ndarray) -> Optional[Tuple[np.ndarray, np.ndarray]]: ... + + @staticmethod + def _perspective_transform(im_source: Image, im_search: Image, src: np.ndarray, dst: np.ndarray) -> Image: ... + + @staticmethod + def _get_perspective_area_rect(im_source: Image, src: np.ndarray) -> Rect: \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py new file mode 100644 index 00000000..1c9fb3f5 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py @@ -0,0 +1 @@ +from .orb import ORB, CUDA_ORB diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py new file mode 100644 index 00000000..06109704 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py @@ -0,0 +1,129 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from baseImage.constant import Place + +from image_registration.matching.template import CudaMatchTemplate +from image_registration.matching.keypoint.base import BaseKeypoint +from image_registration.exceptions import NoEnoughPointsError, CudaOrbDetectorError +from typing import Union + + +class ORB(BaseKeypoint): + METHOD_NAME = 'ORB' + Dtype = np.uint8 + Place = (Place.UMat, Place.Ndarray) + + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, **kwargs): + super().__init__(threshold=threshold, rgb=rgb, **kwargs) + self.descriptor = self.create_descriptor() + + def create_matcher(self, **kwargs): + """ + 创建特征点匹配器 + + Returns: + cv2.FlannBasedMatcher + """ + matcher = cv2.DescriptorMatcher_create(cv2.DescriptorMatcher_BRUTEFORCE_HAMMING) + return matcher + + def create_detector(self, **kwargs): + nfeatures = kwargs.get('nfeatures', 50000) + scaleFactor = kwargs.get('scaleFactor', 1.2) + nlevels = kwargs.get('nlevels', 8) + edgeThreshold = kwargs.get('edgeThreshold', 31) + firstLevel = kwargs.get('firstLevel', 0) + WTA_K = kwargs.get('WTA_K', 2) + scoreType = kwargs.get('scoreType', cv2.ORB_HARRIS_SCORE) + patchSize = kwargs.get('patchSize', 31) + fastThreshold = kwargs.get('fastThreshold', 20) + + params = dict( + nfeatures=nfeatures, scaleFactor=scaleFactor, nlevels=nlevels, + edgeThreshold=edgeThreshold, firstLevel=firstLevel, WTA_K=WTA_K, + scoreType=scoreType, patchSize=patchSize, fastThreshold=fastThreshold, + ) + detector = cv2.ORB_create(**params) + return detector + + @staticmethod + def create_descriptor(): + # https://docs.opencv.org/master/d7/d99/classcv_1_1xfeatures2d_1_1BEBLID.html + # https://github.com/iago-suarez/beblid-opencv-demo + descriptor = cv2.xfeatures2d.BEBLID_create(0.75) + return descriptor + + def get_keypoint_and_descriptor(self, image): + if image.channels == 3: + image = image.cvtColor(cv2.COLOR_BGR2GRAY).data + else: + image = image.data + + keypoints = self.detector.detect(image, None) + keypoints, descriptors = self.descriptor.compute(image, keypoints) + + if len(keypoints) < 2: + raise NoEnoughPointsError('{} detect not enough feature points in input images'.format(self.METHOD_NAME)) + return keypoints, descriptors + + +class CUDA_ORB(BaseKeypoint): + METHOD_NAME = 'CUDA_ORB' + Dtype = np.uint8 + Place = (Place.GpuMat,) + + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, **kwargs): + super().__init__(threshold=threshold, rgb=rgb, **kwargs) + self.template = CudaMatchTemplate(threshold=threshold, rgb=rgb) + + def create_matcher(self, **kwargs): + """ + 创建特征点匹配器 + + Returns: + cv2.FlannBasedMatcher + """ + matcher = cv2.cuda.DescriptorMatcher_createBFMatcher(cv2.NORM_HAMMING) + return matcher + + def create_detector(self, **kwargs): + nfeatures = kwargs.get('nfeatures', 50000) + scaleFactor = kwargs.get('scaleFactor', 1.2) + nlevels = kwargs.get('nlevels', 8) + edgeThreshold = kwargs.get('edgeThreshold', 31) + firstLevel = kwargs.get('firstLevel', 0) + WTA_K = kwargs.get('WTA_K', 2) + scoreType = kwargs.get('scoreType', cv2.ORB_HARRIS_SCORE) + patchSize = kwargs.get('patchSize', 31) + fastThreshold = kwargs.get('fastThreshold', 20) + blurForDescriptor = kwargs.get('blurForDescriptor', False) + + params = dict( + nfeatures=nfeatures, scaleFactor=scaleFactor, nlevels=nlevels, + edgeThreshold=edgeThreshold, firstLevel=firstLevel, WTA_K=WTA_K, + scoreType=scoreType, patchSize=patchSize, fastThreshold=fastThreshold, + blurForDescriptor=blurForDescriptor + ) + + detector = cv2.cuda.ORB_create(**params) + return detector + + def get_keypoint_and_descriptor(self, image): + if image.channels == 3: + image = image.cvtColor(cv2.COLOR_BGR2GRAY).data + else: + image = image.data + + try: + keypoints, descriptors = self.detector.detectAndComputeAsync(image, None) + except cv2.error: + # https://github.com/opencv/opencv/issues/10573 + raise CudaOrbDetectorError('{} detect error, Try adjust detector params'.format(self.METHOD_NAME)) + else: + keypoints = self.detector.convert(keypoints) + + if len(keypoints) < 2: + raise NoEnoughPointsError('{} detect not enough feature points in input images'.format(self.METHOD_NAME)) + return keypoints, descriptors diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi new file mode 100644 index 00000000..364d3e4a --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi @@ -0,0 +1,38 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from typing import Union, Tuple, List +from baseImage import Image + +from image_registration.matching.keypoint.base import BaseKeypoint + + +image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] +keypoint_type = Tuple[cv2.KeyPoint, ...] +matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] +good_match_type = List[cv2.DMatch] + + +class ORB(BaseKeypoint): + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, + nfeatures: int = 50000 ,scaleFactor: Union[int, float] = 1.2, nlevels: int = 8, + edgeThreshold: int = 31, firstLevel: int = 0, WTA_K: int = 2, + scoreType: int = cv2.ORB_HARRIS_SCORE, patchSize: int = 31, fastThreshold: int = 20): + self.descriptor = cv2.xfeatures2d.BEBLID + ... + + @staticmethod + def create_descriptor() -> cv2.xfeatures2d.BEBLID: ... + + +class CUDA_ORB(BaseKeypoint): + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, + nfeatures: int = 50000 ,scaleFactor: Union[int, float] = 1.2, nlevels: int = 8, + edgeThreshold: int = 31, firstLevel: int = 0, WTA_K: int = 2, + scoreType: int = cv2.ORB_HARRIS_SCORE, patchSize: int = 31, fastThreshold: int = 20, + blurForDescriptor: bool = False): ... + + def create_detector(self, **kwargs) -> cv2.cuda.Feature2DAsync: ... + + def create_matcher(self, **kwargs) -> cv2.cuda.DescriptorMatcher: ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py new file mode 100644 index 00000000..3b395981 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py @@ -0,0 +1 @@ +from .sift import SIFT diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py new file mode 100644 index 00000000..dcb8a2ff --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py @@ -0,0 +1,38 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from baseImage.constant import Place + +from image_registration.matching.keypoint.base import BaseKeypoint + + +class SIFT(BaseKeypoint): + FLANN_INDEX_KDTREE = 0 + METHOD_NAME = 'SIFT' + Dtype = np.uint8 + Place = (Place.UMat, Place.Ndarray) + + def create_matcher(self, **kwargs) -> cv2.DescriptorMatcher: + """ + 创建特征点匹配器 + + Returns: + cv2.FlannBasedMatcher + """ + index_params = {'algorithm': self.FLANN_INDEX_KDTREE, 'tree': 5} + # 指定递归遍历的次数. 值越高结果越准确，但是消耗的时间也越多 + search_params = {'checks': 50} + matcher = cv2.FlannBasedMatcher(index_params, search_params) + return matcher + + def create_detector(self, **kwargs) -> cv2.SIFT: + nfeatures = kwargs.get('nfeatures', 0) + nOctaveLayers = kwargs.get('nOctaveLayers', 3) + contrastThreshold = kwargs.get('contrastThreshold', 0.04) + edgeThreshold = kwargs.get('edgeThreshold', 10) + sigma = kwargs.get('sigma', 1.6) + + detector = cv2.SIFT_create(nfeatures=nfeatures, nOctaveLayers=nOctaveLayers, contrastThreshold=contrastThreshold, + edgeThreshold=edgeThreshold, sigma=sigma) + return detector diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi new file mode 100644 index 00000000..fa7bcfc9 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi @@ -0,0 +1,23 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from typing import Union, Tuple, List + +from baseImage import Image + +from image_registration.matching.keypoint.base import BaseKeypoint + + +image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] +keypoint_type = Tuple[cv2.KeyPoint, ...] +matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] +good_match_type = List[cv2.DMatch] + + +class SIFT(BaseKeypoint): + FLANN_INDEX_KDTREE: int = 0 + + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, + nfeatures: int = 0, nOctaveLayers: int = 3, contrastThreshold: float = 0.04, + edgeThreshold: int = 10, sigma: float = 1.6): ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py new file mode 100644 index 00000000..15f433a0 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py @@ -0,0 +1 @@ +from .surf import SURF diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py new file mode 100644 index 00000000..7dc6a767 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py @@ -0,0 +1,38 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from baseImage.constant import Place + +from image_registration.matching.keypoint.base import BaseKeypoint + + +class SURF(BaseKeypoint): + FLANN_INDEX_KDTREE = 0 + METHOD_NAME = "SURF" + Dtype = np.uint8 + Place = (Place.UMat, Place.Ndarray) + + def create_matcher(self, **kwargs) -> cv2.DescriptorMatcher: + """ + 创建特征点匹配器 + + Returns: + cv2.FlannBasedMatcher + """ + index_params = {'algorithm': self.FLANN_INDEX_KDTREE, 'tree': 5} + # 指定递归遍历的次数. 值越高结果越准确，但是消耗的时间也越多 + search_params = {'checks': 50} + matcher = cv2.FlannBasedMatcher(index_params, search_params) + return matcher + + def create_detector(self, **kwargs): + hessianThreshold = kwargs.get('hessianThreshold', 400) + nOctaves = kwargs.get('nOctaves', 4) + nOctaveLayers = kwargs.get('nOctaveLayers', 3) + extended = kwargs.get('extended', True) + upright = kwargs.get('upright', False) + + detector = cv2.xfeatures2d.SURF_create(hessianThreshold=hessianThreshold, nOctaves=nOctaves, nOctaveLayers=nOctaveLayers, + extended=extended, upright=upright) + return detector diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi new file mode 100644 index 00000000..4d8b6caa --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi @@ -0,0 +1,25 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import cv2 +import numpy as np +from typing import Union, Tuple, List + +from baseImage import Image + +from image_registration.matching.keypoint.base import BaseKeypoint + + +image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] +keypoint_type = Tuple[cv2.KeyPoint, ...] +matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] +good_match_type = List[cv2.DMatch] + + +class SURF(BaseKeypoint): + FLANN_INDEX_KDTREE: int = 0 + + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, + hessianThreshold: int = 400, nOctaves: int = 4, nOctaveLayers: int = 3, + extended: bool = True, upright: bool = False): ... + + def create_detector(self, **kwargs) -> cv2.xfeatures2d.SURF: ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py new file mode 100644 index 00000000..96abfc8e --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py @@ -0,0 +1,3 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +from .matchTemplate import MatchTemplate, CudaMatchTemplate diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py new file mode 100644 index 00000000..a1a8a584 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py @@ -0,0 +1,249 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +""" opencv matchTemplate""" +import warnings +import cv2 +import numpy as np +from baseImage import Image, Rect +from baseImage.constant import Place + +from image_registration.exceptions import MatchResultError, NoModuleError, InputImageError +from image_registration.utils import generate_result +from typing import Union + + +class MatchTemplate(object): + METHOD_NAME = 'tpl' + Dtype = np.uint8 + Place = (Place.Ndarray, ) + + def __init__(self, threshold=0.8, rgb=True): + """ + 初始化 + + Args: + threshold: 识别阈值(0~1) + rgb: 是否使用rgb通道进行校验 + """ + assert 0 <= threshold <= 1, 'threshold 取值在0到1直接' + + self.threshold = threshold + self.rgb = rgb + self.matcher = cv2.matchTemplate + + def find_best_result(self, im_source, im_search, threshold=None, rgb=None): + """ + 模板匹配, 返回匹配度最高且大于阈值的范围 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + threshold: 识别阈值(0~1) + rgb: 是否使用rgb通道进行校验 + + Returns: + generate_result + """ + threshold = threshold or self.threshold + rgb = rgb or self.rgb + + im_source, im_search = self.input_image_check(im_source, im_search) + if im_source.channels == 1: + rgb = False + + result = self._get_template_result_matrix(im_source=im_source, im_search=im_search) + # 找到最佳匹配项 + + min_val, max_val, min_loc, max_loc = self.minMaxLoc(result.data) + + h, w = im_search.size + # 求可信度 + crop_rect = Rect(max_loc[0], max_loc[1], w, h) + + confidence = self.cal_confidence(im_source, im_search, crop_rect, max_val, rgb) + # 如果可信度小于threshold,则返回None + if confidence < (threshold or self.threshold): + return None + x, y = max_loc + rect = Rect(x=x, y=y, width=w, height=h) + return generate_result(rect, confidence) + + def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_count=10): + """ + 模板匹配, 返回匹配度大于阈值的范围, 且最大数量不超过max_count + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + threshold:: 识别阈值(0~1) + rgb: 是否使用rgb通道进行校验 + max_count: 最多匹配数量 + + Returns: + + """ + threshold = threshold or self.threshold + rgb = rgb or self.rgb + + im_source, im_search = self.input_image_check(im_source, im_search) + if im_source.channels == 1: + rgb = False + + result = self._get_template_result_matrix(im_source=im_source, im_search=im_search) + results = [] + # 找到最佳匹配项 + h, w = im_search.size + while True: + min_val, max_val, min_loc, max_loc = self.minMaxLoc(result.data) + img_crop = im_source.crop(Rect(max_loc[0], max_loc[1], w, h)) + confidence = self._get_confidence_from_matrix(img_crop, im_search, max_val=max_val, rgb=rgb) + x, y = max_loc + rect = Rect(x, y, w, h) + + if (confidence < (threshold or self.threshold)) or len(results) >= max_count: + break + results.append(generate_result(rect, confidence)) + result.rectangle(rect=Rect(int(max_loc[0] - w / 2), int(max_loc[1] - h / 2), w, h), color=(0, 0, 0), thickness=-1) + + return results if results else None + + def _get_template_result_matrix(self, im_source, im_search): + """求取模板匹配的结果矩阵.""" + if im_source.channels == 3: + i_gray = im_source.cvtColor(cv2.COLOR_BGR2GRAY).data + s_gray = im_search.cvtColor(cv2.COLOR_BGR2GRAY).data + else: + i_gray = im_source.data + s_gray = im_search.data + + result = self.match(i_gray, s_gray) + result = Image(data=result, dtype=np.float32, clone=False, place=im_source.place) + return result + + def input_image_check(self, im_source, im_search): + im_source = self._image_check(im_source) + im_search = self._image_check(im_search) + + if im_source.place != im_search.place: + raise InputImageError('输入图片类型必须相同, source={}, search={}'.format(im_source.place, im_search.place)) + elif im_source.dtype != im_search.dtype: + raise InputImageError('输入图片数据类型必须相同, source={}, search={}'.format(im_source.dtype, im_search.dtype)) + elif im_source.channels != im_search.channels: + raise InputImageError('输入图片通道必须相同, source={}, search={}'.format(im_source.channels, im_search.channels)) + + if im_source.place == Place.UMat: + warnings.warn('Umat has error,will clone new image with np.ndarray ' + '(https://github.com/opencv/opencv/issues/21788)') + im_source = Image(im_source, place=Place.Ndarray, dtype=im_source.dtype) + im_search = Image(im_search, place=Place.Ndarray, dtype=im_search.dtype) + + return im_source, im_search + + def _image_check(self, data): + if not isinstance(data, Image): + data = Image(data, dtype=self.Dtype) + + if data.place not in self.Place: + raise TypeError('Image类型必须为(Place.UMat, Place.Ndarray)') + return data + + @staticmethod + def minMaxLoc(result): + return cv2.minMaxLoc(result) + + def match(self, img1, img2): + return self.matcher(img1, img2, cv2.TM_CCOEFF_NORMED) + + def cal_confidence(self, im_source, im_search, crop_rect, max_val, rgb): + """ + 将截图和识别结果缩放到大小一致,并计算可信度 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + crop_rect: 需要在im_source截取的区域 + max_val: matchTemplate得到的最大值 + rgb: 是否使用rgb通道进行校验 + + Returns: + float: 可信度(0~1) + """ + try: + target_img = im_source.crop(crop_rect) + except OverflowError: + raise MatchResultError(f"Target area({crop_rect}) out of screen{im_source.size}") + confidence = self._get_confidence_from_matrix(target_img, im_search, max_val, rgb) + return confidence + + def cal_rgb_confidence(self, im_source, im_search): + """ + 计算两张图片图片rgb三通道的置信度 + + Args: + im_source: 待匹配图像 + im_search: 图片模板 + + Returns: + float: 最小置信度 + """ + # im_search = im_search.copyMakeBorder(10, 10, 10, 10, cv2.BORDER_REPLICATE) + # + # img_src_hsv = im_source.cvtColor(cv2.COLOR_BGR2HSV) + # img_sch_hsv = im_search.cvtColor(cv2.COLOR_BGR2HSV) + + src_split = im_source.split() + sch_split = im_search.split() + + # 计算BGR三通道的confidence，存入bgr_confidence: + bgr_confidence = [0, 0, 0] + for i in range(3): + res_temp = self.match(sch_split[i], src_split[i]) + min_val, max_val, min_loc, max_loc = self.minMaxLoc(res_temp) + bgr_confidence[i] = max_val + + return min(bgr_confidence) + + def cal_ccoeff_confidence(self, im_source, im_search): + if im_source.channels == 3: + img_src_gray = im_source.cvtColor(cv2.COLOR_BGR2GRAY).data + else: + img_src_gray = im_source.data + + if im_search.channels == 3: + img_sch_gray = im_search.cvtColor(cv2.COLOR_BGR2GRAY).data + else: + img_sch_gray = im_search.data + + res_temp = self.match(img_sch_gray, img_src_gray) + min_val, max_val, min_loc, max_loc = self.minMaxLoc(res_temp) + return max_val + + def _get_confidence_from_matrix(self, img_crop, im_search, max_val, rgb): + """根据结果矩阵求出confidence.""" + # 求取可信度: + if rgb: + # 如果有颜色校验,对目标区域进行BGR三通道校验: + confidence = self.cal_rgb_confidence(img_crop, im_search) + else: + confidence = max_val + return confidence + + +class CudaMatchTemplate(MatchTemplate): + METHOD_NAME = 'cuda_tpl' + Dtype = np.uint8 + Place = (Place.GpuMat, ) + + def __init__(self, threshold=0.8, rgb=True): + super(CudaMatchTemplate, self).__init__(threshold=threshold, rgb=rgb) + try: + self.matcher = cv2.cuda.createTemplateMatching(cv2.CV_8U, cv2.TM_CCOEFF_NORMED) + except AttributeError: + raise NoModuleError('create CUDA TemplateMatching Error') + + @staticmethod + def minMaxLoc(result): + return cv2.cuda.minMaxLoc(result) + + def match(self, img1, img2): + return self.matcher.match(img1, img2) diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi new file mode 100644 index 00000000..5cb2ddfd --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi @@ -0,0 +1,56 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +""" opencv matchTemplate""" +import warnings +import cv2 +import numpy as np +from baseImage import Image, Rect +from baseImage.constant import Place + + +from typing import Union, Any, Tuple, Type + + +class MatchTemplate(object): + METHOD_NAME: str = 'tpl' + Dtype: Type[np.uint8] = np.uint8 + Place: tuple = (Place.Ndarray, ) + + def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True): + self.threshold: Union[int, float] = threshold + self.rgb: bool = rgb + self.matcher: cv2.matchTemplate = cv2.matchTemplate + + def find_best_result(self, im_source: Image, im_search: Image, threshold: Union[int, float] = None, rgb: bool = None) -> dict: ... + + def find_all_results(self, im_source: Image, im_search: Image, threshold: Union[int, float] = None, rgb: bool = None, max_count: int = 10) -> list: ... + + def _get_template_result_matrix(self, im_source: Image, im_search: Image) -> Image: ... + + def input_image_check(self, im_source: Any, im_search: Any) -> Image: ... + + def _image_check(self, data: Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image]) -> Image: ... + + @staticmethod + def minMaxLoc(result: np.ndarray) -> Tuple[float, float, Tuple[int, int], Tuple[int, int]]: ... + + def match(self, img1: np.ndarray, img2: np.ndarray) -> np.ndarray: ... + + def cal_confidence(self, im_source: Image, im_search: Image, crop_rect: Rect, max_val: float, rgb: bool) -> float: ... + + def cal_rgb_confidence(self, im_source: Image, im_search: Image) -> float: ... + + def cal_ccoeff_confidence(self, im_source: Image, im_search: Image) -> float: ... + + def _get_confidence_from_matrix(self, img_crop: Image, im_search: Image, max_val: float, rgb: bool) -> float: ... + + +class CudaMatchTemplate(MatchTemplate): + METHOD_NAME: str = 'tpl' + Dtype: Type[np.uint8] = np.uint8 + Place: tuple = Place.GpuMat + + @staticmethod + def minMaxLoc(result: cv2.cuda.GpuMat) -> tuple:... + + def match(self, img1: cv2.cuda.GpuMat, img2: cv2.cuda.GpuMat) -> cv2.cuda.GpuMat: ... diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py new file mode 100644 index 00000000..5300f15a --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py @@ -0,0 +1,159 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +import math +import cv2 +import numpy as np +from typing import Union + + +def generate_result(rect, confi): + """Format the result: 定义图像识别结果格式.""" + ret = { + 'rect': rect, + 'confidence': confi, + } + return ret + + +def keypoint_distance(kp1, kp2): + """求两个keypoint的两点之间距离""" + if isinstance(kp1, cv2.KeyPoint): + kp1 = kp1.pt + elif isinstance(kp1, (list, tuple)): + kp1 = kp1 + else: + raise ValueError('kp1需要时keypoint或直接是坐标, kp1={}'.format(kp1)) + + if isinstance(kp2, cv2.KeyPoint): + kp2 = kp2.pt + elif isinstance(kp2, (list, tuple)): + kp2 = kp2 + else: + raise ValueError('kp2需要时keypoint或直接是坐标, kp1={}'.format(kp2)) + + x = kp1[0] - kp2[0] + y = kp1[1] - kp2[1] + return math.sqrt((x ** 2) + (y ** 2)) + + +def keypoint_angle(kp1, kp2): + """求两个keypoint的夹角 """ + k = [ + (kp1.angle - 180) if kp1.angle >= 180 else kp1.angle, + (kp2.angle - 180) if kp2.angle >= 180 else kp2.angle + ] + if k[0] == k[1]: + return 0 + else: + return abs(k[0] - k[1]) + + +def get_keypoint_from_matches(kp, matches, mode): + res = [] + if mode == 'query': + for match in matches: + res.append(kp[match.queryIdx]) + elif mode == 'train': + for match in matches: + res.append(kp[match.trainIdx]) + + return res + + +def keypoint_origin_angle(kp1, kp2): + """ + 以kp1为原点,计算kp2的旋转角度 + """ + origin_point = kp1.pt + train_point = kp2.pt + + point = (abs(origin_point[0] - train_point[0]), abs(origin_point[1] - train_point[1])) + + x_quadrant = (1, 4) + y_quadrant = (3, 4) + if origin_point[0] > train_point[0]: + x_quadrant = (2, 3) + + if origin_point[1] > train_point[1]: + y_quadrant = (1, 2) + point_quadrant = list(set(x_quadrant).intersection(set(y_quadrant)))[0] + + x, y = point[::-1] + angle = math.degrees(math.atan2(x, y)) + if point_quadrant == 4: + angle = angle + elif point_quadrant == 3: + angle = 180 - angle + elif point_quadrant == 2: + angle = 180 + angle + elif point_quadrant == 1: + angle = 360 - angle + + return angle + + +def _mapping_angle_distance(distance, origin_angle, angle): + """ + + Args: + distance: 距离 + origin_angle: 对应原点的角度 + angle: 旋转角度 + + """ + _angle = origin_angle + angle + _y = distance * math.cos((math.pi * _angle) / 180) + _x = distance * math.sin((math.pi * _angle) / 180) + return round(_x, 3), round(_y, 3) + + +def rectangle_transform(point, size, mapping_point, mapping_size, angle): + """ + 根据point,找出mapping_point映射的矩形顶点坐标 + + Args: + point: 坐标在矩形中的坐标 + size: 矩形的大小(h, w) + mapping_point: 映射矩形的坐标 + mapping_size: 映射矩形的大小(h, w) + angle: 旋转角度 + + Returns: + + """ + h, w = size[0], size[1] + _h, _w = mapping_size[0], mapping_size[1] + + h_scale = _h / h + w_scale = _w / w + + tl = keypoint_distance((0, 0), point) # 左上 + tr = keypoint_distance((w, 0), point) # 右上 + bl = keypoint_distance((0, h), point) # 左下 + br = keypoint_distance((w, h), point) # 右下 + + # x = np.float32([point[1], point[1], (h - point[1]), (h - point[1])]) + # y = np.float32([point[0], (w - point[0]), point[0], (w - point[0])]) + # A, B, C, D = cv2.phase(x, y, angleInDegrees=True) + A = math.degrees(math.atan2(point[0], point[1])) + B = math.degrees(math.atan2((w - point[0]), point[1])) + C = math.degrees(math.atan2(point[0], (h - point[1]))) + D = math.degrees(math.atan2((w - point[0]), (h - point[1]))) + + new_tl = _mapping_angle_distance(tl, A, angle=angle) + new_tl = (-new_tl[0] * w_scale, -new_tl[1] * h_scale) + new_tl = (mapping_point[0] + new_tl[0], mapping_point[1] + new_tl[1]) + + new_tr = _mapping_angle_distance(tr, B, angle=angle) + new_tr = (new_tr[0] * w_scale, -new_tr[1] * h_scale) + new_tr = (mapping_point[0] + new_tr[0], mapping_point[1] + new_tr[1]) + + new_bl = _mapping_angle_distance(bl, C, angle=angle) + new_bl = (-new_bl[0] * w_scale, new_bl[1] * h_scale) + new_bl = (mapping_point[0] + new_bl[0], mapping_point[1] + new_bl[1]) + + new_br = _mapping_angle_distance(br, D, angle=angle) + new_br = (new_br[0] * w_scale, new_br[1] * h_scale) + new_br = (mapping_point[0] + new_br[0], mapping_point[1] + new_br[1]) + + return [new_tl, new_tr, new_bl, new_br] diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi new file mode 100644 index 00000000..a607bbb6 --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi @@ -0,0 +1,27 @@ +# from __future__ import annotations +from baseImage import Rect +from typing import Union, TypedDict, List, Tuple +import cv2 + + +class ResultType(TypedDict): + rect: Rect + confidence: Union[int, float] +point_type = Tuple[Union[float, int], Union[float, int]] +keypoint_type = Union[cv2.KeyPoint, List[float, float], Tuple[float, float]] + + +def generate_result(rect: Rect, confi: Union[float, int]) -> ResultType: ... + +def keypoint_distance(kp1: keypoint_type, kp2: keypoint_type) -> float: ... + +def keypoint_angle(kp1: cv2.KeyPoint, kp2: cv2.KeyPoint) -> float: ... + +def get_keypoint_from_matches(kp: Tuple[cv2.KeyPoint, ...], matches: List[cv2.DMatch, ...], mode: str) -> List[cv2.KeyPoint]: ... + +def keypoint_origin_angle(kp1: cv2.KeyPoint, kp2: cv2.KeyPoint) -> Union[int, float]: ... + +def _mapping_angle_distance(distance: float, origin_angle: Union[int, float], angle: Union[int, float]) -> Tuple[float, float]: ... + +def rectangle_transform(point: point_type, size: point_type, mapping_point: point_type, mapping_size: point_type, angle: Union[float, int])\ + -> Tuple[point_type, point_type, point_type, point_type]: ... \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f30d2e12..80b3d1eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ jsonpath_ng>=1.5.3 deepdiff>=5.8.1 paddleocr>=2.5.0 paddlepaddle>=2.3.0 -protobuf==3.20.1 \ No newline at end of file +protobuf==3.20.1 +baseImage==2.1.1 \ No newline at end of file From 8d98872f586059683428bc42d97b9106c429ad03 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Fri, 29 Jul 2022 12:51:45 +0800 Subject: [PATCH 02/10] reactor:opencv --- .../core/plugin/plugins/default/__init__.py | 3 + .../core/plugin/plugins/default/screen.py | 3 +- .../plugins/default/ui_driver/__init__.py | 3 + .../ui_driver/image_registration/__init__.py | 1 - .../image_registration/matching/__init__.py | 4 - .../matching/keypoint/__init__.py | 6 - .../matching/keypoint/akaze/__init__.py | 1 - .../matching/keypoint/akaze/akaze.py | 37 ----- .../matching/keypoint/akaze/akaze.pyi | 21 --- .../matching/keypoint/base.pyi | 80 ----------- .../matching/keypoint/orb/__init__.py | 1 - .../matching/keypoint/orb/orb.py | 129 ------------------ .../matching/keypoint/orb/orb.pyi | 38 ------ .../matching/keypoint/sift/__init__.py | 1 - .../matching/keypoint/sift/sift.pyi | 23 ---- .../matching/keypoint/surf/__init__.py | 1 - .../matching/keypoint/surf/surf.py | 38 ------ .../matching/keypoint/surf/surf.pyi | 25 ---- .../matching/template/__init__.py | 3 - .../matching/template/matchTemplate.pyi | 56 -------- .../image_registration/utils/__init__.pyi | 27 ---- .../default/ui_driver/opencv/__init__.py | 7 + .../matching/keypoint => opencv}/base.py | 8 +- .../exceptions.py | 0 .../template => opencv}/matchTemplate.py | 23 +--- .../matching/keypoint/sift => opencv}/sift.py | 2 +- .../utils/__init__.py => opencv/utils.py} | 2 - requirements.txt | 9 +- 28 files changed, 30 insertions(+), 522 deletions(-) delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi delete mode 100644 flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi create mode 100644 flybirds/core/plugin/plugins/default/ui_driver/opencv/__init__.py rename flybirds/core/plugin/plugins/default/ui_driver/{image_registration/matching/keypoint => opencv}/base.py (98%) rename flybirds/core/plugin/plugins/default/ui_driver/{image_registration => opencv}/exceptions.py (100%) rename flybirds/core/plugin/plugins/default/ui_driver/{image_registration/matching/template => opencv}/matchTemplate.py (91%) rename flybirds/core/plugin/plugins/default/ui_driver/{image_registration/matching/keypoint/sift => opencv}/sift.py (94%) rename flybirds/core/plugin/plugins/default/ui_driver/{image_registration/utils/__init__.py => opencv/utils.py} (99%) diff --git a/flybirds/core/plugin/plugins/default/__init__.py b/flybirds/core/plugin/plugins/default/__init__.py index e69de29b..af51f261 100644 --- a/flybirds/core/plugin/plugins/default/__init__.py +++ b/flybirds/core/plugin/plugins/default/__init__.py @@ -0,0 +1,3 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +from .ui_driver import SIFT diff --git a/flybirds/core/plugin/plugins/default/screen.py b/flybirds/core/plugin/plugins/default/screen.py index 180becf2..f567a3bb 100644 --- a/flybirds/core/plugin/plugins/default/screen.py +++ b/flybirds/core/plugin/plugins/default/screen.py @@ -8,7 +8,8 @@ import baseImage from base64 import b64decode from PIL import Image -from flybirds.core.plugin.plugins.default.ui_driver.image_registration.matching.keypoint.sift import SIFT +# from flybirds.core.plugin.plugins.default.ui_driver.image_registration.matching.keypoint.sift import SIFT +from .ui_driver import SIFT import flybirds.core.global_resource as gr import flybirds.utils.file_helper as file_helper diff --git a/flybirds/core/plugin/plugins/default/ui_driver/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/__init__.py index e69de29b..d9f0ce05 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/__init__.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/__init__.py @@ -0,0 +1,3 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +from .opencv import SIFT diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py deleted file mode 100644 index 3f77c355..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -from .template import MatchTemplate, CudaMatchTemplate -from .keypoint import SIFT, ORB, AKAZE, SURF, CUDA_ORB diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py deleted file mode 100644 index 1ee1c973..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -from .akaze import AKAZE -from .sift import SIFT -from .orb import ORB, CUDA_ORB -from .surf import SURF diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py deleted file mode 100644 index 62a878ee..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .akaze import AKAZE diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py deleted file mode 100644 index 64be81be..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.py +++ /dev/null @@ -1,37 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from baseImage.constant import Place - -from image_registration.matching.keypoint.base import BaseKeypoint - - -class AKAZE(BaseKeypoint): - METHOD_NAME = 'AKAZE' - Dtype = np.uint8 - Place = (Place.UMat, Place.Ndarray) - - def create_matcher(self, **kwargs) -> cv2.DescriptorMatcher: - """ - 创建特征点匹配器 - - Returns: - cv2.FlannBasedMatcher - """ - matcher = cv2.BFMatcher_create(cv2.NORM_L1) - return matcher - - def create_detector(self, **kwargs): - descriptor_type = kwargs.get('descriptor_type', cv2.AKAZE_DESCRIPTOR_MLDB) - descriptor_size = kwargs.get('descriptor_size', 0) - descriptor_channels = kwargs.get('descriptor_channels', 3) - threshold = kwargs.get('_threshold', 0.001) - diffusivity = kwargs.get('diffusivity', cv2.KAZE_DIFF_PM_G2) - nOctaveLayers = kwargs.get('nOctaveLayers', 4) - nOctaves = kwargs.get('nOctaves', 4) - - detector = cv2.AKAZE_create(descriptor_type=descriptor_type, descriptor_size=descriptor_size, - descriptor_channels=descriptor_channels, threshold=threshold, - diffusivity=diffusivity, nOctaves=nOctaves, nOctaveLayers=nOctaveLayers,) - return detector diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi deleted file mode 100644 index f5c3f7dd..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/akaze/akaze.pyi +++ /dev/null @@ -1,21 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from typing import Union, Tuple, List -from baseImage import Image - -from image_registration.matching.keypoint.base import BaseKeypoint - - -image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] -keypoint_type = Tuple[cv2.KeyPoint, ...] -matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] -good_match_type = List[cv2.DMatch] - - -class AKAZE(BaseKeypoint): - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, - descriptor_type: int = cv2.AKAZE_DESCRIPTOR_MLDB, descriptor_size: int = 0, - descriptor_channels: int = 3, _threshold: float = 0.001, diffusivity: int = cv2.KAZE_DIFF_PM_G2, - nOctaveLayers: int = 4, nOctaves: int = 4): ... diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi deleted file mode 100644 index 4903930f..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.pyi +++ /dev/null @@ -1,80 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from typing import Union, Optional, Type, Any, Tuple, List -from baseImage.constant import Place -from baseImage import Image, Rect - -from image_registration.matching import MatchTemplate, CudaMatchTemplate - - -image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] -keypoint_type = List[cv2.KeyPoint, ...] -matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] -good_match_type = List[cv2.DMatch] -Detector = Union[cv2.Feature2D, cv2.cuda.Feature2DAsync] -Matcher = Union[cv2.DescriptorMatcher, cv2.cuda.DescriptorMatcher] - - -class BaseKeypoint(object): - FILTER_RATIO: int = 0.59 - METHOD_NAME: str - Dtype: Union[Type[np.uint8], Type[np.int8], Type[np.uint16], Type[np.int16], Type[np.int32], Type[np.float32], Type[np.float64]] - Place: Place - template: Union[MatchTemplate, CudaMatchTemplate] - - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, **kwargs): - self.detector: Detector = None - self.matcher: Matcher = None - self.threshold: Union[int, float] = 0.8 - self.rgb: bool = True - ... - - def create_matcher(self, **kwargs) -> Matcher: ... - - def create_detector(self, **kwargs) -> Detector: ... - - def find_best_result(self, im_source: image_type, im_search: image_type, threshold: Union[int, float] = None, rgb: bool = None) -> Optional[Rect]: ... - - def find_all_results(self, im_source: image_type, im_search: image_type, threshold: Union[int, float] = None, rgb: bool = None, max_count: int = 10, max_iter_counts: int = 20, distance_threshold: int = 150): ... - - def get_keypoint_and_descriptor(self, image: Image) -> Tuple[keypoint_type, np.ndarray]: ... - - def filter_good_point(self, matches: matches_type, kp_src: keypoint_type, kp_sch: keypoint_type, - kp_sch_point: np.ndarray, kp_src_matches_point: np.ndarray) -> Tuple[good_match_type, float, cv2.DMatch]: ... - - def get_rect_from_good_matches(self, im_source: Image, im_search: Image, kp_src: keypoint_type, - des_src: np.ndarray, kp_sch:keypoint_type, des_sch: np.ndarray) -> \ - Tuple[Optional[Rect], matches_type, good_match_type]: ... - - def match_keypoint(self, des_sch: np.ndarray, des_src: np.ndarray, k: int = 2) -> matches_type: ... - - def get_good_in_matches(self, matches: matches_type) -> good_match_type: ... - - def extract_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: float, rgb: bool) -> Union[Optional[Rect], Union[None, int, float]]: ... - - def _handle_one_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: Union[float, int]) -> Tuple[Image, Rect]: ... - - def _handle_two_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: Union[float, int]) -> Tuple[Image, Rect]: ... - - def _handle_three_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type, angle: Union[float, int]) -> Tuple[Image, Rect]: ... - - def _handle_many_good_points(self, im_source: Image, im_search: Image, kp_src: keypoint_type, kp_sch: keypoint_type, good: good_match_type) -> Tuple[Image, Rect]: ... - - def _target_image_crop(self, img: Image, rect: Rect) -> Image: ... - - def _cal_confidence(self, im_source: Image, im_search: Image, rgb: bool) -> Union[int, float]: ... - - def input_image_check(self, im_source: image_type, im_search: image_type) -> Tuple[Image, Image]: ... - - def _image_check(self, data: Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image]) -> Image: ... - - @staticmethod - def _find_homography(sch_pts: np.ndarray, src_pts: np.ndarray) -> Optional[Tuple[np.ndarray, np.ndarray]]: ... - - @staticmethod - def _perspective_transform(im_source: Image, im_search: Image, src: np.ndarray, dst: np.ndarray) -> Image: ... - - @staticmethod - def _get_perspective_area_rect(im_source: Image, src: np.ndarray) -> Rect: \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py deleted file mode 100644 index 1c9fb3f5..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .orb import ORB, CUDA_ORB diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py deleted file mode 100644 index 06109704..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.py +++ /dev/null @@ -1,129 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from baseImage.constant import Place - -from image_registration.matching.template import CudaMatchTemplate -from image_registration.matching.keypoint.base import BaseKeypoint -from image_registration.exceptions import NoEnoughPointsError, CudaOrbDetectorError -from typing import Union - - -class ORB(BaseKeypoint): - METHOD_NAME = 'ORB' - Dtype = np.uint8 - Place = (Place.UMat, Place.Ndarray) - - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, **kwargs): - super().__init__(threshold=threshold, rgb=rgb, **kwargs) - self.descriptor = self.create_descriptor() - - def create_matcher(self, **kwargs): - """ - 创建特征点匹配器 - - Returns: - cv2.FlannBasedMatcher - """ - matcher = cv2.DescriptorMatcher_create(cv2.DescriptorMatcher_BRUTEFORCE_HAMMING) - return matcher - - def create_detector(self, **kwargs): - nfeatures = kwargs.get('nfeatures', 50000) - scaleFactor = kwargs.get('scaleFactor', 1.2) - nlevels = kwargs.get('nlevels', 8) - edgeThreshold = kwargs.get('edgeThreshold', 31) - firstLevel = kwargs.get('firstLevel', 0) - WTA_K = kwargs.get('WTA_K', 2) - scoreType = kwargs.get('scoreType', cv2.ORB_HARRIS_SCORE) - patchSize = kwargs.get('patchSize', 31) - fastThreshold = kwargs.get('fastThreshold', 20) - - params = dict( - nfeatures=nfeatures, scaleFactor=scaleFactor, nlevels=nlevels, - edgeThreshold=edgeThreshold, firstLevel=firstLevel, WTA_K=WTA_K, - scoreType=scoreType, patchSize=patchSize, fastThreshold=fastThreshold, - ) - detector = cv2.ORB_create(**params) - return detector - - @staticmethod - def create_descriptor(): - # https://docs.opencv.org/master/d7/d99/classcv_1_1xfeatures2d_1_1BEBLID.html - # https://github.com/iago-suarez/beblid-opencv-demo - descriptor = cv2.xfeatures2d.BEBLID_create(0.75) - return descriptor - - def get_keypoint_and_descriptor(self, image): - if image.channels == 3: - image = image.cvtColor(cv2.COLOR_BGR2GRAY).data - else: - image = image.data - - keypoints = self.detector.detect(image, None) - keypoints, descriptors = self.descriptor.compute(image, keypoints) - - if len(keypoints) < 2: - raise NoEnoughPointsError('{} detect not enough feature points in input images'.format(self.METHOD_NAME)) - return keypoints, descriptors - - -class CUDA_ORB(BaseKeypoint): - METHOD_NAME = 'CUDA_ORB' - Dtype = np.uint8 - Place = (Place.GpuMat,) - - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, **kwargs): - super().__init__(threshold=threshold, rgb=rgb, **kwargs) - self.template = CudaMatchTemplate(threshold=threshold, rgb=rgb) - - def create_matcher(self, **kwargs): - """ - 创建特征点匹配器 - - Returns: - cv2.FlannBasedMatcher - """ - matcher = cv2.cuda.DescriptorMatcher_createBFMatcher(cv2.NORM_HAMMING) - return matcher - - def create_detector(self, **kwargs): - nfeatures = kwargs.get('nfeatures', 50000) - scaleFactor = kwargs.get('scaleFactor', 1.2) - nlevels = kwargs.get('nlevels', 8) - edgeThreshold = kwargs.get('edgeThreshold', 31) - firstLevel = kwargs.get('firstLevel', 0) - WTA_K = kwargs.get('WTA_K', 2) - scoreType = kwargs.get('scoreType', cv2.ORB_HARRIS_SCORE) - patchSize = kwargs.get('patchSize', 31) - fastThreshold = kwargs.get('fastThreshold', 20) - blurForDescriptor = kwargs.get('blurForDescriptor', False) - - params = dict( - nfeatures=nfeatures, scaleFactor=scaleFactor, nlevels=nlevels, - edgeThreshold=edgeThreshold, firstLevel=firstLevel, WTA_K=WTA_K, - scoreType=scoreType, patchSize=patchSize, fastThreshold=fastThreshold, - blurForDescriptor=blurForDescriptor - ) - - detector = cv2.cuda.ORB_create(**params) - return detector - - def get_keypoint_and_descriptor(self, image): - if image.channels == 3: - image = image.cvtColor(cv2.COLOR_BGR2GRAY).data - else: - image = image.data - - try: - keypoints, descriptors = self.detector.detectAndComputeAsync(image, None) - except cv2.error: - # https://github.com/opencv/opencv/issues/10573 - raise CudaOrbDetectorError('{} detect error, Try adjust detector params'.format(self.METHOD_NAME)) - else: - keypoints = self.detector.convert(keypoints) - - if len(keypoints) < 2: - raise NoEnoughPointsError('{} detect not enough feature points in input images'.format(self.METHOD_NAME)) - return keypoints, descriptors diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi deleted file mode 100644 index 364d3e4a..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/orb/orb.pyi +++ /dev/null @@ -1,38 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from typing import Union, Tuple, List -from baseImage import Image - -from image_registration.matching.keypoint.base import BaseKeypoint - - -image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] -keypoint_type = Tuple[cv2.KeyPoint, ...] -matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] -good_match_type = List[cv2.DMatch] - - -class ORB(BaseKeypoint): - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, - nfeatures: int = 50000 ,scaleFactor: Union[int, float] = 1.2, nlevels: int = 8, - edgeThreshold: int = 31, firstLevel: int = 0, WTA_K: int = 2, - scoreType: int = cv2.ORB_HARRIS_SCORE, patchSize: int = 31, fastThreshold: int = 20): - self.descriptor = cv2.xfeatures2d.BEBLID - ... - - @staticmethod - def create_descriptor() -> cv2.xfeatures2d.BEBLID: ... - - -class CUDA_ORB(BaseKeypoint): - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, - nfeatures: int = 50000 ,scaleFactor: Union[int, float] = 1.2, nlevels: int = 8, - edgeThreshold: int = 31, firstLevel: int = 0, WTA_K: int = 2, - scoreType: int = cv2.ORB_HARRIS_SCORE, patchSize: int = 31, fastThreshold: int = 20, - blurForDescriptor: bool = False): ... - - def create_detector(self, **kwargs) -> cv2.cuda.Feature2DAsync: ... - - def create_matcher(self, **kwargs) -> cv2.cuda.DescriptorMatcher: ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py deleted file mode 100644 index 3b395981..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .sift import SIFT diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi deleted file mode 100644 index fa7bcfc9..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.pyi +++ /dev/null @@ -1,23 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from typing import Union, Tuple, List - -from baseImage import Image - -from image_registration.matching.keypoint.base import BaseKeypoint - - -image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] -keypoint_type = Tuple[cv2.KeyPoint, ...] -matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] -good_match_type = List[cv2.DMatch] - - -class SIFT(BaseKeypoint): - FLANN_INDEX_KDTREE: int = 0 - - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, - nfeatures: int = 0, nOctaveLayers: int = 3, contrastThreshold: float = 0.04, - edgeThreshold: int = 10, sigma: float = 1.6): ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py deleted file mode 100644 index 15f433a0..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .surf import SURF diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py deleted file mode 100644 index 7dc6a767..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.py +++ /dev/null @@ -1,38 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from baseImage.constant import Place - -from image_registration.matching.keypoint.base import BaseKeypoint - - -class SURF(BaseKeypoint): - FLANN_INDEX_KDTREE = 0 - METHOD_NAME = "SURF" - Dtype = np.uint8 - Place = (Place.UMat, Place.Ndarray) - - def create_matcher(self, **kwargs) -> cv2.DescriptorMatcher: - """ - 创建特征点匹配器 - - Returns: - cv2.FlannBasedMatcher - """ - index_params = {'algorithm': self.FLANN_INDEX_KDTREE, 'tree': 5} - # 指定递归遍历的次数. 值越高结果越准确，但是消耗的时间也越多 - search_params = {'checks': 50} - matcher = cv2.FlannBasedMatcher(index_params, search_params) - return matcher - - def create_detector(self, **kwargs): - hessianThreshold = kwargs.get('hessianThreshold', 400) - nOctaves = kwargs.get('nOctaves', 4) - nOctaveLayers = kwargs.get('nOctaveLayers', 3) - extended = kwargs.get('extended', True) - upright = kwargs.get('upright', False) - - detector = cv2.xfeatures2d.SURF_create(hessianThreshold=hessianThreshold, nOctaves=nOctaves, nOctaveLayers=nOctaveLayers, - extended=extended, upright=upright) - return detector diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi deleted file mode 100644 index 4d8b6caa..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/surf/surf.pyi +++ /dev/null @@ -1,25 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -import cv2 -import numpy as np -from typing import Union, Tuple, List - -from baseImage import Image - -from image_registration.matching.keypoint.base import BaseKeypoint - - -image_type = Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image] -keypoint_type = Tuple[cv2.KeyPoint, ...] -matches_type = Tuple[Tuple[cv2.DMatch, ...], ...] -good_match_type = List[cv2.DMatch] - - -class SURF(BaseKeypoint): - FLANN_INDEX_KDTREE: int = 0 - - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True, - hessianThreshold: int = 400, nOctaves: int = 4, nOctaveLayers: int = 3, - extended: bool = True, upright: bool = False): ... - - def create_detector(self, **kwargs) -> cv2.xfeatures2d.SURF: ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py deleted file mode 100644 index 96abfc8e..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -from .matchTemplate import MatchTemplate, CudaMatchTemplate diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi deleted file mode 100644 index 5cb2ddfd..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.pyi +++ /dev/null @@ -1,56 +0,0 @@ -#! usr/bin/python -# -*- coding:utf-8 -*- -""" opencv matchTemplate""" -import warnings -import cv2 -import numpy as np -from baseImage import Image, Rect -from baseImage.constant import Place - - -from typing import Union, Any, Tuple, Type - - -class MatchTemplate(object): - METHOD_NAME: str = 'tpl' - Dtype: Type[np.uint8] = np.uint8 - Place: tuple = (Place.Ndarray, ) - - def __init__(self, threshold: Union[int, float] = 0.8, rgb: bool = True): - self.threshold: Union[int, float] = threshold - self.rgb: bool = rgb - self.matcher: cv2.matchTemplate = cv2.matchTemplate - - def find_best_result(self, im_source: Image, im_search: Image, threshold: Union[int, float] = None, rgb: bool = None) -> dict: ... - - def find_all_results(self, im_source: Image, im_search: Image, threshold: Union[int, float] = None, rgb: bool = None, max_count: int = 10) -> list: ... - - def _get_template_result_matrix(self, im_source: Image, im_search: Image) -> Image: ... - - def input_image_check(self, im_source: Any, im_search: Any) -> Image: ... - - def _image_check(self, data: Union[str, bytes, np.ndarray, cv2.cuda.GpuMat, cv2.Mat, cv2.UMat, Image]) -> Image: ... - - @staticmethod - def minMaxLoc(result: np.ndarray) -> Tuple[float, float, Tuple[int, int], Tuple[int, int]]: ... - - def match(self, img1: np.ndarray, img2: np.ndarray) -> np.ndarray: ... - - def cal_confidence(self, im_source: Image, im_search: Image, crop_rect: Rect, max_val: float, rgb: bool) -> float: ... - - def cal_rgb_confidence(self, im_source: Image, im_search: Image) -> float: ... - - def cal_ccoeff_confidence(self, im_source: Image, im_search: Image) -> float: ... - - def _get_confidence_from_matrix(self, img_crop: Image, im_search: Image, max_val: float, rgb: bool) -> float: ... - - -class CudaMatchTemplate(MatchTemplate): - METHOD_NAME: str = 'tpl' - Dtype: Type[np.uint8] = np.uint8 - Place: tuple = Place.GpuMat - - @staticmethod - def minMaxLoc(result: cv2.cuda.GpuMat) -> tuple:... - - def match(self, img1: cv2.cuda.GpuMat, img2: cv2.cuda.GpuMat) -> cv2.cuda.GpuMat: ... diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi b/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi deleted file mode 100644 index a607bbb6..00000000 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.pyi +++ /dev/null @@ -1,27 +0,0 @@ -# from __future__ import annotations -from baseImage import Rect -from typing import Union, TypedDict, List, Tuple -import cv2 - - -class ResultType(TypedDict): - rect: Rect - confidence: Union[int, float] -point_type = Tuple[Union[float, int], Union[float, int]] -keypoint_type = Union[cv2.KeyPoint, List[float, float], Tuple[float, float]] - - -def generate_result(rect: Rect, confi: Union[float, int]) -> ResultType: ... - -def keypoint_distance(kp1: keypoint_type, kp2: keypoint_type) -> float: ... - -def keypoint_angle(kp1: cv2.KeyPoint, kp2: cv2.KeyPoint) -> float: ... - -def get_keypoint_from_matches(kp: Tuple[cv2.KeyPoint, ...], matches: List[cv2.DMatch, ...], mode: str) -> List[cv2.KeyPoint]: ... - -def keypoint_origin_angle(kp1: cv2.KeyPoint, kp2: cv2.KeyPoint) -> Union[int, float]: ... - -def _mapping_angle_distance(distance: float, origin_angle: Union[int, float], angle: Union[int, float]) -> Tuple[float, float]: ... - -def rectangle_transform(point: point_type, size: point_type, mapping_point: point_type, mapping_size: point_type, angle: Union[float, int])\ - -> Tuple[point_type, point_type, point_type, point_type]: ... \ No newline at end of file diff --git a/flybirds/core/plugin/plugins/default/ui_driver/opencv/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/__init__.py new file mode 100644 index 00000000..3049b31f --- /dev/null +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/__init__.py @@ -0,0 +1,7 @@ +#! usr/bin/python +# -*- coding:utf-8 -*- +from .matchTemplate import MatchTemplate +from .sift import SIFT +from .utils import generate_result, get_keypoint_from_matches, keypoint_distance, rectangle_transform +from .exceptions import NoEnoughPointsError, PerspectiveTransformError, HomographyError, MatchResultError, InputImageError +from .base import BaseKeypoint diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py similarity index 98% rename from flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py rename to flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py index 43db9f62..2fb59a95 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/base.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py @@ -4,9 +4,9 @@ import numpy as np from baseImage import Image, Rect -from image_registration.matching import MatchTemplate -from image_registration.utils import (generate_result, get_keypoint_from_matches, keypoint_distance, rectangle_transform) -from image_registration.exceptions import (NoEnoughPointsError, PerspectiveTransformError, HomographyError, MatchResultError, +from .matchTemplate import MatchTemplate +from .utils import (generate_result, get_keypoint_from_matches, keypoint_distance, rectangle_transform) +from .exceptions import (NoEnoughPointsError, PerspectiveTransformError, HomographyError, MatchResultError, InputImageError) from typing import List @@ -516,7 +516,7 @@ def _find_homography(sch_pts, src_pts): """ try: # M, mask = cv2.findHomography(sch_pts, src_pts, cv2.RANSAC) - M, mask = cv2.findHomography(sch_pts, src_pts, cv2.USAC_MAGSAC, 4.0, None, 2000, 0.99) + M, mask = cv2.findHomography(sch_pts, src_pts, cv2.RANSAC, 4.0, None, 2000, 0.99) except cv2.error: import traceback traceback.print_exc() diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/exceptions.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/exceptions.py similarity index 100% rename from flybirds/core/plugin/plugins/default/ui_driver/image_registration/exceptions.py rename to flybirds/core/plugin/plugins/default/ui_driver/opencv/exceptions.py diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py similarity index 91% rename from flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py rename to flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py index a1a8a584..9552b097 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/template/matchTemplate.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py @@ -7,9 +7,8 @@ from baseImage import Image, Rect from baseImage.constant import Place -from image_registration.exceptions import MatchResultError, NoModuleError, InputImageError -from image_registration.utils import generate_result -from typing import Union +from .exceptions import MatchResultError, InputImageError +from .utils import generate_result class MatchTemplate(object): @@ -229,21 +228,3 @@ def _get_confidence_from_matrix(self, img_crop, im_search, max_val, rgb): return confidence -class CudaMatchTemplate(MatchTemplate): - METHOD_NAME = 'cuda_tpl' - Dtype = np.uint8 - Place = (Place.GpuMat, ) - - def __init__(self, threshold=0.8, rgb=True): - super(CudaMatchTemplate, self).__init__(threshold=threshold, rgb=rgb) - try: - self.matcher = cv2.cuda.createTemplateMatching(cv2.CV_8U, cv2.TM_CCOEFF_NORMED) - except AttributeError: - raise NoModuleError('create CUDA TemplateMatching Error') - - @staticmethod - def minMaxLoc(result): - return cv2.cuda.minMaxLoc(result) - - def match(self, img1, img2): - return self.matcher.match(img1, img2) diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py similarity index 94% rename from flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py rename to flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py index dcb8a2ff..42e7d7a3 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/matching/keypoint/sift/sift.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py @@ -4,7 +4,7 @@ import numpy as np from baseImage.constant import Place -from image_registration.matching.keypoint.base import BaseKeypoint +from .base import BaseKeypoint class SIFT(BaseKeypoint): diff --git a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py similarity index 99% rename from flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py rename to flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py index 5300f15a..46266e9c 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/image_registration/utils/__init__.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py @@ -2,8 +2,6 @@ # -*- coding:utf-8 -*- import math import cv2 -import numpy as np -from typing import Union def generate_result(rect, confi): diff --git a/requirements.txt b/requirements.txt index 80b3d1eb..b793ea37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,11 @@ deepdiff>=5.8.1 paddleocr>=2.5.0 paddlepaddle>=2.3.0 protobuf==3.20.1 -baseImage==2.1.1 \ No newline at end of file +baseImage==2.1.1 +pydantic + +six~=1.16.0 +Pillow~=9.0.1 +opencv-python~=4.5.5.64 +numpy~=1.22.3 +requests~=2.27.1 \ No newline at end of file From 3171d903cc7a9cdf522b2d6223ed47b96cc23248 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Fri, 29 Jul 2022 13:12:53 +0800 Subject: [PATCH 03/10] reactor:remove unused require --- requirements.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index b793ea37..19a890b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,11 +14,4 @@ deepdiff>=5.8.1 paddleocr>=2.5.0 paddlepaddle>=2.3.0 protobuf==3.20.1 -baseImage==2.1.1 -pydantic - -six~=1.16.0 -Pillow~=9.0.1 -opencv-python~=4.5.5.64 -numpy~=1.22.3 -requests~=2.27.1 \ No newline at end of file +baseImage>=2.1.1 From 49e42dee7c8c622d88f73b6d5caa2b7e47ce9616 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Fri, 29 Jul 2022 14:31:16 +0800 Subject: [PATCH 04/10] chire:fix --- .../core/plugin/plugins/default/screen.py | 6 +- .../plugins/default/ui_driver/opencv/base.py | 257 +++++++++--------- .../default/ui_driver/opencv/exceptions.py | 18 -- .../default/ui_driver/opencv/matchTemplate.py | 82 +++--- .../plugins/default/ui_driver/opencv/sift.py | 5 +- .../plugins/default/ui_driver/opencv/utils.py | 81 ++---- 6 files changed, 200 insertions(+), 249 deletions(-) diff --git a/flybirds/core/plugin/plugins/default/screen.py b/flybirds/core/plugin/plugins/default/screen.py index f567a3bb..67bfd85d 100644 --- a/flybirds/core/plugin/plugins/default/screen.py +++ b/flybirds/core/plugin/plugins/default/screen.py @@ -8,7 +8,6 @@ import baseImage from base64 import b64decode from PIL import Image -# from flybirds.core.plugin.plugins.default.ui_driver.image_registration.matching.keypoint.sift import SIFT from .ui_driver import SIFT import flybirds.core.global_resource as gr @@ -127,5 +126,6 @@ def image_verify(img_source_path, img_search_path): start = time.time() result = match.find_all_results(img_source, img_search) - print(time.time() - start) - print(result) + log.info(f"[image verify] cost time:{time.time() - start}") + log.info(f"[image verify] result:{result}") + diff --git a/flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py index 2fb59a95..d854e976 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/base.py @@ -7,7 +7,7 @@ from .matchTemplate import MatchTemplate from .utils import (generate_result, get_keypoint_from_matches, keypoint_distance, rectangle_transform) from .exceptions import (NoEnoughPointsError, PerspectiveTransformError, HomographyError, MatchResultError, - InputImageError) + InputImageError) from typing import List @@ -20,11 +20,11 @@ class BaseKeypoint(object): def __init__(self, threshold=0.8, rgb=True, **kwargs): """ - 初始化 + init Args: - threshold: 识别阈值(0~1) - rgb: 是否使用rgb通道进行校验 + threshold: recognition threshold(0~1) + rgb: Whether to use rgb channel for verification """ self.threshold = threshold self.rgb = rgb @@ -39,13 +39,13 @@ def create_detector(self, **kwargs): def find_best_result(self, im_source, im_search, threshold=None, rgb=None, **kwargs): """ - 通过特征点匹配,在im_source中找到最符合im_search的范围 + Through feature point matching, find the range that best matches im_search in im_source Args: - im_source: 待匹配图像 - im_search: 图片模板 - threshold: 识别阈值(0~1) - rgb: 是否使用rgb通道进行校验 + im_source: image to be matched + im_search: image template + threshold: recognition threshold(0~1) + rgb: Whether to use rgb channel for verification Returns: @@ -57,18 +57,20 @@ def find_best_result(self, im_source, im_search, threshold=None, rgb=None, **kwa return ret[0] return None - def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_count=10, max_iter_counts=20, distance_threshold=150): + def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_count=10, max_iter_counts=20, + distance_threshold=150): """ - 通过特征点匹配,在im_source中找到全部符合im_search的范围 + Through feature point matching, find all the ranges that match im_search in im_source Args: - im_source: 待匹配图像 - im_search: 图片模板 - threshold: 识别阈值(0~1) - rgb: 是否使用rgb通道进行校验 - max_count: 最多可以返回的匹配数量 - max_iter_counts: 最大的搜索次数,需要大于max_count - distance_threshold: 距离阈值,特征点(first_point)大于该阈值后,不做后续筛选 + im_source: the image to be matched + im_search: image template + threshold: recognition threshold (0~1) + rgb: whether to use the rgb channel for verification + max_count: the maximum number of matches that can be returned + max_iter_counts: The maximum number of searches, which needs to be greater than max_count + distance_threshold: distance threshold, after the feature point (first_point) is greater than + the threshold, no subsequent screening will be done Returns: @@ -85,7 +87,7 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c kp_sch, des_sch = self.get_keypoint_and_descriptor(image=im_search) kp_src, kp_sch = list(kp_src), list(kp_sch) - # 在特征点集中,匹配最接近的特征点 + # In the feature point set, match the closest feature point matches = np.array(self.match_keypoint(des_sch=des_sch, des_src=des_src)) kp_sch_point = np.array([(kp.pt[0], kp.pt[1], kp.angle) for kp in kp_sch]) kp_src_matches_point = np.array([[(*kp_src[dMatch.trainIdx].pt, kp_src[dMatch.trainIdx].angle) @@ -93,8 +95,9 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c _max_iter_counts = 0 src_pop_list = [] while True: - # 这里没有用matches判断nan, 是因为类型不对 - if (np.count_nonzero(~np.isnan(kp_src_matches_point)) == 0) or (len(result) == max_count) or (_max_iter_counts >= max_iter_counts): + # not use of matches to judge nan, because the type is wrong + if (np.count_nonzero(~np.isnan(kp_src_matches_point)) == 0) or (len(result) == max_count) or ( + _max_iter_counts >= max_iter_counts): break _max_iter_counts += 1 filtered_good_point, angle, first_point = self.filter_good_point(matches=matches, kp_src=kp_src, @@ -107,7 +110,8 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c rect, confidence = None, 0 try: rect, confidence = self.extract_good_points(im_source=im_source, im_search=im_search, kp_src=kp_src, - kp_sch=kp_sch, good=filtered_good_point, angle=angle, rgb=rgb) + kp_sch=kp_sch, good=filtered_good_point, angle=angle, + rgb=rgb) # print(f'good:{len(filtered_good_point)}, rect={rect}, confidence={confidence}') except PerspectiveTransformError: pass @@ -115,7 +119,6 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c if rect and confidence >= threshold: br, tl = rect.br, rect.tl - # 移除改范围内的所有特征点 ??有可能因为透视变换的原因，删除了多余的特征点 for index, match in enumerate(kp_src_matches_point): x, y = match[:, 0], match[:, 1] flag = np.argwhere((x < br.x) & (x > tl.x) & (y < br.y) & (y > tl.y)) @@ -130,24 +133,14 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c for _index in flags: kp_src_matches_point[match.queryIdx, _index, :] = np.nan matches[match.queryIdx, _index] = np.nan - - # 一下代码用于删除目标图片中的特征点,以后会用 - # src_pop_list = list(set(src_pop_list)) - # src_pop_list.sort(reverse=True) - # mask = np.ones(len(des_src), dtype=bool) - # for v in src_pop_list: - # kp_src.pop(v) - # mask[v] = False - # - # des_src = des_src[mask, ...] return result def get_keypoint_and_descriptor(self, image): """ - 获取图像关键点(keypoint)与描述符(descriptor) + Get image keypoint (keypoint) and descriptor (descriptor) Args: - image: 待检测的灰度图像 + image: Grayscale image to be detected Returns: @@ -164,8 +157,8 @@ def get_keypoint_and_descriptor(self, image): @staticmethod def filter_good_point(matches, kp_src, kp_sch, kp_sch_point, kp_src_matches_point): - """ 筛选最佳点 """ - # 假设第一个点,及distance最小的点,为基准点 + """ Filter the sweet spot """ + # Assume that the first point and the point with the smallest distance are the reference points sort_list = [sorted(match, key=lambda x: x is np.nan and float('inf') or x.distance)[0] for match in matches] sort_list = [v for v in sort_list if v is not np.nan] @@ -191,11 +184,11 @@ def get_points_origin_angle(point_x, point_y, offset): ) return points_origin_angle - # 计算模板图像上,该点与其他特征点的旋转角 + # Calculate the rotation angle between this point and other feature points on the template image first_good_point_sch_origin_angle = get_points_origin_angle(kp_sch_point[:, 0], kp_sch_point[:, 1], first_good_point_query) - # 计算目标图像中,该点与其他特征点的夹角 + # Calculate the angle between the point and other feature points in the target image kp_sch_rotate_angle = kp_sch_point[:, 2] + first_good_point_angle kp_sch_rotate_angle = np.where(kp_sch_rotate_angle >= 360, kp_sch_rotate_angle - 360, kp_sch_rotate_angle) kp_sch_rotate_angle = kp_sch_rotate_angle.reshape(kp_sch_rotate_angle.shape + (1,)) @@ -204,10 +197,10 @@ def get_points_origin_angle(point_x, point_y, offset): good_point = np.array([matches[index][array[0]] for index, array in enumerate(np.argsort(np.abs(kp_src_angle - kp_sch_rotate_angle)))]) - # 计算各点以first_good_point为原点的旋转角 + # Calculate the rotation angle of each point with first_good_point as the origin good_point_nan = (np.nan, np.nan) - good_point_pt = np.array([good_point_nan if dMatch is np.nan else (*kp_src[dMatch.trainIdx].pt, ) - for dMatch in good_point]) + good_point_pt = np.array([good_point_nan if dMatch is np.nan else (*kp_src[dMatch.trainIdx].pt,) + for dMatch in good_point]) good_point_origin_angle = get_points_origin_angle(good_point_pt[:, 0], good_point_pt[:, 1], first_good_point_train) threshold = round(5 / 360, 2) * 100 @@ -219,29 +212,29 @@ def get_points_origin_angle(point_x, point_y, offset): def match_keypoint(self, des_sch, des_src, k=10): """ - 特征点匹配 + Feature point matching Args: - des_src: 待匹配图像的描述符集 - des_sch: 图片模板的描述符集 - k(int): 获取多少匹配点 + des_src: descriptor set of the image to be matched + des_sch: descriptor set for image templates + k(int): how many matching points to get Returns: - List[List[cv2.DMatch]]: 包含最匹配的描述符 + List[List[cv2.DMatch]]: contains the best matching descriptor """ - # k=2表示每个特征点取出2个最匹配的对应点 + # k=2 means that each feature point takes out the 2 most matching corresponding points matches = self.matcher.knnMatch(des_sch, des_src, k) return matches def get_good_in_matches(self, matches): """ - 特征点过滤 + Feature point filtering Args: - matches: 特征点集 + matches: Feature point set Returns: - List[cv2.DMatch]: 过滤后的描述符集 + List[cv2.DMatch]: Filtered descriptor set """ if not matches: return None @@ -255,19 +248,19 @@ def get_good_in_matches(self, matches): def extract_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle, rgb): """ - 根据匹配点(good)数量,提取识别区域 + According to the number of matching points (good), extract the recognition area Args: - im_source: 待匹配图像 - im_search: 图片模板 - kp_src: 关键点集 - kp_sch: 关键点集 - good: 描述符集 - angle: 旋转角度 - rgb: 是否使用rgb通道进行校验 + im_source: the image to be matched + im_search: image template + kp_src: keypoint set + kp_sch: keypoint set + good: descriptor set + angle: rotation angle + rgb: whether to use the rgb channel for verification Returns: - 范围,和置信度 + range, and confidence """ len_good = len(good) confidence, rect, target_img = None, None, None @@ -294,18 +287,20 @@ def extract_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle, def _handle_one_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle): """ - 特征点匹配数量等于1时,根据特征点的大小,对矩形进行缩放,并根据旋转角度,获取识别的目标图片 + When the number of feature point matching is equal to 1, the rectangle is scaled according + to the size of the feature point, and the recognized target image is obtained according to + the rotation angle. Args: - im_source: 待匹配图像 - im_search: 图片模板 - kp_sch: 关键点集 - kp_src: 关键点集 - good: 描述符集 - angle: 旋转角度 + im_source: the image to be matched + im_search: image template + kp_sch: keypoint set + kp_src: keypoint set + good: descriptor set + angle: rotation angle Returns: - 待验证的图片 + Image to be verified """ sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query')[0] src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train')[0] @@ -322,18 +317,20 @@ def _handle_one_good_points(self, im_source, im_search, kp_src, kp_sch, good, an def _handle_two_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle): """ - 特征点匹配数量等于2时,根据两点距离差,对矩形进行缩放,并根据旋转角度,获取识别的目标图片 + When the number of feature points matching is equal to 2, the rectangle is scaled + according to the distance difference between the two points, and the recognized target + image is obtained according to the rotation angle. Args: - im_source: 待匹配图像 - im_search: 图片模板 - kp_sch: 关键点集 - kp_src: 关键点集 - good: 描述符集 - angle: 旋转角度 + im_source: the image to be matched + im_search: image template + kp_sch: keypoint set + kp_src: keypoint set + good: descriptor set + angle: rotation angle Returns: - 待验证的图片 + Image to be verified """ sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query') src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train') @@ -342,7 +339,7 @@ def _handle_two_good_points(self, im_source, im_search, kp_src, kp_sch, good, an src_distance = keypoint_distance(src_point[0], src_point[1]) try: - scale = src_distance / sch_distance # 计算缩放大小 + scale = src_distance / sch_distance # Calculate the zoom size except ZeroDivisionError: if src_distance == sch_distance: scale = 1 @@ -360,18 +357,20 @@ def _handle_two_good_points(self, im_source, im_search, kp_src, kp_sch, good, an def _handle_three_good_points(self, im_source, im_search, kp_src, kp_sch, good, angle): """ - 特征点匹配数量等于3时,根据三个点组成的三角面积差,对矩形进行缩放,并根据旋转角度,获取识别的目标图片 + When the number of feature points matching is equal to 3, the rectangle is scaled + according to the difference in the area of the triangle formed by the three points, + and the recognized target image is obtained according to the rotation angle. Args: - im_source: 待匹配图像 - im_search: 图片模板 - kp_sch: 关键点集 - kp_src: 关键点集 - good: 描述符集 - angle: 旋转角度 + im_source: the image to be matched + im_search: image template + kp_sch: keypoint set + kp_src: keypoint set + good: descriptor set + angle: rotation angle Returns: - 待验证的图片 + Image to be verified """ sch_point = get_keypoint_from_matches(kp=kp_sch, matches=good, mode='query') src_point = get_keypoint_from_matches(kp=kp_src, matches=good, mode='train') @@ -389,7 +388,7 @@ def _area(point_list): src_area = _area(src_point) try: - scale = src_area / sch_area # 计算缩放大小 + scale = src_area / sch_area # Calculate the zoom size except ZeroDivisionError: if sch_area == src_area: scale = 1 @@ -407,24 +406,26 @@ def _area(point_list): def _handle_many_good_points(self, im_source, im_search, kp_src, kp_sch, good): """ - 特征点匹配数量>=4时,使用单矩阵映射,获取识别的目标图片 + When the number of feature point matching is >= 4, + use single matrix mapping to obtain the recognized target image Args: - im_source: 待匹配图像 - im_search: 图片模板 - kp_sch: 关键点集 - kp_src: 关键点集 - good: 描述符集 + im_source: the image to be matched + im_search: image template + kp_sch: keypoint set + kp_src: keypoint set + good: descriptor set Returns: - 透视变换后的图片 + Perspective transformed image """ sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape( -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) - # M是转化矩阵 + # M is the transformation matrix M, mask = self._find_homography(sch_pts, img_pts) - # 计算四个角矩阵变换后的坐标，也就是在大图中的目标区域的顶点坐标: + # Calculate the transformed coordinates of the four corner matrices, + # that is, the vertex coordinates of the target area in the large image: h, w = im_search.size h_s, w_s = im_source.size pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2) @@ -451,14 +452,14 @@ def _handle_many_good_points(self, im_source, im_search, kp_src, kp_sch, good): @staticmethod def _target_image_crop(img, rect): """ - 截取目标图像 + Capture target image Args: - img: 图像 - rect: 图像范围 + img: image + rect: Image range Returns: - 裁剪后的图像 + cropped image """ try: target_img = img.crop(rect) @@ -468,12 +469,12 @@ def _target_image_crop(img, rect): def _cal_confidence(self, im_source, im_search, rgb): """ - 将截图和识别结果缩放到大小一致,并计算可信度 + Scale the screenshot and the recognition result to the same size, and calculate the reliability Args: - im_source: 待匹配图像 - im_search: 图片模板 - rgb:是否使用rgb通道进行校验 + im_source: the image to be matched + im_search: image template + rgb: whether to use the rgb channel for verification Returns: @@ -493,11 +494,14 @@ def input_image_check(self, im_source, im_search): im_search = self._image_check(im_search) if im_source.place != im_search.place: - raise InputImageError('输入图片类型必须相同, source={}, search={}'.format(im_source.place, im_search.place)) + raise InputImageError( + 'image type must be same, source={}, search={}'.format(im_source.place, im_search.place)) elif im_source.dtype != im_search.dtype: - raise InputImageError('输入图片数据类型必须相同, source={}, search={}'.format(im_source.dtype, im_search.dtype)) + raise InputImageError( + 'image data type must be same, source={}, search={}'.format(im_source.dtype, im_search.dtype)) elif im_source.channels != im_search.channels: - raise InputImageError('输入图片通道必须相同, source={}, search={}'.format(im_source.channels, im_search.channels)) + raise InputImageError( + 'image channel must be same, source={}, search={}'.format(im_source.channels, im_search.channels)) return im_source, im_search @@ -506,13 +510,13 @@ def _image_check(self, data): data = Image(data, dtype=self.Dtype) if data.place not in self.Place: - raise TypeError(f'{self.METHOD_NAME}方法,Image类型必须为(Place.UMat, Place.Ndarray)') + raise TypeError(f'{self.METHOD_NAME}method,Image type must be(Place.UMat, Place.Ndarray)') return data @staticmethod def _find_homography(sch_pts, src_pts): """ - 多组特征点对时，求取单向性矩阵 + When there are multiple sets of feature point pairs, obtain the unidirectional matrix """ try: # M, mask = cv2.findHomography(sch_pts, src_pts, cv2.RANSAC) @@ -530,13 +534,16 @@ def _find_homography(sch_pts, src_pts): @staticmethod def _perspective_transform(im_source, im_search, src, dst): """ - 根据四对对应点计算透视变换, 并裁剪相应图片 + Calculate the perspective transformation according to the + four pairs of corresponding points, and crop the corresponding picture Args: - im_source: 待匹配图像 - im_search: 待匹配模板 - src: 目标图像中相应四边形顶点的坐标 (左上,右上,左下,右下) - dst: 源图像中四边形顶点的坐标 (左上,右上,左下,右下) + im_source: the image to be matched + im_search: template to be matched + src: The coordinates of the corresponding quad vertices in the target + image (upper left, upper right, lower left, lower right) + dst: coordinates of the quad vertices in the source image (upper left, + upper right, lower left, lower right) Returns: @@ -551,14 +558,15 @@ def _perspective_transform(im_source, im_search, src, dst): @staticmethod def _get_perspective_area_rect(im_source, src): """ - 根据矩形四个顶点坐标,获取在原图中的最大外接矩形 + According to the coordinates of the four vertices + of the rectangle, obtain the largest circumscribed rectangle in the original image Args: - im_source: 待匹配图像 - src: 目标图像中相应四边形顶点的坐标 + im_source: image to be matched + src: the coordinates of the corresponding quadrilateral vertices in the target image Returns: - 最大外接矩形 + Maximum circumscribed rectangle """ h, w = im_source.size @@ -566,15 +574,18 @@ def _get_perspective_area_rect(im_source, src): y = [int(i[1]) for i in src] x_min, x_max = min(x), max(x) y_min, y_max = min(y), max(y) - # 挑选出目标矩形区域可能会有越界情况，越界时直接将其置为边界： - # 超出左边界取0，超出右边界取w_s-1，超出下边界取0，超出上边界取h_s-1 - # 当x_min小于0时，取0。 x_max小于0时，取0。 + # Selecting the target rectangular area may be out of bounds, + # and directly set it as the boundary when the boundary is exceeded: + # If it exceeds the left boundary, it takes 0, if it exceeds the right boundary, + # it takes w_s-1, if it exceeds the lower boundary, it takes 0, and if it exceeds + # the upper boundary, it takes h_s-1. + # When x_min is less than 0, take 0. When x_max is less than 0, take 0. x_min, x_max = int(max(x_min, 0)), int(max(x_max, 0)) - # 当x_min大于w_s时，取值w_s-1。 x_max大于w_s-1时，取w_s-1。 + # When x_min is greater than w_s, the value is w_s-1. When x_max is greater than w_s-1, take w_s-1。 x_min, x_max = int(min(x_min, w - 1)), int(min(x_max, w - 1)) - # 当y_min小于0时，取0。 y_max小于0时，取0。 + # When y_min is less than 0, take 0. When y_max is less than 0, take 0. y_min, y_max = int(max(y_min, 0)), int(max(y_max, 0)) - # 当y_min大于h_s时，取值h_s-1。 y_max大于h_s-1时，取h_s-1。 + # When y_min is greater than h_s, the value is h_s-1. When y_max is greater than h_s-1, take h_s-1. y_min, y_max = int(min(y_min, h - 1)), int(min(y_max, h - 1)) rect = Rect(x=x_min, y=y_min, width=(x_max - x_min), height=(y_max - y_min)) return rect diff --git a/flybirds/core/plugin/plugins/default/ui_driver/opencv/exceptions.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/exceptions.py index dcf67cc5..eeaef088 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/opencv/exceptions.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/exceptions.py @@ -8,28 +8,10 @@ def __repr__(self): return repr(self.message) -class NoModuleError(BaseError): - """ Missing dependent module """ - - -class CreateExtractorError(BaseError): - """ An error occurred while create Extractor """ - - class NoEnoughPointsError(BaseError): """ detect not enough feature points in input images""" -class CudaSurfInputImageError(BaseError): - """ The image size does not conform to CUDA standard """ - # https://stackoverflow.com/questions/42492060/surf-cuda-error-while-computing-descriptors-and-keypoints - # https://github.com/opencv/opencv_contrib/blob/master/modules/xfeatures2d/src/surf.cuda.cpp#L151 - - -class CudaOrbDetectorError(BaseError): - """ An CvError when orb detector error occurred """ - - class HomographyError(BaseError): """ An error occurred while findHomography """ diff --git a/flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py index 9552b097..bf3ffe2a 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/matchTemplate.py @@ -18,13 +18,13 @@ class MatchTemplate(object): def __init__(self, threshold=0.8, rgb=True): """ - 初始化 + init Args: - threshold: 识别阈值(0~1) - rgb: 是否使用rgb通道进行校验 + threshold: recognition threshold (0~1) + rgb: whether to use the rgb channel for verification """ - assert 0 <= threshold <= 1, 'threshold 取值在0到1直接' + assert 0 <= threshold <= 1, 'threshold value between 0 and 1' self.threshold = threshold self.rgb = rgb @@ -32,13 +32,14 @@ def __init__(self, threshold=0.8, rgb=True): def find_best_result(self, im_source, im_search, threshold=None, rgb=None): """ - 模板匹配, 返回匹配度最高且大于阈值的范围 + Template matching, return the range with the highest + matching degree and greater than the threshold Args: - im_source: 待匹配图像 - im_search: 图片模板 - threshold: 识别阈值(0~1) - rgb: 是否使用rgb通道进行校验 + im_source: the image to be matched + im_search: image template + threshold: recognition threshold (0~1) + rgb: whether to use the rgb channel for verification Returns: generate_result @@ -51,16 +52,16 @@ def find_best_result(self, im_source, im_search, threshold=None, rgb=None): rgb = False result = self._get_template_result_matrix(im_source=im_source, im_search=im_search) - # 找到最佳匹配项 + # Find the best match min_val, max_val, min_loc, max_loc = self.minMaxLoc(result.data) h, w = im_search.size - # 求可信度 + # Seek credibility crop_rect = Rect(max_loc[0], max_loc[1], w, h) confidence = self.cal_confidence(im_source, im_search, crop_rect, max_val, rgb) - # 如果可信度小于threshold,则返回None + # Returns None if the confidence is less than the threshold if confidence < (threshold or self.threshold): return None x, y = max_loc @@ -69,14 +70,15 @@ def find_best_result(self, im_source, im_search, threshold=None, rgb=None): def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_count=10): """ - 模板匹配, 返回匹配度大于阈值的范围, 且最大数量不超过max_count + Template matching, return the range with matching degree greater + than the threshold, and the maximum number does not exceed max_count Args: - im_source: 待匹配图像 - im_search: 图片模板 - threshold:: 识别阈值(0~1) - rgb: 是否使用rgb通道进行校验 - max_count: 最多匹配数量 + im_source: the image to be matched + im_search: image template + threshold:: recognition threshold (0~1) + rgb: whether to use the rgb channel for verification + max_count: maximum number of matches Returns: @@ -90,7 +92,7 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c result = self._get_template_result_matrix(im_source=im_source, im_search=im_search) results = [] - # 找到最佳匹配项 + # Find the best match h, w = im_search.size while True: min_val, max_val, min_loc, max_loc = self.minMaxLoc(result.data) @@ -107,7 +109,7 @@ def find_all_results(self, im_source, im_search, threshold=None, rgb=None, max_c return results if results else None def _get_template_result_matrix(self, im_source, im_search): - """求取模板匹配的结果矩阵.""" + """Get the result matrix of template matching.""" if im_source.channels == 3: i_gray = im_source.cvtColor(cv2.COLOR_BGR2GRAY).data s_gray = im_search.cvtColor(cv2.COLOR_BGR2GRAY).data @@ -124,11 +126,11 @@ def input_image_check(self, im_source, im_search): im_search = self._image_check(im_search) if im_source.place != im_search.place: - raise InputImageError('输入图片类型必须相同, source={}, search={}'.format(im_source.place, im_search.place)) + raise InputImageError('image type must be same, source={}, search={}'.format(im_source.place, im_search.place)) elif im_source.dtype != im_search.dtype: - raise InputImageError('输入图片数据类型必须相同, source={}, search={}'.format(im_source.dtype, im_search.dtype)) + raise InputImageError('image data type must be same, source={}, search={}'.format(im_source.dtype, im_search.dtype)) elif im_source.channels != im_search.channels: - raise InputImageError('输入图片通道必须相同, source={}, search={}'.format(im_source.channels, im_search.channels)) + raise InputImageError('image channel must be same, source={}, search={}'.format(im_source.channels, im_search.channels)) if im_source.place == Place.UMat: warnings.warn('Umat has error,will clone new image with np.ndarray ' @@ -143,7 +145,7 @@ def _image_check(self, data): data = Image(data, dtype=self.Dtype) if data.place not in self.Place: - raise TypeError('Image类型必须为(Place.UMat, Place.Ndarray)') + raise TypeError('Image type must be(Place.UMat, Place.Ndarray)') return data @staticmethod @@ -155,17 +157,18 @@ def match(self, img1, img2): def cal_confidence(self, im_source, im_search, crop_rect, max_val, rgb): """ - 将截图和识别结果缩放到大小一致,并计算可信度 + Scale the screenshot and the recognition result + to the same size, and calculate the reliability Args: - im_source: 待匹配图像 - im_search: 图片模板 - crop_rect: 需要在im_source截取的区域 - max_val: matchTemplate得到的最大值 - rgb: 是否使用rgb通道进行校验 + im_source: the image to be matched + im_search: image template + crop_rect: The area that needs to be intercepted in im_source + max_val: the maximum value obtained by matchTemplate + rgb: whether to use the rgb channel for verification Returns: - float: 可信度(0~1) + float: credibility(0~1) """ try: target_img = im_source.crop(crop_rect) @@ -176,14 +179,14 @@ def cal_confidence(self, im_source, im_search, crop_rect, max_val, rgb): def cal_rgb_confidence(self, im_source, im_search): """ - 计算两张图片图片rgb三通道的置信度 + Calculate the confidence of two picture rgb three-channel Args: - im_source: 待匹配图像 - im_search: 图片模板 + im_source: the image to be matched + im_search: image template Returns: - float: 最小置信度 + float: minimum confidence """ # im_search = im_search.copyMakeBorder(10, 10, 10, 10, cv2.BORDER_REPLICATE) # @@ -193,7 +196,7 @@ def cal_rgb_confidence(self, im_source, im_search): src_split = im_source.split() sch_split = im_search.split() - # 计算BGR三通道的confidence，存入bgr_confidence: + # Calculate the confidence of the BGR three channels and store it in bgr_confidence: bgr_confidence = [0, 0, 0] for i in range(3): res_temp = self.match(sch_split[i], src_split[i]) @@ -218,10 +221,11 @@ def cal_ccoeff_confidence(self, im_source, im_search): return max_val def _get_confidence_from_matrix(self, img_crop, im_search, max_val, rgb): - """根据结果矩阵求出confidence.""" - # 求取可信度: + """Find confidence from the result matrix.""" + # Seek credibility: if rgb: - # 如果有颜色校验,对目标区域进行BGR三通道校验: + # If there is color verification, perform + # BGR three-channel verification on the target area: confidence = self.cal_rgb_confidence(img_crop, im_search) else: confidence = max_val diff --git a/flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py index 42e7d7a3..1d5d797a 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/sift.py @@ -15,13 +15,14 @@ class SIFT(BaseKeypoint): def create_matcher(self, **kwargs) -> cv2.DescriptorMatcher: """ - 创建特征点匹配器 + Create a feature point matcher Returns: cv2.FlannBasedMatcher """ index_params = {'algorithm': self.FLANN_INDEX_KDTREE, 'tree': 5} - # 指定递归遍历的次数. 值越高结果越准确，但是消耗的时间也越多 + # Specifies the number of recursive traversals. + # The higher the value, the more accurate the result, but the more time it takes search_params = {'checks': 50} matcher = cv2.FlannBasedMatcher(index_params, search_params) return matcher diff --git a/flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py b/flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py index 46266e9c..3dfee6ba 100644 --- a/flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py +++ b/flybirds/core/plugin/plugins/default/ui_driver/opencv/utils.py @@ -5,7 +5,7 @@ def generate_result(rect, confi): - """Format the result: 定义图像识别结果格式.""" + """Format the result: Define the image recognition result format.""" ret = { 'rect': rect, 'confidence': confi, @@ -14,38 +14,26 @@ def generate_result(rect, confi): def keypoint_distance(kp1, kp2): - """求两个keypoint的两点之间距离""" + """Find the distance between two keypoints""" if isinstance(kp1, cv2.KeyPoint): kp1 = kp1.pt elif isinstance(kp1, (list, tuple)): kp1 = kp1 else: - raise ValueError('kp1需要时keypoint或直接是坐标, kp1={}'.format(kp1)) + raise ValueError('When kp1 needs keypoint or direct coordinates, kp1={}'.format(kp1)) if isinstance(kp2, cv2.KeyPoint): kp2 = kp2.pt elif isinstance(kp2, (list, tuple)): kp2 = kp2 else: - raise ValueError('kp2需要时keypoint或直接是坐标, kp1={}'.format(kp2)) + raise ValueError('When kp2 needs keypoint or direct coordinates, kp1={}'.format(kp2)) x = kp1[0] - kp2[0] y = kp1[1] - kp2[1] return math.sqrt((x ** 2) + (y ** 2)) -def keypoint_angle(kp1, kp2): - """求两个keypoint的夹角 """ - k = [ - (kp1.angle - 180) if kp1.angle >= 180 else kp1.angle, - (kp2.angle - 180) if kp2.angle >= 180 else kp2.angle - ] - if k[0] == k[1]: - return 0 - else: - return abs(k[0] - k[1]) - - def get_keypoint_from_matches(kp, matches, mode): res = [] if mode == 'query': @@ -58,45 +46,13 @@ def get_keypoint_from_matches(kp, matches, mode): return res -def keypoint_origin_angle(kp1, kp2): - """ - 以kp1为原点,计算kp2的旋转角度 - """ - origin_point = kp1.pt - train_point = kp2.pt - - point = (abs(origin_point[0] - train_point[0]), abs(origin_point[1] - train_point[1])) - - x_quadrant = (1, 4) - y_quadrant = (3, 4) - if origin_point[0] > train_point[0]: - x_quadrant = (2, 3) - - if origin_point[1] > train_point[1]: - y_quadrant = (1, 2) - point_quadrant = list(set(x_quadrant).intersection(set(y_quadrant)))[0] - - x, y = point[::-1] - angle = math.degrees(math.atan2(x, y)) - if point_quadrant == 4: - angle = angle - elif point_quadrant == 3: - angle = 180 - angle - elif point_quadrant == 2: - angle = 180 + angle - elif point_quadrant == 1: - angle = 360 - angle - - return angle - - def _mapping_angle_distance(distance, origin_angle, angle): """ Args: - distance: 距离 - origin_angle: 对应原点的角度 - angle: 旋转角度 + distance: distance + origin_angle: The angle corresponding to the origin + angle: Rotation angle """ _angle = origin_angle + angle @@ -107,14 +63,14 @@ def _mapping_angle_distance(distance, origin_angle, angle): def rectangle_transform(point, size, mapping_point, mapping_size, angle): """ - 根据point,找出mapping_point映射的矩形顶点坐标 + According to the point, find the rectangle vertex coordinates mapped by mapping_point Args: - point: 坐标在矩形中的坐标 - size: 矩形的大小(h, w) - mapping_point: 映射矩形的坐标 - mapping_size: 映射矩形的大小(h, w) - angle: 旋转角度 + point: the coordinates of the coordinates in the rectangle + size: the size of the rectangle (h, w) + mapping_point: the coordinates of the mapping rectangle + mapping_size: the size of the mapping rectangle (h, w) + angle: rotation angle Returns: @@ -125,14 +81,11 @@ def rectangle_transform(point, size, mapping_point, mapping_size, angle): h_scale = _h / h w_scale = _w / w - tl = keypoint_distance((0, 0), point) # 左上 - tr = keypoint_distance((w, 0), point) # 右上 - bl = keypoint_distance((0, h), point) # 左下 - br = keypoint_distance((w, h), point) # 右下 + tl = keypoint_distance((0, 0), point) # upper left + tr = keypoint_distance((w, 0), point) # top right + bl = keypoint_distance((0, h), point) # lower left + br = keypoint_distance((w, h), point) # lower right - # x = np.float32([point[1], point[1], (h - point[1]), (h - point[1])]) - # y = np.float32([point[0], (w - point[0]), point[0], (w - point[0])]) - # A, B, C, D = cv2.phase(x, y, angleInDegrees=True) A = math.degrees(math.atan2(point[0], point[1])) B = math.degrees(math.atan2((w - point[0]), point[1])) C = math.degrees(math.atan2(point[0], (h - point[1]))) From cbd44c3ecaa64b39592fcf3986e3590b18566ee0 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Fri, 29 Jul 2022 17:00:48 +0800 Subject: [PATCH 05/10] chore:report img --- flybirds/core/plugin/plugins/default/screen.py | 7 +++---- .../core/plugin/plugins/default/step/common.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/flybirds/core/plugin/plugins/default/screen.py b/flybirds/core/plugin/plugins/default/screen.py index 67bfd85d..972f1821 100644 --- a/flybirds/core/plugin/plugins/default/screen.py +++ b/flybirds/core/plugin/plugins/default/screen.py @@ -123,9 +123,8 @@ def image_verify(img_source_path, img_search_path): img_source = baseImage.Image(img_source_path) img_search = baseImage.Image(img_search_path) - start = time.time() - result = match.find_all_results(img_source, img_search) - log.info(f"[image verify] cost time:{time.time() - start}") - log.info(f"[image verify] result:{result}") + return result + + diff --git a/flybirds/core/plugin/plugins/default/step/common.py b/flybirds/core/plugin/plugins/default/step/common.py index 4cbd82ab..5e1497d7 100644 --- a/flybirds/core/plugin/plugins/default/step/common.py +++ b/flybirds/core/plugin/plugins/default/step/common.py @@ -89,5 +89,17 @@ def prev_fail_scenario_relevance(context, param1, param2): def img_verify(context, search_image_path): step_index = context.cur_step_index - 1 source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_") - BaseScreen.image_verify(source_image_path, search_image_path) - + start = time.time() + result = BaseScreen.image_verify(source_image_path, search_image_path) + if len(result) == 0: + src_path = "../../../{}".format(search_image_path) + data = ( + 'embeddingsTags, stepIndex={}, '.format(step_index, src_path) + ) + context.scenario.description.append(data) + context.cur_step_index += 1 + raise Exception("[image verify] image not found !") + else: + log.info(f"[image verify] cost time:{time.time() - start}") + log.info(f"[image verify] result:{result}") From 60e9bfe21c71ccca063d282628dd3b9bbaf2683f Mon Sep 17 00:00:00 2001 From: clgwlg Date: Sat, 30 Jul 2022 23:17:14 +0800 Subject: [PATCH 06/10] chore:cur_step --- flybirds/core/plugin/plugins/default/screen_record.py | 2 +- flybirds/core/plugin/plugins/default/step/common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flybirds/core/plugin/plugins/default/screen_record.py b/flybirds/core/plugin/plugins/default/screen_record.py index 2440598d..4e82718f 100644 --- a/flybirds/core/plugin/plugins/default/screen_record.py +++ b/flybirds/core/plugin/plugins/default/screen_record.py @@ -168,7 +168,7 @@ def stop_record(self): return "No recording service" message = "" - proc.wait() + proc.wait(timeout=30) proc_code = proc.poll() if proc_code is None: proc.terminate() diff --git a/flybirds/core/plugin/plugins/default/step/common.py b/flybirds/core/plugin/plugins/default/step/common.py index 5e1497d7..476d72dd 100644 --- a/flybirds/core/plugin/plugins/default/step/common.py +++ b/flybirds/core/plugin/plugins/default/step/common.py @@ -98,7 +98,7 @@ def img_verify(context, search_image_path): ' width="375" src="{}" />'.format(step_index, src_path) ) context.scenario.description.append(data) - context.cur_step_index += 1 + # context.cur_step_index += 1 raise Exception("[image verify] image not found !") else: log.info(f"[image verify] cost time:{time.time() - start}") From dcd8b8788e89ca7edb5600cd13f119226d27b1ca Mon Sep 17 00:00:00 2001 From: clgwlg Date: Wed, 3 Aug 2022 15:45:10 +0800 Subject: [PATCH 07/10] feat:image click --- flybirds/core/dsl/globalization/i18n.py | 1 + flybirds/core/dsl/step/element.py | 11 +++++++++++ .../core/plugin/plugins/default/app_base_step.py | 3 +++ flybirds/core/plugin/plugins/default/screen.py | 15 ++++++++++----- .../core/plugin/plugins/default/step/click.py | 15 +++++++++++++-- .../core/plugin/plugins/default/step/common.py | 6 +++++- flybirds/template/config/flybirds_config.json | 2 +- 7 files changed, 44 insertions(+), 9 deletions(-) diff --git a/flybirds/core/dsl/globalization/i18n.py b/flybirds/core/dsl/globalization/i18n.py index a466c11f..8733e78f 100644 --- a/flybirds/core/dsl/globalization/i18n.py +++ b/flybirds/core/dsl/globalization/i18n.py @@ -101,6 +101,7 @@ "click[{selector}]": ["点击[{selector}]"], "click text[{selector}]": ["点击文案[{selector}]"], "click ocr text[{selector}]": ["点击扫描文案[{selector}]"], + "click image[{selector}]": ["点击图像[{selector}]"], "click position[{x},{y}]": ["点击屏幕位置[{x},{y}]"], "in[{selector}]input[{param2}]": ["在[{selector}]中输入[{param2}]"], "in ocr[{selector}]input[{param2}]": ["在扫描文字[{selector}]中输入[{param2}]"], diff --git a/flybirds/core/dsl/step/element.py b/flybirds/core/dsl/step/element.py index 7de2cdcd..b0451c79 100644 --- a/flybirds/core/dsl/step/element.py +++ b/flybirds/core/dsl/step/element.py @@ -71,6 +71,17 @@ def click_ocr_text(context, selector=None): g_Context.step.click_ocr_text(context, selector) +@step("click image[{selector}]") +@ele_wrap +def click_image(context, selector=None): + """ + Click on the image + :param context: step context + :param selector: locator string for text element (or None). + """ + g_Context.step.click_image(context, selector) + + @step("click position[{x},{y}]") @ele_wrap def click_coordinates(context, x=None, y=None): diff --git a/flybirds/core/plugin/plugins/default/app_base_step.py b/flybirds/core/plugin/plugins/default/app_base_step.py index 46edfee3..1883619e 100644 --- a/flybirds/core/plugin/plugins/default/app_base_step.py +++ b/flybirds/core/plugin/plugins/default/app_base_step.py @@ -64,6 +64,9 @@ def click_coordinates(self, context, x, y): def click_ocr_text(self, context, selector): step_click.click_ocr_text(context, selector) + def click_image(self, context, selector): + step_click.click_image(context, selector) + def sleep(self, context, param): step_common.sleep(context, param) diff --git a/flybirds/core/plugin/plugins/default/screen.py b/flybirds/core/plugin/plugins/default/screen.py index 972f1821..36b6551f 100644 --- a/flybirds/core/plugin/plugins/default/screen.py +++ b/flybirds/core/plugin/plugins/default/screen.py @@ -5,9 +5,8 @@ import os import time import traceback -import baseImage +from baseImage import Image from base64 import b64decode -from PIL import Image from .ui_driver import SIFT import flybirds.core.global_resource as gr @@ -99,10 +98,13 @@ def screen_link_to_behave(scenario, step_index, tag=None): @staticmethod def image_ocr(img_path): + """ + Take a screenshot and ocr + """ log.info(f"[image ocr path] image path is:{img_path}") ocr = g_Context.ocr_driver_instance g_Context.ocr_result = ocr.ocr(img_path, cls=True) - g_Context.image_size = Image.open(img_path).size + g_Context.image_size = Image(img_path).size log.info(f"[image ocr path] image size is:{g_Context.image_size}") for line in g_Context.ocr_result: log.info(f"[image ocr result] scan line info is:{line}") @@ -119,9 +121,12 @@ def image_ocr(img_path): @staticmethod def image_verify(img_source_path, img_search_path): + """ + Take a screenshot and verify image + """ match = SIFT() - img_source = baseImage.Image(img_source_path) - img_search = baseImage.Image(img_search_path) + img_source = Image(img_source_path) + img_search = Image(img_search_path) result = match.find_all_results(img_source, img_search) return result diff --git a/flybirds/core/plugin/plugins/default/step/click.py b/flybirds/core/plugin/plugins/default/step/click.py index cd8c61ca..cd71c337 100644 --- a/flybirds/core/plugin/plugins/default/step/click.py +++ b/flybirds/core/plugin/plugins/default/step/click.py @@ -8,6 +8,7 @@ import flybirds.utils.dsl_helper as dsl_helper from flybirds.core.global_context import GlobalContext as g_Context from flybirds.core.plugin.plugins.default.step.verify import ocr_txt_exist +from flybirds.core.plugin.plugins.default.step.common import img_verify def click_ele(context, param): @@ -115,7 +116,17 @@ def click_ocr_text(context, param): x = (box[0][0] + box[1][0]) / 2 y = (box[0][1] + box[2][1]) / 2 poco_instance = gr.get_value("pocoInstance") - x_coordinate = float(x) / g_Context.image_size[0] - y_coordinate = float(y) / g_Context.image_size[1] + x_coordinate = float(x) / g_Context.image_size[1] + y_coordinate = float(y) / g_Context.image_size[0] poco_instance.click([x_coordinate, y_coordinate]) + +def click_image(context, param): + result = img_verify(context, param) + x = result[0].get('rect').x + result[0].get('rect').width / 2 + y = result[0].get('rect').y + result[0].get('rect').height / 2 + poco_instance = gr.get_value("pocoInstance") + x_coordinate = float(x) / g_Context.image_size[1] + y_coordinate = float(y) / g_Context.image_size[0] + poco_instance.click([x_coordinate, y_coordinate]) + diff --git a/flybirds/core/plugin/plugins/default/step/common.py b/flybirds/core/plugin/plugins/default/step/common.py index 476d72dd..4571e69f 100644 --- a/flybirds/core/plugin/plugins/default/step/common.py +++ b/flybirds/core/plugin/plugins/default/step/common.py @@ -31,7 +31,7 @@ def ocr(context): def change_ocr_lang(context,lang=None): """ - init ocr + change ocr language """ ocr_instance = ui_driver.init_ocr(lang) gr.set_value("ocrInstance", ocr_instance) @@ -87,6 +87,9 @@ def prev_fail_scenario_relevance(context, param1, param2): def img_verify(context, search_image_path): + """ + verify image exist or not + """ step_index = context.cur_step_index - 1 source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_") start = time.time() @@ -103,3 +106,4 @@ def img_verify(context, search_image_path): else: log.info(f"[image verify] cost time:{time.time() - start}") log.info(f"[image verify] result:{result}") + return result diff --git a/flybirds/template/config/flybirds_config.json b/flybirds/template/config/flybirds_config.json index 1ad48ece..7b27d79d 100644 --- a/flybirds/template/config/flybirds_config.json +++ b/flybirds/template/config/flybirds_config.json @@ -48,7 +48,7 @@ "swipeReadyTime": 3, "verifyPosNotChangeCount": 5, "screenRecordTime": 90, - "useSnap": true + "useSnap": false }, "report": {}, "log": {} From 1cf03bc1f33b0c7941476d283f8cbd7baef76837 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Wed, 3 Aug 2022 17:43:50 +0800 Subject: [PATCH 08/10] feat:image not exist --- flybirds/core/dsl/globalization/i18n.py | 3 +- flybirds/core/dsl/step/common.py | 5 --- flybirds/core/dsl/step/element.py | 10 +++++ .../plugin/plugins/default/app_base_step.py | 7 +++- .../plugin/plugins/default/step/common.py | 16 +------- .../plugin/plugins/default/step/verify.py | 39 ++++++++++++++++++- 6 files changed, 57 insertions(+), 23 deletions(-) diff --git a/flybirds/core/dsl/globalization/i18n.py b/flybirds/core/dsl/globalization/i18n.py index 8733e78f..6905e956 100644 --- a/flybirds/core/dsl/globalization/i18n.py +++ b/flybirds/core/dsl/globalization/i18n.py @@ -89,7 +89,8 @@ "screenshot": ["全屏截图"], "ocr": ["全屏扫描"], "change ocr lang [{param}]": ["切换OCR语言[{param}]"], - "image search [{param}]": ["查找图像[{param}]"], + "image exist [{param}]": ["存在图像[{param}]"], + "image not exist [{param}]": ["不存在图像[{param}]"], "information association of failed operation, run the {param1} time" " :[{param2}]": ["失败运行的信息关联,运行第{param1}次:[{param2}]"], "text[{selector}]property[{param2}]is {param3}": [ diff --git a/flybirds/core/dsl/step/common.py b/flybirds/core/dsl/step/common.py index ceb49696..992c7c89 100644 --- a/flybirds/core/dsl/step/common.py +++ b/flybirds/core/dsl/step/common.py @@ -29,11 +29,6 @@ def scan(context): g_Context.step.ocr(context) -@step("image search [{param}]") -def img_verify(context, param): - g_Context.step.img_verify(context, param) - - @step("information association of failed operation," " run the {param1} time :[{param2}]" ) diff --git a/flybirds/core/dsl/step/element.py b/flybirds/core/dsl/step/element.py index b0451c79..2471f972 100644 --- a/flybirds/core/dsl/step/element.py +++ b/flybirds/core/dsl/step/element.py @@ -394,3 +394,13 @@ def find_text_from_parent(context, p_selector=None, c_selector=None, """ g_Context.step.find_text_from_parent(context, p_selector, c_selector, param3) + +@step("image exist [{param}]") +def img_exist(context, param): + g_Context.step.img_exist(context, param) + + +@step("image not exist [{param}]") +def img_not_exist(context, param): + g_Context.step.img_not_exist(context, param) + diff --git a/flybirds/core/plugin/plugins/default/app_base_step.py b/flybirds/core/plugin/plugins/default/app_base_step.py index 1883619e..7aabba40 100644 --- a/flybirds/core/plugin/plugins/default/app_base_step.py +++ b/flybirds/core/plugin/plugins/default/app_base_step.py @@ -31,8 +31,11 @@ def init_device(self, context, param=None): def change_ocr_lang(self, context, param=None): step_common.change_ocr_lang(context, lang=param) - def img_verify(self, context, param): - step_common.img_verify(context, param) + def img_exist(self, context, param): + step_verify.img_exist(context, param) + + def img_not_exist(self, context, param): + step_verify.img_not_exist(context, param) def connect_device(self, context, param): step_app.connect_device(context, param) diff --git a/flybirds/core/plugin/plugins/default/step/common.py b/flybirds/core/plugin/plugins/default/step/common.py index 4571e69f..e21c1113 100644 --- a/flybirds/core/plugin/plugins/default/step/common.py +++ b/flybirds/core/plugin/plugins/default/step/common.py @@ -92,18 +92,6 @@ def img_verify(context, search_image_path): """ step_index = context.cur_step_index - 1 source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_") - start = time.time() result = BaseScreen.image_verify(source_image_path, search_image_path) - if len(result) == 0: - src_path = "../../../{}".format(search_image_path) - data = ( - 'embeddingsTags, stepIndex={}, '.format(step_index, src_path) - ) - context.scenario.description.append(data) - # context.cur_step_index += 1 - raise Exception("[image verify] image not found !") - else: - log.info(f"[image verify] cost time:{time.time() - start}") - log.info(f"[image verify] result:{result}") - return result + return result + diff --git a/flybirds/core/plugin/plugins/default/step/verify.py b/flybirds/core/plugin/plugins/default/step/verify.py index e7122a8a..50d029b1 100644 --- a/flybirds/core/plugin/plugins/default/step/verify.py +++ b/flybirds/core/plugin/plugins/default/step/verify.py @@ -6,6 +6,7 @@ import time import flybirds.core.global_resource as gr +import flybirds.utils.flybirds_log as log from flybirds.core.global_context import GlobalContext as g_Context import flybirds.core.plugin.plugins.default.ui_driver.poco.poco_ele \ as poco_ele @@ -16,7 +17,7 @@ import flybirds.utils.dsl_helper as dsl_helper import flybirds.utils.verify_helper as verify from flybirds.core.exceptions import FlybirdVerifyException -from flybirds.core.plugin.plugins.default.step.common import ocr +from flybirds.core.plugin.plugins.default.step.common import ocr,img_verify def wait_text_exist(context, param): @@ -333,3 +334,39 @@ def paddle_fix_txt(txt): return txt +def img_exist(context, param): + start = time.time() + step_index = context.cur_step_index - 1 + result = img_verify(context, param) + if len(result) == 0: + src_path = "../../../{}".format(param) + data = ( + 'embeddingsTags, stepIndex={}, '.format(step_index, src_path) + ) + context.scenario.description.append(data) + # context.cur_step_index += 1 + raise Exception("[image exist verify] image not found !") + else: + log.info(f"[image exist verify] cost time:{time.time() - start}") + log.info(f"[image exist verify] result:{result}") + + +def img_not_exist(context, param): + start = time.time() + step_index = context.cur_step_index - 1 + result = img_verify(context, param) + if len(result) == 0: + log.info(f"[image not exist verify] cost time:{time.time() - start}") + log.info(f"[image not exist verify] result:{result}") + else: + src_path = "../../../{}".format(param) + data = ( + 'embeddingsTags, stepIndex={}, '.format(step_index, src_path) + ) + context.scenario.description.append(data) + # context.cur_step_index += 1 + raise Exception("[image not exist verify] image found !") + + From 3ee18457993d75a04d404047151afc1ee2bbd431 Mon Sep 17 00:00:00 2001 From: clgwlg Date: Wed, 3 Aug 2022 17:45:15 +0800 Subject: [PATCH 09/10] chore:step name change --- flybirds/core/dsl/globalization/i18n.py | 4 ++-- flybirds/core/dsl/step/element.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flybirds/core/dsl/globalization/i18n.py b/flybirds/core/dsl/globalization/i18n.py index 6905e956..c576e502 100644 --- a/flybirds/core/dsl/globalization/i18n.py +++ b/flybirds/core/dsl/globalization/i18n.py @@ -89,8 +89,8 @@ "screenshot": ["全屏截图"], "ocr": ["全屏扫描"], "change ocr lang [{param}]": ["切换OCR语言[{param}]"], - "image exist [{param}]": ["存在图像[{param}]"], - "image not exist [{param}]": ["不存在图像[{param}]"], + "exist image [{param}]": ["存在图像[{param}]"], + "not exist image [{param}]": ["不存在图像[{param}]"], "information association of failed operation, run the {param1} time" " :[{param2}]": ["失败运行的信息关联,运行第{param1}次:[{param2}]"], "text[{selector}]property[{param2}]is {param3}": [ diff --git a/flybirds/core/dsl/step/element.py b/flybirds/core/dsl/step/element.py index 2471f972..90487292 100644 --- a/flybirds/core/dsl/step/element.py +++ b/flybirds/core/dsl/step/element.py @@ -395,12 +395,12 @@ def find_text_from_parent(context, p_selector=None, c_selector=None, g_Context.step.find_text_from_parent(context, p_selector, c_selector, param3) -@step("image exist [{param}]") +@step("exist image [{param}]") def img_exist(context, param): g_Context.step.img_exist(context, param) -@step("image not exist [{param}]") +@step("not exist image [{param}]") def img_not_exist(context, param): g_Context.step.img_not_exist(context, param) From 9c8cbdb47cde9d0c4c34186b865663a7f60a05fc Mon Sep 17 00:00:00 2001 From: clgwlg Date: Wed, 3 Aug 2022 21:36:17 +0800 Subject: [PATCH 10/10] chore:add switch to screen link to behave --- flybirds/core/plugin/plugins/default/screen.py | 5 +++-- flybirds/core/plugin/plugins/default/step/common.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/flybirds/core/plugin/plugins/default/screen.py b/flybirds/core/plugin/plugins/default/screen.py index 36b6551f..6fd07813 100644 --- a/flybirds/core/plugin/plugins/default/screen.py +++ b/flybirds/core/plugin/plugins/default/screen.py @@ -47,7 +47,7 @@ def screen_shot(path): log.info("[screen_shot] screen shot end!") @staticmethod - def screen_link_to_behave(scenario, step_index, tag=None): + def screen_link_to_behave(scenario, step_index, tag=None, link=True): """ screenshot address and linked to the tag The label information is placed in the description of the scene, @@ -91,7 +91,8 @@ def screen_link_to_behave(scenario, step_index, tag=None): 'embeddingsTags, stepIndex={}, '.format(step_index, src_path) ) - scenario.description.append(data) + if link is True: + scenario.description.append(data) screen_path = os.path.join(current_screen_dir, file_name) g_Context.screen.screen_shot(screen_path) return screen_path diff --git a/flybirds/core/plugin/plugins/default/step/common.py b/flybirds/core/plugin/plugins/default/step/common.py index e21c1113..2c0252c9 100644 --- a/flybirds/core/plugin/plugins/default/step/common.py +++ b/flybirds/core/plugin/plugins/default/step/common.py @@ -25,7 +25,7 @@ def screenshot(context): def ocr(context): step_index = context.cur_step_index - 1 - image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_") + image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_", False) BaseScreen.image_ocr(image_path) @@ -91,7 +91,7 @@ def img_verify(context, search_image_path): verify image exist or not """ step_index = context.cur_step_index - 1 - source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_") + source_image_path = BaseScreen.screen_link_to_behave(context.scenario, step_index, "screen_", False) result = BaseScreen.image_verify(source_image_path, search_image_path) return result