diff --git a/pyproject.toml b/pyproject.toml index ed5313f..22ee2a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "funnylog", "click", "pexpect", - "youqu-html", ] dynamic = [ "version", @@ -31,6 +30,7 @@ desktop-ui = [ "youqu-mousekey", "youqu-button-center", "pdocr-rpc", + "youqu-html", # "wdotool", ] webui = [ diff --git a/youqu3/_setting/_setting.py b/youqu3/_setting/_setting.py index 446fe7c..464b3bd 100644 --- a/youqu3/_setting/_setting.py +++ b/youqu3/_setting/_setting.py @@ -32,9 +32,6 @@ class _Setting(_DynamicSetting): IMAGE_SERVER_HOST = "10.8.11.139" # REMOTE - MODE = "parallel" - BUILD_ENV = True - SEND = True # PYPI_MIRROR = "https://pypi.tuna.tsinghua.edu.cn/simple" diff --git a/youqu3/dogtail.py b/youqu3/dogtail.py index 8b1f0dc..2af7e4b 100644 --- a/youqu3/dogtail.py +++ b/youqu3/dogtail.py @@ -9,4 +9,3 @@ if HAS_DOGTAIL is False: raise YouQuPluginInstalledError("youqu-dogtail") - diff --git a/youqu3/driver/cli.py b/youqu3/driver/cli.py index 7994ca3..870535c 100644 --- a/youqu3/driver/cli.py +++ b/youqu3/driver/cli.py @@ -1,4 +1,5 @@ import click + from youqu3 import version @@ -16,16 +17,19 @@ def cli(): ... help="指定用例关键词执行，支持 'and/or/not' 逻辑表达式") @click.option("-t", "--tags", default=None, type=click.STRING, help="指定用例标签执行，支持 'and/or/not' 逻辑表达式") +@click.option("--setup-plan", is_flag=True, default=False, help="") def run( filepath, keywords, tags, + setup_plan, ): """本地执行""" args = { "filepath": filepath, "keywords": keywords, "tags": tags, + "setup_plan": setup_plan, } from youqu3.driver.run import Run Run(**args).run() @@ -35,12 +39,6 @@ def run( @click.help_option("-h", "--help", help="查看帮助信息") @click.option("-c", "--clients", default=None, type=click.STRING, help="远程机器信息:user@ip:password，多个机器之间用 '/' 连接") -@click.option("-s", "--send", default=None, type=click.STRING, - help="发送代码到远程机器") -@click.option("-e", "--build-env", default=None, type=click.STRING, - help="远程机器安装环境") -@click.option("-m", "--mode", default="parallel", type=click.Choice(["parallel", "nginx"]), - help="远程控制驱动模式 (parallel or nginx)") @click.option("-f", "--filepath", default=None, type=click.STRING, help="指定用例文件路径执行") @click.option("-k", "--keywords", default=None, type=click.STRING, @@ -49,9 +47,6 @@ def run( help="指定用例标签执行，支持 'and/or/not' 逻辑表达式") def remote( clients, - send, - build_env, - mode, filepath, keywords, tags, @@ -59,9 +54,6 @@ def remote( """远程控制执行""" args = { "clients": clients, - "send": send, - "build_env": build_env, - "mode": mode, "filepath": filepath, "keywords": keywords, "tags": tags, diff --git a/youqu3/driver/remote.py b/youqu3/driver/remote.py index fc7b62e..f54812b 100644 --- a/youqu3/driver/remote.py +++ b/youqu3/driver/remote.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: GPL-2.0-only import pathlib import re -import time from concurrent.futures import ALL_COMPLETED from concurrent.futures import ThreadPoolExecutor from concurrent.futures import wait @@ -21,9 +20,6 @@ class Remote: def __init__( self, clients=None, - send=None, - build_env=None, - mode=None, filepath=None, keywords=None, tags=None, @@ -35,12 +31,15 @@ def __init__( self.keywords = keywords self.tags = tags self.clients = clients - self.send = send or setting.SEND - self.build_env = build_env or setting.BUILD_ENV - self.mode = mode or setting.MODE - cli_clients = {} - if self.clients: + if not self.clients: + raise ValueError("REMOTE驱动模式, 未传入远程客户端信息：-c/--clients user@ip:pwd") + self.group_type = False + if "{" in self.clients and "}" in self.clients: + self.group_type = True + + if self.group_type is False: + self.cli_clients = {} _cli_clients = self.clients.split("/") for index, _client in enumerate(_cli_clients): _cli_client_info = re.findall(r"^(.+?)@(\d+\.\d+\.\d+\.\d+):{0,1}(.*?)$", _client) @@ -48,29 +47,33 @@ def __init__( _c = list(_cli_client_info[0]) if _c[2] == "": _c[2] = setting.PASSWORD - cli_clients[f"client{index + 1}"] = _c - + self.cli_clients[f"client{index + 1}"] = _c else: - raise ValueError("REMOTE驱动模式, 未传入远程客户端信息：-c/--clients user@ip:pwd") - - self.clients = cli_clients + self.cli_groups = {} + groups = re.findall(r'\{(.*?)\}', self.clients) + for group_index, group in enumerate(groups): + cli_clients = {} + for client_index, _client in enumerate(group.split("/")): + _cli_client_info = re.findall(r"^(.+?)@(\d+\.\d+\.\d+\.\d+):{0,1}(.*?)$", _client) + if _cli_client_info: + _c = list(_cli_client_info[0]) + if _c[2] == "": + _c[2] = setting.PASSWORD + cli_clients[f"client{client_index + 1}"] = _c + self.cli_groups[f"group{group_index + 1}"] = cli_clients self.server_rootdir = pathlib.Path(".").absolute() self.rootdir_name = self.server_rootdir.name - self.client_rootdir = lambda x: f"/home/{x}/{self.rootdir_name}_{setting.TIME_STRING}" self.client_report_path = lambda x: f"{self.client_rootdir(x)}/report" self.client_html_report_path = lambda x: f"{self.client_report_path(x)}/html" self.client_json_report_path = lambda x: f"{self.client_report_path(x)}/json" - self.strf_time = time.strftime("%m%d%p%I%M%S") self.rsync = "rsync -av -e ssh -o StrictHostKeyChecking=no" self.empty = "> /dev/null 2>&1" self.collection_json = False self.server_json_dir_id = None - self.pms_user = None - self.pms_password = None from funnylog.conf import setting as log_setting @@ -99,7 +102,7 @@ def send_code(self, user, _ip, password): exclude += f"--exclude='{i}' " _, return_code = Cmd.expect_run( f"{self.rsync} {exclude} {self.server_rootdir}/* {user}@{_ip}:{self.client_rootdir(user)}/", - events={'(?i)password':f'{password}\\n'} + events={'(?i)password': f'{password}\\n'} ) _, return_code = Cmd.expect_run( f"{self.rsync} {exclude} {self.server_rootdir}/.env {user}@{_ip}:{self.client_rootdir(user)}/", @@ -117,7 +120,8 @@ def _remote_run(cmd): if return_code != 0: _remote_run("curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py") _remote_run(f"pip3 install -U youqu3 -i {setting.PYPI_MIRROR}") - _, return_code = _remote_run(f"export PATH=$PATH:$HOME/.local/bin;cd {self.client_rootdir(user)} && youqu3 envx") + _, return_code = _remote_run( + f"export PATH=$PATH:$HOME/.local/bin;cd {self.client_rootdir(user)} && youqu3 envx") logger.info(f"环境安装{'成功' if return_code == 0 else '失败'} - < {user}@{_ip} >") def send_code_and_env(self, user, _ip, password): @@ -128,30 +132,34 @@ def send_code_and_env(self, user, _ip, password): def makedirs(dirs): pathlib.Path(dirs).mkdir(parents=True, exist_ok=True) - def get_back_all_report(self, client_list): + def get_back_all_report(self, client_list, clients): def get_back(user, _ip, password): - server_html_path = f"{self.server_rootdir}/report/remote/{self.strf_time}_{_ip}_{self.rootdir_name}" + server_html_path = f"{self.server_rootdir}/report/remote/{setting.TIME_STRING}_{_ip}_{self.rootdir_name}" self.makedirs(server_html_path) Cmd.run( f"{self.rsync % password} {user}@{_ip}:{self.client_report_path(user)}/* {server_html_path}/ {self.empty}") - if len(self.clients) >= 2: + if len(clients) >= 2: _ps = [] executor = ThreadPoolExecutor() for client in client_list[:-1]: - user, _ip, password = self.clients.get(client) + user, _ip, password = clients.get(client) _p4 = executor.submit(get_back, user, _ip, password) _ps.append(_p4) sleep(2) - user, _ip, password = self.clients.get(client_list[-1]) + user, _ip, password = clients.get(client_list[-1]) get_back(user, _ip, password) wait(_ps, return_when=ALL_COMPLETED) else: - user, _ip, password = self.clients.get(client_list[0]) + user, _ip, password = clients.get(client_list[0]) get_back(user, _ip, password) - def generate_remote_cmd(self, user): - cmd = ["cd", f"{self.client_rootdir(user)}/", "&&", "youqu3-cargo", "run"] + def changdir_remote_cmd(self, user): + return ["cd", f"{self.client_rootdir(user)}/", "&&"] + + @property + def generate_cmd(self): + cmd = ["youqu3-cargo", "run"] if self.filepath: cmd.append(self.filepath) @@ -160,69 +168,103 @@ def generate_remote_cmd(self, user): if self.tags: cmd.extend(["-m", f"'{self.tags}'"]) - cmd.extend([ - f"--maxfail={setting.MAX_FAIL}", - f"--reruns={setting.RERUNS}", - f"--timeout={setting.TIMEOUT}", - "--json-report", - "--json-report-indent=2", - f"--json-report-file={self.client_json_report_path(user)}/report_{setting.TIME_STRING}.json", - f"--alluredir={self.client_html_report_path(user)}", - "--clean-alluredir", - ]) - return cmd def run_test(self, user, _ip, password): - RemoteCmd(user, _ip, password).remote_run(" ".join(self.generate_remote_cmd(user))) + RemoteCmd(user, _ip, password).remote_run( + " ".join( + self.changdir_remote_cmd(user) + self.generate_cmd + ) + ) - def parallel_run(self): + @property + def collection_only_cmd(self): + return self.generate_cmd + ["--setup-plan"] + + @property + def get_collection_only_cases(self): + stdout = Cmd.run(f"cd {self.server_rootdir} && {' '.join(self.collection_only_cmd)}", timeout=600) + lines = stdout.split("\n") + _collection_cases = [] + for line in lines: + line = line.strip() + if line and " " not in line: + _collection_cases.append(line.split("::")[0]) + collection_cases = set(_collection_cases) + return collection_cases + + def parallel_run(self, clients): _ps = [] executor = ThreadPoolExecutor() - for client in list(self.clients.keys())[:-1]: - user, _ip, password = self.clients.get(client) + for client in list(clients.keys())[:-1]: + user, _ip, password = clients.get(client) _p3 = executor.submit(self.run_test, user, _ip, password) _ps.append(_p3) sleep(1) - user, _ip, password = list(self.clients.values())[-1] + user, _ip, password = list(clients.values())[-1] self.run_test(user, _ip, password) wait(_ps, return_when=ALL_COMPLETED) sleep(5) - def mul_do(self, func_obj, client_list): + def mul_do(self, func_obj, client_list, clients): if len(client_list) >= 2: executor = ThreadPoolExecutor() _ps = [] for client in client_list[:-1]: - user, _ip, password = self.clients.get(client) + user, _ip, password = clients.get(client) _p1 = executor.submit(func_obj, user, _ip, password) _ps.append(_p1) - user, _ip, password = self.clients.get(client_list[-1]) + user, _ip, password = clients.get(client_list[-1]) func_obj(user, _ip, password) wait(_ps, return_when=ALL_COMPLETED) else: - user, _ip, password = self.clients.get(client_list[0]) + user, _ip, password = clients.get(client_list[0]) func_obj(user, _ip, password) def run(self): - client_list = list(self.clients.keys()) + Cmd.run(f"rm -rf ~/.ssh/known_hosts {self.empty}") - print("-" * 51) - print(f"|{'CLIENTS'.center(11)}|{'USER'.center(10)}|{'IP'.center(15)}|{'PASSWORD'.center(10)}|") - print("-" * 51) - for c, (user, _ip, password) in self.clients.items(): - print(f"|{c.center(11)}|{user.center(10)}|{_ip.center(15)}|{password.center(10)}|") - print("-" * 51) + if self.group_type: + print("远程测试机列表".center(54)) + print("-" * 58) + print( + f"|{'GROUPS'.center(8)}|{'CLIENTS'.center(9)}|{'USER'.center(10)}|{'IP'.center(15)}|{'PASSWORD'.center(10)}|") + print("-" * 58) + for group, clients in self.cli_groups.items(): + for c, (user, _ip, password) in clients.items(): + print(f"|{group.center(8)}|{c.center(9)}|{user.center(10)}|{_ip.center(15)}|{password.center(10)}|") + print("-" * 58) + + for group, clients in self.cli_groups.items(): + client_list = list(clients.keys()) + if len(client_list) > 1: + # TODO + cases = self.get_collection_only_cases + def split_list(lst, n): + k, m = divmod(len(lst), n) + return [lst[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n)] + + client_case_list = split_list(list(cases), len(client_list)) + client_case_map = dict(zip(client_list, client_case_list)) + print(1) + + + else: + self.mul_do(self.send_code_and_env, client_list, clients) + self.parallel_run(clients) + self.get_back_all_report(client_list, clients) - Cmd.run(f"rm -rf ~/.ssh/known_hosts {self.empty}") - if self.build_env: - self.mul_do(self.send_code_and_env, client_list) - else: - if self.send: - self.mul_do(self.send_code, client_list) - if self.mode == "parallel": - self.parallel_run() else: - ... - self.get_back_all_report(client_list) + print("远程测试机列表".center(47)) + print("-" * 49) + print(f"|{'CLIENTS'.center(9)}|{'USER'.center(10)}|{'IP'.center(15)}|{'PASSWORD'.center(10)}|") + print("-" * 49) + for c, (user, _ip, password) in self.cli_clients.items(): + print(f"|{c.center(9)}|{user.center(10)}|{_ip.center(15)}|{password.center(10)}|") + print("-" * 49) + + client_list = list(self.cli_clients.keys()) + self.mul_do(self.send_code_and_env, client_list, self.cli_clients) + self.parallel_run(self.cli_clients) + self.get_back_all_report(client_list, self.cli_clients) diff --git a/youqu3/driver/run.py b/youqu3/driver/run.py index cba8b75..6db4206 100644 --- a/youqu3/driver/run.py +++ b/youqu3/driver/run.py @@ -19,6 +19,7 @@ def __init__( filepath=None, keywords=None, tags=None, + setup_plan=None, **kwargs, ): logger("INFO") @@ -26,6 +27,7 @@ def __init__( self.filepath = filepath self.keywords = keywords self.tags = tags + self.setup_plan = setup_plan self.rootdir = pathlib.Path(".").absolute() self.report_path = self.rootdir / "report" @@ -60,15 +62,21 @@ def generate_cmd(self): self.set_recursion_limit(self.tags) cmd.extend(["-m", f"'{self.tags}'"]) - cmd.extend([ - f"--maxfail={setting.MAX_FAIL}", - f"--reruns={setting.RERUNS}", - f"--timeout={setting.TIMEOUT}", + if self.setup_plan: + cmd.append("--setup-plan") + else: + cmd.extend([ "--json-report", "--json-report-indent=2", f"--json-report-file={self.json_report_path / f'report_{setting.TIME_STRING}.json'}", f"--alluredir={self.allure_data_path}", "--clean-alluredir", + ]) + + cmd.extend([ + f"--maxfail={setting.MAX_FAIL}", + f"--reruns={setting.RERUNS}", + f"--timeout={setting.TIMEOUT}", ]) return cmd @@ -77,7 +85,8 @@ def run(self): pytest.main( [i.strip("'") for i in self.generate_cmd()[1:]] ) - + if self.setup_plan: + return from youqu_html import YouQuHtml YouQuHtml.gen(str(self.allure_data_path), str(self.allure_html_path), clean=True)