diff --git a/python/paddle_serving_server/env_check/run.py b/python/paddle_serving_server/env_check/run.py index 6b2955547..4268a9873 100644 --- a/python/paddle_serving_server/env_check/run.py +++ b/python/paddle_serving_server/env_check/run.py @@ -30,18 +30,40 @@ inference_test_cases = ["test_fit_a_line.py::TestFitALine::test_inference"] cpp_test_cases = ["test_fit_a_line.py::TestFitALine::test_cpu", "test_fit_a_line.py::TestFitALine::test_gpu"] pipeline_test_cases = ["test_uci_pipeline.py::TestUCIPipeline::test_cpu", "test_uci_pipeline.py::TestUCIPipeline::test_gpu"] +log_files = ["PipelineServingLogs", "log", "stderr.log", "stdout.log"] -def run_test_cases(cases_list, case_type): +def set_serving_log_path(): + if 'SERVING_LOG_PATH' not in os.environ: + serving_log_path = os.path.expanduser(os.getcwd()) + os.environ['SERVING_LOG_PATH']=serving_log_path + +def mv_log_to_new_dir(dir_path): + import shutil + if not os.path.exists(dir_path): + os.mkdir(dir_path) + serving_log_path = os.environ['SERVING_LOG_PATH'] + for file_name in log_files: + file_path = os.path.join(serving_log_path, file_name) + if os.path.exists(file_path): + shutil.move(file_path, dir_path) + + +def run_test_cases(cases_list, case_type, is_open_std): old_stdout, old_stderr = sys.stdout, sys.stderr real_path = os.path.dirname(os.path.realpath(__file__)) for case in cases_list: - sys.stdout = open('/dev/null', 'w') - sys.stderr = open('/dev/null', 'w') + if is_open_std is False: + sys.stdout = open('/dev/null', 'w') + sys.stderr = open('/dev/null', 'w') args_str = "--disable-warnings " + str(real_path) + "/" + case args = args_str.split(" ") res = pytest.main(args) sys.stdout, sys.stderr = old_stdout, old_stderr case_name = case.split('_')[-1] + serving_log_path = os.environ['SERVING_LOG_PATH'] + dir_name = str(case_type) + '_' + case.split(':')[-1] + new_dir_path = os.path.join(serving_log_path, dir_name) + mv_log_to_new_dir(new_dir_path) if res == 0: print("{} {} environment running success".format(case_type, case_name)) elif res == 1: @@ -49,15 +71,27 @@ def run_test_cases(cases_list, case_type): print("{} {} environment running failure. Please refer to https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/pip/linux-pip.html to configure environment".format(case_type, case_name)) os._exit(0) else: - print("{} {} environment running failure, if you need this environment, please refer to https://github.com/PaddlePaddle/Serving/blob/HEAD/doc/Compile_CN.md to configure environment".format(case_type, case_name)) + print("{} {} environment running failure, if you need this environment, please refer to https://github.com/PaddlePaddle/Serving/blob/develop/doc/Install_CN.md".format(case_type, case_name)) -def unset_proxy(key): - os.unsetenv(key) +def unset_env(key): + del os.environ[key] -def check_env(): +def check_env(mode): + set_serving_log_path() if 'https_proxy' in os.environ or 'http_proxy' in os.environ: - unset_proxy("https_proxy") - unset_proxy("http_proxy") - run_test_cases(inference_test_cases, "PaddlePaddle") - run_test_cases(cpp_test_cases, "C++") - run_test_cases(pipeline_test_cases, "Pipeline") + unset_env("https_proxy") + unset_env("http_proxy") + if 'GREP_OPTIONS' in os.environ: + unset_env("GREP_OPTIONS") + is_open_std = False + if mode is "debug": + is_open_std = True + if mode is "all" or mode is "inference" or mode is "debug": + run_test_cases(inference_test_cases, "PaddlePaddle", is_open_std) + if mode is "all" or mode is "cpp" or mode is "debug": + run_test_cases(cpp_test_cases, "C++", is_open_std) + if mode is "all" or mode is "pipeline" or mode is "debug": + run_test_cases(pipeline_test_cases, "Pipeline", is_open_std) + +if __name__ == '__main__': + check_env("debug") diff --git a/python/paddle_serving_server/env_check/test_fit_a_line.py b/python/paddle_serving_server/env_check/test_fit_a_line.py index ad8c21de3..dcf2ece87 100644 --- a/python/paddle_serving_server/env_check/test_fit_a_line.py +++ b/python/paddle_serving_server/env_check/test_fit_a_line.py @@ -20,12 +20,14 @@ def setup_class(self): serving_util.check_model_data_exist() self.get_truth_val_by_inference(self) self.serving_util = serving_util + self.serving_util.release('service') + kill_process(9494) def teardown_method(self): print_log(["stderr.log", "stdout.log", - "log/serving.ERROR", "PipelineServingLogs/pipeline.log"], iden="after predict") + "log/serving.ERROR", "PipelineServingLogs/pipeline.log"]) kill_process(9494) - self.serving_util.release() + self.serving_util.release('service') def get_truth_val_by_inference(self): try: @@ -58,11 +60,9 @@ def get_truth_val_by_inference(self): output_data = output_handle.copy_to_cpu() output_data_dict[output_data_name] = output_data # convert to the same format of Serving output - print(output_data_dict) output_data_dict["price"] = output_data_dict["fc_0.tmp_1"] del output_data_dict["fc_0.tmp_1"] self.truth_val = output_data_dict - print(self.truth_val, self.truth_val["price"].shape) def predict_brpc(self, batch_size=1): data = np.array( @@ -74,7 +74,6 @@ def predict_brpc(self, batch_size=1): fetch_list = client.get_fetch_names() fetch_map = client.predict( feed={"x": data}, fetch=fetch_list, batch=True) - print(fetch_map) return fetch_map def predict_http(self, batch_size=1): @@ -87,12 +86,12 @@ def predict_http(self, batch_size=1): fetch_list = client.get_fetch_names() fetch_map = client.predict( feed={"x": data}, fetch=fetch_list, batch=True) - print(fetch_map) output_dict = self.serving_util.parse_http_result(fetch_map) return output_dict def test_inference(self): - assert self.truth_val['price'].size != 0 + self.serving_util.start_server_by_shell(cmd="", sleep=1) + assert self.truth_val['price'].size != 0, "The result of inference is empty" def test_cpu(self): @@ -103,7 +102,7 @@ def test_cpu(self): ) # 2.resource check - assert count_process_num_on_port(9494) == 1 + assert count_process_num_on_port(9494) == 1, "Error occured when Paddle Server started" # 4.predict by brpc # batch_size 1 @@ -123,7 +122,7 @@ def test_gpu(self): ) # 2.resource check - assert count_process_num_on_port(9494) == 1 + assert count_process_num_on_port(9494) == 1, "Error occured when Paddle Server started" # 4.predict by brpc # batch_size 1 diff --git a/python/paddle_serving_server/env_check/test_uci_pipeline.py b/python/paddle_serving_server/env_check/test_uci_pipeline.py index 1d32f4e7c..a48c5cda5 100644 --- a/python/paddle_serving_server/env_check/test_uci_pipeline.py +++ b/python/paddle_serving_server/env_check/test_uci_pipeline.py @@ -21,12 +21,14 @@ def setup_class(self): serving_util.check_model_data_exist() self.get_truth_val_by_inference(self) self.serving_util = serving_util + self.serving_util.release('web_service') def teardown_method(self): print_log(["stderr.log", "stdout.log", - "log/serving.ERROR", "PipelineServingLogs/pipeline.log"], iden="after predict") + "PipelineServingLogs/pipeline.log"]) kill_process(9998) - self.serving_util.release() + kill_process(18082) + self.serving_util.release('web_service') def get_truth_val_by_inference(self): try: @@ -62,7 +64,6 @@ def get_truth_val_by_inference(self): output_data_dict["prob"] = output_data_dict["fc_0.tmp_1"] del output_data_dict["fc_0.tmp_1"] self.truth_val = output_data_dict - print(self.truth_val, self.truth_val["prob"].shape) def predict_pipeline_rpc(self, batch_size=1): # 1.prepare feed_data @@ -74,10 +75,8 @@ def predict_pipeline_rpc(self, batch_size=1): # 3.predict for fetch_map ret = client.predict(feed_dict=feed_dict) - print(ret) # 4.convert dict to numpy result = {"prob": np.array(eval(ret.value[0]))} - print(result) return result def predict_pipeline_http(self, batch_size=1): @@ -91,7 +90,6 @@ def predict_pipeline_http(self, batch_size=1): # 2.predict for fetch_map url = "http://127.0.0.1:18082/uci/prediction" r = requests.post(url=url, data=json.dumps(feed_dict)) - print(r.json()) # 3.convert dict to numpy array result = {"prob": np.array(eval(r.json()["value"][0]))} return result @@ -104,11 +102,8 @@ def test_cpu(self): ) # 2.resource check - assert count_process_num_on_port(9998) == 1 # gRPC Server - assert count_process_num_on_port(18082) == 1 # gRPC gateway - - # 3.keywords check - check_keywords_in_server_log("MKLDNN is enabled", filename="stderr.log") + assert count_process_num_on_port(9998) == 1, "Error occured when Paddle Server started" # gRPC Server + assert count_process_num_on_port(18082) == 1, "Error occured when Paddle Server started" # gRPC gateway # 4.predict by rpc result = self.predict_pipeline_rpc(batch_size=1) @@ -130,8 +125,8 @@ def test_gpu(self): ) # 2.resource check - assert count_process_num_on_port(9998) == 1 # gRPC Server - assert count_process_num_on_port(18082) == 1 # gRPC gateway + assert count_process_num_on_port(9998) == 1, "Error occured when Paddle Server started" # gRPC Server + assert count_process_num_on_port(18082) == 1, "Error occured when Paddle Server started" # gRPC gateway # 3.predict by rpc result = self.predict_pipeline_rpc(batch_size=1) diff --git a/python/paddle_serving_server/env_check/util.py b/python/paddle_serving_server/env_check/util.py index ef758c525..7add6c0c6 100644 --- a/python/paddle_serving_server/env_check/util.py +++ b/python/paddle_serving_server/env_check/util.py @@ -1,9 +1,9 @@ import os -import pynvml import argparse import base64 import subprocess import numpy as np +import sys class ServingTest(object): def __init__(self, data_path: str, example_path: str, model_dir: str, client_dir: str): @@ -13,16 +13,17 @@ def __init__(self, data_path: str, example_path: str, model_dir: str, client_dir DATA_PATH: 数据集根目录 py_version: python版本 python3.6~3.8 """ + self.serving_log_path = os.environ['SERVING_LOG_PATH'] code_path = os.path.dirname(os.path.realpath(__file__)) self.data_path = f"{code_path}/{data_path}/" self.example_path = f"{code_path}/{example_path}/" - self.py_version = os.environ.get("PYTHON_EXECUTABLE") + self.py_version = sys.executable + if 'PYTHON_EXECUTABLE' in os.environ: + self.py_version = os.environ.get("PYTHON_EXECUTABLE") self.model_dir = model_dir self.client_config = f"{client_dir}/serving_client_conf.prototxt" os.chdir(self.example_path) - print("======================cur path======================") - print(os.getcwd()) self.check_model_data_exist() def check_model_data_exist(self): @@ -37,6 +38,9 @@ def check_model_data_exist(self): os.system(f"ln -s {abs_path} {file}") def start_server_by_shell(self, cmd: str, sleep: int = 5, err="stderr.log", out="stdout.log", wait=False): + + err = os.path.join(self.serving_log_path, err) + out = os.path.join(self.serving_log_path, out) self.err = open(err, "w") self.out = open(out, "w") p = subprocess.Popen(cmd, shell=True, stdout=self.out, stderr=self.err) @@ -44,7 +48,6 @@ def start_server_by_shell(self, cmd: str, sleep: int = 5, err="stderr.log", out= if wait: p.wait() - print_log([err, out]) @staticmethod def check_result(result_data: dict, truth_data: dict, batch_size=1, delta=1e-3): @@ -87,20 +90,9 @@ def kill_process(port, sleep_time=0): # 解决端口占用 os.system(f"sleep {sleep_time}") - -def check_gpu_memory(gpu_id): - pynvml.nvmlInit() - handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_id) - mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) - mem_used = mem_info.used / 1024 ** 2 - print(f"GPU-{gpu_id} memory used:", mem_used) - return mem_used > 100 - - def count_process_num_on_port(port): command = "netstat -nlp | grep :" + str(port) + " | wc -l" count = eval(os.popen(command).read()) - print(f"port-{port} processes num:", count) return count @@ -140,17 +132,15 @@ def diff_compare(array1, array2): def print_log(file_list, iden=""): + serving_log_path = os.environ['SERVING_LOG_PATH'] for file in file_list: - print(f"======================{file} {iden}=====================") - if os.path.exists(file): - with open(file, "r") as f: + print(f"======================{file}=====================") + file_path = os.path.join(serving_log_path, file) + if os.path.exists(file_path): + with open(file_path, "r") as f: print(f.read()) - if file.startswith("log") or file.startswith("PipelineServingLogs"): - os.remove(file) else: - print(f"{file} not exist") - print("======================================================") - + pass def parse_prototxt(file): with open(file, "r") as f: diff --git a/python/paddle_serving_server/serve.py b/python/paddle_serving_server/serve.py index 1340933df..0bdfe7490 100755 --- a/python/paddle_serving_server/serve.py +++ b/python/paddle_serving_server/serve.py @@ -35,6 +35,7 @@ import signal from paddle_serving_server.util import * from paddle_serving_server.env_check.run import check_env +import cmd # web_service.py is still used by Pipeline. @@ -474,6 +475,47 @@ def stop_serving(command: str, port: int=None): os.remove(filepath) return True +class Check_Env_Shell(cmd.Cmd): + intro = "Welcome to the check env shell.Type help to list commands.\n" + # ----- basic commands ----- + def do_help(self, arg): + print("\nCommand list\t\tDescription\n"\ + "check_all\t\tCheck Environment of Paddle Inference, Pipeline Serving, C++ Serving. "\ + "If failed, using debug command to debug\n"\ + "check_pipeline\t\tCheck Environment of Pipeline Serving. "\ + "If failed, using debug command to debug\n"\ + "check_cpp\t\tCheck Environment of C++ Serving. "\ + "If failed, using debug command to debug\n"\ + "check_inference\t\tCheck Environment of Paddle Inference. "\ + "If failed, using debug command to debug\n"\ + "debug\t\t\tWhen checking was failed, open log to debug\n"\ + "exit\t\t\tExit Check Env Shell\n") + + def do_check_all(self, arg): + "Check Environment of Paddle Inference, Pipeline Serving, C++ Serving" + check_env("all") + + def do_check_pipeline(self, arg): + "Check Environment of Pipeline Serving" + check_env("pipeline") + + def do_check_cpp(self, arg): + "Check Environment of C++ Serving" + check_env("cpp") + + def do_check_inference(self, arg): + "Check Environment of Paddle Inference" + check_env("inference") + + def do_debug(self, arg): + "Open pytest log to debug" + check_env("debug") + + def do_exit(self, arg): + "Exit Check Env Shell" + print('Check Environment Shell Exit') + os._exit(0) + return True if __name__ == "__main__": # args.device is not used at all. @@ -491,8 +533,7 @@ def stop_serving(command: str, port: int=None): else: os._exit(-1) elif args.server == "check": - check_env() - os._exit(0) + Check_Env_Shell().cmdloop() for single_model_config in args.model: if os.path.isdir(single_model_config): pass diff --git a/python/pipeline/logger.py b/python/pipeline/logger.py index ebf31564b..24761b104 100644 --- a/python/pipeline/logger.py +++ b/python/pipeline/logger.py @@ -24,8 +24,11 @@ def __init__(self, levels): def filter(self, logRecord): return logRecord.levelno in self._levels - log_dir = "PipelineServingLogs" +if 'SERVING_LOG_PATH' in os.environ: + serving_log_path = os.environ['SERVING_LOG_PATH'] + log_dir = os.path.join(serving_log_path, log_dir) + if not os.path.exists(log_dir): os.makedirs(log_dir)