diff --git a/tests/stream/ci_runner.py b/tests/stream/ci_runner.py index 718922b9af6..87d7fa64f42 100644 --- a/tests/stream/ci_runner.py +++ b/tests/stream/ci_runner.py @@ -14,11 +14,7 @@ ) PROTON_PYTHON_DRIVER_S3_BUCKET_NAME = "tp-internal" -PROTON_PYTHON_DIRVER_S3_OBJ_NAME = ( - "proton/proton-python-driver/clickhouse-driver-0.2.4.tar.gz" -) -PROTON_PYTHON_DRIVER_FILE_NAME = "clickhouse-driver-0.2.4.tar.gz" -PROTON_PYTHON_DRIVER_NANME = "clickhouse-driver" +PROTON_PYTHON_DRIVER_NANME = "proton-driver" cur_dir = os.path.dirname(os.path.abspath(__file__)) config_file_path = f"{cur_dir}/test_stream_smoke/configs/config.json" @@ -101,41 +97,6 @@ def upload_proton_logs(s3_client, proton_log_folder, pr_number="0", commit_sha=" return proton_log_url -def proton_python_driver_install(): - s3_helper = S3Helper("https://s3.amazonaws.com") - command = "pip3 list | grep clickhouse-driver" - ret = subprocess.run( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - timeout=600, - ) - logger.debug(f"ret.stdout == {ret.stdout}") - if PROTON_PYTHON_DRIVER_NANME not in ret.stdout: - s3_helper.client.download_file( - PROTON_PYTHON_DRIVER_S3_BUCKET_NAME, - PROTON_PYTHON_DIRVER_S3_OBJ_NAME, - PROTON_PYTHON_DRIVER_FILE_NAME, - ) - logger.debug(f"{PROTON_PYTHON_DRIVER_FILE_NAME} is downloaded") - command = "pip3 install ./" + PROTON_PYTHON_DRIVER_FILE_NAME - ret = ret = subprocess.run( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - timeout=600, - ) - logger.debug(f"ret of subprocess.run({command}) = {ret}") - else: - logger.debug( - f"{PROTON_PYTHON_DRIVER_NANME} exists bypass s3 download and install" - ) - time.sleep(1) - def ci_runner( local_all_results_folder_path, setting_config, @@ -511,8 +472,7 @@ def ci_runner( print( f"one of TIMEPLUS_API_KEY,TIMEPLUS_ADDRESS,TIMEPLUS_WORKSPACE is not found in ENV" ) - logger.info(f"Check proton_python_driver and install...") - proton_python_driver_install() + if ( "test_production_compatibility" in test_folders_list ): # todo: hardcode now, refactor later to have a system_test_runner to run all pytests folders like test_production_compatibility diff --git a/tests/stream/helpers/bucks.py b/tests/stream/helpers/bucks.py index 603bb50a0a2..a3d77d2e8c9 100755 --- a/tests/stream/helpers/bucks.py +++ b/tests/stream/helpers/bucks.py @@ -2,8 +2,8 @@ import os, sys, getopt, json, random, copy, traceback from re import sub import logging, logging.config -from clickhouse_driver import Client -from clickhouse_driver import errors +from proton_driver import Client +from proton_driver import errors import csv import datetime import time diff --git a/tests/stream/helpers/easy_json_util/json_input.py b/tests/stream/helpers/easy_json_util/json_input.py index c925234d64a..b4a01955283 100755 --- a/tests/stream/helpers/easy_json_util/json_input.py +++ b/tests/stream/helpers/easy_json_util/json_input.py @@ -6,8 +6,8 @@ import requests import csv import multiprocessing as mp -from clickhouse_driver import Client -from clickhouse_driver import errors +from proton_driver import Client +from proton_driver import errors from github import (Github,enable_console_debug_logging,GithubException,RateLimitExceededException) from requests.exceptions import ReadTimeout import requests diff --git a/tests/stream/helpers/input_walk.py b/tests/stream/helpers/input_walk.py index 8dc4d2d6f14..b29257f62c4 100755 --- a/tests/stream/helpers/input_walk.py +++ b/tests/stream/helpers/input_walk.py @@ -12,8 +12,8 @@ import requests import multiprocessing as mp -from clickhouse_driver import Client -from clickhouse_driver import errors +from proton_driver import Client +from proton_driver import errors from requests.api import request sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tests/stream/helpers/pgmz.py b/tests/stream/helpers/pgmz.py index c25a2b2bfed..0d84a7134c1 100755 --- a/tests/stream/helpers/pgmz.py +++ b/tests/stream/helpers/pgmz.py @@ -2,8 +2,8 @@ import os, sys, getopt, json, random from re import sub import logging, logging.config -from clickhouse_driver import Client -from clickhouse_driver import errors +from proton_driver import Client +from proton_driver import errors import csv import datetime import time diff --git a/tests/stream/helpers/query_runner.py b/tests/stream/helpers/query_runner.py index d4ca0e1b50b..9bda8f9fbd0 100755 --- a/tests/stream/helpers/query_runner.py +++ b/tests/stream/helpers/query_runner.py @@ -2,8 +2,8 @@ import os, sys, getopt, json, random from re import sub import logging, logging.config -from clickhouse_driver import Client -from clickhouse_driver import errors +from proton_driver import Client +from proton_driver import errors import csv import datetime import time diff --git a/tests/stream/helpers/requirements.txt b/tests/stream/helpers/requirements.txt index 48ade7d3ae2..0dc06695300 100755 --- a/tests/stream/helpers/requirements.txt +++ b/tests/stream/helpers/requirements.txt @@ -10,4 +10,5 @@ pytest-html==3.1.1 psycopg2-binary==2.9.3 kafka-python==2.0.2 timeplus==1.1.2 -sseclient-py>=1.7.2 \ No newline at end of file +sseclient-py>=1.7.2 +proton-driver>=0.2.4 diff --git a/tests/stream/helpers/rockets.py b/tests/stream/helpers/rockets.py index b700f4a6c3b..9bf7e930774 100755 --- a/tests/stream/helpers/rockets.py +++ b/tests/stream/helpers/rockets.py @@ -2,7 +2,7 @@ # _*_ coding: utf-8 _*_ import datetime, yaml, json, getopt, logging, logging.config, math, os, platform, random, requests,signal,subprocess, sys, threading, time, traceback, uuid import multiprocessing as mp -from clickhouse_driver import Client, errors +from proton_driver import Client, errors from timeplus import Stream, Environment from helpers.event_util import ( Event, @@ -57,7 +57,7 @@ TIME_STR_FORMAT = "%y/%m/%d, %H:%M:%S" -# TestException and errors, steal some codes from clickhouse_driver.errors +# TestException and errors, steal some codes from proton_driver.errors class ErrorCodes: STREAM_CREATE_FAILED = 100 QUERY_ID_NOT_EXIST = 101 diff --git a/tests/stream/helpers/tpexporter/tpclient.py b/tests/stream/helpers/tpexporter/tpclient.py index 936031ee917..79184fe8cfe 100644 --- a/tests/stream/helpers/tpexporter/tpclient.py +++ b/tests/stream/helpers/tpexporter/tpclient.py @@ -13,7 +13,7 @@ import timeplus from timeplus import Environment, Stream import sseclient -from clickhouse_driver import Client +from proton_driver import Client logger = logging.getLogger(__name__) formatter = logging.Formatter( @@ -23,7 +23,7 @@ DEFAULT_REST_GET_TIMEOUT = 5 DEFAULT_REST_TIMEOUT_RETRY = 3 DEFAULT_REST_TIMEOUT_RETRY_INTERVAL = 1 -TPNATIVE_DRIVER = 'clickhouse_driver' +TPNATIVE_DRIVER = 'proton_driver' @unique diff --git a/tests/stream/helpers/tpexporter/tpexporter.py b/tests/stream/helpers/tpexporter/tpexporter.py index 679b61d4c5d..44f2e856b59 100644 --- a/tests/stream/helpers/tpexporter/tpexporter.py +++ b/tests/stream/helpers/tpexporter/tpexporter.py @@ -13,7 +13,7 @@ from enum import Enum, unique # import swagger_client # from swagger_client.rest import ApiException -from clickhouse_driver import Client +from proton_driver import Client sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # from timeplus import Environment, Stream