manual_trigger_test #41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: manual_trigger_test | |
| on: # yamllint disable-line rule:truthy | |
| workflow_dispatch: | |
| inputs: | |
| arch: | |
| description: 'Hardware architecture' | |
| required: true | |
| default: x64 | |
| type: choice | |
| options: | |
| - x64 | |
| - arm | |
| tag: | |
| description: 'Docker image tag' | |
| required: true | |
| type: string | |
| default: 'sanitizer' | |
| has_smoke_test: | |
| description: 'Run smoke test' | |
| required: true | |
| type: boolean | |
| default: true | |
| has_unit_test: | |
| description: 'Run unit test' | |
| required: true | |
| type: boolean | |
| default: false | |
| has_stateless_test: | |
| description: 'Run stateless test' | |
| required: true | |
| type: boolean | |
| default: true | |
| has_stateful_test: | |
| description: 'Run stateful test' | |
| required: true | |
| type: boolean | |
| default: true | |
| has_query_compatibility_test: | |
| description: 'Run query compatibility test' | |
| required: true | |
| type: boolean | |
| default: true | |
| jobs: | |
| prepare_smoke_test: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.inputs.has_smoke_test == 'true' }} | |
| outputs: | |
| command: | | |
| # pull docker image | |
| docker pull ghcr.io/timeplus-io/proton:$TAG | |
| echo "run stream test" | |
| cd $GITHUB_WORKSPACE/tests/stream | |
| export PROTON_VERSION=$TAG | |
| env | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| # FIXME: remove this line after pyyaml community fixed install bug | |
| pip install pyyaml==5.3.1 | |
| pip install -r helpers/requirements.txt | |
| python ci_runner.py --settings=redp,nativelog | |
| echo "check test result" | |
| cat $GITHUB_WORKSPACE/tests/stream/.status | |
| cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK | |
| steps: | |
| - name: display command | |
| run: | | |
| echo 'command: ${{ steps.set_command.outputs.command }}' | |
| smoke_test_on_x64: | |
| needs: prepare_smoke_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'x64' | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '60' | |
| submodules: false | |
| sanitizer: '' | |
| arch: ${{ vars.X64_ARCH }} | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_smoke_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| smoke_test_on_arm: | |
| needs: prepare_smoke_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'arm' | |
| with: | |
| ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
| ec2-volume-size: '50' | |
| submodules: false | |
| arch: ${{ vars.ARM_ARCH }} | |
| sanitizer: '' | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_smoke_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| prepare_query_compability_test: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.inputs.has_query_compatibility_test == 'true' }} | |
| outputs: | |
| command: | | |
| # pull docker image | |
| docker pull ghcr.io/timeplus-io/proton:$TAG | |
| echo "run quary compatibility test" | |
| # cd $GITHUB_WORKSPACE/tests/stream | |
| mkdir -p output | |
| cd quark/api | |
| export PROTON_VERSION=$TAG | |
| env | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| # FIXME: remove this line after pyyaml community fixed install bug | |
| pip install pyyaml==5.3.1 | |
| pip install -r helpers/requirements.txt | |
| aws s3 cp s3://tp-internal/proton/proton-python-driver/clickhouse-driver-0.2.4.tar.gz ./ | |
| pip install ./clickhouse-driver-0.2.4.tar.gz | |
| #python ci_runner.py --debug --settings=default --test_folders=test_production_compatibility | |
| docker-compose -f test_production_compatibility/configs/docker/docker-compose.yaml up -d | |
| sleep 10 | |
| bash ./api_test_run_proton.sh | |
| #echo "check test result" | |
| #cat $GITHUB_WORKSPACE/tests/stream/.status | |
| #cat $GITHUB_WORKSPACE/tests/stream/.status | grep default:ExitCode.OK | |
| steps: | |
| - name: display command | |
| run: | | |
| echo 'command: ${{ steps.set_command.outputs.command }}' | |
| query_compability_test_on_x64: | |
| needs: prepare_query_compability_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'x64' | |
| with: | |
| ec2-instance-type: ${{ vars.X64_TEST_ONLY_INSTANCE_TYPE_SMALL }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '30' | |
| submodules: false | |
| sanitizer: '' | |
| arch: ${{ vars.X64_ARCH }} | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_query_compability_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| TP_DEMO_API_KEY: ${{ secrets.TP_DEMO_API_KEY }} | |
| TP_DEMO_SITE_API_KEY: ${{ secrets.TP_DEMO_SITE_API_KEY }} | |
| query_compability_on_arm: | |
| needs: prepare_query_compability_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'arm' | |
| with: | |
| ec2-instance-type: ${{ vars.ARM_TEST_ONLY_INSTANCE_TYPE_SMALL }} | |
| ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
| ec2-volume-size: '30' | |
| submodules: false | |
| arch: ${{ vars.ARM_ARCH }} | |
| sanitizer: '' | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_query_compability_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| TP_DEMO_API_KEY: ${{ secrets.TP_DEMO_API_KEY }} | |
| TP_DEMO_SITE_API_KEY: ${{ secrets.TP_DEMO_SITE_API_KEY }} | |
| prepare_unit_test: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.inputs.has_unit_test == 'true' }} | |
| outputs: | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=$TAG | |
| python unit_tests_check.py | |
| steps: | |
| - name: display command | |
| run: | | |
| echo 'command: ${{ steps.set_command.outputs.command }}' | |
| unit_test_on_x64: | |
| needs: prepare_unit_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'x64' | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '30' | |
| submodules: false | |
| sanitizer: '' | |
| arch: ${{ vars.X64_ARCH }} | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_unit_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| unit_test_on_arm: | |
| needs: prepare_unit_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'arm' | |
| with: | |
| ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
| ec2-volume-size: '30' | |
| submodules: false | |
| arch: ${{ vars.ARM_ARCH }} | |
| sanitizer: '' | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_unit_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| prepare_stateless_test: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.inputs.has_stateless_test == 'true' }} | |
| outputs: | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=$TAG | |
| python functional_tests_check.py stateless | |
| steps: | |
| - name: display command | |
| run: | | |
| echo 'command: ${{ steps.set_command.outputs.command }}' | |
| stateless_test_on_x64: | |
| needs: prepare_stateless_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'x64' | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| sanitizer: '' | |
| arch: ${{ vars.X64_ARCH }} | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_stateless_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| stateless_test_on_arm: | |
| needs: prepare_stateless_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'arm' | |
| with: | |
| ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| arch: ${{ vars.ARM_ARCH }} | |
| sanitizer: '' | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_stateless_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| prepare_stateful_test: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.inputs.has_stateful_test == 'true' }} | |
| outputs: | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=$TAG | |
| python functional_tests_check.py stateful | |
| steps: | |
| - name: display command | |
| run: | | |
| echo 'command: ${{ steps.set_command.outputs.command }}' | |
| stateful_test_on_x64: | |
| needs: prepare_stateful_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'x64' | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| sanitizer: '' | |
| arch: ${{ vars.X64_ARCH }} | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_stateful_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| stateful_test_on_arm: | |
| needs: prepare_stateful_test | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.inputs.arch == 'arm' | |
| with: | |
| ec2-instance-type: ${{ vars.ARM_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.ARM_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| arch: ${{ vars.ARM_ARCH }} | |
| sanitizer: '' | |
| tag: ${{ github.event.inputs.tag }} | |
| command: | | |
| ${{ needs.prepare_stateful_test.outputs.command }} | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} |