support force drop big stream #1465
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ProtonCI | |
| on: # yamllint disable-line rule:truthy | |
| pull_request: | |
| types: | |
| - synchronize | |
| - reopened | |
| - opened | |
| - ready_for_review | |
| branches: | |
| - develop | |
| paths-ignore: | |
| - '**.md' | |
| - 'docker/docs/**' | |
| - 'docs/**' | |
| - 'spec/**' | |
| - 'utils/check-style/aspell-ignore/**' | |
| - 'examples/**' | |
| jobs: | |
| verify: | |
| name: Labels check | |
| runs-on: ubuntu-latest | |
| # CI is triggered for untrusted contributors only when the `pr-test` label is present and a new commit is pushed. | |
| if: github.event.pull_request.draft == false | |
| env: | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| with: | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| submodules: false | |
| - name: Label Check | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/proton_ci" | |
| pip install -r requirements.txt | |
| python3 run_check.py | |
| build_address_x64: | |
| needs: verify | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_AMI }} | |
| ec2-volume-size: ${{ vars.VOLUME_SIZE }} | |
| submodules: 'recursive' | |
| run_mode: 'start' # start ec2 on demand instance | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal/proton/ci_cache/cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --sanitizer $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| # clear compiling footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf build_docker | |
| # build unit test docker image | |
| cp $GITHUB_WORKSPACE/output/unit_tests_dbms $GITHUB_WORKSPACE/docker/test/proton_unit_test/ | |
| cd $GITHUB_WORKSPACE/docker/test/proton_unit_test/ | |
| docker build . -t ghcr.io/timeplus-io/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| docker push ghcr.io/timeplus-io/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # build docker image | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| # push docker image | |
| docker push ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| echo 'docker pull ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA' | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| unit_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: c5.xlarge | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '30' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 30 | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| python unit_tests_check.py | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| smoke_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: c7i.8xlarge | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '80' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 40 | |
| command: | | |
| # run stream test | |
| cd $GITHUB_WORKSPACE/tests/stream | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| # FIXME: remove this line after pyyaml community fixed install bug | |
| pip install pyyaml==5.3.1 | |
| # FIXME(yokofly): docker 7.0.0 introduce a breaking change | |
| # https://github.com/docker/docker-py/issues/3194 | |
| pip install docker==6.1.3 | |
| pip install -r helpers/requirements.txt | |
| df -h | |
| python ci_runner.py --settings=redp,nativelog | |
| df -h | |
| echo "check test result" | |
| cat $GITHUB_WORKSPACE/tests/stream/.status | |
| cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| stateless_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: c5.4xlarge | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 45 | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| python functional_tests_check.py stateless | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| stateful_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: c5.2xlarge | |
| ec2-image-id: ${{vars.X64_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 40 | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| python functional_tests_check.py stateful | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| clean_passed_test_image: | |
| needs: | |
| - unit_test_address_x64 | |
| - smoke_test_address_x64 | |
| - stateless_test_address_x64 | |
| - stateful_test_address_x64 | |
| if: github.event.pull_request.draft == false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: delete testing image | |
| run: | | |
| echo "try to delete image with tag testing-$SANITIZER-$ARCH-$GITHUB_SHA" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer $GH_PERSONAL_ACCESS_TOKEN" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_delete_image.yml/dispatches \ | |
| -d "{\"ref\":\"develop\",\"inputs\":{\"tag\":\"testing-$SANITIZER-$ARCH-$GITHUB_SHA\"}}" | |
| env: | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| SANITIZER: "address" | |
| ARCH: ${{ vars.X64_ARCH }} | |
| SkipDraft: | |
| if: github.event.pull_request.draft == true | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| - name: Skip | |
| run: echo "Skip" |