Proton-Server will coredump directly while inserting data into the stream using 'changelog' over 'tumble' stream without group by #151
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ProtonCI | |
| on: | |
| issue_comment: | |
| types: | |
| - created | |
| jobs: | |
| verify: | |
| name: Verify PR comment | |
| permissions: | |
| actions: read # for downloading artifact | |
| id-token: write # This is required for requesting the JWT (needed for assume AWS IAM role) | |
| issues: write # for posting comments | |
| runs-on: ubuntu-latest | |
| # because `issue_comment` does not support filters other than `types`, | |
| # here we need a bunch of checks to make sure that this comment is from a valid deployment issue. | |
| # | |
| # Note: 41898282 is the github-action[bot] | |
| # TODO: figure out a way not to hard-code the approver list | |
| if: >- | |
| ${{ github.event.pull_request.draft == false && github.event.issue.pull_request && github.event_name == 'issue_comment' && github.event.issue.state == 'open' && startsWith(github.event.comment.body, '/TODO') }} | |
| steps: | |
| - name: Report Started | |
| run: | | |
| curl --show-error --silent -X POST \ | |
| -H 'Content-Type: application/json' \ | |
| -H 'Accept: application/vnd.github+json' \ | |
| -H "Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}" \ | |
| -d '{"body": "github workflow triggered: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \ | |
| ${{ github.event.issue.comments_url }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: verify | |
| run: | | |
| echo verify started | |
| BODY='${{ format('{0}', github.event.comment.body) }}' | |
| parts=($BODY) | |
| echo parts[0] = ${parts[0]} | |
| echo parts[1] = ${parts[1]} | |
| if [ "${parts[0]}" != '/test' ]; then | |
| curl --show-error --silent -X POST \ | |
| -H 'Content-Type: application/json' \ | |
| -H 'Accept: application/vnd.github+json' \ | |
| -H 'Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' \ | |
| -d '{"body": "Unrecognized command."}' \ | |
| ${{ github.event.issue.comments_url }} | |
| exit 250 | |
| fi | |
| planfile='${{ steps.vars.outputs.environment }}_${{ steps.vars.outputs.gitsha }}.tfplan' | |
| if [ ! -f $planfile ]; then | |
| echo Plan file not found. | |
| exit 251 | |
| fi | |
| filehash=($(sha256sum $planfile)) | |
| echo aws kms verify-mac --key-id 'alias/github-deployment' --message "${filehash[0]}" --mac-algorithm HMAC_SHA_256 --mac "${parts[1]}" | |
| if ! aws kms verify-mac --key-id 'alias/github-deployment' --message "${filehash[0]}" --mac-algorithm HMAC_SHA_256 --mac "${parts[1]}"; then | |
| curl --show-error --silent -X POST \ | |
| -H 'Content-Type: application/json' \ | |
| -H 'Accept: application/vnd.github+json' \ | |
| -H 'Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' \ | |
| -d '{"body": "Invalid MAC hash."}' \ | |
| ${{ github.event.issue.comments_url }} | |
| exit 252 | |
| fi | |
| curl --show-error --silent -X POST \ | |
| -H 'Content-Type: application/json' \ | |
| -H 'Accept: application/vnd.github+json' \ | |
| -H 'Authorization: token ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}' \ | |
| -d '{"body": "MAC hash verified, start deployment, please wait."}' \ | |
| ${{ github.event.issue.comments_url }} | |
| build_address_x64: | |
| needs: verify | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_AMI }} | |
| ec2-volume-size: ${{ vars.VOLUME_SIZE }} | |
| submodules: 'recursive' | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "[email protected]" | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal/proton/ci_cache/cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-16 --sanitizer $SANITIZER --proton-build --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| # clear compiling footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf build_docker | |
| # build unit test docker image | |
| cp $GITHUB_WORKSPACE/output/unit_tests_dbms $GITHUB_WORKSPACE/docker/test/proton_unit_test/ | |
| cd $GITHUB_WORKSPACE/docker/test/proton_unit_test/ | |
| docker build . -t timeplus/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| docker push timeplus/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # build docker image | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t timeplus/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| # push docker image | |
| docker push timeplus/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| echo 'docker pull timeplus/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA' | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| unit_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '30' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 30 | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| python unit_tests_check.py | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| smoke_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '60' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 30 | |
| command: | | |
| # run stream test | |
| cd $GITHUB_WORKSPACE/tests/stream | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| # FIXME: remove this line after pyyaml community fixed install bug | |
| pip install pyyaml==5.3.1 | |
| pip install -r helpers/requirements.txt | |
| df -h | |
| python ci_runner.py --settings=redp,nativelog | |
| df -h | |
| echo "check test result" | |
| cat $GITHUB_WORKSPACE/tests/stream/.status | |
| cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| stateless_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{ vars.X64_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 30 | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| python functional_tests_check.py stateless | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| stateful_test_address_x64: | |
| needs: build_address_x64 | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| if: github.event.pull_request.draft == false | |
| with: | |
| ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }} | |
| ec2-image-id: ${{vars.X64_TEST_AMI }} | |
| ec2-volume-size: '40' | |
| submodules: false | |
| sanitizer: "address" | |
| arch: ${{ vars.X64_ARCH }} | |
| timeout: 30 | |
| command: | | |
| cd $GITHUB_WORKSPACE/tests/proton_ci | |
| # make virtualenv | |
| ln -s /usr/bin/python3 /usr/bin/python | |
| apt-get update | |
| systemctl stop unattended-upgrades | |
| apt install python3-venv -y | |
| python -m venv env | |
| source env/bin/activate | |
| pip install -r requirements.txt | |
| export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA | |
| python functional_tests_check.py stateful | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }} | |
| TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }} | |
| TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }} | |
| SkipDraft: | |
| if: github.event.pull_request.draft == true | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| # Steps represent a sequence of tasks that will be executed as part of the job | |
| steps: | |
| - name: Skip | |
| run: echo "Skip" |