Skip to content

support force drop big stream #1465

support force drop big stream

support force drop big stream #1465

Workflow file for this run

name: ProtonCI
on: # yamllint disable-line rule:truthy
pull_request:
types:
- synchronize
- reopened
- opened
- ready_for_review
branches:
- develop
paths-ignore:
- '**.md'
- 'docker/docs/**'
- 'docs/**'
- 'spec/**'
- 'utils/check-style/aspell-ignore/**'
- 'examples/**'
jobs:
verify:
name: Labels check
runs-on: ubuntu-latest
# CI is triggered for untrusted contributors only when the `pr-test` label is present and a new commit is pushed.
if: github.event.pull_request.draft == false
env:
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
submodules: false
- name: Label Check
run: |
cd "$GITHUB_WORKSPACE/tests/proton_ci"
pip install -r requirements.txt
python3 run_check.py
build_address_x64:
needs: verify
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
with:
ec2-instance-type: ${{ vars.X64_INSTANCE_TYPE }}
ec2-image-id: ${{ vars.X64_AMI }}
ec2-volume-size: ${{ vars.VOLUME_SIZE }}
submodules: 'recursive'
run_mode: 'start' # start ec2 on demand instance
sanitizer: "address"
arch: ${{ vars.X64_ARCH }}
command: |
cd $GITHUB_WORKSPACE
# git config
git config user.name "proton-robot"
git config user.email "[email protected]"
# prepare build cache
aws s3 cp --no-progress s3://tp-internal/proton/ci_cache/cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz .
mkdir $GITHUB_WORKSPACE/ccache
tar -zxf ./cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz -C $GITHUB_WORKSPACE/ccache
rm cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz
# compiling
./docker/packager/packager --package-type binary --docker-image-version clang-16 --sanitizer $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
# clear compiling footprint
rm -rf $GITHUB_WORKSPACE/ccache
rm -rf build_docker
# build unit test docker image
cp $GITHUB_WORKSPACE/output/unit_tests_dbms $GITHUB_WORKSPACE/docker/test/proton_unit_test/
cd $GITHUB_WORKSPACE/docker/test/proton_unit_test/
docker build . -t ghcr.io/timeplus-io/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA
docker push ghcr.io/timeplus-io/proton-unit-test:testing-$SANITIZER-$ARCH-$GITHUB_SHA
docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
cd $GITHUB_WORKSPACE/docker/server
# build docker image
docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
# push docker image
docker push ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
echo 'docker pull ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA'
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
unit_test_address_x64:
needs: build_address_x64
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.pull_request.draft == false
with:
ec2-instance-type: c5.xlarge
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '30'
submodules: false
sanitizer: "address"
arch: ${{ vars.X64_ARCH }}
timeout: 30
command: |
cd $GITHUB_WORKSPACE/tests/proton_ci
export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install -r requirements.txt
python unit_tests_check.py
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
smoke_test_address_x64:
needs: build_address_x64
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.pull_request.draft == false
with:
ec2-instance-type: c7i.8xlarge
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '80'
submodules: false
sanitizer: "address"
arch: ${{ vars.X64_ARCH }}
timeout: 40
command: |
# run stream test
cd $GITHUB_WORKSPACE/tests/stream
export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install --upgrade pip
# FIXME: remove this line after pyyaml community fixed install bug
pip install pyyaml==5.3.1
# FIXME(yokofly): docker 7.0.0 introduce a breaking change
# https://github.com/docker/docker-py/issues/3194
pip install docker==6.1.3
pip install -r helpers/requirements.txt
df -h
python ci_runner.py --settings=redp,nativelog
df -h
echo "check test result"
cat $GITHUB_WORKSPACE/tests/stream/.status
cat $GITHUB_WORKSPACE/tests/stream/.status | grep nativelog:ExitCode.OK | grep redp:ExitCode.OK
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
stateless_test_address_x64:
needs: build_address_x64
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.pull_request.draft == false
with:
ec2-instance-type: c5.4xlarge
ec2-image-id: ${{ vars.X64_TEST_AMI }}
ec2-volume-size: '40'
submodules: false
sanitizer: "address"
arch: ${{ vars.X64_ARCH }}
timeout: 45
command: |
cd $GITHUB_WORKSPACE/tests/proton_ci
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install -r requirements.txt
export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA
python functional_tests_check.py stateless
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
stateful_test_address_x64:
needs: build_address_x64
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
if: github.event.pull_request.draft == false
with:
ec2-instance-type: c5.2xlarge
ec2-image-id: ${{vars.X64_TEST_AMI }}
ec2-volume-size: '40'
submodules: false
sanitizer: "address"
arch: ${{ vars.X64_ARCH }}
timeout: 40
command: |
cd $GITHUB_WORKSPACE/tests/proton_ci
# make virtualenv
ln -s /usr/bin/python3 /usr/bin/python
apt-get update
systemctl stop unattended-upgrades
apt install python3-venv -y
python -m venv env
source env/bin/activate
pip install -r requirements.txt
export PROTON_VERSION=testing-$SANITIZER-$ARCH-$GITHUB_SHA
python functional_tests_check.py stateful
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
clean_passed_test_image:
needs:
- unit_test_address_x64
- smoke_test_address_x64
- stateless_test_address_x64
- stateful_test_address_x64
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: delete testing image
run: |
echo "try to delete image with tag testing-$SANITIZER-$ARCH-$GITHUB_SHA"
curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $GH_PERSONAL_ACCESS_TOKEN" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_delete_image.yml/dispatches \
-d "{\"ref\":\"develop\",\"inputs\":{\"tag\":\"testing-$SANITIZER-$ARCH-$GITHUB_SHA\"}}"
env:
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
SANITIZER: "address"
ARCH: ${{ vars.X64_ARCH }}
SkipDraft:
if: github.event.pull_request.draft == true
runs-on: ubuntu-latest
timeout-minutes: 1
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- name: Skip
run: echo "Skip"