Skip to content

release_build

release_build #54

Workflow file for this run

# FIXME: update this workflow
name: release_build
on: # yamllint disable-line rule:truthy
workflow_dispatch:
jobs:
Build_Linux_X86_64:
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
with:
ec2-instance-type: c5.4xlarge
ec2-image-id: ami-042a37e33a285c22b
submodules: 'recursive'
run_mode: 'start' # start ec2 on demand instance
upload_files: proton-*-Linux-x86_64
command: |
cd $GITHUB_WORKSPACE
# git config
git config user.name "proton-robot"
git config user.email "[email protected]"
# update version
./release --version patch
# prepare build cache
aws s3 cp --no-progress s3://tp-internal/proton/cache.tar.gz .
mkdir $GITHUB_WORKSPACE/ccache
tar -zxf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache
rm cache.tar.gz
# compiling
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
echo "Compiling proton Failed"
exit 127
fi
# get proton tag
PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
echo "Proton tag: $PROTON_TAG"
echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64
echo "Proton Binary Name: $PROTON_BINARY"
docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
cd $GITHUB_WORKSPACE/docker/server
# build docker image
docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64
# push docker image
docker push ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64
# upload build cache
tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache .
aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal/proton/
mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
Build_Linux_Arm64:
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
with:
ec2-instance-type: c6g.4xlarge
ec2-image-id: ami-0f3dbc4cc9994fdee
submodules: 'recursive'
run_mode: 'start' # start ec2 on demand instance
upload_files: proton-*-Linux-aarch64
command: |
cd $GITHUB_WORKSPACE
# git config
git config user.name "proton-robot"
git config user.email "[email protected]"
# update version
./release --version patch
# prepare build cache
aws s3 cp --no-progress s3://tp-internal/proton/cache-arm.tar.gz .
mkdir $GITHUB_WORKSPACE/ccache
tar -zxf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache
rm cache-arm.tar.gz
# compiling
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
# get proton tag
PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
echo "Proton tag: $PROTON_TAG"
echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64
echo "Proton Binary Name: $PROTON_BINARY"
docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
cd $GITHUB_WORKSPACE/docker/server
# build docker image
docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8
# push docker image
docker push ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8
# upload build cache
tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache .
aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal/proton/
mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
CommitTag:
needs: [Build_Linux_X86_64, Build_Linux_Arm64]
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
with:
ec2-instance-type: c5.4xlarge
ec2-image-id: ami-042a37e33a285c22b
submodules: 'recursive'
command: |
cd $GITHUB_WORKSPACE
# git config
git config user.name "proton-robot"
git config user.email "[email protected]"
docker pull ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64
docker pull ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8
docker manifest create ghcr.io/timeplus-io/proton:latest \
ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 \
ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8
docker manifest push ghcr.io/timeplus-io/proton:latest
PROTON_TAG="$(docker history --no-trunc ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 | grep -Eo 'ARG version=[0-9]+.[0-9]+.[0-9]+' -o | grep -Eo '[0-9]+.[0-9]+.[0-9]+')"
docker manifest create ghcr.io/timeplus-io/proton:$PROTON_TAG-rc \
ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64 \
ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8
docker manifest push ghcr.io/timeplus-io/proton:$PROTON_TAG-rc
curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \
-d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG-rc\"}}\""
curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \
-d "{\"ref\":\"develop\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG-rc\"}}\""
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
Build_Darwin_X86_64:
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
with:
ec2-instance-type: c5.4xlarge
ec2-image-id: ami-042a37e33a285c22b
submodules: 'recursive'
run_mode: 'start' # start ec2 on demand instance
upload_files: proton-*-Darwin-x86_64
command: |
cd $GITHUB_WORKSPACE
# git config
git config user.name "proton-robot"
git config user.email "[email protected]"
# update version
./release --version patch
# download the pre-built binary of v8 (this is only for cross-compile)
aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz
chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
# prepare build cache
mkdir $GITHUB_WORKSPACE/ccache
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
# compiling
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
echo "Compiling proton Failed"
exit 127
fi
# clean ccache and build footprint
rm -rf $GITHUB_WORKSPACE/ccache
rm -rf $GITHUB_WORKSPACE/build_docker
# get proton tag
PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
echo "Proton tag: $PROTON_TAG"
echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64
echo "Proton Binary Name: $PROTON_BINARY"
sha256sum $GITHUB_WORKSPACE/output/proton
mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
Build_Darwin_Arm64:
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
with:
ec2-instance-type: c6g.4xlarge
ec2-image-id: ami-0f3dbc4cc9994fdee
submodules: 'recursive'
run_mode: 'start' # start ec2 on demand instance
upload_files: proton-*-Darwin-arm64
command: |
cd $GITHUB_WORKSPACE
# git config
git config user.name "proton-robot"
git config user.email "[email protected]"
# update version
./release --version patch
# download the pre-built binary of v8 (this is only for cross-compile)
aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz
chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
# prepare build cache
mkdir $GITHUB_WORKSPACE/ccache
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
# compiling
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
echo "Compiling proton Failed"
exit 127
fi
# clean ccache and build footprint
rm -rf $GITHUB_WORKSPACE/ccache
rm -rf $GITHUB_WORKSPACE/build_docker
# get proton tag
PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
echo "Proton tag: $PROTON_TAG"
echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64
echo "Proton Binary Name: $PROTON_BINARY"
sha256sum $GITHUB_WORKSPACE/output/proton
mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}