Skip to content

update to build native docker image #22806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
majetideepak merged 3 commits into from
May 29, 2024
Merged

update to build native docker image #22806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e24936f
Build native worker image
wanglinsong May 23, 2024
37be4a5
fix var name
wanglinsong May 23, 2024
0f0c2d3
increase agent size
wanglinsong May 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 81 additions & 13 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ pipeline {
parameters {
booleanParam(name: 'PUBLISH_ARTIFACTS_ON_CURRENT_BRANCH',
defaultValue: false,
description: 'whether to publish tar and docker image even if current branch is not master'
)
description: 'whether to publish tar and docker image even if current branch is not master')
booleanParam(name: 'BUILD_PRESTISSIMO_DEPENDENCY',
defaultValue: false,
description: 'Check to build a new native dependency image because of the build dependency updates')
}

stages {
Expand All @@ -36,7 +38,7 @@ pipeline {
}

stages {
stage('Setup') {
stage('Maven Agent Setup') {
steps {
sh 'apt update && apt install -y awscli git tree'
sh 'git config --global --add safe.directory ${WORKSPACE}'
Expand Down Expand Up @@ -94,6 +96,8 @@ pipeline {
sh(script: "git show -s --format=%cd --date=format:'%Y%m%d%H%M%S'", returnStdout: true).trim() + "-" +
env.PRESTO_COMMIT_SHA.substring(0, 7)
env.DOCKER_IMAGE = env.AWS_ECR + "/${IMG_NAME}:${PRESTO_BUILD_VERSION}"
env.NATIVE_DOCKER_IMAGE_DEPENDENCY = env.AWS_ECR + "/presto-native-dependency:${PRESTO_BUILD_VERSION}"
env.NATIVE_DOCKER_IMAGE = env.AWS_ECR + "/presto-native:${PRESTO_BUILD_VERSION}"
}
sh 'printenv | sort'

Expand Down Expand Up @@ -133,28 +137,30 @@ pipeline {
}

stages {
stage('Docker') {
stage('Docker Agent Setup') {
steps {
echo 'build docker image'
sh 'apk update && apk add aws-cli bash git'
sh 'apk update && apk add aws-cli bash git make'
sh 'git config --global --add safe.directory ${WORKSPACE}'
sh 'git config --global --add safe.directory ${WORKSPACE}/presto-native-execution/velox'
sh '''
docker run --privileged --rm tonistiigi/binfmt --install all
docker context ls
docker buildx create --name="container" --driver=docker-container --bootstrap
docker buildx ls
docker buildx inspect container
'''
}
}

stage('Docker Java') {
steps {
echo 'build docker image'
withCredentials([[
$class: 'AmazonWebServicesCredentialsBinding',
credentialsId: "${AWS_CREDENTIAL_ID}",
accessKeyVariable: 'AWS_ACCESS_KEY_ID',
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh '''#!/bin/bash -ex
for dir in /home/jenkins/agent/workspace/*/; do
echo "${dir}"
git config --global --add safe.directory "${dir:0:-1}"
done

cd docker/
aws s3 cp ${AWS_S3_PREFIX}/${PRESTO_BUILD_VERSION}/${PRESTO_PKG} . --no-progress
aws s3 cp ${AWS_S3_PREFIX}/${PRESTO_BUILD_VERSION}/${PRESTO_CLI_JAR} . --no-progress
Expand All @@ -166,13 +172,12 @@ pipeline {
}
}

stage('Publish Docker') {
stage('Publish Docker Java') {
when {
anyOf {
expression { params.PUBLISH_ARTIFACTS_ON_CURRENT_BRANCH }
branch "master"
}
beforeAgent true
}

steps {
Expand All @@ -191,6 +196,69 @@ pipeline {
}
}
}

stage ('Build Docker Native Dependency') {
when {
environment name: 'BUILD_PRESTISSIMO_DEPENDENCY', value: 'true'
}
steps {
sh '''
cd presto-native-execution/
make submodules
docker buildx build --load --platform "linux/amd64" \
-t "${NATIVE_DOCKER_IMAGE_DEPENDENCY}" \
-f scripts/dockerfiles/centos-8-stream-dependency.dockerfile \
.
docker tag "${NATIVE_DOCKER_IMAGE_DEPENDENCY}" "${AWS_ECR}/presto-native-dependency:latest"
docker image ls
'''
}
}

stage ('Build Docker Native') {
steps {
sh '''
cd presto-native-execution/
make submodules
docker buildx build --load --platform "linux/amd64" \
-t "${NATIVE_DOCKER_IMAGE}" \
--build-arg BUILD_TYPE=Release \
--build-arg DEPENDENCY_IMAGE=${AWS_ECR}/presto-native-dependency:latest \
--build-arg "EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON" \
-f scripts/dockerfiles/prestissimo-runtime.dockerfile \
.
docker image ls
'''
}
}

stage('Publish Docker Native') {
when {
anyOf {
expression { params.PUBLISH_ARTIFACTS_ON_CURRENT_BRANCH }
branch "master"
}
}

steps {
echo 'Publish docker image'
withCredentials([[
$class: 'AmazonWebServicesCredentialsBinding',
credentialsId: "${AWS_CREDENTIAL_ID}",
accessKeyVariable: 'AWS_ACCESS_KEY_ID',
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh '''
aws ecr-public get-login-password | docker login --username AWS --password-stdin ${AWS_ECR}
if ${BUILD_PRESTISSIMO_DEPENDENCY}
then
docker push "${NATIVE_DOCKER_IMAGE_DEPENDENCY}"
docker push "${AWS_ECR}/presto-native-dependency:latest"
fi
docker push "${NATIVE_DOCKER_IMAGE}"
'''
}
}
}
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions jenkins/agent-dind.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ spec:
tty: true
resources:
requests:
memory: "4Gi"
cpu: "2000m"
memory: "29Gi"
cpu: "7500m"
limits:
memory: "4Gi"
cpu: "2000m"
memory: "29Gi"
cpu: "7500m"