From 80e69fc44e7dceac90e69edaeb26fafcc6e5a3d0 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Thu, 4 Oct 2018 12:57:27 -0700 Subject: [PATCH] Seperate Build, Test, and Deploy Stages with parallel --- Makefile | 16 ++-- ci/Jenkinsfile_utils.groovy | 9 +- ci/build.py | 20 +++-- ci/docker/install/ubuntu_scala.sh | 3 + ci/docker/runtime_functions.sh | 41 +++++++-- ci/docker_cache.py | 3 +- ci/publish/Jenkinsfile | 100 +++++++++++++-------- scala-package/.gitignore | 2 +- scala-package/dev/build.sh | 46 +++------- scala-package/dev/buildkey.py | 84 ++++++++++------- scala-package/dev/compile-mxnet-backend.sh | 7 +- scala-package/dev/deploy.sh | 53 +++++++++++ scala-package/dev/test.sh | 23 +++++ scala-package/packageTest/Makefile | 12 ++- 14 files changed, 283 insertions(+), 136 deletions(-) create mode 100755 scala-package/dev/deploy.sh create mode 100755 scala-package/dev/test.sh diff --git a/Makefile b/Makefile index 4fb518e591bc..a4095240b73c 100644 --- a/Makefile +++ b/Makefile @@ -454,6 +454,10 @@ else CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0 endif +ifeq ($(CI), 1) + MAVEN_ARGS := -B +endif + # For quick compile test, used smaller subset ALLX_DEP= $(ALL_DEP) @@ -618,7 +622,7 @@ scalatestcompile: scalapkg: (cd $(ROOTDIR)/scala-package && \ - mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + mvn package $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dcurrent_libdir="$(ROOTDIR)/lib" \ @@ -626,19 +630,19 @@ scalapkg: scalaunittest: (cd $(ROOTDIR)/scala-package && \ - mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ + mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalaintegrationtest: (cd $(ROOTDIR)/scala-package && \ - mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ + mvn integration-test $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalainstall: (cd $(ROOTDIR)/scala-package && \ - mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ + mvn install $(MAVEN_ARGS) -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -663,14 +667,14 @@ scalarelease-perform: scaladeploy: (cd $(ROOTDIR)/scala-package && \ - mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + mvn deploy $(MAVEN_ARGS) -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scaladeploylocal: (cd $(ROOTDIR)/scala-package && \ - mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + mvn deploy $(MAVEN_ARGS) -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ -Dgpg.skip \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index f82c238ed075..8291bae1f7b7 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -64,7 +64,7 @@ def init_git_win() { // pack libraries for later use def pack_lib(name, libs, include_gcov_data = false) { - sh """ + sh returnStatus: true, script: """ set +e echo "Packing ${libs} into ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -83,7 +83,7 @@ return 0 def unpack_and_init(name, libs, include_gcov_data = false) { init_git() unstash name - sh """ + sh returnStatus: true, script: """ set +e echo "Unpacked ${libs} from ${name}" echo ${libs} | sed -e 's/,/ /g' | xargs md5sum @@ -147,8 +147,9 @@ def collect_test_results_windows(original_file_name, new_file_name) { } -def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" +def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "") { + def command = "ci/build.py %ENV_VARS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '') command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') command = command.replaceAll('%PLATFORM%', platform) command = command.replaceAll('%FUNCTION_NAME%', function_name) diff --git a/ci/build.py b/ci/build.py index 0069392d9a2a..e5cf933d2fd7 100755 --- a/ci/build.py +++ b/ci/build.py @@ -215,20 +215,21 @@ def container_run(platform: str, local_ccache_dir: str, command: List[str], cleanup: Cleanup, + environment: Dict[str, str], dry_run: bool = False) -> int: """Run command in a container""" container_wait_s = 600 # # Environment setup # - environment = { + environment.update({ 'CCACHE_MAXSIZE': '500G', 'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared 'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is # mounted 'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache # verification. - } + }) # These variables are passed to the container to the process tree killer can find runaway # process inside the container # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller @@ -446,6 +447,10 @@ def main() -> int: parser.add_argument("--no-cache", action="store_true", help="passes --no-cache to docker build") + parser.add_argument("-e", "--environment", nargs="*", default=[], + help="Environment variables for the docker container. " + "Specify with a list containing either names or name=value") + parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) @@ -474,6 +479,9 @@ def signal_handler(signum, _): signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) + environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e])) + for e in args.environment]) + if args.list: print(list_platforms()) elif args.platform: @@ -493,13 +501,13 @@ def signal_handler(signum, _): ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) elif args.print_docker_run: command = [] ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment) else: # With no commands, execute a build function for the target platform command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)] @@ -507,7 +515,7 @@ def signal_handler(signum, _): ret = container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) if ret != 0: logging.critical("Execution of %s failed with status: %d", command, ret) @@ -535,7 +543,7 @@ def signal_handler(signum, _): container_run( platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, - local_ccache_dir=args.ccache_dir, cleanup=cleanup) + local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 6ecb8d801186..22be230efdd3 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -33,4 +33,7 @@ apt-get install -y openjdk-8-jre apt-get update || true apt-get install -y \ maven \ + gnupg \ + gnupg2 \ + gnupg-agent \ scala diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 2223d3cb5f84..9add6bf0baa0 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -36,6 +36,11 @@ clean_repo() { git submodule update --init --recursive } +scala_prepare() { + # Clean up maven logs + export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" +} + build_ccache_wrappers() { set -ex @@ -835,21 +840,24 @@ unittest_ubuntu_python3_quantization_gpu() { unittest_ubuntu_cpu_scala() { set -ex - make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 } unittest_centos7_cpu_scala() { set -ex cd /work/mxnet + scala_prepare make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 } unittest_ubuntu_cpu_clojure() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 ./contrib/clojure-package/ci-test.sh } @@ -993,8 +1001,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() { integrationtest_ubuntu_gpu_scala() { set -ex - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 - make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + scala_prepare + make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1 + make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 } integrationtest_ubuntu_gpu_dist_kvstore() { @@ -1246,14 +1255,30 @@ deploy_jl_docs() { # ... } -deploy_nightly_maven() { +publish_scala_build() { set -ex pushd . - cd /work/mxnet + scala_prepare ./scala-package/dev/build.sh popd } +publish_scala_test() { + set -ex + pushd . + scala_prepare + ./scala-package/dev/test.sh + popd +} + +publish_scala_deploy() { + set -ex + pushd . + scala_prepare + ./scala-package/dev/deploy.sh + popd +} + # broken_link_checker broken_link_checker() { diff --git a/ci/docker_cache.py b/ci/docker_cache.py index fe1882a567aa..24a806c7dc97 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -211,8 +211,7 @@ def _get_dockerhub_credentials(): # pragma: no cover logging.exception("The request was invalid due to:") elif client_error.response['Error']['Code'] == 'InvalidParameterException': logging.exception("The request had invalid params:") - else: - raise + raise else: secret = get_secret_value_response['SecretString'] secret_dict = json.loads(secret) diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index b03c42d75162..49e7d7a88d40 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -20,58 +20,86 @@ // Jenkins pipeline // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ +//mxnet libraries +mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**' + // timeout in minutes max_time = 120 -node('restricted-mxnetlinux-cpu') { +node('restricted-utility') { // Loading the utilities requires a node context unfortunately checkout scm utils = load('ci/Jenkinsfile_utils.groovy') } -utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') +utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') -utils.main_wrapper( -core_logic: { - stage('Deploy Nightly Maven') { - environment { - MVN_DEPLOY_USER = credentials('mvn-deploy-user') - MVN_DEPLOY_PASSWORD = credentials('mvn-deploy-password') - MVN_DEPLOY_PASSPHRASE = credentials('mvn-deploy-passphrase') - MVN_DEPLOY_MASTERPASS = credentials('mvn-deploy-masterpass') - MVN_DEPLOY_GPG_KEY = credentials('mvn-deploy-gpg-key') - } - parallel 'linux-cpu': { - node(NODE_LINUX_CPU) { - ws('workspace/linux-cpu') { - environment { - MVN_DEPLOY_OS_TYPE = linux-x86_64-cpu - } - utils.init_git() - timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven" - } - } - } - }, - 'linux-gpu': { - node(NODE_LINUX_CPU) { - ws('workspace/linux-gpu') { - environment { - MVN_DEPLOY_OS_TYPE = linux-x86_64-gpu - } - utils.init_git() - timeout(time: max_time, unit: 'MINUTES') { - sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven" - } +// CPU and GPU. OSX nodes are not currently supported by Jenkins +def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU] +def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu'] + +def wrapStep(nodeToRun, workspaceName, step) { + return { + node(nodeToRun) { + ws("workspace/${workspaceName}") { + timeout(time: max_time, unit: 'MINUTES') { + step() } } } } } + +def toBuild = [:] +def labels = ['cpu'] // , 'gpu'] +for (x in labels) { + def label = x // Required due to language + toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { + env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] + utils.init_git() + utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.pack_lib("scala_${label}", mx_scala_pub, false) + } +} + +def toTest = [:] +def systems = ['ubuntu'] // , 'centos7'] +for (x in labels) { + def label = x // Required due to language + for (y in systems) { + def system = y // Required due to language + toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu') + } + } +} + +def toDeploy = [:] +for (x in labels) { + def label = x // Required due to language + toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { + env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + } +} + +utils.main_wrapper( +core_logic: { + stage('Build Packages') { + parallel toBuild + } + stage('Test Packages') { + parallel toTest + } + stage('Deploy Packages') { + parallel toDeploy + } +} , failure_handler: { if (currentBuild.result == "FAILURE") { - emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' + // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' } } ) diff --git a/scala-package/.gitignore b/scala-package/.gitignore index 9a89bef324bc..f75826b29281 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -8,4 +8,4 @@ core/src/main/scala/org/apache/mxnet/NDArrayRandomAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ -local-snapshot \ No newline at end of file +local-snapshot diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh index bb7da2a85371..013ee5739f84 100755 --- a/scala-package/dev/build.sh +++ b/scala-package/dev/build.sh @@ -16,43 +16,21 @@ # limitations under the License. # -# Install Dependencies -sudo bash ci/docker/install/ubuntu_core.sh -sudo bash ci/docker/install/ubuntu_scala.sh +set -ex # Setup Environment Variables -# MVN_DEPLOY_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu -# export MVN_DEPLOY_OS_TYPE=linux-x86_64-cpu +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu -# This script is used to build the base dependencies of MXNet Scala Env -# git clone --recursive https://github.com/apache/incubator-mxnet -# cd incubator-mxnet/ -# git checkout origin/master -b maven -sudo apt-get install -y libssl-dev +bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ -# This part is used to build the gpg dependencies: -sudo apt-get install -y maven gnupg gnupg2 gnupg-agent -# Mitigation for Ubuntu versions before 18.04 -if ! [gpg --version | grep -q "gpg (GnuPG) 2" ]; then - sudo mv /usr/bin/gpg /usr/bin/gpg1 - sudo ln -s /usr/bin/gpg2 /usr/bin/gpg -fi - -# Run python to configure keys -python3 $PWD/scala-package/dev/buildkey.py +# Scala steps to deploy +make scalapkg CI=1 +make scalaunittest CI=1 +make scalaintegrationtest CI=1 -# Updating cache -mkdir -p ~/.gnupg -echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf -echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf +# Compile tests for discovery later export GPG_TTY=$(tty) - -# Build the Scala MXNet backend -bash scala-package/dev/compile-mxnet-backend.sh $MVN_DEPLOY_OS_TYPE ./ - -# Scala steps to deploy -make scalapkg -make scalaunittest -make scalaintegrationtest -echo "\n\n\n" | make scalarelease-dryrun -# make scaladeploy +make scalatestcompile CI=1 +# make scalainstall CI=1 +make scaladeploylocal CI=1 diff --git a/scala-package/dev/buildkey.py b/scala-package/dev/buildkey.py index 7892185b05c6..8a1b7bf63286 100644 --- a/scala-package/dev/buildkey.py +++ b/scala-package/dev/buildkey.py @@ -37,8 +37,9 @@ def getCredentials(): import boto3 import botocore - secret_name = os.environ['DOCKERHUB_SECRET_NAME'] - endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL'] + endpoint_url = os.environ['MAVEN_PUBLISH_SECRET_ENDPOINT_URL'] + secret_creds_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS'] + secret_key_name = os.environ['MAVEN_PUBLISH_SECRET_NAME_GPG'] region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION'] session = boto3.Session() @@ -49,39 +50,62 @@ def getCredentials(): ) try: get_secret_value_response = client.get_secret_value( - SecretId=secret_name + SecretId=secret_creds_name + ) + get_secret_key_response = client.get_secret_value( + SecretId=secret_key_name ) except botocore.exceptions.ClientError as client_error: if client_error.response['Error']['Code'] == 'ResourceNotFoundException': - logging.exception("The requested secret %s was not found", secret_name) + name = (secret_key_name if get_secret_value_response + else secret_creds_name) + logging.exception("The requested secret %s was not found", name) elif client_error.response['Error']['Code'] == 'InvalidRequestException': logging.exception("The request was invalid due to:") elif client_error.response['Error']['Code'] == 'InvalidParameterException': logging.exception("The request had invalid params:") - else: - raise + raise else: secret = get_secret_value_response['SecretString'] secret_dict = json.loads(secret) - return secret_dict + secret_key = get_secret_key_response['SecretString'] + return secret_dict, secret_key -def importASC(path, passPhrase): - subprocess.run(['gpg', '--batch', '--yes', - '--passphrase=\"{}\"'.format(passPhrase), - "--import", "{}".format(os.environ['MVN_DEPLOY_GPG_KEY'])]) +def importASC(key, gpgPassphrase): + filename = os.path.join(KEY_PATH, "key.asc") + with open(filename, 'w') as f: + f.write(key) + subprocess.check_output(['gpg2', '--batch', '--yes', + '--passphrase-fd', '0', + "--import", "{}".format(filename)], + input=str.encode(gpgPassphrase)) def encryptMasterPSW(password): - result = subprocess.run(['mvn', '--encrypt-master-password', password], - stdout=subprocess.PIPE) - return str(result.stdout)[2:-3] + filename = os.path.join(KEY_PATH, "encryptMasterPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-master-password + expect -exact "Master password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] def encryptPSW(password): - result = subprocess.run(['mvn', '--encrypt-password', password], - stdout=subprocess.PIPE) - return str(result.stdout)[2:-3] + filename = os.path.join(KEY_PATH, "encryptPassword.exp") + with open(filename, 'w') as f: + f.write(''' + spawn mvn --encrypt-password + expect -exact "Password: " + send -- "{}\r" + expect eof + '''.format(password)) + result = subprocess.check_output(['expect', filename]) + return str(result).split('\r\n')[-1][2:-3] def masterPSW(password): @@ -90,7 +114,7 @@ def masterPSW(password): .format(password)) -def severPSW(username, password, passPhrase): +def serverPSW(username, password, gpgPassphrase): with open(os.path.join(KEY_PATH, "settings.xml"), "w") as f: settingsString = ''' gpg - gpg + gpg2 {} + true gpg - '''.format(username, password, username, password, passPhrase) + '''.format(username, password, username, password, gpgPassphrase) f.write(settingsString) if __name__ == "__main__": if not os.path.exists(KEY_PATH): os.makedirs(KEY_PATH) - userCredential = { - "username": os.environ['MVN_DEPLOY_USER'], - "password": os.environ['MVN_DEPLOY_PASSWORD'] - } - keyCredential = { - "passPhrase": os.environ['MVN_DEPLOY_GPG_PASSPHRASE'], - "masterPass": os.environ['MVN_DEPLOY_MASTERPASS'] - } - masterPass = encryptMasterPSW(keyCredential["masterPass"]) + credentials, gpgKey = getCredentials() + masterPass = encryptMasterPSW(credentials['masterpass']) masterPSW(masterPass) - passwordEncrypted = encryptPSW(userCredential["password"]) - severPSW(userCredential["username"], passwordEncrypted, - keyCredential["passPhrase"]) - importASC(HOME, keyCredential["passPhrase"]) + passwordEncrypted = encryptPSW(credentials['password']) + serverPSW(credentials['user'], passwordEncrypted, + credentials['gpgPassphrase']) + importASC(gpgKey, credentials['gpgPassphrase']) diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh index b065e01afc8e..114bf0766444 100755 --- a/scala-package/dev/compile-mxnet-backend.sh +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -33,7 +33,7 @@ MXNETDIR=$2 # below routine shamelessly copied from # https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh -# This routine executes a command, +# This routine executes a command, # prints error message on the console on non-zero exit codes and # returns the exit code to the caller. chkret() { @@ -51,7 +51,10 @@ chkret() { UNAME=`uname -s` chkret pushd $MXNETDIR -chkret git submodule update --init --recursive + +set +e +git submodule update --init --recursive +set -e # don't want to overwrite an existing config file cp make/config.mk ./config.mk diff --git a/scala-package/dev/deploy.sh b/scala-package/dev/deploy.sh new file mode 100755 index 000000000000..fde7c96af594 --- /dev/null +++ b/scala-package/dev/deploy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Setup Environment Variables +# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu + + +if [[ $MAVEN_PUBLISH_OS_TYPE == "linux-x86_64-cpu" ]]; +then + MAKE_FLAGS="USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1" +elif [[ $MAVEN_PUBLISH_OS_TYPE == "linux-x86_64-gpu" ]] +then + MAKE_FLAGS="USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_ON_GPU=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1" +fi + +# Run python to configure keys +python3 $PWD/scala-package/dev/buildkey.py + +# Updating cache +mkdir -p ~/.gnupg +echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf +echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf +echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf +echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf +export GPG_TTY=$(tty) + +cd scala-package +VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) +cd .. + +# echo "\n\n$VERSION\n" | make scalarelease-dryrun $MAKE_FLAGS +make scaladeploy $MAKE_FLAGS CI=1 + +# Clear all password .xml files, exp files, and gpg key files +rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp diff --git a/scala-package/dev/test.sh b/scala-package/dev/test.sh new file mode 100755 index 000000000000..03810fbf8d55 --- /dev/null +++ b/scala-package/dev/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Test +cd scala-package/packageTest +make scalaintegrationtestlocal CI=1 diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile index 6073ff8a722f..8c12c1d04189 100644 --- a/scala-package/packageTest/Makefile +++ b/scala-package/packageTest/Makefile @@ -43,6 +43,10 @@ else endif endif +ifeq ($(CI), 1) + MAVEN_ARGS := -B +endif + PROFILES := -Ptest ifeq ($(UNIT), 1) PROFILES := "$(PROFILES),unittest" @@ -59,27 +63,27 @@ endif clean: - (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) testinstall: - (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ $(PROFILES) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) testlocal: - (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ $(PROFILES),fromLocal \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) testsnapshot: - (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + (mvn $(MAVEN_ARGS) integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ $(PROFILES),fromSnapshots \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.repo=$(MXNET_REPO) \