diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d12958fa613e..2223d3cb5f84 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -308,7 +308,6 @@ build_amzn_linux_cpu() { ninja -v } - build_centos7_mkldnn() { set -ex cd /work/mxnet @@ -1059,7 +1058,6 @@ build_docs() { popd } - # Functions that run the nightly Tests: #Runs Apache RAT Check on MXNet Source for License Headers @@ -1248,6 +1246,14 @@ deploy_jl_docs() { # ... } +deploy_nightly_maven() { + set -ex + pushd . + cd /work/mxnet + ./scala-package/dev/build.sh + popd +} + # broken_link_checker broken_link_checker() { diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile new file mode 100644 index 000000000000..b03c42d75162 --- /dev/null +++ b/ci/publish/Jenkinsfile @@ -0,0 +1,77 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 120 + +node('restricted-mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') +} +utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + stage('Deploy Nightly Maven') { + environment { + MVN_DEPLOY_USER = credentials('mvn-deploy-user') + MVN_DEPLOY_PASSWORD = credentials('mvn-deploy-password') + MVN_DEPLOY_PASSPHRASE = credentials('mvn-deploy-passphrase') + MVN_DEPLOY_MASTERPASS = credentials('mvn-deploy-masterpass') + MVN_DEPLOY_GPG_KEY = credentials('mvn-deploy-gpg-key') + } + parallel 'linux-cpu': { + node(NODE_LINUX_CPU) { + ws('workspace/linux-cpu') { + environment { + MVN_DEPLOY_OS_TYPE = linux-x86_64-cpu + } + utils.init_git() + timeout(time: max_time, unit: 'MINUTES') { + sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven" + } + } + } + }, + 'linux-gpu': { + node(NODE_LINUX_CPU) { + ws('workspace/linux-gpu') { + environment { + MVN_DEPLOY_OS_TYPE = linux-x86_64-gpu + } + utils.init_git() + timeout(time: max_time, unit: 'MINUTES') { + sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven" + } + } + } + } + } +} +, +failure_handler: { + if (currentBuild.result == "FAILURE") { + emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' + } +} +) diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh new file mode 100755 index 000000000000..bb7da2a85371 --- /dev/null +++ b/scala-package/dev/build.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Install Dependencies +sudo bash ci/docker/install/ubuntu_core.sh +sudo bash ci/docker/install/ubuntu_scala.sh + +# Setup Environment Variables +# MVN_DEPLOY_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu +# export MVN_DEPLOY_OS_TYPE=linux-x86_64-cpu + +# This script is used to build the base dependencies of MXNet Scala Env +# git clone --recursive https://github.com/apache/incubator-mxnet +# cd incubator-mxnet/ +# git checkout origin/master -b maven +sudo apt-get install -y libssl-dev + +# This part is used to build the gpg dependencies: +sudo apt-get install -y maven gnupg gnupg2 gnupg-agent +# Mitigation for Ubuntu versions before 18.04 +if ! [gpg --version | grep -q "gpg (GnuPG) 2" ]; then + sudo mv /usr/bin/gpg /usr/bin/gpg1 + sudo ln -s /usr/bin/gpg2 /usr/bin/gpg +fi + +# Run python to configure keys +python3 $PWD/scala-package/dev/buildkey.py + +# Updating cache +mkdir -p ~/.gnupg +echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf +echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf +export GPG_TTY=$(tty) + +# Build the Scala MXNet backend +bash scala-package/dev/compile-mxnet-backend.sh $MVN_DEPLOY_OS_TYPE ./ + +# Scala steps to deploy +make scalapkg +make scalaunittest +make scalaintegrationtest +echo "\n\n\n" | make scalarelease-dryrun +# make scaladeploy diff --git a/scala-package/dev/buildkey.py b/scala-package/dev/buildkey.py new file mode 100644 index 000000000000..7892185b05c6 --- /dev/null +++ b/scala-package/dev/buildkey.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import json +import logging +import subprocess + +HOME = os.environ['HOME'] +KEY_PATH = os.path.join(HOME, ".m2") + + +''' +This file would do the following items: + Import keys from AWS Credential services + Create settings.xml in .m2 with pass phrase + Create security-settings.xml in .m2 with master password + Import keys.asc the encrypted keys in gpg +''' + + +def getCredentials(): + import boto3 + import botocore + secret_name = os.environ['DOCKERHUB_SECRET_NAME'] + endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL'] + region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION'] + + session = boto3.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name, + endpoint_url=endpoint_url + ) + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + except botocore.exceptions.ClientError as client_error: + if client_error.response['Error']['Code'] == 'ResourceNotFoundException': + logging.exception("The requested secret %s was not found", secret_name) + elif client_error.response['Error']['Code'] == 'InvalidRequestException': + logging.exception("The request was invalid due to:") + elif client_error.response['Error']['Code'] == 'InvalidParameterException': + logging.exception("The request had invalid params:") + else: + raise + else: + secret = get_secret_value_response['SecretString'] + secret_dict = json.loads(secret) + return secret_dict + + +def importASC(path, passPhrase): + subprocess.run(['gpg', '--batch', '--yes', + '--passphrase=\"{}\"'.format(passPhrase), + "--import", "{}".format(os.environ['MVN_DEPLOY_GPG_KEY'])]) + + +def encryptMasterPSW(password): + result = subprocess.run(['mvn', '--encrypt-master-password', password], + stdout=subprocess.PIPE) + return str(result.stdout)[2:-3] + + +def encryptPSW(password): + result = subprocess.run(['mvn', '--encrypt-password', password], + stdout=subprocess.PIPE) + return str(result.stdout)[2:-3] + + +def masterPSW(password): + with open(os.path.join(KEY_PATH, "settings-security.xml"), "w") as f: + f.write("\n {}\n" + .format(password)) + + +def severPSW(username, password, passPhrase): + with open(os.path.join(KEY_PATH, "settings.xml"), "w") as f: + settingsString = ''' + + + + + + apache.snapshots.https + {} + {} + + + + apache.releases.https + {} + {} + + + + + + gpg + + gpg + {} + + + + + gpg + + '''.format(username, password, username, password, passPhrase) + f.write(settingsString) + + +if __name__ == "__main__": + if not os.path.exists(KEY_PATH): + os.makedirs(KEY_PATH) + userCredential = { + "username": os.environ['MVN_DEPLOY_USER'], + "password": os.environ['MVN_DEPLOY_PASSWORD'] + } + keyCredential = { + "passPhrase": os.environ['MVN_DEPLOY_GPG_PASSPHRASE'], + "masterPass": os.environ['MVN_DEPLOY_MASTERPASS'] + } + masterPass = encryptMasterPSW(keyCredential["masterPass"]) + masterPSW(masterPass) + passwordEncrypted = encryptPSW(userCredential["password"]) + severPSW(userCredential["username"], passwordEncrypted, + keyCredential["passPhrase"]) + importASC(HOME, keyCredential["passPhrase"])