diff --git a/tools/build/build_lib.sh b/tools/build/build_lib.sh new file mode 100755 index 000000000000..032fcb92045f --- /dev/null +++ b/tools/build/build_lib.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script builds the libraries of mxnet. +make_config=config/pip_${PLATFORM}_${VARIANT}.mk +if [[ ! -f $make_config ]]; then + >&2 echo "Couldn't find make config $make_config for the current settings." + exit 1 +fi + +git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build + +>&2 echo "Now building mxnet modules..." +cp $make_config mxnet-build/config.mk + +cd mxnet-build + +make DEPS_PATH=$DEPS_PATH DMLCCORE +make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a +make DEPS_PATH=$DEPS_PATH PSLITE + +if [[ $VARIANT == *mkl ]]; then + MKLDNN_LICENSE='license.txt' + if [[ $PLATFORM == 'linux' ]]; then + IOMP_LIBFILE='libiomp5.so' + MKLML_LIBFILE='libmklml_intel.so' + MKLDNN_LIBFILE='libmkldnn.so.0' + else + IOMP_LIBFILE='libiomp5.dylib' + MKLML_LIBFILE='libmklml.dylib' + MKLDNN_LIBFILE='libmkldnn.0.dylib' + fi + make DEPS_PATH=$DEPS_PATH mkldnn + cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE +fi + +if [[ $VARIANT == *mkl ]]; then + >&2 echo "Copying MKL license." + rm lib/libmkldnn.{so,dylib} + rm lib/libmkldnn.0.*.dylib + rm lib/libmkldnn.so.0.* +fi + +>&2 echo "Now building mxnet..." +make DEPS_PATH=$DEPS_PATH || exit 1; + +if [[ $PLATFORM == 'linux' ]]; then + cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 + cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0 +fi + +# Print the linked objects on libmxnet.so +>&2 echo "Checking linked objects on libmxnet.so..." +if [[ ! -z $(command -v readelf) ]]; then + readelf -d lib/libmxnet.so + strip --strip-unneeded lib/libmxnet.so +elif [[ ! -z $(command -v otool) ]]; then + otool -L lib/libmxnet.so + strip -u -r -x lib/libmxnet.so +else + >&2 echo "Not available" +fi + +cd ../ diff --git a/tools/pip_package/MANIFEST.in b/tools/build/build_wheel.sh old mode 100644 new mode 100755 similarity index 69% rename from tools/pip_package/MANIFEST.in rename to tools/build/build_wheel.sh index 5c6a72377e9f..a79634117c21 --- a/tools/pip_package/MANIFEST.in +++ b/tools/build/build_wheel.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +17,15 @@ # specific language governing permissions and limitations # under the License. -include README -recursive-include * *.py -recursive-include * *.so +# This script builds the wheel for binary distribution and performs sanity check. + +cd mxnet-build +echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH +cd - + +# Make wheel for testing +python setup.py bdist_wheel + +wheel_name=$(ls -t dist | head -n 1) +pip install -U --user --force-reinstall dist/$wheel_name +python sanity_test.py diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh new file mode 100755 index 000000000000..d678fddcc02d --- /dev/null +++ b/tools/dependencies/make_shared_dependencies.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This is a convenience script for calling the build scripts of all dependency libraries. +# Environment variables should be set beforehand. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + + +if [[ ! $PLATFORM == 'darwin' ]]; then + source $DIR/openblas.sh +fi +source $DIR/libz.sh +source $DIR/libturbojpeg.sh +source $DIR/libpng.sh +source $DIR/libtiff.sh +source $DIR/openssl.sh +source $DIR/curl.sh +source $DIR/eigen.sh +source $DIR/opencv.sh +source $DIR/protobuf.sh +source $DIR/cityhash.sh +source $DIR/zmq.sh +source $DIR/lz4.sh diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 98ff115f1765..99d0ecb71c36 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -20,6 +20,7 @@ # This script builds the static library of opencv that can be used as dependency of mxnet. # It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. OPENCV_VERSION=3.4.2 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" if [[ $PLATFORM == 'linux' ]]; then OPENCV_LAPACK_OPTIONS=" \ -D OpenBLAS_HOME=$DEPS_PATH \ @@ -181,7 +182,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. if [[ $PLATFORM == 'linux' ]]; then - cp $DEPS_PATH/../patch/opencv_lapack.h ./ + cp $DIR/patch/opencv_lapack.h ./ fi make make install diff --git a/tools/dependencies/patch/opencv_lapack.h b/tools/dependencies/patch/opencv_lapack.h new file mode 100644 index 000000000000..97af9d67ea31 --- /dev/null +++ b/tools/dependencies/patch/opencv_lapack.h @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +extern "C" { +#include "cblas.h" +#include "lapacke.h" +} diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh index dfa3d71f3750..1564701042af 100755 --- a/tools/dependencies/protobuf.sh +++ b/tools/dependencies/protobuf.sh @@ -39,3 +39,5 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then make install cd - fi + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/pip/MANIFEST.in b/tools/pip/MANIFEST.in new file mode 100644 index 000000000000..5e072064193c --- /dev/null +++ b/tools/pip/MANIFEST.in @@ -0,0 +1,11 @@ +include README +include mxnet/COMMIT_HASH +recursive-include mxnet/tools * +recursive-include mxnet *.py +recursive-include mxnet *.so +recursive-include mxnet *.so.* +recursive-include mxnet *.dylib +recursive-include mxnet *_LICENSE +recursive-include mxnet *.h +recursive-include mxnet *.cuh +recursive-include dmlc_tracker *.py diff --git a/tools/pip/sanity_test.py b/tools/pip/sanity_test.py new file mode 100644 index 000000000000..dc51e479906b --- /dev/null +++ b/tools/pip/sanity_test.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""Sanity test.""" +from __future__ import print_function +import sys +from base64 import b64decode + +try: + import mxnet as mx + mx.img.imdecode(b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')).asnumpy() + print('Test succeeded') +except: + import traceback + print('Test failed') + traceback.print_exc() + sys.exit(1) diff --git a/tools/pip/setup.py b/tools/pip/setup.py new file mode 100644 index 000000000000..d5db6d87fc1d --- /dev/null +++ b/tools/pip/setup.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +# pylint: disable=invalid-name, exec-used +"""Setup mxnet package for pip.""" +from __future__ import absolute_import +from datetime import datetime +import os +import sys +import shutil +import platform + +if platform.system() == 'Linux': + sys.argv.append('--universal') + sys.argv.append('--plat-name=manylinux1_x86_64') + +from setuptools import setup, find_packages +from setuptools.dist import Distribution + +# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py +# Will be invoked which introduces dependences +CURRENT_DIR = os.path.dirname(__file__) +libinfo_py = os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet/libinfo.py') +libinfo = {'__file__': libinfo_py} +exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) + +LIB_PATH = libinfo['find_lib_path']() +__version__ = libinfo['__version__'] +if 'TRAVIS_TAG' not in os.environ or not os.environ['TRAVIS_TAG'].strip(): + __version__ += 'b{0}'.format(datetime.today().strftime('%Y%m%d')) +elif 'TRAVIS_TAG' in os.environ and os.environ['TRAVIS_TAG'].startswith('patch-'): + __version__ = os.environ['TRAVIS_TAG'].split('-')[1] + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return platform.system() == 'Darwin' + + +DEPENDENCIES = [ + 'numpy<1.15.0,>=1.8.2', + 'requests>=2.20.0', + 'graphviz<0.9.0,>=0.8.1' +] + +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) +shutil.rmtree(os.path.join(CURRENT_DIR, 'dmlc_tracker'), ignore_errors=True) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/python/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/tracker/dmlc_tracker'), + os.path.join(CURRENT_DIR, 'dmlc_tracker')) +shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) + +# copy tools to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/tools'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/launch.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/im2rec.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/kill-mxnet.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/parse_log.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copy(os.path.join(CURRENT_DIR, 'mxnet-build/tools/diagnose.py'), os.path.join(CURRENT_DIR, 'mxnet/tools')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/caffe_converter'), os.path.join(CURRENT_DIR, 'mxnet/tools/caffe_converter')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/tools/bandwidth'), os.path.join(CURRENT_DIR, 'mxnet/tools/bandwidth')) + +# copy headers to mxnet package +shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet/include'), ignore_errors=True) +os.mkdir(os.path.join(CURRENT_DIR, 'mxnet/include')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/include/mxnet'), + os.path.join(CURRENT_DIR, 'mxnet/include/mxnet')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dlpack/include/dlpack'), + os.path.join(CURRENT_DIR, 'mxnet/include/dlpack')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/dmlc-core/include/dmlc'), + os.path.join(CURRENT_DIR, 'mxnet/include/dmlc')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/mshadow/mshadow'), + os.path.join(CURRENT_DIR, 'mxnet/include/mshadow')) +shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/tvm/nnvm/include/nnvm'), + os.path.join(CURRENT_DIR, 'mxnet/include/nnvm')) + +package_name = 'mxnet' + +variant = os.environ['mxnet_variant'].upper() +if variant != 'CPU': + package_name = 'mxnet_{0}'.format(variant.lower()) + +with open('doc/PYPI_README.md') as readme_file: + long_description = readme_file.read() + +with open('doc/{0}_ADDITIONAL.md'.format(variant)) as variant_doc: + long_description = long_description + variant_doc.read() + +# pypi only supports rst, so use pandoc to convert +import pypandoc +if platform.system() == 'Darwin': + pypandoc.download_pandoc() +long_description = pypandoc.convert_text(long_description, 'rst', 'md') +short_description = 'MXNet is an ultra-scalable deep learning framework.' +libraries = [] +if variant == 'CPU': + libraries.append('openblas') +else: + if variant.startswith('CU92'): + libraries.append('CUDA-9.2') + elif variant.startswith('CU91'): + libraries.append('CUDA-9.1') + elif variant.startswith('CU90'): + libraries.append('CUDA-9.0') + elif variant.startswith('CU80'): + libraries.append('CUDA-8.0') + elif variant.startswith('CU75'): + libraries.append('CUDA-7.5') + if variant.endswith('MKL'): + libraries.append('MKLDNN') + +short_description += ' This version uses {0}.'.format(' and '.join(libraries)) + +package_data = {'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))], + 'dmlc_tracker': []} +if variant.endswith('MKL'): + if platform.system() == 'Darwin': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.0.dylib'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml.dylib') + package_data['mxnet'].append('mxnet/libiomp5.dylib') + package_data['mxnet'].append('mxnet/libmkldnn.0.dylib') + else: + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmklml_intel.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libiomp5.so'), os.path.join(CURRENT_DIR, 'mxnet')) + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libmkldnn.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libmklml_intel.so') + package_data['mxnet'].append('mxnet/libiomp5.so') + package_data['mxnet'].append('mxnet/libmkldnn.so.0') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), '../MKLML_LICENSE'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/MKLML_LICENSE') +if platform.system() == 'Linux': + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libgfortran.so.3'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libgfortran.so.3') + shutil.copy(os.path.join(os.path.dirname(LIB_PATH[0]), 'libquadmath.so.0'), os.path.join(CURRENT_DIR, 'mxnet')) + package_data['mxnet'].append('mxnet/libquadmath.so.0') + +from mxnet.base import _generate_op_module_signature +from mxnet.ndarray.register import _generate_ndarray_function_code +from mxnet.symbol.register import _generate_symbol_function_code +_generate_op_module_signature('mxnet', 'symbol', _generate_symbol_function_code) +_generate_op_module_signature('mxnet', 'ndarray', _generate_ndarray_function_code) + +setup(name=package_name, + version=__version__, + long_description=long_description, + description=short_description, + zip_safe=False, + packages=find_packages(), + package_data=package_data, + include_package_data=True, + install_requires=DEPENDENCIES, + distclass=BinaryDistribution, + license='Apache 2.0', + classifiers=[ # https://pypi.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: C++', + 'Programming Language :: Cython', + 'Programming Language :: Other', # R, Scala + 'Programming Language :: Perl', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + url='https://github.com/apache/incubator-mxnet') diff --git a/tools/pip_package/README.md b/tools/pip_package/README.md deleted file mode 100644 index 6d044167fcf8..000000000000 --- a/tools/pip_package/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - -MXNet Python Package -==================== -MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. -It allows you to mix the flavours of deep learning programs together to maximize the efficiency and your productivity. - - -Installation ------------- -To install, check [Build Instruction](http://mxnet.io/get_started/setup.html) diff --git a/tools/pip_package/make_pip_package.sh b/tools/pip_package/make_pip_package.sh deleted file mode 100755 index 46b4938b0785..000000000000 --- a/tools/pip_package/make_pip_package.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# Assuming the script is run at mxnet/tools/pip_package -# This script builds from scratch the dependencies of mxnet into static -# librareis and statically links them to produce a (mostly) standalone -# libmxnet.so, then packages it into the python wheel. -# It assumes the build environment to be a sandbox that doesn't have the .so -# objects for the dependencies, i.e. zlib, openblas, libjpeg, libpng, libtiff -# and opencv. - -# Install necessary build tools -if [ -n "$(command -v apt-get)" ]; then - sudo apt-get update; - sudo apt-get install -y build-essential git python-pip zip pkg-config cmake -elif [ -n "$(command -v yum)" ]; then - sudo yum install -y cmake - sudo yum groupinstall -y "Development Tools" - sudo yum install -y python27 python27-setuptools python27-tools python-pip -else - echo "Need a package manager to install build tools, e.g. apt/yum" - exit 1 -fi -sudo pip install -U pip setuptools wheel - -# Set up path as temporary working directory -DEPS_PATH=$PWD/../../deps -mkdir $DEPS_PATH - -# Dependencies can be updated here. Be sure to verify the download link before -# changing. The dependencies are: -ZLIB_VERSION=1.2.6 -OPENBLAS_VERSION=0.2.19 -JPEG_VERSION=8.4.0 -PNG_VERSION=1.5.10 -TIFF_VERSION=3.8.2 -OPENCV_VERSION=2.4.13 - -# Setup path to dependencies -export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$PKG_CONFIG_PATH -export CPATH=$DEPS_PATH/include:$CPATH - -# Position Independent code must be turned on for statically linking .a -export CC="gcc -fPIC" -export CXX="g++ -fPIC" - -# Download and build zlib -curl -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip -unzip $DEPS_PATH/zlib.zip -d $DEPS_PATH -mkdir $DEPS_PATH/zlib-$ZLIB_VERSION/build -cd $DEPS_PATH/zlib-$ZLIB_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build openblas -curl -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip -unzip $DEPS_PATH/openblas.zip -d $DEPS_PATH -cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION -make FC=gfortran -j $(($(nproc) + 1)) -make PREFIX=$DEPS_PATH install -cd - -ln -s $DEPS_PATH/lib/libopenblas_haswellp-r0.2.19.a $DEPS_PATH/lib/libcblas.a - -# download and build libjpeg -curl -L https://github.com/LuaDist/libjpeg/archive/$JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip -unzip $DEPS_PATH/libjpeg.zip -d $DEPS_PATH -cd $DEPS_PATH/libjpeg-$JPEG_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make test -make install -cd - - -# download and build libpng -curl -L https://github.com/LuaDist/libpng/archive/$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip -unzip $DEPS_PATH/libpng.zip -d $DEPS_PATH -mkdir $DEPS_PATH/libpng-$PNG_VERSION/build -cd $DEPS_PATH/libpng-$PNG_VERSION/build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ - -D PNG_CONFIGURE_LIBPNG=-fPIC \ - -D BUILD_SHARED_LIBS=OFF .. -make -j$(nproc) -make install -cd - - -# download and build libtiff -curl -L https://github.com/LuaDist/libtiff/archive/$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip -unzip $DEPS_PATH/libtiff.zip -d $DEPS_PATH -cd $DEPS_PATH/libtiff-$TIFF_VERSION -./configure --disable-shared --prefix=$DEPS_PATH -make -j$(nproc) -make install -cd - - -# download and build opencv since we need the static library -curl -L https://github.com/Itseez/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip -unzip $DEPS_PATH/opencv.zip -d $DEPS_PATH -mkdir $DEPS_PATH/opencv-$OPENCV_VERSION/build -cd $DEPS_PATH/opencv-$OPENCV_VERSION/build -cmake -D WITH_1394=OFF \ - -D WITH_AVFOUNDATION=OFF \ - -D WITH_CUDA=OFF \ - -D WITH_VTK=OFF \ - -D WITH_CUFFT=OFF \ - -D WITH_CUBLAS=OFF \ - -D WITH_NVCUVID=OFF \ - -D WITH_EIGEN=ON \ - -D WITH_VFW=OFF \ - -D WITH_FFMPEG=OFF \ - -D WITH_GSTREAMER=OFF \ - -D WITH_GTK=OFF \ - -D WITH_JASPER=OFF \ - -D WITH_JPEG=ON \ - -D WITH_PNG=ON \ - -D WITH_QUICKTIME=OFF \ - -D WITH_TBB=ON \ - -D WITH_TIFF=OFF \ - -D WITH_V4L=OFF \ - -D WITH_LIBV4L=OFF \ - -D WITH_DSHOW=OFF \ - -D WITH_MSMF=OFF \ - -D WITH_OPENCL=OFF \ - -D WITH_OPENCLAMDFFT=OFF \ - -D WITH_OPENCLAMDBLAS=OFF \ - -D BUILD_SHARED_LIBS=OFF \ - -D BUILD_opencv_apps=OFF \ - -D BUILD_opencv_gpu=OFF \ - -D BUILD_opencv_video=OFF \ - -D BUILD_opencv_contrib=OFF \ - -D BUILD_opencv_nonfree=OFF \ - -D BUILD_opencv_flann=OFF \ - -D BUILD_opencv_features2d=OFF \ - -D BUILD_opencv_calib3d=OFF \ - -D BUILD_opencv_objdetect=OFF \ - -D BUILD_opencv_ml=OFF \ - -D BUILD_opencv_photo=OFF \ - -D BUILD_DOCS=OFF \ - -D BUILD_PACKAGE=OFF \ - -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. -make -j $(nproc) -make install # user will always have access to home, so no sudo needed -cd - - -# Although .so building is explicitly turned off for most libraries, sometimes -# they still get created. So, remove them just to make sure they don't -# interfere, or otherwise we might get libmxnet.so that is not self-contained. -rm $DEPS_PATH/{lib,lib64}/*.{so,so.0} - -# Go to the parent path and build mxnet -cd ../../ -cp make/pip_$(uname | tr '[:upper:]' '[:lower:]')_cpu.mk config.mk -make -j $(nproc) - -# Generate wheel. The output is in the mxnet/tools/pip_package/dist path. -cd tools/pip_package -python setup.py bdist_wheel diff --git a/tools/pip_package/setup.py b/tools/pip_package/setup.py deleted file mode 100644 index e4bf48236bde..000000000000 --- a/tools/pip_package/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# pylint: disable=invalid-name, exec-used -"""Setup mxnet package.""" -from __future__ import absolute_import -import os -import shutil - -from setuptools import setup, find_packages -from setuptools.dist import Distribution - -# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py -# Will be invoked which introduces dependences -CURRENT_DIR = os.path.dirname(__file__) -libinfo_py = os.path.join(CURRENT_DIR, '../../python/mxnet/libinfo.py') -libinfo = {'__file__': libinfo_py} -exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) - -LIB_PATH = libinfo['find_lib_path']() -__version__ = libinfo['__version__'] - -class BinaryDistribution(Distribution): - def has_ext_modules(self): - return True - - -DEPENDENCIES = [ - 'numpy', -] - -shutil.rmtree(os.path.join(CURRENT_DIR, 'mxnet'), ignore_errors=True) -shutil.copytree(os.path.join(CURRENT_DIR, '../../python/mxnet'), - os.path.join(CURRENT_DIR, 'mxnet')) -shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet')) - -setup(name='mxnet', - version=__version__, - description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), - zip_safe=False, - packages=find_packages(), - package_data={'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))]}, - include_package_data=True, - install_requires=DEPENDENCIES, - distclass=BinaryDistribution, - url='https://github.com/dmlc/mxnet')