From 16fcf34d5340449d22660a7f10855ddf47419cd6 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Fri, 2 Nov 2018 17:37:21 -0700 Subject: [PATCH] Dockerfiles for Publish Testing --- ci/build.py | 11 +++-- ci/docker/Dockerfile.publish.test.centos7_cpu | 38 +++++++++++++++++ ci/docker/Dockerfile.publish.test.centos7_gpu | 38 +++++++++++++++++ .../Dockerfile.publish.test.ubuntu1404_cpu | 39 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1404_gpu | 40 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1604_cpu | 39 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1604_gpu | 39 ++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1804_cpu | 41 +++++++++++++++++++ .../Dockerfile.publish.test.ubuntu1804_gpu | 41 +++++++++++++++++++ ci/docker/Dockerfile.publish.ubuntu1404_cpu | 36 ++++++++++++++++ ci/docker/Dockerfile.publish.ubuntu1404_gpu | 36 ++++++++++++++++ ci/docker/install/centos7_base.sh | 33 +++++++++++++++ ci/docker/install/centos7_scala.sh | 10 ++++- ci/docker/install/ubuntu_base.sh | 36 ++++++++++++++++ ci/docker/install/ubuntu_core.sh | 9 +++- ci/docker/install/ubuntu_publish.sh | 25 +++++++++-- ci/docker/install/ubuntu_scala.sh | 27 +++++++++--- ci/docker/runtime_functions.sh | 6 +-- ci/publish/Jenkinsfile | 22 +++++----- .../dev => ci/publish/scala}/build.sh | 9 +--- .../dev => ci/publish/scala}/buildkey.py | 0 .../dev => ci/publish/scala}/deploy.sh | 7 +--- ci/publish/scala/fullDeploy.sh | 23 +++++++++++ .../dev => ci/publish/scala}/test.sh | 7 +++- ci/test_docker_cache.py | 4 +- scala-package/.gitignore | 1 - scala-package/packageTest/pom.xml | 2 +- 27 files changed, 573 insertions(+), 46 deletions(-) create mode 100644 ci/docker/Dockerfile.publish.test.centos7_cpu create mode 100644 ci/docker/Dockerfile.publish.test.centos7_gpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1404_cpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1404_gpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1604_cpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1604_gpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1804_cpu create mode 100644 ci/docker/Dockerfile.publish.test.ubuntu1804_gpu create mode 100644 ci/docker/Dockerfile.publish.ubuntu1404_cpu create mode 100644 ci/docker/Dockerfile.publish.ubuntu1404_gpu create mode 100755 ci/docker/install/centos7_base.sh create mode 100755 ci/docker/install/ubuntu_base.sh rename {scala-package/dev => ci/publish/scala}/build.sh (88%) rename {scala-package/dev => ci/publish/scala}/buildkey.py (100%) rename {scala-package/dev => ci/publish/scala}/deploy.sh (86%) create mode 100644 ci/publish/scala/fullDeploy.sh rename {scala-package/dev => ci/publish/scala}/test.sh (88%) diff --git a/ci/build.py b/ci/build.py index e5cf933d2fd7..1c7a4f8b3231 100755 --- a/ci/build.py +++ b/ci/build.py @@ -92,22 +92,24 @@ def get_dockerfiles_path(): def get_platforms(path: str = get_dockerfiles_path()) -> List[str]: """Get a list of architectures given our dockerfiles""" - dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*")) + dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles)) - files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles)) + files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles)) platforms = list(map(lambda x: os.path.split(x)[1], sorted(files))) return platforms def get_docker_tag(platform: str, registry: str) -> str: """:return: docker tag to be used for the container""" + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) if not registry: registry = "mxnet_local" - return "{0}/build.{1}".format(registry, platform) + return "{0}/{1}".format(registry, platform) def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str: - return os.path.join(path, "Dockerfile.build.{0}".format(platform)) + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) + return os.path.join(path, "Dockerfile.{0}".format(platform)) def get_docker_binary(use_nvidia_docker: bool) -> str: @@ -523,6 +525,7 @@ def signal_handler(signum, _): elif args.all: platforms = get_platforms() + platforms = [platform for platform in platforms if 'build.' in platform] logging.info("Building for all architectures: %s", platforms) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: diff --git a/ci/docker/Dockerfile.publish.test.centos7_cpu b/ci/docker/Dockerfile.publish.test.centos7_cpu new file mode 100644 index 000000000000..7d284452971b --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.centos7_cpu @@ -0,0 +1,38 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM centos:7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh + +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh + +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet + +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.centos7_gpu b/ci/docker/Dockerfile.publish.test.centos7_gpu new file mode 100644 index 000000000000..e7f584683109 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.centos7_gpu @@ -0,0 +1,38 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM nvidia/cuda:9.2-cudnn7-devel-centos7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh + +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh + +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet + +COPY runtime_functions.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu new file mode 100644 index 000000000000..035837686554 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu new file mode 100644 index 000000000000..854dd68a63c1 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu @@ -0,0 +1,40 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +# Use CPU with setup_gpu script +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu new file mode 100644 index 000000000000..bbb7b6a0d7bd --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu new file mode 100644 index 000000000000..660461dc0cfa --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu @@ -0,0 +1,39 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu new file mode 100644 index 000000000000..e3a8c193f234 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu @@ -0,0 +1,41 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU + +FROM ubuntu:18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu new file mode 100644 index 000000000000..99f7e0d3eff9 --- /dev/null +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu @@ -0,0 +1,41 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 18.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_cpu b/ci/docker/Dockerfile.publish.ubuntu1404_cpu new file mode 100644 index 000000000000..04ce94f95eae --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1404_cpu @@ -0,0 +1,36 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1404_gpu b/ci/docker/Dockerfile.publish.ubuntu1404_gpu new file mode 100644 index 000000000000..9855986a2891 --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1404_gpu @@ -0,0 +1,36 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_base.sh b/ci/docker/install/centos7_base.sh new file mode 100755 index 000000000000..3b84aeb57b06 --- /dev/null +++ b/ci/docker/install/centos7_base.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +# Multipackage installation does not fail in yum +yum -y install epel-release +yum -y install git +yum -y install wget +yum -y install make +yum -y install cmake +yum -y install unzip +yum -y install ninja-build +yum -y install gcc-gfortran diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh index ea46de9b9311..5c43f011cbf1 100755 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/install/centos7_scala.sh @@ -23,9 +23,17 @@ set -ex yum install -y java-1.8.0-openjdk-devel +export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk +export PATH=$JAVA_HOME/bin:$PATH # Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz tar xzf apache-maven-3.3.9-bin.tar.gz mkdir /usr/local/maven mv apache-maven-3.3.9/ /usr/local/maven/ alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + +echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh +echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh +echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh +chmod +x /etc/profile.d/maven.sh +source /etc/profile.d/maven.sh diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh new file mode 100755 index 000000000000..b34c0b3e18f1 --- /dev/null +++ b/ci/docker/install/ubuntu_base.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex +apt-get update || true +apt-get install -y \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + ninja-build \ + libgfortran3 \ + software-properties-common \ + sudo \ + unzip \ + wget diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 64f8af3e0444..4382aa6aefd0 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -26,7 +26,6 @@ apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ - cmake \ curl \ git \ libatlas-base-dev \ @@ -41,3 +40,11 @@ apt-get install -y \ sudo \ unzip \ wget + + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + apt-get install -y cmake3 +else + apt-get install -y cmake +fi diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh index bc3513dd13e5..1ad6ab947842 100644 --- a/ci/docker/install/ubuntu_publish.sh +++ b/ci/docker/install/ubuntu_publish.sh @@ -35,14 +35,31 @@ apt-get install -y git \ nasm \ libtool \ curl \ + wget \ + sudo \ + gnupg \ + gnupg2 \ + gnupg-agent \ pandoc \ python3-pip \ automake \ pkg-config \ openjdk-8-jdk -curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ +curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +tar xzf apache-maven-3.3.9-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.3.9/ /usr/local/maven/ update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 update-ca-certificates -f + +apt-get install -y python python3 + +# the version of the pip shipped with ubuntu may be too lower, install a recent version here +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +python2 get-pip.py + +apt-get remove -y python3-urllib3 + +pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 22be230efdd3..5bade47463e2 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -24,16 +24,31 @@ set -ex cd "$(dirname "$0")" # install libraries for mxnet's scala package on ubuntu echo 'Installing Scala...' -apt-get update || true -apt-get install -y software-properties-common -apt-get update || true -apt-get install -y openjdk-8-jdk -apt-get install -y openjdk-8-jre +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + add-apt-repository -y ppa:openjdk-r/ppa +fi + +# All Ubuntu apt-get update || true apt-get install -y \ - maven \ + openjdk-8-jdk \ + openjdk-8-jre \ + software-properties-common \ gnupg \ gnupg2 \ gnupg-agent \ scala + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz + tar xzf apache-maven-3.3.9-bin.tar.gz + mkdir /usr/local/maven + mv apache-maven-3.3.9/ /usr/local/maven/ + update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + update-ca-certificates -f +else + apt-get install -y maven +fi diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4ec6b76d6267..94803ce2751f 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1274,7 +1274,7 @@ publish_scala_build() { set -ex pushd . scala_prepare - ./scala-package/dev/build.sh + ./ci/publish/scala/build.sh popd } @@ -1282,7 +1282,7 @@ publish_scala_test() { set -ex pushd . scala_prepare - ./scala-package/dev/test.sh + ./ci/publish/scala/test.sh popd } @@ -1290,7 +1290,7 @@ publish_scala_deploy() { set -ex pushd . scala_prepare - ./scala-package/dev/deploy.sh + ./ci/publish/scala/deploy.sh popd } diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index 49e7d7a88d40..9a360c6b5bed 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -21,7 +21,7 @@ // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ //mxnet libraries -mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**' +mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**' // timeout in minutes max_time = 120 @@ -54,22 +54,23 @@ def labels = ['cpu'] // , 'gpu'] for (x in labels) { def label = x // Required due to language toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { - env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] - utils.init_git() - utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') - utils.pack_lib("scala_${label}", mx_scala_pub, false) + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + utils.init_git() + utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.pack_lib("scala_${label}", mx_scala_pub, false) + } } } def toTest = [:] -def systems = ['ubuntu'] // , 'centos7'] +def systems = ['ubuntu1604', 'ubuntu1804', 'centos7'] for (x in labels) { def label = x // Required due to language for (y in systems) { def system = y // Required due to language toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu') + utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu') } } } @@ -78,9 +79,10 @@ def toDeploy = [:] for (x in labels) { def label = x // Required due to language toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { - env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] - utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + } } } diff --git a/scala-package/dev/build.sh b/ci/publish/scala/build.sh similarity index 88% rename from scala-package/dev/build.sh rename to ci/publish/scala/build.sh index c336fd84e5e2..17f969afe142 100755 --- a/scala-package/dev/build.sh +++ b/ci/publish/scala/build.sh @@ -24,11 +24,6 @@ set -ex bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ -# Scala steps to deploy -make scalapkg CI=1 - # Compile tests for discovery later -export GPG_TTY=$(tty) -make scalatestcompile CI=1 -# make scalainstall CI=1 -make scaladeploylocal CI=1 +cd scala-package +mvn -B deploy diff --git a/scala-package/dev/buildkey.py b/ci/publish/scala/buildkey.py similarity index 100% rename from scala-package/dev/buildkey.py rename to ci/publish/scala/buildkey.py diff --git a/scala-package/dev/deploy.sh b/ci/publish/scala/deploy.sh similarity index 86% rename from scala-package/dev/deploy.sh rename to ci/publish/scala/deploy.sh index 6f845ba5d78f..4eb33907eeb5 100755 --- a/scala-package/dev/deploy.sh +++ b/ci/publish/scala/deploy.sh @@ -23,7 +23,7 @@ set -ex # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu # Run python to configure keys -python3 $PWD/scala-package/dev/buildkey.py +python3 ci/publish/scala/buildkey.py # Updating cache mkdir -p ~/.gnupg @@ -34,11 +34,8 @@ echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf export GPG_TTY=$(tty) cd scala-package -VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) -cd .. -# echo "\n\n$VERSION\n" | make scalarelease-dryrun -make scaladeploy CI=1 +mvn -B deploy -Pnightly # Clear all password .xml files, exp files, and gpg key files rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh new file mode 100644 index 000000000000..69d674a97497 --- /dev/null +++ b/ci/publish/scala/fullDeploy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +./ci/publish/scala/build.sh +./ci/publish/scala/test.sh +./ci/publish/scala/deploy.sh diff --git a/scala-package/dev/test.sh b/ci/publish/scala/test.sh similarity index 88% rename from scala-package/dev/test.sh rename to ci/publish/scala/test.sh index 03810fbf8d55..5cef35ca3c2b 100755 --- a/scala-package/dev/test.sh +++ b/ci/publish/scala/test.sh @@ -18,6 +18,11 @@ set -ex +if [ -z "$JAVA_HOME" ]; then + source /etc/profile +fi + # Test cd scala-package/packageTest -make scalaintegrationtestlocal CI=1 +# make testlocal CI=1 +make testsnapshot UNIT=1 CI=1 diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py index 358d54985aca..0a3bc4640c05 100644 --- a/ci/test_docker_cache.py +++ b/ci/test_docker_cache.py @@ -135,7 +135,7 @@ def test_full_cache(self): """ platform = 'test_full_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: with open(dockerfile_path, 'w') as dockerfile_handle: dockerfile_handle.write(dockerfile_content) @@ -196,7 +196,7 @@ def test_partial_cache(self): """ platform = 'test_partial_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: # Write initial Dockerfile with open(dockerfile_path, 'w') as dockerfile_handle: diff --git a/scala-package/.gitignore b/scala-package/.gitignore index bf60e7bef0c5..22b12b31d050 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -8,4 +8,3 @@ core/src/main/scala/org/apache/mxnet/SymbolBase.scala core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ -local-snapshot diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index 9c5c11cf2779..f7d9e3b180bc 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -42,7 +42,7 @@ local-snapshot - file://${basedir}/../local-snapshot + file://${basedir}/../deploy/target/repo true