Fix testing pipeline

apache · Dec 8, 2018 · 88cd89e · 88cd89e
1 parent a9f17a0
commit 88cd89e
Show file tree

Hide file tree

Showing 12 changed files with 47 additions and 28 deletions.
diff --git a/ci/build.py b/ci/build.py
@@ -101,14 +101,14 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
 
 def get_docker_tag(platform: str, registry: str) -> str:
     """:return: docker tag to be used for the container"""
-    platform = 'build.{}'.format(platform) if '.' not in platform else platform
+    platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
     if not registry:
         registry = "mxnet_local"
     return "{0}/{1}".format(registry, platform)
 
 
 def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
-    platform = 'build.{}'.format(platform) if '.' not in platform else platform
+    platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
     return os.path.join(path, "Dockerfile.{0}".format(platform))
 
 

diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
@@ -24,8 +24,6 @@ WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
 RUN /work/centos7_core.sh
-COPY install/centos7_build.sh /work/
-RUN /work/centos7_build.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
@@ -24,8 +24,6 @@ WORKDIR /work/deps
 
 COPY install/centos7_core.sh /work/
 RUN /work/centos7_core.sh
-COPY install/centos7_build.sh /work/
-RUN /work/centos7_build.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu
@@ -22,6 +22,8 @@ FROM ubuntu:18.04
 
 WORKDIR /work/deps
 
+ENV DEBIAN_FRONTEND noninteractive
+
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
 

diff --git a/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu
@@ -22,6 +22,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04
 
 WORKDIR /work/deps
 
+ENV DEBIAN_FRONTEND noninteractive
+
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
 

diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh
@@ -26,8 +26,14 @@ yum install -y java-1.8.0-openjdk-devel
 export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk
 export PATH=$JAVA_HOME/bin:$PATH
 # Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
 tar xzf apache-maven-3.3.9-bin.tar.gz
 mkdir /usr/local/maven
 mv apache-maven-3.3.9/ /usr/local/maven/
 alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+
+echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh
+echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh
+echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh
+chmod +x /etc/profile.d/maven.sh
+source /etc/profile.d/maven.sh
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
@@ -36,6 +36,7 @@ apt-get install -y \
     libopenblas-dev \
     libopencv-dev \
     libzmq3-dev \
+    libgfortran3 \
     ninja-build \
     software-properties-common \
     sudo \

diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh
@@ -41,3 +41,8 @@ apt-get install -y \
     gnupg2 \
     gnupg-agent \
     scala
+
+# Ubuntu 14.04
+if [[ $(lsb_release -r | grep 14.04) ]]; then
+    sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+fi
diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh
@@ -32,5 +32,4 @@ make scalaintegrationtest CI=1
 # Compile tests for discovery later
 export GPG_TTY=$(tty)
 make scalatestcompile CI=1
-# make scalainstall CI=1
 make scaladeploylocal CI=1
diff --git a/scala-package/dev/fullDeploy.sh b/scala-package/dev/fullDeploy.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+./scala-package/dev/build.sh
+./scala-package/dev/test.sh
+./scala-package/dev/deploy.sh
diff --git a/scala-package/dev/test.sh b/scala-package/dev/test.sh
@@ -18,14 +18,11 @@
 
 set -ex
 
-# Install
-# make scalainstall
-# MXNET_FULL=$(find assembly/ -name "mxnet-full*-SNAPSHOT.jar" | grep -v "archive-tmp")
-# MXNET_EXAMPLES=$(find . -name "mxnet-examples*-SNAPSHOT.jar")
-# mvn install:install-file -Dfile=$MXNET_FULL
-# mvn install:install-file -Dfile=$MXNET_EXAMPLES
-# make scalainstallonly CI=1
+if [ -z "$JAVA_HOME" ]; then
+    source /etc/profile
+fi
 
 # Test
 cd scala-package/packageTest
-make scalaintegrationtestsnapshot CI=1
+# make testlocal CI=1
+make testsnapshot UNIT=1 CI=1
diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml
@@ -29,11 +29,7 @@
         <repositories>
           <repository>
             <id>apache-snapshots</id>
-<<<<<<< HEAD
             <url>${mxnet.repo}</url>
-=======
-            <url>https://repository.apache.org/content/repositories/snapshots</url>
->>>>>>> 5503e4e79... Seperate Build, Test, and Deploy Stages with parallel
             <layout>default</layout>
             <snapshots>
               <enabled>true</enabled>
@@ -64,14 +60,6 @@
             <groupId>org.apache.mxnet</groupId>
             <artifactId>mxnet-full_${mxnet.scalaprofile}-${mxnet.profile}</artifactId>
             <version>${mxnet.version}</version>
-<<<<<<< HEAD
-=======
-            <!--
-            <classifier>tests</classifier>
-            <type>test-jar</type>
-            <scope>test</scope>
-            -->
->>>>>>> 5503e4e79... Seperate Build, Test, and Deploy Stages with parallel
         </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>