CI: Simplify CentOS7 CI/CD config (#18093)

* CI: Unify centos7_base.sh and centos7_core.sh (delete centos7_base.sh) * Simplify centos7.publish.test Dockerfiles used for testing binary artifacts publish.test Dockerfiles should have as few system packages present as possible to catch potential dependency issues in release artifacts.
apache · Apr 20, 2020 · 586c8ab · 586c8ab
1 parent dcada9b
commit 586c8ab
Show file tree

Hide file tree

Showing 16 changed files with 43 additions and 84 deletions.
diff --git a/ci/docker/Dockerfile.publish.centos7_cpu b/ci/docker/Dockerfile.publish.centos7_cpu
@@ -22,8 +22,8 @@ FROM centos:7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu100 b/ci/docker/Dockerfile.publish.centos7_gpu_cu100
@@ -20,8 +20,8 @@ FROM nvidia/cuda:10.0-cudnn7-devel-centos7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu101 b/ci/docker/Dockerfile.publish.centos7_gpu_cu101
@@ -20,8 +20,8 @@ FROM nvidia/cuda:10.1-cudnn7-devel-centos7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu102 b/ci/docker/Dockerfile.publish.centos7_gpu_cu102
@@ -20,8 +20,8 @@ FROM nvidia/cuda:10.2-cudnn7-devel-centos7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu92 b/ci/docker/Dockerfile.publish.centos7_gpu_cu92
@@ -20,8 +20,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/

diff --git a/ci/docker/Dockerfile.publish.test.centos7_cpu b/ci/docker/Dockerfile.publish.test.centos7_cpu
@@ -22,12 +22,18 @@ FROM centos:7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
-
 COPY install/centos7_scala.sh /work/
 RUN /work/centos7_scala.sh
 
+# Install runtime dependencies for publish tests
+# - make is used to run tests ci/publish/scala/test.sh
+# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite
+# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship
+#   inside jar)
+RUN yum -y check-update || true && \
+    yum install -y make gcc unzip && \
+    yum clean all
+
 ARG USER_ID=0
 COPY install/centos7_adduser.sh /work/
 RUN /work/centos7_adduser.sh

diff --git a/ci/docker/Dockerfile.publish.test.centos7_gpu b/ci/docker/Dockerfile.publish.test.centos7_gpu
@@ -22,12 +22,18 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7
 
 WORKDIR /work/deps
 
-COPY install/centos7_base.sh /work/
-RUN /work/centos7_base.sh
-
 COPY install/centos7_scala.sh /work/
 RUN /work/centos7_scala.sh
 
+# Install runtime dependencies for publish tests
+# - make is used to run tests ci/publish/scala/test.sh
+# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite
+# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship
+#   inside jar)
+RUN yum -y check-update || true && \
+    yum install -y make gcc unzip && \
+    yum clean all
+
 ARG USER_ID=0
 COPY install/centos7_adduser.sh /work/
 RUN /work/centos7_adduser.sh

diff --git a/ci/docker/install/centos7_base.sh b/ci/docker/install/centos7_base.sh
diff --git a/ci/docker/install/centos7_core.sh b/ci/docker/install/centos7_core.sh
@@ -35,6 +35,7 @@ yum -y install protobuf-devel
 yum -y install zeromq-devel
 yum -y install openssl-devel
 yum -y install gcc-c++-4.8.*
+yum -y install gcc-gfortran
 yum -y install make
 yum -y install wget
 yum -y install unzip
@@ -43,6 +44,7 @@ yum -y install automake
 yum -y install autoconf
 yum -y install libtool
 yum -y install patchelf
+yum -y install pandoc
 
 # gcc7
 yum -y install centos-release-scl

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
@@ -734,7 +734,7 @@ def compile_windows_gpu_mkldnn() {
 }
 
 def test_static_scala_cpu() {
-  return ['Static build CPU 14.04 Scala' : {
+  return ['Static build CPU CentOS7 Scala' : {
     node(NODE_LINUX_CPU) {
         ws('workspace/ut-publish-scala-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {

diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu
@@ -41,7 +41,10 @@ core_logic: {
 
   utils.parallel_stage('Tests', [
     custom_steps.test_centos7_python3_cpu(),
-    custom_steps.test_centos7_scala_cpu()
+    custom_steps.test_centos7_scala_cpu(),
+    custom_steps.test_static_scala_cpu(),
+    custom_steps.test_static_python_cpu(),
+    custom_steps.test_static_python_cpu_cmake(),
   ])
 }
 ,

diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu
@@ -35,11 +35,13 @@ utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
     custom_steps.compile_centos7_gpu()
-  ]) 
-  
+  ])
+
   utils.parallel_stage('Tests', [
-    custom_steps.test_centos7_python3_gpu()
-  ]) 
+    custom_steps.test_centos7_python3_gpu(),
+    custom_steps.test_static_python_gpu(),
+    custom_steps.test_static_python_gpu_cmake(),
+  ])
 }
 ,
 failure_handler: {

diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -62,9 +62,6 @@ core_logic: {
     custom_steps.test_unix_julia10_cpu(),
     custom_steps.test_unix_onnx_cpu(),
     custom_steps.test_unix_cpp_cpu(),
-    custom_steps.test_static_scala_cpu(),
-    custom_steps.test_static_python_cpu(),
-    custom_steps.test_static_python_cpu_cmake(),
     /*  Disabled due to master build failure:
      *  http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
      *  https://github.com/apache/incubator-mxnet/issues/11801

diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu
@@ -60,8 +60,6 @@ core_logic: {
     custom_steps.test_unix_cpp_package_gpu(),
     custom_steps.test_unix_scala_gpu(),
     custom_steps.test_unix_distributed_kvstore_gpu(),
-    custom_steps.test_static_python_gpu(),
-    custom_steps.test_static_python_gpu_cmake(),
     custom_steps.test_unix_python3_gpu_no_tvm_op(),
     custom_steps.test_unix_capi_cpp_package(),
 

diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml
@@ -55,7 +55,7 @@
       <includes>
         <include>libmxnet.so</include>
         <include>libtvm_runtime.so</include>
-        <include>libgfortran.so.3</include>
+        <include>libgfortran.so.4</include>
         <include>libquadmath.so.0</include>
       </includes>
       <outputDirectory>lib/native</outputDirectory>

diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala
@@ -87,7 +87,7 @@ private[mxnet] object NativeLibraryLoader {
     val libFileInJar = libPathInJar + loadLibname
     saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true)
     saveLibraryToTemp("libtvm_runtime.so", "/lib/native/libtvm_runtime.so", false)
-    saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false)
+    saveLibraryToTemp("libgfortran.so.4", "/lib/native/libgfortran.so.4", false)
     saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false)
     val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true)