Simplify publish.test.centos7 images

These images are used for testing the static builds. The idea is to have as few system packages present as possible to catch potential dependency issues in our release artifacts.
apache · Apr 18, 2020 · 48eb33f · 48eb33f
1 parent c234074
commit 48eb33f
Show file tree

Hide file tree

Showing 9 changed files with 28 additions and 18 deletions.
diff --git a/ci/docker/Dockerfile.publish.test.centos7_cpu b/ci/docker/Dockerfile.publish.test.centos7_cpu
@@ -22,12 +22,17 @@ FROM centos:7
 
 WORKDIR /work/deps
 
-COPY install/centos7_core.sh /work/
-RUN /work/centos7_core.sh
-
 COPY install/centos7_scala.sh /work/
 RUN /work/centos7_scala.sh
 
+# Install runtime dependencies for publish tests
+# - make is used to run tests ci/publish/scala/test.sh
+# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship
+#   inside jar)
+RUN yum -y check-update || true && \
+    yum install -y make gcc && \
+    yum clean all
+
 ARG USER_ID=0
 COPY install/centos7_adduser.sh /work/
 RUN /work/centos7_adduser.sh

diff --git a/ci/docker/Dockerfile.publish.test.centos7_gpu b/ci/docker/Dockerfile.publish.test.centos7_gpu
@@ -22,12 +22,17 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7
 
 WORKDIR /work/deps
 
-COPY install/centos7_core.sh /work/
-RUN /work/centos7_core.sh
-
 COPY install/centos7_scala.sh /work/
 RUN /work/centos7_scala.sh
 
+# Install runtime dependencies for publish tests
+# - make is used to run tests ci/publish/scala/test.sh
+# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship
+#   inside jar)
+RUN yum -y check-update || true && \
+    yum install -y make gcc && \
+    yum clean all
+
 ARG USER_ID=0
 COPY install/centos7_adduser.sh /work/
 RUN /work/centos7_adduser.sh

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
@@ -734,7 +734,7 @@ def compile_windows_gpu_mkldnn() {
 }
 
 def test_static_scala_cpu() {
-  return ['Static build CPU 14.04 Scala' : {
+  return ['Static build CPU CentOS7 Scala' : {
     node(NODE_LINUX_CPU) {
         ws('workspace/ut-publish-scala-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {

diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu
@@ -41,7 +41,10 @@ core_logic: {
 
   utils.parallel_stage('Tests', [
     custom_steps.test_centos7_python3_cpu(),
-    custom_steps.test_centos7_scala_cpu()
+    custom_steps.test_centos7_scala_cpu(),
+    custom_steps.test_static_scala_cpu(),
+    custom_steps.test_static_python_cpu(),
+    custom_steps.test_static_python_cpu_cmake(),
   ])
 }
 ,

diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu
@@ -35,11 +35,13 @@ utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
     custom_steps.compile_centos7_gpu()
-  ]) 
-  
+  ])
+
   utils.parallel_stage('Tests', [
     custom_steps.test_centos7_python3_gpu()
-  ]) 
+    custom_steps.test_static_python_gpu(),
+    custom_steps.test_static_python_gpu_cmake(),
+  ])
 }
 ,
 failure_handler: {

diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -62,9 +62,6 @@ core_logic: {
     custom_steps.test_unix_julia10_cpu(),
     custom_steps.test_unix_onnx_cpu(),
     custom_steps.test_unix_cpp_cpu(),
-    custom_steps.test_static_scala_cpu(),
-    custom_steps.test_static_python_cpu(),
-    custom_steps.test_static_python_cpu_cmake(),
     /*  Disabled due to master build failure:
      *  http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
      *  https://github.com/apache/incubator-mxnet/issues/11801

diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu
@@ -60,8 +60,6 @@ core_logic: {
     custom_steps.test_unix_cpp_package_gpu(),
     custom_steps.test_unix_scala_gpu(),
     custom_steps.test_unix_distributed_kvstore_gpu(),
-    custom_steps.test_static_python_gpu(),
-    custom_steps.test_static_python_gpu_cmake(),
     custom_steps.test_unix_python3_gpu_no_tvm_op(),
     custom_steps.test_unix_capi_cpp_package(),
 

diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml
@@ -55,7 +55,7 @@
       <includes>
         <include>libmxnet.so</include>
         <include>libtvm_runtime.so</include>
-        <include>libgfortran.so.3</include>
+        <include>libgfortran.so.4</include>
         <include>libquadmath.so.0</include>
       </includes>
       <outputDirectory>lib/native</outputDirectory>

diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala
@@ -87,7 +87,7 @@ private[mxnet] object NativeLibraryLoader {
     val libFileInJar = libPathInJar + loadLibname
     saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true)
     saveLibraryToTemp("libtvm_runtime.so", "/lib/native/libtvm_runtime.so", false)
-    saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false)
+    saveLibraryToTemp("libgfortran.so.4", "/lib/native/libgfortran.so.4", false)
     saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false)
     val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true)