From 92be7ec5915615e2a9bd486ccd9e834ac6250d64 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Sun, 19 Apr 2020 17:56:22 -0700 Subject: [PATCH] CI: Simplify CentOS7 CI/CD config (#18093) * CI: Unify centos7_base.sh and centos7_core.sh (delete centos7_base.sh) * Simplify centos7.publish.test Dockerfiles used for testing binary artifacts publish.test Dockerfiles should have as few system packages present as possible to catch potential dependency issues in release artifacts. --- ci/docker/Dockerfile.publish.centos7_cpu | 4 +- .../Dockerfile.publish.centos7_gpu_cu100 | 4 +- .../Dockerfile.publish.centos7_gpu_cu101 | 4 +- .../Dockerfile.publish.centos7_gpu_cu102 | 4 +- ci/docker/Dockerfile.publish.centos7_gpu_cu92 | 4 +- ci/docker/Dockerfile.publish.test.centos7_cpu | 12 +++- ci/docker/Dockerfile.publish.test.centos7_gpu | 12 +++- ci/docker/install/centos7_base.sh | 55 ------------------- ci/docker/install/centos7_core.sh | 2 + ci/jenkins/Jenkins_steps.groovy | 2 +- ci/jenkins/Jenkinsfile_centos_cpu | 5 +- ci/jenkins/Jenkinsfile_centos_gpu | 10 ++-- ci/jenkins/Jenkinsfile_unix_cpu | 3 - ci/jenkins/Jenkinsfile_unix_gpu | 2 - .../assembly/src/main/assembly/assembly.xml | 2 +- .../mxnet/util/NativeLibraryLoader.scala | 2 +- 16 files changed, 43 insertions(+), 84 deletions(-) delete mode 100755 ci/docker/install/centos7_base.sh diff --git a/ci/docker/Dockerfile.publish.centos7_cpu b/ci/docker/Dockerfile.publish.centos7_cpu index 2010238cb71d..3a87f8e45fff 100644 --- a/ci/docker/Dockerfile.publish.centos7_cpu +++ b/ci/docker/Dockerfile.publish.centos7_cpu @@ -22,8 +22,8 @@ FROM centos:7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh +COPY install/centos7_core.sh /work/ +RUN /work/centos7_core.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu100 b/ci/docker/Dockerfile.publish.centos7_gpu_cu100 index 41291f21ac72..ce5aa3c8d468 100644 --- a/ci/docker/Dockerfile.publish.centos7_gpu_cu100 +++ b/ci/docker/Dockerfile.publish.centos7_gpu_cu100 @@ -20,8 +20,8 @@ FROM nvidia/cuda:10.0-cudnn7-devel-centos7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh +COPY install/centos7_core.sh /work/ +RUN /work/centos7_core.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu101 b/ci/docker/Dockerfile.publish.centos7_gpu_cu101 index a1e90b9eafc3..2b207748db95 100644 --- a/ci/docker/Dockerfile.publish.centos7_gpu_cu101 +++ b/ci/docker/Dockerfile.publish.centos7_gpu_cu101 @@ -20,8 +20,8 @@ FROM nvidia/cuda:10.1-cudnn7-devel-centos7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh +COPY install/centos7_core.sh /work/ +RUN /work/centos7_core.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu102 b/ci/docker/Dockerfile.publish.centos7_gpu_cu102 index 851d63029a84..564ea01b887f 100644 --- a/ci/docker/Dockerfile.publish.centos7_gpu_cu102 +++ b/ci/docker/Dockerfile.publish.centos7_gpu_cu102 @@ -20,8 +20,8 @@ FROM nvidia/cuda:10.2-cudnn7-devel-centos7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh +COPY install/centos7_core.sh /work/ +RUN /work/centos7_core.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.publish.centos7_gpu_cu92 b/ci/docker/Dockerfile.publish.centos7_gpu_cu92 index 75277f0f1fd2..397954d63597 100644 --- a/ci/docker/Dockerfile.publish.centos7_gpu_cu92 +++ b/ci/docker/Dockerfile.publish.centos7_gpu_cu92 @@ -20,8 +20,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh +COPY install/centos7_core.sh /work/ +RUN /work/centos7_core.sh COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ diff --git a/ci/docker/Dockerfile.publish.test.centos7_cpu b/ci/docker/Dockerfile.publish.test.centos7_cpu index 7d284452971b..9f5f50f9c6bf 100644 --- a/ci/docker/Dockerfile.publish.test.centos7_cpu +++ b/ci/docker/Dockerfile.publish.test.centos7_cpu @@ -22,12 +22,18 @@ FROM centos:7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh - COPY install/centos7_scala.sh /work/ RUN /work/centos7_scala.sh +# Install runtime dependencies for publish tests +# - make is used to run tests ci/publish/scala/test.sh +# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite +# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship +# inside jar) +RUN yum -y check-update || true && \ + yum install -y make gcc unzip && \ + yum clean all + ARG USER_ID=0 COPY install/centos7_adduser.sh /work/ RUN /work/centos7_adduser.sh diff --git a/ci/docker/Dockerfile.publish.test.centos7_gpu b/ci/docker/Dockerfile.publish.test.centos7_gpu index e7f584683109..0faafcdaa223 100644 --- a/ci/docker/Dockerfile.publish.test.centos7_gpu +++ b/ci/docker/Dockerfile.publish.test.centos7_gpu @@ -22,12 +22,18 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7 WORKDIR /work/deps -COPY install/centos7_base.sh /work/ -RUN /work/centos7_base.sh - COPY install/centos7_scala.sh /work/ RUN /work/centos7_scala.sh +# Install runtime dependencies for publish tests +# - make is used to run tests ci/publish/scala/test.sh +# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite +# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship +# inside jar) +RUN yum -y check-update || true && \ + yum install -y make gcc unzip && \ + yum clean all + ARG USER_ID=0 COPY install/centos7_adduser.sh /work/ RUN /work/centos7_adduser.sh diff --git a/ci/docker/install/centos7_base.sh b/ci/docker/install/centos7_base.sh deleted file mode 100755 index 93e2801481c5..000000000000 --- a/ci/docker/install/centos7_base.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex - -# Multipackage installation does not fail in yum -yum -y install epel-release -yum -y install git -yum -y install wget -yum -y install make -yum -y install unzip -yum -y install ninja-build -yum -y install gcc-gfortran -yum -y install automake -yum -y install autoconf -yum -y install libtool -yum -y install protobuf-compiler -yum -y install protobuf-devel -yum -y install zeromq-devel -yum -y install patchelf -yum -y install pandoc - -# gcc7 -yum -y install centos-release-scl -yum -y install devtoolset-7 - -# Centos 7 only provides ninja-build -ln -s /usr/bin/ninja-build /usr/bin/ninja - -# CMake 3.13.2+ is required -mkdir /opt/cmake && cd /opt/cmake -wget -nv https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh -sh cmake-3.13.5-Linux-x86_64.sh --prefix=/opt/cmake --skip-license -ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -rm cmake-3.13.5-Linux-x86_64.sh -cmake --version diff --git a/ci/docker/install/centos7_core.sh b/ci/docker/install/centos7_core.sh index 7f1c3d70aebc..119d172ff70f 100755 --- a/ci/docker/install/centos7_core.sh +++ b/ci/docker/install/centos7_core.sh @@ -35,6 +35,7 @@ yum -y install protobuf-devel yum -y install zeromq-devel yum -y install openssl-devel yum -y install gcc-c++-4.8.* +yum -y install gcc-gfortran yum -y install make yum -y install wget yum -y install unzip @@ -43,6 +44,7 @@ yum -y install automake yum -y install autoconf yum -y install libtool yum -y install patchelf +yum -y install pandoc # gcc7 yum -y install centos-release-scl diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index eb4c0099579f..450036ce811b 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -734,7 +734,7 @@ def compile_windows_gpu_mkldnn() { } def test_static_scala_cpu() { - return ['Static build CPU 14.04 Scala' : { + return ['Static build CPU CentOS7 Scala' : { node(NODE_LINUX_CPU) { ws('workspace/ut-publish-scala-cpu') { timeout(time: max_time, unit: 'MINUTES') { diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu index 793d1f12e8d3..e05d32633194 100644 --- a/ci/jenkins/Jenkinsfile_centos_cpu +++ b/ci/jenkins/Jenkinsfile_centos_cpu @@ -41,7 +41,10 @@ core_logic: { utils.parallel_stage('Tests', [ custom_steps.test_centos7_python3_cpu(), - custom_steps.test_centos7_scala_cpu() + custom_steps.test_centos7_scala_cpu(), + custom_steps.test_static_scala_cpu(), + custom_steps.test_static_python_cpu(), + custom_steps.test_static_python_cpu_cmake(), ]) } , diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu index cad77a9a7dd8..31fad5ca8883 100644 --- a/ci/jenkins/Jenkinsfile_centos_gpu +++ b/ci/jenkins/Jenkinsfile_centos_gpu @@ -35,11 +35,13 @@ utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_centos7_gpu() - ]) - + ]) + utils.parallel_stage('Tests', [ - custom_steps.test_centos7_python3_gpu() - ]) + custom_steps.test_centos7_python3_gpu(), + custom_steps.test_static_python_gpu(), + custom_steps.test_static_python_gpu_cmake(), + ]) } , failure_handler: { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 5bfad60ef605..cccbdd5729a3 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -62,9 +62,6 @@ core_logic: { custom_steps.test_unix_julia10_cpu(), custom_steps.test_unix_onnx_cpu(), custom_steps.test_unix_cpp_cpu(), - custom_steps.test_static_scala_cpu(), - custom_steps.test_static_python_cpu(), - custom_steps.test_static_python_cpu_cmake(), /* Disabled due to master build failure: * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ * https://github.com/apache/incubator-mxnet/issues/11801 diff --git a/ci/jenkins/Jenkinsfile_unix_gpu b/ci/jenkins/Jenkinsfile_unix_gpu index 66d3c1391944..34ee5af1ce76 100644 --- a/ci/jenkins/Jenkinsfile_unix_gpu +++ b/ci/jenkins/Jenkinsfile_unix_gpu @@ -60,8 +60,6 @@ core_logic: { custom_steps.test_unix_cpp_package_gpu(), custom_steps.test_unix_scala_gpu(), custom_steps.test_unix_distributed_kvstore_gpu(), - custom_steps.test_static_python_gpu(), - custom_steps.test_static_python_gpu_cmake(), custom_steps.test_unix_python3_gpu_no_tvm_op(), custom_steps.test_unix_capi_cpp_package(), diff --git a/scala-package/assembly/src/main/assembly/assembly.xml b/scala-package/assembly/src/main/assembly/assembly.xml index 6b8480c53f8f..655c4fdb6ef2 100644 --- a/scala-package/assembly/src/main/assembly/assembly.xml +++ b/scala-package/assembly/src/main/assembly/assembly.xml @@ -55,7 +55,7 @@ libmxnet.so libtvm_runtime.so - libgfortran.so.3 + libgfortran.so.4 libquadmath.so.0 lib/native diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index 4b03fd2ef82f..a523e2d35d19 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -87,7 +87,7 @@ private[mxnet] object NativeLibraryLoader { val libFileInJar = libPathInJar + loadLibname saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true) saveLibraryToTemp("libtvm_runtime.so", "/lib/native/libtvm_runtime.so", false) - saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false) + saveLibraryToTemp("libgfortran.so.4", "/lib/native/libgfortran.so.4", false) saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false) val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true)