Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

CI: unify centos7_base.sh and centos7_core.sh #18093

Merged
merged 3 commits into from
Apr 20, 2020
Merged

CI: unify centos7_base.sh and centos7_core.sh #18093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c234074
CI: unify centos7_base.sh and centos7_core.sh
leezu Apr 17, 2020
33cd408
Simplify publish.test.centos7 images
leezu Apr 18, 2020
e4f90cc
syntax fix
leezu Apr 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ci/docker/Dockerfile.publish.centos7_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ FROM centos:7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_ccache.sh /work/
RUN /work/centos7_ccache.sh
COPY install/centos7_python.sh /work/
Expand Down
4 changes: 2 additions & 2 deletions ci/docker/Dockerfile.publish.centos7_gpu_cu100
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ FROM nvidia/cuda:10.0-cudnn7-devel-centos7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_ccache.sh /work/
RUN /work/centos7_ccache.sh
COPY install/centos7_python.sh /work/
Expand Down
4 changes: 2 additions & 2 deletions ci/docker/Dockerfile.publish.centos7_gpu_cu101
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ FROM nvidia/cuda:10.1-cudnn7-devel-centos7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_ccache.sh /work/
RUN /work/centos7_ccache.sh
COPY install/centos7_python.sh /work/
Expand Down
4 changes: 2 additions & 2 deletions ci/docker/Dockerfile.publish.centos7_gpu_cu102
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ FROM nvidia/cuda:10.2-cudnn7-devel-centos7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_ccache.sh /work/
RUN /work/centos7_ccache.sh
COPY install/centos7_python.sh /work/
Expand Down
4 changes: 2 additions & 2 deletions ci/docker/Dockerfile.publish.centos7_gpu_cu92
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_ccache.sh /work/
RUN /work/centos7_ccache.sh
COPY install/centos7_python.sh /work/
Expand Down
12 changes: 9 additions & 3 deletions ci/docker/Dockerfile.publish.test.centos7_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,18 @@ FROM centos:7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh

COPY install/centos7_scala.sh /work/
RUN /work/centos7_scala.sh

# Install runtime dependencies for publish tests
# - make is used to run tests ci/publish/scala/test.sh
# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite
# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship
# inside jar)
RUN yum -y check-update || true && \
yum install -y make gcc unzip && \
yum clean all

ARG USER_ID=0
COPY install/centos7_adduser.sh /work/
RUN /work/centos7_adduser.sh
Expand Down
12 changes: 9 additions & 3 deletions ci/docker/Dockerfile.publish.test.centos7_gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,18 @@ FROM nvidia/cuda:9.2-cudnn7-devel-centos7

WORKDIR /work/deps

COPY install/centos7_base.sh /work/
RUN /work/centos7_base.sh

COPY install/centos7_scala.sh /work/
RUN /work/centos7_scala.sh

# Install runtime dependencies for publish tests
# - make is used to run tests ci/publish/scala/test.sh
# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite
# - gcc to provide libgomp.so.1 (may want to drop this in the future and ship
# inside jar)
RUN yum -y check-update || true && \
yum install -y make gcc unzip && \
yum clean all

ARG USER_ID=0
COPY install/centos7_adduser.sh /work/
RUN /work/centos7_adduser.sh
Expand Down
55 changes: 0 additions & 55 deletions ci/docker/install/centos7_base.sh
View file
Open in desktop

This file was deleted.

2 changes: 2 additions & 0 deletions ci/docker/install/centos7_core.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ yum -y install protobuf-devel
yum -y install zeromq-devel
yum -y install openssl-devel
yum -y install gcc-c++-4.8.*
yum -y install gcc-gfortran
yum -y install make
yum -y install wget
yum -y install unzip
Expand All @@ -43,6 +44,7 @@ yum -y install automake
yum -y install autoconf
yum -y install libtool
yum -y install patchelf
yum -y install pandoc

# gcc7
yum -y install centos-release-scl
Expand Down
2 changes: 1 addition & 1 deletion ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -734,7 +734,7 @@ def compile_windows_gpu_mkldnn() {
}

def test_static_scala_cpu() {
return ['Static build CPU 14.04 Scala' : {
return ['Static build CPU CentOS7 Scala' : {
node(NODE_LINUX_CPU) {
ws('workspace/ut-publish-scala-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
Expand Down
5 changes: 4 additions & 1 deletion ci/jenkins/Jenkinsfile_centos_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ core_logic: {

utils.parallel_stage('Tests', [
custom_steps.test_centos7_python3_cpu(),
custom_steps.test_centos7_scala_cpu()
custom_steps.test_centos7_scala_cpu(),
custom_steps.test_static_scala_cpu(),
custom_steps.test_static_python_cpu(),
custom_steps.test_static_python_cpu_cmake(),
])
}
,
Expand Down
10 changes: 6 additions & 4 deletions ci/jenkins/Jenkinsfile_centos_gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,13 @@ utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_centos7_gpu()
])
])

utils.parallel_stage('Tests', [
custom_steps.test_centos7_python3_gpu()
])
custom_steps.test_centos7_python3_gpu(),
custom_steps.test_static_python_gpu(),
custom_steps.test_static_python_gpu_cmake(),
])
}
,
failure_handler: {
Expand Down
3 changes: 0 additions & 3 deletions ci/jenkins/Jenkinsfile_unix_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,6 @@ core_logic: {
custom_steps.test_unix_julia10_cpu(),
custom_steps.test_unix_onnx_cpu(),
custom_steps.test_unix_cpp_cpu(),
custom_steps.test_static_scala_cpu(),
custom_steps.test_static_python_cpu(),
custom_steps.test_static_python_cpu_cmake(),
/* Disabled due to master build failure:
* http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
* https://github.com/apache/incubator-mxnet/issues/11801
Expand Down
2 changes: 0 additions & 2 deletions ci/jenkins/Jenkinsfile_unix_gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ core_logic: {
custom_steps.test_unix_cpp_package_gpu(),
custom_steps.test_unix_scala_gpu(),
custom_steps.test_unix_distributed_kvstore_gpu(),
custom_steps.test_static_python_gpu(),
custom_steps.test_static_python_gpu_cmake(),
custom_steps.test_unix_python3_gpu_no_tvm_op(),
custom_steps.test_unix_capi_cpp_package(),

Expand Down
2 changes: 1 addition & 1 deletion scala-package/assembly/src/main/assembly/assembly.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
<includes>
<include>libmxnet.so</include>
<include>libtvm_runtime.so</include>
<include>libgfortran.so.3</include>
<include>libgfortran.so.4</include>
<include>libquadmath.so.0</include>
</includes>
<outputDirectory>lib/native</outputDirectory>
Expand Down
2 changes: 1 addition & 1 deletion scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ private[mxnet] object NativeLibraryLoader {
val libFileInJar = libPathInJar + loadLibname
saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true)
saveLibraryToTemp("libtvm_runtime.so", "/lib/native/libtvm_runtime.so", false)
saveLibraryToTemp("libgfortran.so.3", "/lib/native/libgfortran.so.3", false)
saveLibraryToTemp("libgfortran.so.4", "/lib/native/libgfortran.so.4", false)
saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false)
val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true)

Expand Down