[MXNET-862] Basic maven jenkins pipeline (apache#13450)

* Jenkins Publish Nightly Maven

Progress

* Seperate Build, Test, and Deploy Stages with parallel

ci/docker/install/ubuntu_scala.sh

@@ -36,6 +36,9 @@ apt-get install -y \
     openjdk-8-jdk \
     openjdk-8-jre \
     software-properties-common \
+    gnupg \
+    gnupg2 \
+    gnupg-agent \
     scala
 
 # Ubuntu 14.04

scala-package/dev/build.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+# Setup Environment Variables
+# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
+# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
+
+bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
+
+# Scala steps to deploy
+make scalapkg CI=1
+
+# Compile tests for discovery later
+export GPG_TTY=$(tty)
+make scalatestcompile CI=1
+# make scalainstall CI=1
+make scaladeploylocal CI=1

scala-package/dev/buildkey.py

@@ -76,22 +76,36 @@ def importASC(key, gpgPassphrase):
     filename = os.path.join(KEY_PATH, "key.asc")
     with open(filename, 'w') as f:
         f.write(key)
-    subprocess.run(['gpg2', '--batch', '--yes',
+    subprocess.check_output(['gpg2', '--batch', '--yes',
                     '--passphrase-fd', '0',
                     "--import", "{}".format(filename)],
                    input=str.encode(gpgPassphrase))
 
 
 def encryptMasterPSW(password):
-    result = subprocess.run(['mvn', '--encrypt-master-password'],
-                            stdout=subprocess.PIPE, input=str.encode(password))
-    return str(result.stdout)[2:-3]
+    filename = os.path.join(KEY_PATH, "encryptMasterPassword.exp")
+    with open(filename, 'w') as f:
+        f.write('''
+        spawn mvn --encrypt-master-password
+        expect -exact "Master password: "
+        send -- "{}\r"
+        expect eof
+        '''.format(password))
+    result = subprocess.check_output(['expect', filename])
+    return str(result).split('\r\n')[-1][2:-3]
 
 
 def encryptPSW(password):
-    result = subprocess.run(['mvn', '--encrypt-password'],
-                            stdout=subprocess.PIPE, input=str.encode(password))
-    return str(result.stdout)[2:-3]
+    filename = os.path.join(KEY_PATH, "encryptPassword.exp")
+    with open(filename, 'w') as f:
+        f.write('''
+        spawn mvn --encrypt-password
+        expect -exact "Password: "
+        send -- "{}\r"
+        expect eof
+        '''.format(password))
+    result = subprocess.check_output(['expect', filename])
+    return str(result).split('\r\n')[-1][2:-3]
 
 
 def masterPSW(password):

scala-package/dev/deploy.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+# Setup Environment Variables
+# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
+# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
+
+# Run python to configure keys
+python3 $PWD/scala-package/dev/buildkey.py
+
+# Updating cache
+mkdir -p ~/.gnupg
+echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf
+echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf
+echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
+echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf
+export GPG_TTY=$(tty)
+
+cd scala-package
+VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
+cd ..
+
+# echo "\n\n$VERSION\n" | make scalarelease-dryrun
+make scaladeploy CI=1
+
+# Clear all password .xml files, exp files, and gpg key files
+rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp

scala-package/dev/test.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+# Test
+cd scala-package/packageTest
+make scalaintegrationtestlocal CI=1