Jenkins Publish Nightly Maven

Progress

ci/docker/runtime_functions.sh

@@ -308,7 +308,6 @@ build_amzn_linux_cpu() {
     ninja -v
 }
 
-
 build_centos7_mkldnn() {
     set -ex
     cd /work/mxnet
@@ -1059,7 +1058,6 @@ build_docs() {
     popd
 }
 
-
 # Functions that run the nightly Tests:
 
 #Runs Apache RAT Check on MXNet Source for License Headers
@@ -1248,6 +1246,14 @@ deploy_jl_docs() {
     # ...
 }
 
+deploy_nightly_maven() {
+    set -ex
+    pushd .
+    cd /work/mxnet
+    ./scala-package/dev/build.sh
+    popd
+}
+
 # broken_link_checker
 
 broken_link_checker() {

ci/publish/Jenkinsfile

@@ -0,0 +1,77 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 120
+
+node('restricted-mxnetlinux-cpu') {
+  // Loading the utilities requires a node context unfortunately
+  checkout scm
+  utils = load('ci/Jenkinsfile_utils.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+  stage('Deploy Nightly Maven') {
+    environment {
+      MVN_DEPLOY_USER = credentials('mvn-deploy-user')
+      MVN_DEPLOY_PASSWORD = credentials('mvn-deploy-password')
+      MVN_DEPLOY_PASSPHRASE = credentials('mvn-deploy-passphrase')
+      MVN_DEPLOY_MASTERPASS = credentials('mvn-deploy-masterpass')
+      MVN_DEPLOY_GPG_KEY = credentials('mvn-deploy-gpg-key')
+    }
+    parallel 'linux-cpu': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/linux-cpu') {
+          environment {
+            MVN_DEPLOY_OS_TYPE = linux-x86_64-cpu
+          }
+          utils.init_git()
+          timeout(time: max_time, unit: 'MINUTES') {
+              sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven"
+          }
+        }
+      }
+    },
+    'linux-gpu': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/linux-gpu') {
+          environment {
+            MVN_DEPLOY_OS_TYPE = linux-x86_64-gpu
+          }
+          utils.init_git()
+          timeout(time: max_time, unit: 'MINUTES') {
+              sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh deploy_nightly_maven"
+          }
+        }
+      }
+    }
+  }
+}
+,
+failure_handler: {
+  if (currentBuild.result == "FAILURE") {
+    emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+  }
+}
+)

scala-package/dev/build.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Install Dependencies
+sudo bash ci/docker/install/ubuntu_core.sh
+sudo bash ci/docker/install/ubuntu_scala.sh
+
+# Setup Environment Variables
+# MVN_DEPLOY_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
+# export MVN_DEPLOY_OS_TYPE=linux-x86_64-cpu
+
+# This script is used to build the base dependencies of MXNet Scala Env
+# git clone --recursive https://github.com/apache/incubator-mxnet
+# cd incubator-mxnet/
+# git checkout origin/master -b maven
+sudo apt-get install -y libssl-dev
+
+# This part is used to build the gpg dependencies:
+sudo apt-get install -y maven gnupg gnupg2 gnupg-agent
+# Mitigation for Ubuntu versions before 18.04
+if ! [gpg --version | grep -q "gpg (GnuPG) 2" ]; then
+    sudo mv /usr/bin/gpg /usr/bin/gpg1
+    sudo ln -s /usr/bin/gpg2 /usr/bin/gpg
+fi
+
+# Run python to configure keys
+python3 $PWD/scala-package/dev/buildkey.py
+
+# Updating cache
+mkdir -p ~/.gnupg
+echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf
+echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf
+export GPG_TTY=$(tty)
+
+# Build the Scala MXNet backend
+bash scala-package/dev/compile-mxnet-backend.sh $MVN_DEPLOY_OS_TYPE ./
+
+# Scala steps to deploy
+make scalapkg
+make scalaunittest
+make scalaintegrationtest
+echo "\n\n\n" | make scalarelease-dryrun
+# make scaladeploy

scala-package/dev/buildkey.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import json
+import logging
+import subprocess
+
+HOME = os.environ['HOME']
+KEY_PATH = os.path.join(HOME, ".m2")
+
+
+'''
+This file would do the following items:
+    Import keys from AWS Credential services
+    Create settings.xml in .m2 with pass phrase
+    Create security-settings.xml in .m2 with master password
+    Import keys.asc the encrypted keys in gpg
+'''
+
+
+def getCredentials():
+    import boto3
+    import botocore
+    secret_name = os.environ['DOCKERHUB_SECRET_NAME']
+    endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL']
+    region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']
+
+    session = boto3.Session()
+    client = session.client(
+        service_name='secretsmanager',
+        region_name=region_name,
+        endpoint_url=endpoint_url
+    )
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name
+        )
+    except botocore.exceptions.ClientError as client_error:
+        if client_error.response['Error']['Code'] == 'ResourceNotFoundException':
+            logging.exception("The requested secret %s was not found", secret_name)
+        elif client_error.response['Error']['Code'] == 'InvalidRequestException':
+            logging.exception("The request was invalid due to:")
+        elif client_error.response['Error']['Code'] == 'InvalidParameterException':
+            logging.exception("The request had invalid params:")
+        else:
+            raise
+    else:
+        secret = get_secret_value_response['SecretString']
+        secret_dict = json.loads(secret)
+        return secret_dict
+
+
+def importASC(path, passPhrase):
+    subprocess.run(['gpg', '--batch', '--yes',
+                    '--passphrase=\"{}\"'.format(passPhrase),
+                    "--import", "{}".format(os.environ['MVN_DEPLOY_GPG_KEY'])])
+
+
+def encryptMasterPSW(password):
+    result = subprocess.run(['mvn', '--encrypt-master-password', password],
+                            stdout=subprocess.PIPE)
+    return str(result.stdout)[2:-3]
+
+
+def encryptPSW(password):
+    result = subprocess.run(['mvn', '--encrypt-password', password],
+                            stdout=subprocess.PIPE)
+    return str(result.stdout)[2:-3]
+
+
+def masterPSW(password):
+    with open(os.path.join(KEY_PATH, "settings-security.xml"), "w") as f:
+        f.write("<settingsSecurity>\n <master>{}</master>\n</settingsSecurity>"
+                .format(password))
+
+
+def severPSW(username, password, passPhrase):
+    with open(os.path.join(KEY_PATH, "settings.xml"), "w") as f:
+        settingsString = '''<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+<pluginGroups></pluginGroups>
+<proxies></proxies>
+<servers>
+<server>
+        <id>apache.snapshots.https</id>
+        <username>{}</username>
+        <password>{}</password>
+</server>
+<!-- To stage a release of some part of Maven -->
+<server>
+        <id>apache.releases.https</id>
+        <username>{}</username>
+        <password>{}</password>
+</server>
+</servers>
+<mirrors></mirrors>
+<profiles>
+<profile>
+        <id>gpg</id>
+        <properties>
+        <gpg.executable>gpg</gpg.executable>
+        <gpg.passphrase>{}</gpg.passphrase>
+        </properties>
+</profile>
+</profiles>
+<activeProfiles>
+        <activeProfile>gpg</activeProfile>
+</activeProfiles>
+</settings> '''.format(username, password, username, password, passPhrase)
+        f.write(settingsString)
+
+
+if __name__ == "__main__":
+    if not os.path.exists(KEY_PATH):
+        os.makedirs(KEY_PATH)
+    userCredential = {
+        "username": os.environ['MVN_DEPLOY_USER'],
+        "password": os.environ['MVN_DEPLOY_PASSWORD']
+    }
+    keyCredential = {
+        "passPhrase": os.environ['MVN_DEPLOY_GPG_PASSPHRASE'],
+        "masterPass": os.environ['MVN_DEPLOY_MASTERPASS']
+    }
+    masterPass = encryptMasterPSW(keyCredential["masterPass"])
+    masterPSW(masterPass)
+    passwordEncrypted = encryptPSW(userCredential["password"])
+    severPSW(userCredential["username"], passwordEncrypted,
+             keyCredential["passPhrase"])
+    importASC(HOME, keyCredential["passPhrase"])