Merge branch 'master' into mvs-improve-chainer-tests

Author: mvsusp

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## v1.30.0 (2019-06-25)
+
+### Features
+
+ * add TensorFlow 1.13 support
+ * add git_config and git_clone, validate method
+
+### Bug fixes and other changes
+
+ * add pytest.mark.local_mode annotation to broken tests
+
+## v1.29.0 (2019-06-24)
+
+### Features
+
+ * network isolation mode in training
+
+### Bug fixes and other changes
+
+ * Integrate black into development process
+ * moving not canary TFS tests to local mode
+
 ## v1.28.3 (2019-06-20)
 
 ### Bug fixes and other changes

README.rst

@@ -173,7 +173,7 @@ TensorFlow SageMaker Estimators
 
 By using TensorFlow SageMaker Estimators, you can train and host TensorFlow models on Amazon SageMaker.
 
-Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``.
+Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``, ``1.13.1``.
 
 Supported versions of TensorFlow for Elastic Inference: ``1.11.0``, ``1.12.0``.
 

VERSION

@@ -1 +1 @@
-1.28.4.dev0
+1.30.1.dev0

buildspec-release.yml

@@ -9,6 +9,9 @@ phases:
       # run linters
       - tox -e flake8,pylint
 
+      # run format verification
+      - tox -e black-check
+
       # run package and docbuild checks
       - tox -e twine
       - tox -e sphinx

buildspec.yml

@@ -14,6 +14,9 @@ phases:
       - tox -e twine
       - tox -e sphinx
 
+      # run format verification
+      - tox -e black-check
+
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=

doc/conf.py

@@ -28,51 +28,65 @@ def __getattr__(cls, name):
             return MagicMock()
 
 
-MOCK_MODULES = ['tensorflow', 'tensorflow.core', 'tensorflow.core.framework', 'tensorflow.python',
-                'tensorflow.python.framework', 'tensorflow_serving', 'tensorflow_serving.apis',
-                'numpy', 'scipy', 'scipy.sparse']
+MOCK_MODULES = [
+    "tensorflow",
+    "tensorflow.core",
+    "tensorflow.core.framework",
+    "tensorflow.python",
+    "tensorflow.python.framework",
+    "tensorflow_serving",
+    "tensorflow_serving.apis",
+    "numpy",
+    "scipy",
+    "scipy.sparse",
+]
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
-project = u'sagemaker'
+project = u"sagemaker"
 version = pkg_resources.require(project)[0].version
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
-              'sphinx.ext.intersphinx', 'sphinx.ext.todo',
-              'sphinx.ext.coverage', 'sphinx.ext.autosummary',
-              'sphinx.ext.napoleon']
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
-source_suffix = '.rst'  # The suffix of source filenames.
-master_doc = 'index'  # The master toctree document.
+source_suffix = ".rst"  # The suffix of source filenames.
+master_doc = "index"  # The master toctree document.
 
-copyright = u'%s, Amazon' % datetime.now().year
+copyright = u"%s, Amazon" % datetime.now().year
 
 # The full version, including alpha/beta/rc tags.
 release = version
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_trees = ['_build']
+exclude_trees = ["_build"]
 
-pygments_style = 'default'
+pygments_style = "default"
 
 autoclass_content = "both"
-autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members']
-autodoc_member_order = 'bysource'
+autodoc_default_flags = ["show-inheritance", "members", "undoc-members"]
+autodoc_member_order = "bysource"
 
-if 'READTHEDOCS' in os.environ:
-    html_theme = 'default'
+if "READTHEDOCS" in os.environ:
+    html_theme = "default"
 else:
-    html_theme = 'haiku'
+    html_theme = "haiku"
 html_static_path = []
-htmlhelp_basename = '%sdoc' % project
+htmlhelp_basename = "%sdoc" % project
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {"http://docs.python.org/": None}
 
 # autosummary
 autosummary_generate = True

doc/overview.rst

@@ -84,6 +84,65 @@ For more `information <https://boto3.amazonaws.com/v1/documentation/api/latest/r
     # Deletes the SageMaker model
     mxnet_predictor.delete_model()
 
+Git Support
+~~~~~~~~~~~
+If you have your training scripts in your GitHub repository, you can use them directly without the trouble to download
+them to local machine. Git support can be enabled simply by providing ``git_config`` parameter when initializing an
+estimator. If Git support is enabled, then ``entry_point``, ``source_dir`` and  ``dependencies`` should all be relative
+paths in the Git repo. Note that if you decided to use Git support, then everything you need for ``entry_point``,
+``source_dir`` and ``dependencies`` should be in a single Git repo.
+
+Here are ways to specify ``git_config``:
+
+.. code:: python
+
+        # Specifies the git_config parameter
+        git_config = {'repo': 'https://github.com/username/repo-with-training-scripts.git',
+                      'branch': 'branch1',
+                      'commit': '4893e528afa4a790331e1b5286954f073b0f14a2'}
+
+        # Alternatively, you can also specify git_config by providing only 'repo' and 'branch'.
+        # If this is the case, the latest commit in the branch will be used.
+        git_config = {'repo': 'https://github.com/username/repo-with-training-scripts.git',
+                      'branch': 'branch1'}
+
+        # Only providing 'repo' is also allowed. If this is the case, latest commit in
+        # 'master' branch will be used.
+        git_config = {'repo': 'https://github.com/username/repo-with-training-scripts.git'
+
+The following are some examples to define estimators with Git support:
+
+.. code:: python
+
+        # In this example, the source directory 'pytorch' contains the entry point 'mnist.py' and other source code.
+        # and it is  relative path inside the Git repo.
+        pytorch_estimator = PyTorch(entry_point='mnist.py',
+                                    role='SageMakerRole',
+                                    source_dir='pytorch',
+                                    git_config=git_config,
+                                    train_instance_count=1,
+                                    train_instance_type='ml.c4.xlarge')
+
+        # In this example, the entry point 'mnist.py' is all we need for source code.
+        # We need to specify the path to it in the Git repo.
+        mx_estimator = MXNet(entry_point='mxnet/mnist.py',
+                             role='SageMakerRole',
+                             git_config=git_config,
+                             train_instance_count=1,
+                             train_instance_type='ml.c4.xlarge')
+
+        # In this example, besides entry point and other source code in source directory, we still need some
+        # dependencies for the training job. Dependencies should also be paths inside the Git repo.
+        pytorch_estimator = PyTorch(entry_point='mnist.py',
+                                    role='SageMakerRole',
+                                    source_dir='pytorch',
+                                    dependencies=['dep.py', 'foo/bar.py'],
+                                    git_config=git_config,
+                                    train_instance_count=1,
+                                    train_instance_type='ml.c4.xlarge')
+
+When Git support is enabled, users can still use local mode in the same way.
+
 Training Metrics
 ~~~~~~~~~~~~~~~~
 The SageMaker Python SDK allows you to specify a name and a regular expression for metrics you want to track for training.
@@ -268,6 +327,7 @@ Currently, the following algorithms support incremental training:
 - Object Detection
 - Semantic Segmentation
 
+
 Using SageMaker AlgorithmEstimators
 -----------------------------------
 

doc/using_tf.rst

@@ -443,20 +443,10 @@ After a TensorFlow estimator has been fit, it saves a TensorFlow SavedModel in
 the S3 location defined by ``output_path``. You can call ``deploy`` on a TensorFlow
 estimator to create a SageMaker Endpoint.
 
-SageMaker provides two different options for deploying TensorFlow models to a SageMaker
-Endpoint:
+Your model will be deployed to a TensorFlow Serving-based server. The server provides a super-set of the
+`TensorFlow Serving REST API <https://www.tensorflow.org/serving/api_rest>`_.
 
-- The first option uses a Python-based server that allows you to specify your own custom
-  input and output handling functions in a Python script. This is the default option.
-
-  See `Deploying to Python-based Endpoints <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/deploying_python.rst>`_ to learn how to use this option.
-
-
-- The second option uses a TensorFlow Serving-based server to provide a super-set of the
-  `TensorFlow Serving REST API <https://www.tensorflow.org/serving/api_rest>`_. This option
-  does not require (or allow) a custom python script.
-
-  See `Deploying to TensorFlow Serving Endpoints <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/deploying_tensorflow_serving.rst>`_ to learn how to use this option.
+See `Deploying to TensorFlow Serving Endpoints <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/tensorflow/deploying_tensorflow_serving.rst>`_ to learn how to deploy your model and make inference requests.
 
 
 SageMaker TensorFlow Docker containers

examples/cli/host/script.py

@@ -12,12 +12,12 @@ def model_fn(model_dir):
     :param: model_dir The directory where model files are stored.
     :return: a model (in this case a Gluon network)
     """
-    symbol = mx.sym.load('%s/model.json' % model_dir)
-    outputs = mx.symbol.softmax(data=symbol, name='softmax_label')
-    inputs = mx.sym.var('data')
-    param_dict = gluon.ParameterDict('model_')
+    symbol = mx.sym.load("%s/model.json" % model_dir)
+    outputs = mx.symbol.softmax(data=symbol, name="softmax_label")
+    inputs = mx.sym.var("data")
+    param_dict = gluon.ParameterDict("model_")
     net = gluon.SymbolBlock(outputs, inputs, param_dict)
-    net.load_params('%s/model.params' % model_dir, ctx=mx.cpu())
+    net.load_params("%s/model.params" % model_dir, ctx=mx.cpu())
     return net
 
 

examples/cli/train/download_training_data.py

@@ -2,8 +2,8 @@
 
 
 def download_training_data():
-    gluon.data.vision.MNIST('./data/training', train=True)
-    gluon.data.vision.MNIST('./data/training', train=False)
+    gluon.data.vision.MNIST("./data/training", train=True)
+    gluon.data.vision.MNIST("./data/training", train=False)
 
 
 if __name__ == "__main__":