feature: Amazon SageMaker Model Monitoring

tl;dr: Added Amazon SageMaker Model Monitoring feature support for sagemaker-python-sdk.

Specifically:
* Added ModelMonitor and DefaultModelMonitor objects capable of running BaseliningJobs and create/updating schedules. These classes have many convenience functions to suggest baseline statistics and constraints, view monitor-generated statistics and violations, and many more.
* Customers are able to pull data generated by these jobs and analyze/modify them.
* Customers can attach Monitor objects to existing schedules in order to update them or review the data they generated.
* Customers can enable Data Capture on their existing or new endpoints to allow for model monitoring. This can be done by utilizing a DataCaptureConfig object, or by called .enable_data_capture() on a predictor object.
* Customers can specify a DatasetFormat for their monitoring schedules utilizing a DatasetFormat object.
* Added a CronExpressionGenerator to generate cron expressions supported by the feature.
* Added an output capturer as part of the unit tests, in order to test print statements to the customer.
* Added a convenience method to upload a string to S3: S3Uploader.upload_string_as_file_body(my_string).
* Added a convenience method to read an S3 file into a python string object: S3Downloader.read_file(my_uri).
* Added method to create new endpoint_config from existing endpoint_config to streamline the process of updating endpoints.
* Added schedule cleanup to endpoint cleanup to allow for cleaning up endpoints that have schedules attached.
* Imports greatly simplified for customers to allow them to import all the modules they need from a single namespace: "from sagemaker.model_monitor import *". Note that I do not condone the use of import * in a code-base that is meant to be maintained.

Author: knakad

buildspec.yml

@@ -19,7 +19,7 @@ phases:
       - start_time=`date +%s`
       - |
         if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{"region_name": "us-east-1"}'
+          tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{"region_name": "us-east-2"}'
         fi
       - ./ci-scripts/displaytime.sh 'py36 tests/integ' $start_time
 

doc/conf.py

@@ -41,7 +41,6 @@ def __getattr__(cls, name):
     "tensorflow.python.framework",
     "tensorflow_serving",
     "tensorflow_serving.apis",
-    "numpy",
     "scipy",
     "scipy.sparse",
 ]

src/sagemaker/estimator.py

@@ -557,6 +557,7 @@ def deploy(
         wait=True,
         model_name=None,
         kms_key=None,
+        data_capture_config=None,
         **kwargs
     ):
         """Deploy the trained model to an Amazon SageMaker endpoint and return a
@@ -599,6 +600,9 @@ def deploy(
             kms_key (str): The ARN of the KMS key that is used to encrypt the
                 data on the storage volume attached to the instance hosting the
                 endpoint.
+            data_capture_config (DataCaptureConfig): Specifies configuration
+                related to Endpoint data capture for use with
+                Amazon SageMaker Model Monitoring. Default: None.
             **kwargs: Passed to invocation of ``create_model()``.
                 Implementations may customize ``create_model()`` to accept
                 ``**kwargs`` to customize model creation during deploy.
@@ -624,7 +628,9 @@ def deploy(
         else:
             kwargs["model_kms_key"] = self.output_kms_key
             model = self.create_model(**kwargs)
+
         model.name = model_name
+
         return model.deploy(
             instance_type=instance_type,
             initial_instance_count=initial_instance_count,
@@ -634,6 +640,7 @@ def deploy(
             tags=self.tags,
             wait=wait,
             kms_key=kms_key,
+            data_capture_config=data_capture_config,
         )
 
     @property

src/sagemaker/model.py

@@ -384,6 +384,7 @@ def deploy(
         tags=None,
         kms_key=None,
         wait=True,
+        data_capture_config=None,
     ):
         """Deploy this ``Model`` to an ``Endpoint`` and optionally return a
         ``Predictor``.
@@ -424,6 +425,9 @@ def deploy(
                 endpoint.
             wait (bool): Whether the call should wait until the deployment of
                 this model completes (default: True).
+            data_capture_config (DataCaptureConfig): Specifies configuration
+                related to Endpoint data capture for use with
+                Amazon SageMaker Model Monitoring. Default: None.
 
         Returns:
             callable[string, sagemaker.session.Session] or None: Invocation of
@@ -454,6 +458,10 @@ def deploy(
             if self._is_compiled_model and not self.endpoint_name.endswith(compiled_model_suffix):
                 self.endpoint_name += compiled_model_suffix
 
+        data_capture_config_dict = (
+            data_capture_config.to_request_dict() if data_capture_config else None
+        )
+
         if update_endpoint:
             endpoint_config_name = self.sagemaker_session.create_endpoint_config(
                 name=self.name,
@@ -463,11 +471,17 @@ def deploy(
                 accelerator_type=accelerator_type,
                 tags=tags,
                 kms_key=kms_key,
+                data_capture_config_dict=data_capture_config_dict,
             )
             self.sagemaker_session.update_endpoint(self.endpoint_name, endpoint_config_name)
         else:
             self.sagemaker_session.endpoint_from_production_variants(
-                self.endpoint_name, [production_variant], tags, kms_key, wait
+                name=self.endpoint_name,
+                production_variants=[production_variant],
+                tags=tags,
+                kms_key=kms_key,
+                wait=wait,
+                data_capture_config_dict=data_capture_config_dict,
             )
 
         if self.predictor_cls:

src/sagemaker/model_monitor/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Imports the classes in this module to simplify customer imports
+
+Example:
+    >>> from sagemaker.model_monitor import ModelMonitor
+
+"""
+from __future__ import absolute_import
+
+from sagemaker.model_monitor.model_monitoring import ModelMonitor  # noqa: F401
+from sagemaker.model_monitor.model_monitoring import DefaultModelMonitor  # noqa: F401
+from sagemaker.model_monitor.model_monitoring import MonitoringOutput  # noqa: F401
+
+from sagemaker.model_monitor.cron_expression_generator import CronExpressionGenerator  # noqa: F401
+from sagemaker.model_monitor.monitoring_files import Statistics  # noqa: F401
+from sagemaker.model_monitor.monitoring_files import Constraints  # noqa: F401
+from sagemaker.model_monitor.monitoring_files import ConstraintViolations  # noqa: F401
+
+from sagemaker.model_monitor.data_capture_config import DataCaptureConfig  # noqa: F401
+
+from sagemaker.network import NetworkConfig  # noqa: F401

src/sagemaker/model_monitor/cron_expression_generator.py

@@ -0,0 +1,75 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains code related to the CronExpressionGenerator class, which is used
+for generating cron expressions compatible with Amazon SageMaker Model Monitoring Schedules.
+"""
+from __future__ import print_function, absolute_import
+
+
+class CronExpressionGenerator(object):
+    """Generates cron expression strings for use with the Amazon SageMaker Model Monitoring Schedule
+    API.
+    """
+
+    @staticmethod
+    def hourly():
+        """Generates hourly cron expression that denotes that a job runs at the top of every hour.
+
+        Returns:
+            str: The cron expression format accepted by the Amazon SageMaker Model Monitoring
+                Schedule API.
+
+        """
+        return "cron(0 * ? * * *)"
+
+    @staticmethod
+    def daily(hour=0):
+        """Generates daily cron expression that denotes that a job runs at the top of every hour.
+
+        Args:
+            hour (int): The hour in HH24 format (UTC) to run the job at, on a daily schedule.
+                Examples:
+                    - 00
+                    - 12
+                    - 17
+                    - 23
+
+        Returns:
+            str: The cron expression format accepted by the Amazon SageMaker Model Monitoring
+                Schedule API.
+
+        """
+        return "cron(0 {} ? * * *)".format(hour)
+
+    @staticmethod
+    def daily_every_x_hours(hour_interval, starting_hour=0):
+        """Generates "daily every x hours" cron expression that denotes that a job runs every day
+        at the specified hour, and then every x hours, as specified in hour_interval.
+
+         Example:
+             >>> daily_every_x_hours(hour_interval=2, starting_hour=0)
+             This will run every 2 hours starting at midnight.
+
+             >>> daily_every_x_hours(hour_interval=10, starting_hour=0)
+             This will run at midnight, 10am, and 8pm every day.
+
+        Args:
+            hour_interval (int): The hour interval to run the job at.
+            starting_hour (int): The hour at which to begin in HH24 format (UTC).
+
+        Returns:
+            str: The cron expression format accepted by the Amazon SageMaker Model Monitoring
+                Schedule API.
+
+        """
+        return "cron(0 {}/{} ? * * *)".format(starting_hour, hour_interval)

src/sagemaker/model_monitor/data_capture_config.py

@@ -0,0 +1,98 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains code related to the DataCaptureConfig class, which is used
+for configuring capture, collection, and storage, for prediction requests and responses
+for models hosted on SageMaker Endpoints.
+"""
+from __future__ import print_function, absolute_import
+
+import os
+
+from sagemaker.session import Session
+
+_MODEL_MONITOR_S3_PATH = "model-monitor"
+_DATA_CAPTURE_S3_PATH = "data-capture"
+
+
+class DataCaptureConfig(object):
+    """Configuration object passed in when deploying models to Amazon SageMaker Endpoints.
+    This object specifies configuration related to endpoint data capture for use with
+    Amazon SageMaker Model Monitoring.
+    """
+
+    API_MAPPING = {"REQUEST": "Input", "RESPONSE": "Output"}
+
+    def __init__(
+        self,
+        enable_capture,
+        sampling_percentage=20,
+        destination_s3_uri=None,
+        kms_key_id=None,
+        capture_options=None,
+        csv_content_types=None,
+        json_content_types=None,
+    ):
+        """Initialize a DataCaptureConfig object for capturing data from Amazon SageMaker Endpoints.
+
+        Args:
+            enable_capture (bool): Required. Whether data capture should be enabled or not.
+            sampling_percentage (int): Optional. Default=20. The percentage of data to sample.
+                Must be between 0 and 100.
+            destination_s3_uri (str): Optional. Defaults to "s3://<default-session-bucket>/
+                <model-monitor>/data-capture
+            kms_key_id (str): Optional. Default=None. The kms key to use when writing to S3.
+            capture_options ([str]): Optional. Must be a list containing any combination of the
+                following values: "REQUEST", "RESPONSE". Default=["REQUEST", "RESPONSE"]. Denotes
+                which data to capture between request and response.
+            csv_content_types ([str]): Optional. Default=["text/csv"].
+            json_content_types([str]): Optional. Default=["application/json"].
+
+        """
+        self.enable_capture = enable_capture
+        self.sampling_percentage = sampling_percentage
+        self.destination_s3_uri = destination_s3_uri
+        if self.destination_s3_uri is None:
+            self.destination_s3_uri = os.path.join(
+                "s3://", Session().default_bucket(), _MODEL_MONITOR_S3_PATH, _DATA_CAPTURE_S3_PATH
+            )
+
+        self.kms_key_id = kms_key_id
+        self.capture_options = capture_options or ["REQUEST", "RESPONSE"]
+        self.csv_content_types = csv_content_types or ["text/csv"]
+        self.json_content_types = json_content_types or ["application/json"]
+
+    def to_request_dict(self):
+        """Generates a request dictionary using the parameters provided to the class."""
+        request_dict = {
+            "EnableCapture": self.enable_capture,
+            "InitialSamplingPercentage": self.sampling_percentage,
+            "DestinationS3Uri": self.destination_s3_uri,
+            "CaptureOptions": [
+                {"CaptureMode": dict(self.API_MAPPING).get(capture_option.upper(), capture_option)}
+                for capture_option in list(self.capture_options)
+            ],
+        }
+
+        if self.kms_key_id is not None:
+            request_dict["KmsKeyId"] = self.kms_key_id
+
+        if self.csv_content_types is not None or self.json_content_types is not None:
+            request_dict["CaptureContentTypeHeader"] = {}
+
+        if self.csv_content_types is not None:
+            request_dict["CaptureContentTypeHeader"]["CsvContentTypes"] = self.csv_content_types
+
+        if self.json_content_types is not None:
+            request_dict["CaptureContentTypeHeader"]["JsonContentTypes"] = self.json_content_types
+
+        return request_dict

src/sagemaker/model_monitor/dataset_format.py

@@ -0,0 +1,61 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains code related to the DatasetFormat class, which is used
+for managing the constraints JSON file generated and consumed by Amazon SageMaker Model Monitoring
+Schedules.
+"""
+from __future__ import print_function, absolute_import
+
+
+class DatasetFormat(object):
+    """Represents a Dataset Format that is used when calling a DefaultModelMonitor.
+    """
+
+    @staticmethod
+    def csv(header=True, output_columns_position="START"):
+        """Returns a DatasetFormat JSON string for use with a DefaultModelMonitor.
+
+        Args:
+            header (bool): Whether the csv dataset to baseline and monitor has a header.
+                Default: True.
+            output_columns_position (str): The position of the output columns.
+                Must be one of ("START", "END"). Default: "START".
+
+        Returns:
+            dict: JSON string containing DatasetFormat to be used by DefaultModelMonitor.
+
+        """
+        return {"csv": {"header": header, "output_columns_position": output_columns_position}}
+
+    @staticmethod
+    def json(lines=True):
+        """Returns a DatasetFormat JSON string for use with a DefaultModelMonitor.
+
+        Args:
+            lines (bool): Read the file as a json object per line. Default: True.
+
+        Returns:
+            dict: JSON string containing DatasetFormat to be used by DefaultModelMonitor.
+
+        """
+        return {"json": {"lines": lines}}
+
+    @staticmethod
+    def sagemaker_capture_json():
+        """Returns a DatasetFormat SageMaker Capture Json string for use with a DefaultModelMonitor.
+
+        Returns:
+            dict: JSON string containing DatasetFormat to be used by DefaultModelMonitor.
+
+        """
+        return {"sagemaker_capture_json": {}}