From 8bba81a125c4153e65d6bcb6a98542041d68977e Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Fri, 12 Jul 2019 13:21:49 -0700 Subject: [PATCH 01/28] Add calculate_feature_contrinutions method to Pipeline --- src/python/nimbusml.pyproj | 1 + .../PipelineWithFeatureContributions.py | 37 ++++++ src/python/nimbusml/pipeline.py | 118 ++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 src/python/nimbusml/examples/PipelineWithFeatureContributions.py diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 9c09758d..e564b6e9 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -172,6 +172,7 @@ + diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py new file mode 100644 index 00000000..97385bd5 --- /dev/null +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -0,0 +1,37 @@ +############################################################################### +# Pipeline with feature contributions +from nimbusml import Pipeline, FileDataStream +from nimbusml.datasets import get_dataset +from nimbusml.linear_model import LogisticRegressionBinaryClassifier + +# data input (as a FileDataStream) +path = get_dataset('uciadult_train').as_filepath() + +data = FileDataStream.read_csv(path) +print(data.head()) +# label workclass education ... capital-loss hours-per-week +# 0 0 Private 11th ... 0 40 +# 1 0 Private HS-grad ... 0 50 +# 2 1 Local-gov Assoc-acdm ... 0 40 +# 3 1 Private Some-college ... 0 40 +# 4 0 ? Some-college ... 0 30 +# define the training pipeline +pipeline = Pipeline([LogisticRegressionBinaryClassifier( + feature=['age', 'education-num', 'hours-per-week'], label='label')]) + +# train, predict, and evaluate +# TODO: Replace with CV +model = pipeline.fit(data) + +feature_contributions = model.calculate_feature_contributions( + data, output_scores=True) + +# Print predictions with feature contributions, which give a relative measure +# of how much each feature impacted the Score. +print(feature_contributions.head()) +# label ... PredictedLabel Score ... FeatureContributions.hours-per-week +# 0 ... 0 -2.010687 ... 0.833069 +# 1 ... 0 -1.216163 ... 0.809928 +# 2 ... 0 -1.248412 ... 0.485957 +# 3 ... 0 -1.132419 ... 0.583148 +# 4 ... 0 -1.969522 ... 0.437361 diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 7237ef7a..ebd28163 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -40,6 +40,8 @@ transforms_datasetscorer from .internal.entrypoints.transforms_featurecombiner import \ transforms_featurecombiner +from .internal.entrypoints.transforms_featurecontributioncalculationtransformer import \ + transforms_featurecontributioncalculationtransformer from .internal.entrypoints.transforms_labelcolumnkeybooleanconverter \ import \ transforms_labelcolumnkeybooleanconverter @@ -1693,6 +1695,122 @@ def getn(n): "only fit(X) is allowed or the training becomes " "ambiguous.") + @trace + def calculate_feature_contributions(self, X, y=None, + evaltype='auto', group_id=None, + weight=None, + verbose=0, + top=10, + bottom=10, + normalize=True, + as_binary_data_stream=False, **params): + """ + Apply transforms and test with the final estimator, return metrics + """ + # start the clock! + start_time = time.time() + self.verbose = verbose + + if not self._is_fitted: + raise ValueError( + "Model is not fitted. Train or load a model before test(" + ").") + + if y is not None: + if len(self.steps) > 0: + last_node = self.last_node + if last_node.type == 'transform': + raise ValueError( + "Pipeline needs a trainer as last step for test()") + + X, y_temp, columns_renamed, feature_columns, label_column, \ + schema, weights, weight_column = self._preprocess_X_y( + X, y, w=weight + ) + + if (not isinstance(y, (str, tuple))) or ( + isinstance(X, DataFrame) and isinstance(y, (str, tuple))): + y = y_temp + + all_nodes = [] + inputs = dict([('data', ''), ('predictor_model', self.model)]) + if isinstance(X, FileDataStream): + importtext_node = data_customtextloader( + input_file="$file", + data="$data", + custom_schema=schema.to_string( + add_sep=True)) + all_nodes = [importtext_node] + inputs = dict([('file', ''), ('predictor_model', self.model)]) + + score_node = transforms_datasetscorer( + data="$data", + predictor_model="$predictor_model", + scored_data="$scoredvectordata") + + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$fccData", + top=top, + bottom=bottom, + normalize=normalize) + all_nodes.extend([score_node, fcc_node]) + + if hasattr(self, 'steps') and len(self.steps) > 0 \ + and self.last_node.type == 'classifier': + convert_label_node = \ + transforms_predictedlabelcolumnoriginalvalueconverter( + data="$fccData", + predicted_label_column="PredictedLabel", + output_data="$output_data") + all_nodes.extend([convert_label_node]) + + if y is not None: + evaluate_nodes = self._evaluation_infer( + evaltype, label_column, group_id, **params) + for node in evaluate_nodes: + all_nodes.extend([node]) + output_scores = '' if params.get( + 'output_scores', False) else '' + outputs = OrderedDict( + [('output_metrics', ''), ('output_data', output_scores)]) + else: + outputs = dict(output_data="") + + graph = Graph( + inputs, + outputs, + as_binary_data_stream, + *all_nodes) + + class_name = type(self).__name__ + method_name = inspect.currentframe().f_code.co_name + telemetry_info = ".".join([class_name, method_name]) + + try: + (out_model, out_data, out_metrics) = graph.run( + X=X, + y=y, + random_state=self.random_state, + model=self.model, + verbose=verbose, + telemetry_info=telemetry_info, + **params) + except RuntimeError as e: + self._run_time = time.time() - start_time + raise e + + if y is not None: + # We need to fix the schema for ranking metrics + if evaltype == 'ranking': + out_metrics = self._fix_ranking_metrics_schema(out_metrics) + + # stop the clock + self._run_time = time.time() - start_time + self._write_csv_time = graph._write_csv_time + return out_data + @trace def _predict(self, X, y=None, evaltype='auto', group_id=None, From ff15cf5dbc43a6c6070b96871ce03c8d1ddc7a1a Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 12:42:56 -0700 Subject: [PATCH 02/28] typo --- src/python/nimbusml.pyproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 68780828..4f03e01b 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -177,7 +177,7 @@ - + From 663966902ac841ca86fb1326b99791dc7edf39ce Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 14:50:43 -0700 Subject: [PATCH 03/28] rename to get_feature_contributions(), add docs, improve sample --- src/python/nimbusml.pyproj | 2 +- .../PipelineWithFeatureContributions.py | 60 ++++++++++++++++--- src/python/nimbusml/pipeline.py | 31 ++++++++-- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 4f03e01b..f694be00 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -177,7 +177,7 @@ - + diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 97385bd5..0f7c080b 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -1,8 +1,17 @@ ############################################################################### # Pipeline with feature contributions + +# Scoring a dataset with a trained model produces a score, or prediction, for +# each example. To understand and explain these predictions it can be useful to +# inspect which features influenced them most significantly. This function +# computes a model-specific list of per-feature contributions to the score for +# each example. These contributions can be positive (they make the score +# higher) or negative (they make the score lower). + from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import LogisticRegressionBinaryClassifier +from nimbusml.linear_model import LogisticRegressionBinaryClassifier, PoissonRegressionRegressor +from nimbusml.ensemble import FastTreesBinaryClassifier # data input (as a FileDataStream) path = get_dataset('uciadult_train').as_filepath() @@ -15,23 +24,56 @@ # 2 1 Local-gov Assoc-acdm ... 0 40 # 3 1 Private Some-college ... 0 40 # 4 0 ? Some-college ... 0 30 -# define the training pipeline -pipeline = Pipeline([LogisticRegressionBinaryClassifier( + +# define the training pipeline with a linear model +lr_pipeline = Pipeline([PoissonRegressionRegressor( feature=['age', 'education-num', 'hours-per-week'], label='label')]) -# train, predict, and evaluate -# TODO: Replace with CV -model = pipeline.fit(data) +# train the model +lr_model = lr_pipeline.fit(data) -feature_contributions = model.calculate_feature_contributions( - data, output_scores=True) +# For linear models, the contribution of a given feature is equal to the +# product of feature value times the corresponding weight. Similarly, for +# Generalized Additive Models (GAM), the contribution of a feature is equal to +# the shape function for the given feature evaluated at the feature value. +lr_feature_contributions = lr_model.get_feature_contributions(data) # Print predictions with feature contributions, which give a relative measure # of how much each feature impacted the Score. -print(feature_contributions.head()) +print("========== Feature Contributions for Linear Model ==========") +print(lr_feature_contributions.head()) # label ... PredictedLabel Score ... FeatureContributions.hours-per-week # 0 ... 0 -2.010687 ... 0.833069 # 1 ... 0 -1.216163 ... 0.809928 # 2 ... 0 -1.248412 ... 0.485957 # 3 ... 0 -1.132419 ... 0.583148 # 4 ... 0 -1.969522 ... 0.437361 + +# define the training pipeline with a tree model +tree_pipeline = Pipeline([FastTreesBinaryClassifier( + feature=['age', 'education-num', 'hours-per-week'], label='label')]) + +# train the model +tree_model = tree_pipeline.fit(data) + +# For tree-based models, the calculation of feature contribution essentially +# consists in determining which splits in the tree have the most impact on the +# final score and assigning the value of the impact to the features determining +# the split. More precisely, the contribution of a feature is equal to the +# change in score produced by exploring the opposite sub-tree every time a +# decision node for the given feature is encountered. +# +# Consider a simple case with a single decision tree that has a decision node +# for the binary feature F1. Given an example that has feature F1 equal to +# true, we can calculate the score it would have obtained if we chose the +# subtree corresponding to the feature F1 being equal to false while keeping +# the other features constant. The contribution of feature F1 for the given +# example is the difference between the original score and the score obtained +# by taking the opposite decision at the node corresponding to feature F1. This +# algorithm extends naturally to models with many decision trees. +tree_feature_contributions = tree_model.get_feature_contributions(data) + +# Print predictions with feature contributions, which give a relative measure +# of how much each feature impacted the Score. +print("========== Feature Contributions for Tree Model ==========") +print(tree_feature_contributions.head()) \ No newline at end of file diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 4235ad9a..9ebc7e7e 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1696,16 +1696,35 @@ def getn(n): "ambiguous.") @trace - def calculate_feature_contributions(self, X, y=None, + def get_feature_contributions(self, X, y=None, evaltype='auto', group_id=None, weight=None, - verbose=0, top=10, bottom=10, - normalize=True, + verbose=0, as_binary_data_stream=False, **params): """ - Apply transforms and test with the final estimator, return metrics + Return dataframe with raw data, predictions, and feature contributiuons + for the predictions. + + :param X: {array-like [n_samples, n_features], + :py:class:`nimbusml.FileDataStream` } + :param y: {array-like [n_samples]} + + :param evaltype: the evaluation type for the problem, can be { + 'binary', 'multiclass', 'regression', 'cluster', 'anomaly', + 'ranking'}. The default is 'auto'. If model is loaded using the + load_model() method, evaltype cannot be 'auto', and therefore + must be explicitly specified. + :param group_id: the column name for group_id for ranking problem + :param weight: the column name for the weight column for each + sample. + :param top: the number of positive contributions with highest magnitude + to report. + :param bottom: The number of negative contributions with highest + magnitude to report. + :return: dataframe of containing the raw data, predicted label, score, + probabilities, and feature contributions. """ # start the clock! start_time = time.time() @@ -1754,7 +1773,7 @@ def calculate_feature_contributions(self, X, y=None, output_data="$fccData", top=top, bottom=bottom, - normalize=normalize) + normalize=True) all_nodes.extend([score_node, fcc_node]) if hasattr(self, 'steps') and len(self.steps) > 0 \ @@ -2060,7 +2079,7 @@ def test( otherwise None in the returned tuple. :return: tuple (dataframe of evaluation metrics, dataframe of - scores). Is scores are + scores). If scores are required, set `output_scores`=True, otherwise None is returned by default. """ From 28dad24ee7e4749217a3158331d668528e2ba635 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 15:28:41 -0700 Subject: [PATCH 04/28] Add list of supported models to sample --- .../PipelineWithFeatureContributions.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 0f7c080b..855d2789 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -7,10 +7,36 @@ # computes a model-specific list of per-feature contributions to the score for # each example. These contributions can be positive (they make the score # higher) or negative (they make the score lower). +# +# Feature Contribution Calculation is currently supported for the following +# models: +# - Regression: +# - OrdinaryLeastSquaresRegressor +# - FastLinearRegressor +# - OnlineGradientDescentRegressor +# - PoissonRegressionRegressor +# - GamRegressor +# - LightGbmRegressor +# - FastTreesRegressor +# - FastForestRegressor +# - FastTreesTweedieRegressor +# - Binary Classification: +# - AveragedPerceptronBinaryClassifier +# - LinearSvmBinaryClassifier +# - LogisticRegressionBinaryClassifier +# - FastLinearBinaryClassifier +# - SgdBinaryClassifier +# - SymSgdBinaryClassifier +# - GamBinaryClassifier +# - FastForestBinaryClassifier +# - FastTreesBinaryClassifier +# - LightGbmBinaryClassifier +# - Ranking: +# - LightGbmRanker from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import LogisticRegressionBinaryClassifier, PoissonRegressionRegressor +from nimbusml.linear_model import LogisticRegressionBinaryClassifier from nimbusml.ensemble import FastTreesBinaryClassifier # data input (as a FileDataStream) @@ -26,7 +52,7 @@ # 4 0 ? Some-college ... 0 30 # define the training pipeline with a linear model -lr_pipeline = Pipeline([PoissonRegressionRegressor( +lr_pipeline = Pipeline([LogisticRegressionBinaryClassifier( feature=['age', 'education-num', 'hours-per-week'], label='label')]) # train the model From 41cfb2c5f4a3846623d04a428f3f493fecc942e3 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 16:50:54 -0700 Subject: [PATCH 05/28] Some PR feedback --- src/python/nimbusml/pipeline.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 9ebc7e7e..a530b616 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1726,8 +1726,6 @@ def get_feature_contributions(self, X, y=None, :return: dataframe of containing the raw data, predicted label, score, probabilities, and feature contributions. """ - # start the clock! - start_time = time.time() self.verbose = verbose if not self._is_fitted: @@ -1735,6 +1733,7 @@ def get_feature_contributions(self, X, y=None, "Model is not fitted. Train or load a model before test(" ").") + #print(self.last_node.type) if y is not None: if len(self.steps) > 0: last_node = self.last_node @@ -1785,17 +1784,7 @@ def get_feature_contributions(self, X, y=None, output_data="$output_data") all_nodes.extend([convert_label_node]) - if y is not None: - evaluate_nodes = self._evaluation_infer( - evaltype, label_column, group_id, **params) - for node in evaluate_nodes: - all_nodes.extend([node]) - output_scores = '' if params.get( - 'output_scores', False) else '' - outputs = OrderedDict( - [('output_metrics', ''), ('output_data', output_scores)]) - else: - outputs = dict(output_data="") + outputs = dict(output_data="") graph = Graph( inputs, @@ -1817,17 +1806,8 @@ def get_feature_contributions(self, X, y=None, telemetry_info=telemetry_info, **params) except RuntimeError as e: - self._run_time = time.time() - start_time raise e - if y is not None: - # We need to fix the schema for ranking metrics - if evaltype == 'ranking': - out_metrics = self._fix_ranking_metrics_schema(out_metrics) - - # stop the clock - self._run_time = time.time() - start_time - self._write_csv_time = graph._write_csv_time return out_data @trace From a0d4f7019d3a4606a1ebc40b673a3976dd007e8b Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Tue, 16 Jul 2019 15:27:22 -0700 Subject: [PATCH 06/28] Fix feature contributions for regression and ranking --- src/python/nimbusml/pipeline.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index a530b616..b3f2dd39 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1765,24 +1765,32 @@ def get_feature_contributions(self, X, y=None, data="$data", predictor_model="$predictor_model", scored_data="$scoredvectordata") - - fcc_node = transforms_featurecontributioncalculationtransformer( - data="$scoredvectordata", - predictor_model="$predictor_model", - output_data="$fccData", - top=top, - bottom=bottom, - normalize=True) - all_nodes.extend([score_node, fcc_node]) + all_nodes.extend([score_node]) if hasattr(self, 'steps') and len(self.steps) > 0 \ and self.last_node.type == 'classifier': + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$fccData", + top=top, + bottom=bottom, + normalize=True) convert_label_node = \ transforms_predictedlabelcolumnoriginalvalueconverter( data="$fccData", predicted_label_column="PredictedLabel", output_data="$output_data") - all_nodes.extend([convert_label_node]) + all_nodes.extend([fcc_node, convert_label_node]) + else: + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$output_data", + top=top, + bottom=bottom, + normalize=True) + all_nodes.extend([fcc_node]) outputs = dict(output_data="") From 818c36e6d22b2b5df7df75f680da8b4eda6f9df5 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Tue, 16 Jul 2019 16:00:20 -0700 Subject: [PATCH 07/28] Implement feature contributions in BasePredictor --- src/python/nimbusml/base_predictor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/nimbusml/base_predictor.py b/src/python/nimbusml/base_predictor.py index bfa2813f..068e3120 100644 --- a/src/python/nimbusml/base_predictor.py +++ b/src/python/nimbusml/base_predictor.py @@ -89,6 +89,10 @@ def _invoke_inference_method(self, method, X, **params): data = getattr(pipeline, method)(X, **params) return data + @trace + def get_feature_contributions(self, X, **params): + return self._invoke_inference_method('get_feature_contributions', X, **params) + @trace def predict(self, X, **params): """ From 5a8092961010edc55f0e155378d237e94c205b1f Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Tue, 16 Jul 2019 17:57:19 -0700 Subject: [PATCH 08/28] Add test to check feature contributions with unpickled pipeline --- .../PipelineWithFeatureContributions.py | 5 +++- .../nimbusml/tests/pipeline/test_load_save.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 855d2789..392a5355 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -36,8 +36,8 @@ from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import LogisticRegressionBinaryClassifier from nimbusml.ensemble import FastTreesBinaryClassifier +from nimbusml.linear_model import LogisticRegressionBinaryClassifier # data input (as a FileDataStream) path = get_dataset('uciadult_train').as_filepath() @@ -75,6 +75,9 @@ # 3 ... 0 -1.132419 ... 0.583148 # 4 ... 0 -1.969522 ... 0.437361 +assert 'FeatureContributions.age' in lr_feature_contributions.columns + + # define the training pipeline with a tree model tree_pipeline = Pipeline([FastTreesBinaryClassifier( feature=['age', 'education-num', 'hours-per-week'], label='label')]) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 309650b5..08b655f4 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. # -------------------------------------------------------------------------------------------- +import os import pickle import unittest @@ -119,6 +120,30 @@ def test_model_datastream(self): model_nimbusml_load.sum().sum(), decimal=2) + def test_unpickled_pipeline_has_feature_contributions(self): + features = ['age', 'education-num', 'hours-per-week'] + + model_nimbusml = Pipeline( + steps=[FastLinearBinaryClassifier(feature=features)]) + + model_nimbusml.fit(train, label) + + pickle_filename = 'nimbusml_model.p' + + # Save with pickle + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + os.remove(pickle_filename) + + feature_contributions = model_nimbusml_pickle.get_feature_contributions( + test, test_label) + + assert ['FeatureContributions.' + feature in feature_contributions.columns + for feature in features] if __name__ == '__main__': unittest.main() From 6337475272f99df2151d0579cc6434e229ff3f52 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 17 Jul 2019 16:41:39 -0700 Subject: [PATCH 09/28] Improve doc, simplify entrypoint graph for feature contributions --- .../PipelineWithFeatureContributions.py | 49 ++++--------- src/python/nimbusml/pipeline.py | 69 +++++++++++-------- 2 files changed, 55 insertions(+), 63 deletions(-) diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 392a5355..426241a2 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -1,5 +1,5 @@ ############################################################################### -# Pipeline with feature contributions +# Pipeline with observation level feature contributions # Scoring a dataset with a trained model produces a score, or prediction, for # each example. To understand and explain these predictions it can be useful to @@ -7,32 +7,6 @@ # computes a model-specific list of per-feature contributions to the score for # each example. These contributions can be positive (they make the score # higher) or negative (they make the score lower). -# -# Feature Contribution Calculation is currently supported for the following -# models: -# - Regression: -# - OrdinaryLeastSquaresRegressor -# - FastLinearRegressor -# - OnlineGradientDescentRegressor -# - PoissonRegressionRegressor -# - GamRegressor -# - LightGbmRegressor -# - FastTreesRegressor -# - FastForestRegressor -# - FastTreesTweedieRegressor -# - Binary Classification: -# - AveragedPerceptronBinaryClassifier -# - LinearSvmBinaryClassifier -# - LogisticRegressionBinaryClassifier -# - FastLinearBinaryClassifier -# - SgdBinaryClassifier -# - SymSgdBinaryClassifier -# - GamBinaryClassifier -# - FastForestBinaryClassifier -# - FastTreesBinaryClassifier -# - LightGbmBinaryClassifier -# - Ranking: -# - LightGbmRanker from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset @@ -69,14 +43,11 @@ print("========== Feature Contributions for Linear Model ==========") print(lr_feature_contributions.head()) # label ... PredictedLabel Score ... FeatureContributions.hours-per-week -# 0 ... 0 -2.010687 ... 0.833069 -# 1 ... 0 -1.216163 ... 0.809928 -# 2 ... 0 -1.248412 ... 0.485957 -# 3 ... 0 -1.132419 ... 0.583148 -# 4 ... 0 -1.969522 ... 0.437361 - -assert 'FeatureContributions.age' in lr_feature_contributions.columns - +# 0 0 ... 0 -2.010687 ... 0.833069 +# 1 0 ... 0 -1.216163 ... 0.809928 +# 2 1 ... 0 -1.248412 ... 0.485957 +# 3 1 ... 0 -1.132419 ... 0.583148 +# 4 0 ... 0 -1.969522 ... 0.437361 # define the training pipeline with a tree model tree_pipeline = Pipeline([FastTreesBinaryClassifier( @@ -105,4 +76,10 @@ # Print predictions with feature contributions, which give a relative measure # of how much each feature impacted the Score. print("========== Feature Contributions for Tree Model ==========") -print(tree_feature_contributions.head()) \ No newline at end of file +print(tree_feature_contributions.head()) +# label ... PredictedLabel Score ... FeatureContributions.hours-per-week +# 0 0 ... 0 -16.717360 ... -0.608664 +# 1 0 ... 0 -7.688200 ... -0.541213 +# 2 1 ... 1 1.571164 ... 0.032862 +# 3 1 ... 1 2.115638 ... 0.537077 +# 4 0 ... 0 -23.038410 ... -0.682764 diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index b3f2dd39..7c23afd3 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1704,8 +1704,39 @@ def get_feature_contributions(self, X, y=None, verbose=0, as_binary_data_stream=False, **params): """ - Return dataframe with raw data, predictions, and feature contributiuons - for the predictions. + Calculates observation level feature contributions. Returns dataframe + with raw data, predictions, and feature contributiuons for each + prediction. Observation level feature contriutions are supported for + the following models: + + * Regression: + + * OrdinaryLeastSquaresRegressor + * FastLinearRegressor + * OnlineGradientDescentRegressor + * PoissonRegressionRegressor + * GamRegressor + * LightGbmRegressor + * FastTreesRegressor + * FastForestRegressor + * FastTreesTweedieRegressor + + * Binary Classification: + + * AveragedPerceptronBinaryClassifier + * LinearSvmBinaryClassifier + * LogisticRegressionBinaryClassifier + * FastLinearBinaryClassifier + * SgdBinaryClassifier + * SymSgdBinaryClassifier + * GamBinaryClassifier + * FastForestBinaryClassifier + * FastTreesBinaryClassifier + * LightGbmBinaryClassifier + + * Ranking: + + * LightGbmRanker :param X: {array-like [n_samples, n_features], :py:class:`nimbusml.FileDataStream` } @@ -1765,32 +1796,16 @@ def get_feature_contributions(self, X, y=None, data="$data", predictor_model="$predictor_model", scored_data="$scoredvectordata") - all_nodes.extend([score_node]) - if hasattr(self, 'steps') and len(self.steps) > 0 \ - and self.last_node.type == 'classifier': - fcc_node = transforms_featurecontributioncalculationtransformer( - data="$scoredvectordata", - predictor_model="$predictor_model", - output_data="$fccData", - top=top, - bottom=bottom, - normalize=True) - convert_label_node = \ - transforms_predictedlabelcolumnoriginalvalueconverter( - data="$fccData", - predicted_label_column="PredictedLabel", - output_data="$output_data") - all_nodes.extend([fcc_node, convert_label_node]) - else: - fcc_node = transforms_featurecontributioncalculationtransformer( - data="$scoredvectordata", - predictor_model="$predictor_model", - output_data="$output_data", - top=top, - bottom=bottom, - normalize=True) - all_nodes.extend([fcc_node]) + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$output_data", + top=top, + bottom=bottom, + normalize=True) + + all_nodes.extend([score_node, fcc_node]) outputs = dict(output_data="") From 11c388e451667701151bc6e2cb438c5ddb56c161 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 17 Jul 2019 17:37:36 -0700 Subject: [PATCH 10/28] Add test to check feature contributions with pipeline loaded from zip --- .../nimbusml/tests/pipeline/test_load_save.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 08b655f4..855999d1 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -7,7 +7,7 @@ import pickle import unittest -from nimbusml import Pipeline +from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset from nimbusml.feature_extraction.categorical import OneHotVectorizer from nimbusml.linear_model import FastLinearBinaryClassifier @@ -145,5 +145,29 @@ def test_unpickled_pipeline_has_feature_contributions(self): assert ['FeatureContributions.' + feature in feature_contributions.columns for feature in features] + def test_pipeline_loaded_from_zip_has_feature_contributions(self): + features = ['age', 'education-num', 'hours-per-week'] + + model_nimbusml = Pipeline( + steps=[FastLinearBinaryClassifier(feature=features)]) + + model_nimbusml.fit(train, label) + + # Save the model to zip + model_filename = 'nimbusml_model.zip' + model_nimbusml.save_model(model_filename) + + # Load the model from zip + model_nimbusml_zip = Pipeline() + model_nimbusml_zip.load_model(model_filename) + + feature_contributions = model_nimbusml_zip.get_feature_contributions( + test, test_label) + + os.remove(model_filename) + + assert ['FeatureContributions.' + feature in feature_contributions.columns + for feature in features] + if __name__ == '__main__': unittest.main() From ea3dce4df2c3a5a3653d735168c901e0d4b7970f Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 17 Jul 2019 17:41:56 -0700 Subject: [PATCH 11/28] nit --- src/python/nimbusml/tests/pipeline/test_load_save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 855999d1..8e1cc3b5 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -7,7 +7,7 @@ import pickle import unittest -from nimbusml import Pipeline, FileDataStream +from nimbusml import Pipeline from nimbusml.datasets import get_dataset from nimbusml.feature_extraction.categorical import OneHotVectorizer from nimbusml.linear_model import FastLinearBinaryClassifier From 93a46fa251ec5fb0b19d162a21c8780fb75e659a Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 18 Jul 2019 14:05:32 -0700 Subject: [PATCH 12/28] PR feedback --- src/python/nimbusml/pipeline.py | 48 ++++++++++----------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 7c23afd3..259020d9 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1696,13 +1696,8 @@ def getn(n): "ambiguous.") @trace - def get_feature_contributions(self, X, y=None, - evaltype='auto', group_id=None, - weight=None, - top=10, - bottom=10, - verbose=0, - as_binary_data_stream=False, **params): + def get_feature_contributions(self, X, top=10, bottom=10, verbose=0, + as_binary_data_stream=False, **params): """ Calculates observation level feature contributions. Returns dataframe with raw data, predictions, and feature contributiuons for each @@ -1740,16 +1735,6 @@ def get_feature_contributions(self, X, y=None, :param X: {array-like [n_samples, n_features], :py:class:`nimbusml.FileDataStream` } - :param y: {array-like [n_samples]} - - :param evaltype: the evaluation type for the problem, can be { - 'binary', 'multiclass', 'regression', 'cluster', 'anomaly', - 'ranking'}. The default is 'auto'. If model is loaded using the - load_model() method, evaltype cannot be 'auto', and therefore - must be explicitly specified. - :param group_id: the column name for group_id for ranking problem - :param weight: the column name for the weight column for each - sample. :param top: the number of positive contributions with highest magnitude to report. :param bottom: The number of negative contributions with highest @@ -1761,25 +1746,21 @@ def get_feature_contributions(self, X, y=None, if not self._is_fitted: raise ValueError( - "Model is not fitted. Train or load a model before test(" - ").") + "Model is not fitted. Train or load a model before test().") - #print(self.last_node.type) - if y is not None: - if len(self.steps) > 0: - last_node = self.last_node - if last_node.type == 'transform': - raise ValueError( - "Pipeline needs a trainer as last step for test()") + # BUG: If model is loaded from zip file, self.steps is an empty array + # so this condition will always evaluate to False. Consequently, this + # code will never check if the last node is a transform or not. In any + # case, self.last_node will not exist if a model is loaded from zip so + # we could not check for it outside of this condition. + if len(self.steps) > 0: + last_node = self.last_node + if last_node.type == 'transform': + raise ValueError( + "Pipeline needs a trainer as last step for test()") X, y_temp, columns_renamed, feature_columns, label_column, \ - schema, weights, weight_column = self._preprocess_X_y( - X, y, w=weight - ) - - if (not isinstance(y, (str, tuple))) or ( - isinstance(X, DataFrame) and isinstance(y, (str, tuple))): - y = y_temp + schema, weights, weight_column = self._preprocess_X_y(X) all_nodes = [] inputs = dict([('data', ''), ('predictor_model', self.model)]) @@ -1822,7 +1803,6 @@ def get_feature_contributions(self, X, y=None, try: (out_model, out_data, out_metrics) = graph.run( X=X, - y=y, random_state=self.random_state, model=self.model, verbose=verbose, From c0500d1f67d1afd1d90ef6da14206ddb443324f4 Mon Sep 17 00:00:00 2001 From: pieths Date: Thu, 18 Jul 2019 14:15:57 -0700 Subject: [PATCH 13/28] Save the model file when pickling a NimbusML Pipeline. (#189) * Save the model file when pickling a NimbusML Pipeline. * Add version to the pickled Pipeline. * Add the steps attribute to a pickled Pipeline instance. * Add extra unit test for pickled nimbusml pipelines. * Add export_version to pickled base_pipeline_items. Remove unnecessary export_version attribute from an unpickled Pipeline. --- .../internal/core/base_pipeline_item.py | 7 +- src/python/nimbusml/pipeline.py | 33 ++++++ .../nimbusml/tests/pipeline/test_load_save.py | 111 +++++++++++++++++- .../tests/scikit/test_uci_adult_scikit.py | 6 + 4 files changed, 152 insertions(+), 5 deletions(-) diff --git a/src/python/nimbusml/internal/core/base_pipeline_item.py b/src/python/nimbusml/internal/core/base_pipeline_item.py index e45b3d0c..fa02a3c3 100644 --- a/src/python/nimbusml/internal/core/base_pipeline_item.py +++ b/src/python/nimbusml/internal/core/base_pipeline_item.py @@ -375,6 +375,8 @@ def _get_node(self, **params): def __getstate__(self): "Selects what to pickle." odict = self.__dict__.copy() + odict['export_version'] = 1 + if hasattr(self, 'model_') and \ self.model_ is not None and os.path.isfile(self.model_): with open(self.model_, "rb") as mfile: @@ -387,8 +389,11 @@ def __getstate__(self): def __setstate__(self, state): "Restore a pickled object." for k, v in state.items(): - if k not in {'modelbytes', 'type'}: + if k not in {'modelbytes', 'type', 'export_version'}: setattr(self, k, v) + + # Note: modelbytes and type were + # added before export_version 1 if 'modelbytes' in state: (fd, modelfile) = tempfile.mkstemp() fl = os.fdopen(fd, "wb") diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 5a15bac4..8e9510d1 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -5,6 +5,7 @@ import inspect import itertools import os +import tempfile import time import warnings from collections import OrderedDict, namedtuple, defaultdict @@ -2265,6 +2266,38 @@ def load_model(self, src): self.model = src self.steps = [] + def __getstate__(self): + odict = {'export_version': 1} + + if hasattr(self, 'steps'): + odict['steps'] = self.steps + + if (hasattr(self, 'model') and + self.model is not None and + os.path.isfile(self.model)): + + with open(self.model, "rb") as f: + odict['modelbytes'] = f.read() + + return odict + + def __setstate__(self, state): + self.steps = [] + self.model = None + self.random_state = None + + for k, v in state.items(): + if k not in {'modelbytes', 'export_version'}: + setattr(self, k, v) + + if state.get('export_version', 0) == 1: + if 'modelbytes' in state: + (fd, modelfile) = tempfile.mkstemp() + fl = os.fdopen(fd, "wb") + fl.write(state['modelbytes']) + fl.close() + self.model = modelfile + @trace def score( self, diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 309650b5..7102c066 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. # -------------------------------------------------------------------------------------------- +import os import pickle import unittest @@ -44,8 +45,14 @@ def test_model_dataframe(self): model_nimbusml.fit(train, label) # Save with pickle - pickle.dump(model_nimbusml, open('nimbusml_model.p', 'wb')) - model_nimbusml_pickle = pickle.load(open("nimbusml_model.p", "rb")) + pickle_filename = 'nimbusml_model.p' + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + os.remove(pickle_filename) score1 = model_nimbusml.predict(test).head(5) score2 = model_nimbusml_pickle.predict(test).head(5) @@ -72,6 +79,8 @@ def test_model_dataframe(self): model_nimbusml_load.sum().sum(), decimal=2) + os.remove('model.nimbusml.m') + def test_model_datastream(self): model_nimbusml = Pipeline( steps=[ @@ -85,8 +94,14 @@ def test_model_datastream(self): model_nimbusml.fit(train, label) # Save with pickle - pickle.dump(model_nimbusml, open('nimbusml_model.p', 'wb')) - model_nimbusml_pickle = pickle.load(open("nimbusml_model.p", "rb")) + pickle_filename = 'nimbusml_model.p' + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + os.remove(pickle_filename) score1 = model_nimbusml.predict(test).head(5) score2 = model_nimbusml_pickle.predict(test).head(5) @@ -119,6 +134,94 @@ def test_model_datastream(self): model_nimbusml_load.sum().sum(), decimal=2) + os.remove('model.nimbusml.m') + + def test_pipeline_saves_complete_model_file_when_pickled(self): + model_nimbusml = Pipeline( + steps=[ + ('cat', + OneHotVectorizer() << categorical_columns), + ('linear', + FastLinearBinaryClassifier( + shuffle=False, + number_of_threads=1))]) + + model_nimbusml.fit(train, label) + metrics, score = model_nimbusml.test(test, test_label, output_scores=True) + + pickle_filename = 'nimbusml_model.p' + + # Save with pickle + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + + # Remove the pipeline model from disk so + # that the unpickled pipeline is forced + # to get its model from the pickled file. + os.remove(model_nimbusml.model) + + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + os.remove(pickle_filename) + + metrics_pickle, score_pickle = model_nimbusml_pickle.test( + test, test_label, output_scores=True) + + assert_almost_equal(score.sum().sum(), + score_pickle.sum().sum(), + decimal=2) + + assert_almost_equal(metrics.sum().sum(), + metrics_pickle.sum().sum(), + decimal=2) + + def test_unfitted_pickled_pipeline_can_be_fit(self): + pipeline = Pipeline( + steps=[ + ('cat', + OneHotVectorizer() << categorical_columns), + ('linear', + FastLinearBinaryClassifier( + shuffle=False, + number_of_threads=1))]) + + pipeline.fit(train, label) + metrics, score = pipeline.test(test, test_label, output_scores=True) + + # Create a new unfitted pipeline + pipeline = Pipeline( + steps=[ + ('cat', + OneHotVectorizer() << categorical_columns), + ('linear', + FastLinearBinaryClassifier( + shuffle=False, + number_of_threads=1))]) + + pickle_filename = 'nimbusml_model.p' + + # Save with pickle + with open(pickle_filename, 'wb') as f: + pickle.dump(pipeline, f) + + with open(pickle_filename, "rb") as f: + pipeline_pickle = pickle.load(f) + + os.remove(pickle_filename) + + pipeline_pickle.fit(train, label) + metrics_pickle, score_pickle = pipeline_pickle.test( + test, test_label, output_scores=True) + + assert_almost_equal(score.sum().sum(), + score_pickle.sum().sum(), + decimal=2) + + assert_almost_equal(metrics.sum().sum(), + metrics_pickle.sum().sum(), + decimal=2) + if __name__ == '__main__': unittest.main() diff --git a/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py b/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py index 503c21a6..a08ce3b9 100644 --- a/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py +++ b/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. # -------------------------------------------------------------------------------------------- +import os import pickle import unittest @@ -111,6 +112,7 @@ def test_pickle_predictor(self): # Unpickle model and score. We should get the exact same accuracy as # above s = pickle.dumps(ftree) + os.remove(ftree.model_) ftree2 = pickle.loads(s) scores2 = ftree2.predict(X_test) accu2 = np.mean(y_test.values.ravel() == scores2.values) @@ -130,6 +132,7 @@ def test_pickle_transform(self): # Unpickle transform and generate output. # We should get the exact same output as above s = pickle.dumps(cat) + os.remove(cat.model_) cat2 = pickle.loads(s) out2 = cat2.transform(X_train) assert_equal( @@ -158,7 +161,10 @@ def test_pickle_pipeline(self): # Unpickle model and score. We should get the exact same accuracy as # above s = pickle.dumps(pipe) + os.remove(cat.model_) + os.remove(ftree.model_) pipe2 = pickle.loads(s) + scores2 = pipe2.predict(X_test) accu2 = np.mean(y_test.values.ravel() == scores2.values) assert_equal( From 266d27d239322093eda9062fec8804b203df66c8 Mon Sep 17 00:00:00 2001 From: pieths Date: Thu, 18 Jul 2019 15:13:13 -0700 Subject: [PATCH 14/28] Remove stored references to X and y in BasePredictor. (#195) * Remove stored references to X and y in BasePredictor. * Remove unnecessary scikit-learn import. --- src/python/nimbusml/base_predictor.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/python/nimbusml/base_predictor.py b/src/python/nimbusml/base_predictor.py index bfa2813f..588971b3 100644 --- a/src/python/nimbusml/base_predictor.py +++ b/src/python/nimbusml/base_predictor.py @@ -12,7 +12,6 @@ from sklearn.base import BaseEstimator from sklearn.utils.multiclass import unique_labels -from sklearn.utils.validation import check_is_fitted from . import Pipeline from .internal.core.base_pipeline_item import BasePipelineItem @@ -49,8 +48,6 @@ def fit(self, X, y=None, **params): "Classifier can't train when only one class is " "present.") self.classes_ = unique_classes - self.X_ = X - self.y_ = y # Clear cached summary since it should not # retain its value after a new call to fit @@ -69,13 +66,24 @@ def fit(self, X, y=None, **params): set_shape(self, X) return self + @property + def _is_fitted(self): + """ + Tells if the predictor was trained. + """ + return (hasattr(self, 'model_') and + self.model_ and + os.path.isfile(self.model_)) + @trace def _invoke_inference_method(self, method, X, **params): """ Returns predictions. Can be predicted labels, probabilities or else decision values. """ - check_is_fitted(self, ["X_", "y_"]) + if not self._is_fitted: + raise ValueError("Model is not fitted. " + "fit() must be called before {}.".format(method)) # Check that the input is of the same shape as the one passed # during From d20c3982e8ece24c45e943e8cbae9715ee424b68 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Fri, 12 Jul 2019 13:21:49 -0700 Subject: [PATCH 15/28] Add calculate_feature_contrinutions method to Pipeline --- src/python/nimbusml.pyproj | 1 + .../PipelineWithFeatureContributions.py | 37 ++++++ src/python/nimbusml/pipeline.py | 118 ++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 src/python/nimbusml/examples/PipelineWithFeatureContributions.py diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index a97e8b14..68780828 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -177,6 +177,7 @@ + diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py new file mode 100644 index 00000000..97385bd5 --- /dev/null +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -0,0 +1,37 @@ +############################################################################### +# Pipeline with feature contributions +from nimbusml import Pipeline, FileDataStream +from nimbusml.datasets import get_dataset +from nimbusml.linear_model import LogisticRegressionBinaryClassifier + +# data input (as a FileDataStream) +path = get_dataset('uciadult_train').as_filepath() + +data = FileDataStream.read_csv(path) +print(data.head()) +# label workclass education ... capital-loss hours-per-week +# 0 0 Private 11th ... 0 40 +# 1 0 Private HS-grad ... 0 50 +# 2 1 Local-gov Assoc-acdm ... 0 40 +# 3 1 Private Some-college ... 0 40 +# 4 0 ? Some-college ... 0 30 +# define the training pipeline +pipeline = Pipeline([LogisticRegressionBinaryClassifier( + feature=['age', 'education-num', 'hours-per-week'], label='label')]) + +# train, predict, and evaluate +# TODO: Replace with CV +model = pipeline.fit(data) + +feature_contributions = model.calculate_feature_contributions( + data, output_scores=True) + +# Print predictions with feature contributions, which give a relative measure +# of how much each feature impacted the Score. +print(feature_contributions.head()) +# label ... PredictedLabel Score ... FeatureContributions.hours-per-week +# 0 ... 0 -2.010687 ... 0.833069 +# 1 ... 0 -1.216163 ... 0.809928 +# 2 ... 0 -1.248412 ... 0.485957 +# 3 ... 0 -1.132419 ... 0.583148 +# 4 ... 0 -1.969522 ... 0.437361 diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 8e9510d1..4a70a23d 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -41,6 +41,8 @@ transforms_datasetscorer from .internal.entrypoints.transforms_featurecombiner import \ transforms_featurecombiner +from .internal.entrypoints.transforms_featurecontributioncalculationtransformer import \ + transforms_featurecontributioncalculationtransformer from .internal.entrypoints.transforms_labelcolumnkeybooleanconverter \ import \ transforms_labelcolumnkeybooleanconverter @@ -1694,6 +1696,122 @@ def getn(n): "only fit(X) is allowed or the training becomes " "ambiguous.") + @trace + def calculate_feature_contributions(self, X, y=None, + evaltype='auto', group_id=None, + weight=None, + verbose=0, + top=10, + bottom=10, + normalize=True, + as_binary_data_stream=False, **params): + """ + Apply transforms and test with the final estimator, return metrics + """ + # start the clock! + start_time = time.time() + self.verbose = verbose + + if not self._is_fitted: + raise ValueError( + "Model is not fitted. Train or load a model before test(" + ").") + + if y is not None: + if len(self.steps) > 0: + last_node = self.last_node + if last_node.type == 'transform': + raise ValueError( + "Pipeline needs a trainer as last step for test()") + + X, y_temp, columns_renamed, feature_columns, label_column, \ + schema, weights, weight_column = self._preprocess_X_y( + X, y, w=weight + ) + + if (not isinstance(y, (str, tuple))) or ( + isinstance(X, DataFrame) and isinstance(y, (str, tuple))): + y = y_temp + + all_nodes = [] + inputs = dict([('data', ''), ('predictor_model', self.model)]) + if isinstance(X, FileDataStream): + importtext_node = data_customtextloader( + input_file="$file", + data="$data", + custom_schema=schema.to_string( + add_sep=True)) + all_nodes = [importtext_node] + inputs = dict([('file', ''), ('predictor_model', self.model)]) + + score_node = transforms_datasetscorer( + data="$data", + predictor_model="$predictor_model", + scored_data="$scoredvectordata") + + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$fccData", + top=top, + bottom=bottom, + normalize=normalize) + all_nodes.extend([score_node, fcc_node]) + + if hasattr(self, 'steps') and len(self.steps) > 0 \ + and self.last_node.type == 'classifier': + convert_label_node = \ + transforms_predictedlabelcolumnoriginalvalueconverter( + data="$fccData", + predicted_label_column="PredictedLabel", + output_data="$output_data") + all_nodes.extend([convert_label_node]) + + if y is not None: + evaluate_nodes = self._evaluation_infer( + evaltype, label_column, group_id, **params) + for node in evaluate_nodes: + all_nodes.extend([node]) + output_scores = '' if params.get( + 'output_scores', False) else '' + outputs = OrderedDict( + [('output_metrics', ''), ('output_data', output_scores)]) + else: + outputs = dict(output_data="") + + graph = Graph( + inputs, + outputs, + as_binary_data_stream, + *all_nodes) + + class_name = type(self).__name__ + method_name = inspect.currentframe().f_code.co_name + telemetry_info = ".".join([class_name, method_name]) + + try: + (out_model, out_data, out_metrics) = graph.run( + X=X, + y=y, + random_state=self.random_state, + model=self.model, + verbose=verbose, + telemetry_info=telemetry_info, + **params) + except RuntimeError as e: + self._run_time = time.time() - start_time + raise e + + if y is not None: + # We need to fix the schema for ranking metrics + if evaltype == 'ranking': + out_metrics = self._fix_ranking_metrics_schema(out_metrics) + + # stop the clock + self._run_time = time.time() - start_time + self._write_csv_time = graph._write_csv_time + return out_data + @trace def _predict(self, X, y=None, evaltype='auto', group_id=None, From e21f91d5f1e9bfbe8c1b9f3a39dd2850f1479055 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 12:42:56 -0700 Subject: [PATCH 16/28] typo --- src/python/nimbusml.pyproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 68780828..4f03e01b 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -177,7 +177,7 @@ - + From df3bdc780cd109d505e8f4a67dc8ff67b8f3ad34 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 14:50:43 -0700 Subject: [PATCH 17/28] rename to get_feature_contributions(), add docs, improve sample --- src/python/nimbusml.pyproj | 2 +- .../PipelineWithFeatureContributions.py | 60 ++++++++++++++++--- src/python/nimbusml/pipeline.py | 31 ++++++++-- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 4f03e01b..f694be00 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -177,7 +177,7 @@ - + diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 97385bd5..0f7c080b 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -1,8 +1,17 @@ ############################################################################### # Pipeline with feature contributions + +# Scoring a dataset with a trained model produces a score, or prediction, for +# each example. To understand and explain these predictions it can be useful to +# inspect which features influenced them most significantly. This function +# computes a model-specific list of per-feature contributions to the score for +# each example. These contributions can be positive (they make the score +# higher) or negative (they make the score lower). + from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import LogisticRegressionBinaryClassifier +from nimbusml.linear_model import LogisticRegressionBinaryClassifier, PoissonRegressionRegressor +from nimbusml.ensemble import FastTreesBinaryClassifier # data input (as a FileDataStream) path = get_dataset('uciadult_train').as_filepath() @@ -15,23 +24,56 @@ # 2 1 Local-gov Assoc-acdm ... 0 40 # 3 1 Private Some-college ... 0 40 # 4 0 ? Some-college ... 0 30 -# define the training pipeline -pipeline = Pipeline([LogisticRegressionBinaryClassifier( + +# define the training pipeline with a linear model +lr_pipeline = Pipeline([PoissonRegressionRegressor( feature=['age', 'education-num', 'hours-per-week'], label='label')]) -# train, predict, and evaluate -# TODO: Replace with CV -model = pipeline.fit(data) +# train the model +lr_model = lr_pipeline.fit(data) -feature_contributions = model.calculate_feature_contributions( - data, output_scores=True) +# For linear models, the contribution of a given feature is equal to the +# product of feature value times the corresponding weight. Similarly, for +# Generalized Additive Models (GAM), the contribution of a feature is equal to +# the shape function for the given feature evaluated at the feature value. +lr_feature_contributions = lr_model.get_feature_contributions(data) # Print predictions with feature contributions, which give a relative measure # of how much each feature impacted the Score. -print(feature_contributions.head()) +print("========== Feature Contributions for Linear Model ==========") +print(lr_feature_contributions.head()) # label ... PredictedLabel Score ... FeatureContributions.hours-per-week # 0 ... 0 -2.010687 ... 0.833069 # 1 ... 0 -1.216163 ... 0.809928 # 2 ... 0 -1.248412 ... 0.485957 # 3 ... 0 -1.132419 ... 0.583148 # 4 ... 0 -1.969522 ... 0.437361 + +# define the training pipeline with a tree model +tree_pipeline = Pipeline([FastTreesBinaryClassifier( + feature=['age', 'education-num', 'hours-per-week'], label='label')]) + +# train the model +tree_model = tree_pipeline.fit(data) + +# For tree-based models, the calculation of feature contribution essentially +# consists in determining which splits in the tree have the most impact on the +# final score and assigning the value of the impact to the features determining +# the split. More precisely, the contribution of a feature is equal to the +# change in score produced by exploring the opposite sub-tree every time a +# decision node for the given feature is encountered. +# +# Consider a simple case with a single decision tree that has a decision node +# for the binary feature F1. Given an example that has feature F1 equal to +# true, we can calculate the score it would have obtained if we chose the +# subtree corresponding to the feature F1 being equal to false while keeping +# the other features constant. The contribution of feature F1 for the given +# example is the difference between the original score and the score obtained +# by taking the opposite decision at the node corresponding to feature F1. This +# algorithm extends naturally to models with many decision trees. +tree_feature_contributions = tree_model.get_feature_contributions(data) + +# Print predictions with feature contributions, which give a relative measure +# of how much each feature impacted the Score. +print("========== Feature Contributions for Tree Model ==========") +print(tree_feature_contributions.head()) \ No newline at end of file diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 4a70a23d..adf0e4df 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1697,16 +1697,35 @@ def getn(n): "ambiguous.") @trace - def calculate_feature_contributions(self, X, y=None, + def get_feature_contributions(self, X, y=None, evaltype='auto', group_id=None, weight=None, - verbose=0, top=10, bottom=10, - normalize=True, + verbose=0, as_binary_data_stream=False, **params): """ - Apply transforms and test with the final estimator, return metrics + Return dataframe with raw data, predictions, and feature contributiuons + for the predictions. + + :param X: {array-like [n_samples, n_features], + :py:class:`nimbusml.FileDataStream` } + :param y: {array-like [n_samples]} + + :param evaltype: the evaluation type for the problem, can be { + 'binary', 'multiclass', 'regression', 'cluster', 'anomaly', + 'ranking'}. The default is 'auto'. If model is loaded using the + load_model() method, evaltype cannot be 'auto', and therefore + must be explicitly specified. + :param group_id: the column name for group_id for ranking problem + :param weight: the column name for the weight column for each + sample. + :param top: the number of positive contributions with highest magnitude + to report. + :param bottom: The number of negative contributions with highest + magnitude to report. + :return: dataframe of containing the raw data, predicted label, score, + probabilities, and feature contributions. """ # start the clock! start_time = time.time() @@ -1755,7 +1774,7 @@ def calculate_feature_contributions(self, X, y=None, output_data="$fccData", top=top, bottom=bottom, - normalize=normalize) + normalize=True) all_nodes.extend([score_node, fcc_node]) if hasattr(self, 'steps') and len(self.steps) > 0 \ @@ -2061,7 +2080,7 @@ def test( otherwise None in the returned tuple. :return: tuple (dataframe of evaluation metrics, dataframe of - scores). Is scores are + scores). If scores are required, set `output_scores`=True, otherwise None is returned by default. """ From a09d6ef19137036848e44ea17a25e58dea41fc3a Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 15:28:41 -0700 Subject: [PATCH 18/28] Add list of supported models to sample --- .../PipelineWithFeatureContributions.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 0f7c080b..855d2789 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -7,10 +7,36 @@ # computes a model-specific list of per-feature contributions to the score for # each example. These contributions can be positive (they make the score # higher) or negative (they make the score lower). +# +# Feature Contribution Calculation is currently supported for the following +# models: +# - Regression: +# - OrdinaryLeastSquaresRegressor +# - FastLinearRegressor +# - OnlineGradientDescentRegressor +# - PoissonRegressionRegressor +# - GamRegressor +# - LightGbmRegressor +# - FastTreesRegressor +# - FastForestRegressor +# - FastTreesTweedieRegressor +# - Binary Classification: +# - AveragedPerceptronBinaryClassifier +# - LinearSvmBinaryClassifier +# - LogisticRegressionBinaryClassifier +# - FastLinearBinaryClassifier +# - SgdBinaryClassifier +# - SymSgdBinaryClassifier +# - GamBinaryClassifier +# - FastForestBinaryClassifier +# - FastTreesBinaryClassifier +# - LightGbmBinaryClassifier +# - Ranking: +# - LightGbmRanker from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import LogisticRegressionBinaryClassifier, PoissonRegressionRegressor +from nimbusml.linear_model import LogisticRegressionBinaryClassifier from nimbusml.ensemble import FastTreesBinaryClassifier # data input (as a FileDataStream) @@ -26,7 +52,7 @@ # 4 0 ? Some-college ... 0 30 # define the training pipeline with a linear model -lr_pipeline = Pipeline([PoissonRegressionRegressor( +lr_pipeline = Pipeline([LogisticRegressionBinaryClassifier( feature=['age', 'education-num', 'hours-per-week'], label='label')]) # train the model From c8c851d492b2579e1a7e291bf07a5a2d75becce9 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Mon, 15 Jul 2019 16:50:54 -0700 Subject: [PATCH 19/28] Some PR feedback --- src/python/nimbusml/pipeline.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index adf0e4df..662714dd 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1727,8 +1727,6 @@ def get_feature_contributions(self, X, y=None, :return: dataframe of containing the raw data, predicted label, score, probabilities, and feature contributions. """ - # start the clock! - start_time = time.time() self.verbose = verbose if not self._is_fitted: @@ -1736,6 +1734,7 @@ def get_feature_contributions(self, X, y=None, "Model is not fitted. Train or load a model before test(" ").") + #print(self.last_node.type) if y is not None: if len(self.steps) > 0: last_node = self.last_node @@ -1786,17 +1785,7 @@ def get_feature_contributions(self, X, y=None, output_data="$output_data") all_nodes.extend([convert_label_node]) - if y is not None: - evaluate_nodes = self._evaluation_infer( - evaltype, label_column, group_id, **params) - for node in evaluate_nodes: - all_nodes.extend([node]) - output_scores = '' if params.get( - 'output_scores', False) else '' - outputs = OrderedDict( - [('output_metrics', ''), ('output_data', output_scores)]) - else: - outputs = dict(output_data="") + outputs = dict(output_data="") graph = Graph( inputs, @@ -1818,17 +1807,8 @@ def get_feature_contributions(self, X, y=None, telemetry_info=telemetry_info, **params) except RuntimeError as e: - self._run_time = time.time() - start_time raise e - if y is not None: - # We need to fix the schema for ranking metrics - if evaltype == 'ranking': - out_metrics = self._fix_ranking_metrics_schema(out_metrics) - - # stop the clock - self._run_time = time.time() - start_time - self._write_csv_time = graph._write_csv_time return out_data @trace From 80f46554a5f015037641cafe1068ed5802f9730c Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Tue, 16 Jul 2019 15:27:22 -0700 Subject: [PATCH 20/28] Fix feature contributions for regression and ranking --- src/python/nimbusml/pipeline.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 662714dd..504555d9 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1766,24 +1766,32 @@ def get_feature_contributions(self, X, y=None, data="$data", predictor_model="$predictor_model", scored_data="$scoredvectordata") - - fcc_node = transforms_featurecontributioncalculationtransformer( - data="$scoredvectordata", - predictor_model="$predictor_model", - output_data="$fccData", - top=top, - bottom=bottom, - normalize=True) - all_nodes.extend([score_node, fcc_node]) + all_nodes.extend([score_node]) if hasattr(self, 'steps') and len(self.steps) > 0 \ and self.last_node.type == 'classifier': + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$fccData", + top=top, + bottom=bottom, + normalize=True) convert_label_node = \ transforms_predictedlabelcolumnoriginalvalueconverter( data="$fccData", predicted_label_column="PredictedLabel", output_data="$output_data") - all_nodes.extend([convert_label_node]) + all_nodes.extend([fcc_node, convert_label_node]) + else: + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$output_data", + top=top, + bottom=bottom, + normalize=True) + all_nodes.extend([fcc_node]) outputs = dict(output_data="") From 0c208dc4504fd19346755031a14f6a57b962f86b Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Tue, 16 Jul 2019 16:00:20 -0700 Subject: [PATCH 21/28] Implement feature contributions in BasePredictor --- src/python/nimbusml/base_predictor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/nimbusml/base_predictor.py b/src/python/nimbusml/base_predictor.py index 588971b3..b38d5854 100644 --- a/src/python/nimbusml/base_predictor.py +++ b/src/python/nimbusml/base_predictor.py @@ -97,6 +97,10 @@ def _invoke_inference_method(self, method, X, **params): data = getattr(pipeline, method)(X, **params) return data + @trace + def get_feature_contributions(self, X, **params): + return self._invoke_inference_method('get_feature_contributions', X, **params) + @trace def predict(self, X, **params): """ From 042566fe2327d795ab33f60e82eaa9f328650b21 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Tue, 16 Jul 2019 17:57:19 -0700 Subject: [PATCH 22/28] Add test to check feature contributions with unpickled pipeline --- .../examples/PipelineWithFeatureContributions.py | 5 ++++- src/python/nimbusml/tests/pipeline/test_load_save.py | 11 ----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 855d2789..392a5355 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -36,8 +36,8 @@ from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset -from nimbusml.linear_model import LogisticRegressionBinaryClassifier from nimbusml.ensemble import FastTreesBinaryClassifier +from nimbusml.linear_model import LogisticRegressionBinaryClassifier # data input (as a FileDataStream) path = get_dataset('uciadult_train').as_filepath() @@ -75,6 +75,9 @@ # 3 ... 0 -1.132419 ... 0.583148 # 4 ... 0 -1.969522 ... 0.437361 +assert 'FeatureContributions.age' in lr_feature_contributions.columns + + # define the training pipeline with a tree model tree_pipeline = Pipeline([FastTreesBinaryClassifier( feature=['age', 'education-num', 'hours-per-week'], label='label')]) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 7102c066..3c66253e 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -149,22 +149,11 @@ def test_pipeline_saves_complete_model_file_when_pickled(self): model_nimbusml.fit(train, label) metrics, score = model_nimbusml.test(test, test_label, output_scores=True) - pickle_filename = 'nimbusml_model.p' - - # Save with pickle - with open(pickle_filename, 'wb') as f: - pickle.dump(model_nimbusml, f) - # Remove the pipeline model from disk so # that the unpickled pipeline is forced # to get its model from the pickled file. os.remove(model_nimbusml.model) - with open(pickle_filename, "rb") as f: - model_nimbusml_pickle = pickle.load(f) - - os.remove(pickle_filename) - metrics_pickle, score_pickle = model_nimbusml_pickle.test( test, test_label, output_scores=True) From 9bd8b4e8e7d46d7b85ea59305d2ef38f944625bc Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 17 Jul 2019 16:41:39 -0700 Subject: [PATCH 23/28] Improve doc, simplify entrypoint graph for feature contributions --- .../PipelineWithFeatureContributions.py | 49 ++++--------- src/python/nimbusml/pipeline.py | 69 +++++++++++-------- 2 files changed, 55 insertions(+), 63 deletions(-) diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py index 392a5355..426241a2 100644 --- a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py +++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py @@ -1,5 +1,5 @@ ############################################################################### -# Pipeline with feature contributions +# Pipeline with observation level feature contributions # Scoring a dataset with a trained model produces a score, or prediction, for # each example. To understand and explain these predictions it can be useful to @@ -7,32 +7,6 @@ # computes a model-specific list of per-feature contributions to the score for # each example. These contributions can be positive (they make the score # higher) or negative (they make the score lower). -# -# Feature Contribution Calculation is currently supported for the following -# models: -# - Regression: -# - OrdinaryLeastSquaresRegressor -# - FastLinearRegressor -# - OnlineGradientDescentRegressor -# - PoissonRegressionRegressor -# - GamRegressor -# - LightGbmRegressor -# - FastTreesRegressor -# - FastForestRegressor -# - FastTreesTweedieRegressor -# - Binary Classification: -# - AveragedPerceptronBinaryClassifier -# - LinearSvmBinaryClassifier -# - LogisticRegressionBinaryClassifier -# - FastLinearBinaryClassifier -# - SgdBinaryClassifier -# - SymSgdBinaryClassifier -# - GamBinaryClassifier -# - FastForestBinaryClassifier -# - FastTreesBinaryClassifier -# - LightGbmBinaryClassifier -# - Ranking: -# - LightGbmRanker from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset @@ -69,14 +43,11 @@ print("========== Feature Contributions for Linear Model ==========") print(lr_feature_contributions.head()) # label ... PredictedLabel Score ... FeatureContributions.hours-per-week -# 0 ... 0 -2.010687 ... 0.833069 -# 1 ... 0 -1.216163 ... 0.809928 -# 2 ... 0 -1.248412 ... 0.485957 -# 3 ... 0 -1.132419 ... 0.583148 -# 4 ... 0 -1.969522 ... 0.437361 - -assert 'FeatureContributions.age' in lr_feature_contributions.columns - +# 0 0 ... 0 -2.010687 ... 0.833069 +# 1 0 ... 0 -1.216163 ... 0.809928 +# 2 1 ... 0 -1.248412 ... 0.485957 +# 3 1 ... 0 -1.132419 ... 0.583148 +# 4 0 ... 0 -1.969522 ... 0.437361 # define the training pipeline with a tree model tree_pipeline = Pipeline([FastTreesBinaryClassifier( @@ -105,4 +76,10 @@ # Print predictions with feature contributions, which give a relative measure # of how much each feature impacted the Score. print("========== Feature Contributions for Tree Model ==========") -print(tree_feature_contributions.head()) \ No newline at end of file +print(tree_feature_contributions.head()) +# label ... PredictedLabel Score ... FeatureContributions.hours-per-week +# 0 0 ... 0 -16.717360 ... -0.608664 +# 1 0 ... 0 -7.688200 ... -0.541213 +# 2 1 ... 1 1.571164 ... 0.032862 +# 3 1 ... 1 2.115638 ... 0.537077 +# 4 0 ... 0 -23.038410 ... -0.682764 diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 504555d9..a78a0b44 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1705,8 +1705,39 @@ def get_feature_contributions(self, X, y=None, verbose=0, as_binary_data_stream=False, **params): """ - Return dataframe with raw data, predictions, and feature contributiuons - for the predictions. + Calculates observation level feature contributions. Returns dataframe + with raw data, predictions, and feature contributiuons for each + prediction. Observation level feature contriutions are supported for + the following models: + + * Regression: + + * OrdinaryLeastSquaresRegressor + * FastLinearRegressor + * OnlineGradientDescentRegressor + * PoissonRegressionRegressor + * GamRegressor + * LightGbmRegressor + * FastTreesRegressor + * FastForestRegressor + * FastTreesTweedieRegressor + + * Binary Classification: + + * AveragedPerceptronBinaryClassifier + * LinearSvmBinaryClassifier + * LogisticRegressionBinaryClassifier + * FastLinearBinaryClassifier + * SgdBinaryClassifier + * SymSgdBinaryClassifier + * GamBinaryClassifier + * FastForestBinaryClassifier + * FastTreesBinaryClassifier + * LightGbmBinaryClassifier + + * Ranking: + + * LightGbmRanker :param X: {array-like [n_samples, n_features], :py:class:`nimbusml.FileDataStream` } @@ -1766,32 +1797,16 @@ def get_feature_contributions(self, X, y=None, data="$data", predictor_model="$predictor_model", scored_data="$scoredvectordata") - all_nodes.extend([score_node]) - if hasattr(self, 'steps') and len(self.steps) > 0 \ - and self.last_node.type == 'classifier': - fcc_node = transforms_featurecontributioncalculationtransformer( - data="$scoredvectordata", - predictor_model="$predictor_model", - output_data="$fccData", - top=top, - bottom=bottom, - normalize=True) - convert_label_node = \ - transforms_predictedlabelcolumnoriginalvalueconverter( - data="$fccData", - predicted_label_column="PredictedLabel", - output_data="$output_data") - all_nodes.extend([fcc_node, convert_label_node]) - else: - fcc_node = transforms_featurecontributioncalculationtransformer( - data="$scoredvectordata", - predictor_model="$predictor_model", - output_data="$output_data", - top=top, - bottom=bottom, - normalize=True) - all_nodes.extend([fcc_node]) + fcc_node = transforms_featurecontributioncalculationtransformer( + data="$scoredvectordata", + predictor_model="$predictor_model", + output_data="$output_data", + top=top, + bottom=bottom, + normalize=True) + + all_nodes.extend([score_node, fcc_node]) outputs = dict(output_data="") From b142ab1d936a5e646e99e76af3ff736f9a4bbb10 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 17 Jul 2019 17:37:36 -0700 Subject: [PATCH 24/28] Add test to check feature contributions with pipeline loaded from zip --- .../nimbusml/tests/pipeline/test_load_save.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 3c66253e..448518a9 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -7,7 +7,7 @@ import pickle import unittest -from nimbusml import Pipeline +from nimbusml import Pipeline, FileDataStream from nimbusml.datasets import get_dataset from nimbusml.feature_extraction.categorical import OneHotVectorizer from nimbusml.linear_model import FastLinearBinaryClassifier @@ -212,5 +212,29 @@ def test_unfitted_pickled_pipeline_can_be_fit(self): decimal=2) + def test_pipeline_loaded_from_zip_has_feature_contributions(self): + features = ['age', 'education-num', 'hours-per-week'] + + model_nimbusml = Pipeline( + steps=[FastLinearBinaryClassifier(feature=features)]) + + model_nimbusml.fit(train, label) + + # Save the model to zip + model_filename = 'nimbusml_model.zip' + model_nimbusml.save_model(model_filename) + + # Load the model from zip + model_nimbusml_zip = Pipeline() + model_nimbusml_zip.load_model(model_filename) + + feature_contributions = model_nimbusml_zip.get_feature_contributions( + test, test_label) + + os.remove(model_filename) + + assert ['FeatureContributions.' + feature in feature_contributions.columns + for feature in features] + if __name__ == '__main__': unittest.main() From af7c37cf2665471517dfaca671404e7ac72775f8 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 17 Jul 2019 17:41:56 -0700 Subject: [PATCH 25/28] nit --- src/python/nimbusml/tests/pipeline/test_load_save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 448518a9..880b03d1 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -7,7 +7,7 @@ import pickle import unittest -from nimbusml import Pipeline, FileDataStream +from nimbusml import Pipeline from nimbusml.datasets import get_dataset from nimbusml.feature_extraction.categorical import OneHotVectorizer from nimbusml.linear_model import FastLinearBinaryClassifier From e1f12f441458ee5f103d039d28f589b403a87ed0 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 18 Jul 2019 14:05:32 -0700 Subject: [PATCH 26/28] PR feedback --- src/python/nimbusml/pipeline.py | 48 ++++++++++----------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index a78a0b44..0877ef1e 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1697,13 +1697,8 @@ def getn(n): "ambiguous.") @trace - def get_feature_contributions(self, X, y=None, - evaltype='auto', group_id=None, - weight=None, - top=10, - bottom=10, - verbose=0, - as_binary_data_stream=False, **params): + def get_feature_contributions(self, X, top=10, bottom=10, verbose=0, + as_binary_data_stream=False, **params): """ Calculates observation level feature contributions. Returns dataframe with raw data, predictions, and feature contributiuons for each @@ -1741,16 +1736,6 @@ def get_feature_contributions(self, X, y=None, :param X: {array-like [n_samples, n_features], :py:class:`nimbusml.FileDataStream` } - :param y: {array-like [n_samples]} - - :param evaltype: the evaluation type for the problem, can be { - 'binary', 'multiclass', 'regression', 'cluster', 'anomaly', - 'ranking'}. The default is 'auto'. If model is loaded using the - load_model() method, evaltype cannot be 'auto', and therefore - must be explicitly specified. - :param group_id: the column name for group_id for ranking problem - :param weight: the column name for the weight column for each - sample. :param top: the number of positive contributions with highest magnitude to report. :param bottom: The number of negative contributions with highest @@ -1762,25 +1747,21 @@ def get_feature_contributions(self, X, y=None, if not self._is_fitted: raise ValueError( - "Model is not fitted. Train or load a model before test(" - ").") + "Model is not fitted. Train or load a model before test().") - #print(self.last_node.type) - if y is not None: - if len(self.steps) > 0: - last_node = self.last_node - if last_node.type == 'transform': - raise ValueError( - "Pipeline needs a trainer as last step for test()") + # BUG: If model is loaded from zip file, self.steps is an empty array + # so this condition will always evaluate to False. Consequently, this + # code will never check if the last node is a transform or not. In any + # case, self.last_node will not exist if a model is loaded from zip so + # we could not check for it outside of this condition. + if len(self.steps) > 0: + last_node = self.last_node + if last_node.type == 'transform': + raise ValueError( + "Pipeline needs a trainer as last step for test()") X, y_temp, columns_renamed, feature_columns, label_column, \ - schema, weights, weight_column = self._preprocess_X_y( - X, y, w=weight - ) - - if (not isinstance(y, (str, tuple))) or ( - isinstance(X, DataFrame) and isinstance(y, (str, tuple))): - y = y_temp + schema, weights, weight_column = self._preprocess_X_y(X) all_nodes = [] inputs = dict([('data', ''), ('predictor_model', self.model)]) @@ -1823,7 +1804,6 @@ def get_feature_contributions(self, X, y=None, try: (out_model, out_data, out_metrics) = graph.run( X=X, - y=y, random_state=self.random_state, model=self.model, verbose=verbose, From 1a181bd339cf0c913300530dd7c9a27675b8b005 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 18 Jul 2019 16:44:39 -0700 Subject: [PATCH 27/28] Adding more tests, implement save_model in BasePipelineItem, and PR feedback --- .../internal/core/base_pipeline_item.py | 14 +++ src/python/nimbusml/pipeline.py | 10 +- .../nimbusml/tests/pipeline/test_load_save.py | 100 +++++++++++++++++- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/python/nimbusml/internal/core/base_pipeline_item.py b/src/python/nimbusml/internal/core/base_pipeline_item.py index fa02a3c3..cfd1aee9 100644 --- a/src/python/nimbusml/internal/core/base_pipeline_item.py +++ b/src/python/nimbusml/internal/core/base_pipeline_item.py @@ -15,6 +15,7 @@ from abc import ABCMeta, abstractmethod from collections import OrderedDict from itertools import chain +from shutil import copyfile from textwrap import wrap import six @@ -447,6 +448,19 @@ def get_roles_params(self): res["columns"] = pars return res + @trace + def save_model(self, dst): + """ + Save model to file. For more details, please refer to + `load/save model `_ + + :param dst: filename to be saved with + + """ + if self.model_ is not None: + if os.path.isfile(self.model_): + copyfile(self.model_, dst) + def __getitem__(self, cols): """ Returns a View on this element restricted to the selected column. diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 0877ef1e..93384176 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1702,8 +1702,9 @@ def get_feature_contributions(self, X, top=10, bottom=10, verbose=0, """ Calculates observation level feature contributions. Returns dataframe with raw data, predictions, and feature contributiuons for each - prediction. Observation level feature contriutions are supported for - the following models: + prediction. Feature contributions are not supported for transforms, so + make sure that the last step in a pipeline is a model. Feature + contriutions are supported for the following models: * Regression: @@ -1749,11 +1750,6 @@ def get_feature_contributions(self, X, top=10, bottom=10, verbose=0, raise ValueError( "Model is not fitted. Train or load a model before test().") - # BUG: If model is loaded from zip file, self.steps is an empty array - # so this condition will always evaluate to False. Consequently, this - # code will never check if the last node is a transform or not. In any - # case, self.last_node will not exist if a model is loaded from zip so - # we could not check for it outside of this condition. if len(self.steps) > 0: last_node = self.last_node if last_node.type == 'transform': diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py index 880b03d1..fc112fe5 100644 --- a/src/python/nimbusml/tests/pipeline/test_load_save.py +++ b/src/python/nimbusml/tests/pipeline/test_load_save.py @@ -149,11 +149,22 @@ def test_pipeline_saves_complete_model_file_when_pickled(self): model_nimbusml.fit(train, label) metrics, score = model_nimbusml.test(test, test_label, output_scores=True) + pickle_filename = 'nimbusml_model.p' + + # Save with pickle + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + # Remove the pipeline model from disk so # that the unpickled pipeline is forced # to get its model from the pickled file. os.remove(model_nimbusml.model) + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + os.remove(pickle_filename) + metrics_pickle, score_pickle = model_nimbusml_pickle.test( test, test_label, output_scores=True) @@ -211,30 +222,109 @@ def test_unfitted_pickled_pipeline_can_be_fit(self): metrics_pickle.sum().sum(), decimal=2) + def test_unpickled_pipeline_has_feature_contributions(self): + features = ['age', 'education-num', 'hours-per-week'] + + model_nimbusml = Pipeline( + steps=[FastLinearBinaryClassifier(feature=features)]) + model_nimbusml.fit(train, label) + fc = model_nimbusml.get_feature_contributions(test) + + # Save with pickle + pickle_filename = 'nimbusml_model.p' + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + # Unpickle model + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + fc_pickle = model_nimbusml_pickle.get_feature_contributions(test) + + assert ['FeatureContributions.' + feature in fc_pickle.columns + for feature in features] + + assert [fc['FeatureContributions.' + feature].equals( + fc_pickle['FeatureContributions.' + feature]) + for feature in features] + + os.remove(pickle_filename) + + def test_unpickled_predictor_has_feature_contributions(self): + features = ['age', 'education-num', 'hours-per-week'] + + model_nimbusml = FastLinearBinaryClassifier(feature=features) + model_nimbusml.fit(train, label) + fc = model_nimbusml.get_feature_contributions(test) + + # Save with pickle + pickle_filename = 'nimbusml_model.p' + with open(pickle_filename, 'wb') as f: + pickle.dump(model_nimbusml, f) + # Unpickle model + with open(pickle_filename, "rb") as f: + model_nimbusml_pickle = pickle.load(f) + + fc_pickle = model_nimbusml_pickle.get_feature_contributions(test) + + assert ['FeatureContributions.' + feature in fc_pickle.columns + for feature in features] + + assert [fc['FeatureContributions.' + feature].equals( + fc_pickle['FeatureContributions.' + feature]) + for feature in features] + + os.remove(pickle_filename) def test_pipeline_loaded_from_zip_has_feature_contributions(self): features = ['age', 'education-num', 'hours-per-week'] model_nimbusml = Pipeline( steps=[FastLinearBinaryClassifier(feature=features)]) - model_nimbusml.fit(train, label) + fc = model_nimbusml.get_feature_contributions(test) # Save the model to zip model_filename = 'nimbusml_model.zip' model_nimbusml.save_model(model_filename) - # Load the model from zip model_nimbusml_zip = Pipeline() model_nimbusml_zip.load_model(model_filename) - feature_contributions = model_nimbusml_zip.get_feature_contributions( - test, test_label) + fc_zip = model_nimbusml_zip.get_feature_contributions(test) + + assert ['FeatureContributions.' + feature in fc_zip.columns + for feature in features] + + assert [fc['FeatureContributions.' + feature].equals( + fc_zip['FeatureContributions.' + feature]) + for feature in features] os.remove(model_filename) - assert ['FeatureContributions.' + feature in feature_contributions.columns + def test_predictor_loaded_from_zip_has_feature_contributions(self): + features = ['age', 'education-num', 'hours-per-week'] + + model_nimbusml = FastLinearBinaryClassifier(feature=features) + model_nimbusml.fit(train, label) + fc = model_nimbusml.get_feature_contributions(test) + + # Save the model to zip + model_filename = 'nimbusml_model.zip' + model_nimbusml.save_model(model_filename) + # Load the model from zip + model_nimbusml_zip = Pipeline() + model_nimbusml_zip.load_model(model_filename) + + fc_zip = model_nimbusml_zip.get_feature_contributions(test) + + assert ['FeatureContributions.' + feature in fc_zip.columns for feature in features] + assert [fc['FeatureContributions.' + feature].equals( + fc_zip['FeatureContributions.' + feature]) + for feature in features] + + os.remove(model_filename) + if __name__ == '__main__': unittest.main() From 8546a19843c8d4ce6cdf0fe9ff963a567d0c4808 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 18 Jul 2019 16:52:14 -0700 Subject: [PATCH 28/28] one more conflict --- src/python/nimbusml/pipeline.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py index 4c0515af..93384176 100644 --- a/src/python/nimbusml/pipeline.py +++ b/src/python/nimbusml/pipeline.py @@ -1702,14 +1702,9 @@ def get_feature_contributions(self, X, top=10, bottom=10, verbose=0, """ Calculates observation level feature contributions. Returns dataframe with raw data, predictions, and feature contributiuons for each -<<<<<<< HEAD prediction. Feature contributions are not supported for transforms, so make sure that the last step in a pipeline is a model. Feature contriutions are supported for the following models: -======= - prediction. Observation level feature contriutions are supported for - the following models: ->>>>>>> 93a46fa251ec5fb0b19d162a21c8780fb75e659a * Regression: @@ -1755,14 +1750,6 @@ def get_feature_contributions(self, X, top=10, bottom=10, verbose=0, raise ValueError( "Model is not fitted. Train or load a model before test().") -<<<<<<< HEAD -======= - # BUG: If model is loaded from zip file, self.steps is an empty array - # so this condition will always evaluate to False. Consequently, this - # code will never check if the last node is a transform or not. In any - # case, self.last_node will not exist if a model is loaded from zip so - # we could not check for it outside of this condition. ->>>>>>> 93a46fa251ec5fb0b19d162a21c8780fb75e659a if len(self.steps) > 0: last_node = self.last_node if last_node.type == 'transform':