diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj
index a97e8b14..f694be00 100644
--- a/src/python/nimbusml.pyproj
+++ b/src/python/nimbusml.pyproj
@@ -177,6 +177,7 @@
+
diff --git a/src/python/nimbusml/base_predictor.py b/src/python/nimbusml/base_predictor.py
index bfa2813f..b38d5854 100644
--- a/src/python/nimbusml/base_predictor.py
+++ b/src/python/nimbusml/base_predictor.py
@@ -12,7 +12,6 @@
from sklearn.base import BaseEstimator
from sklearn.utils.multiclass import unique_labels
-from sklearn.utils.validation import check_is_fitted
from . import Pipeline
from .internal.core.base_pipeline_item import BasePipelineItem
@@ -49,8 +48,6 @@ def fit(self, X, y=None, **params):
"Classifier can't train when only one class is "
"present.")
self.classes_ = unique_classes
- self.X_ = X
- self.y_ = y
# Clear cached summary since it should not
# retain its value after a new call to fit
@@ -69,13 +66,24 @@ def fit(self, X, y=None, **params):
set_shape(self, X)
return self
+ @property
+ def _is_fitted(self):
+ """
+ Tells if the predictor was trained.
+ """
+ return (hasattr(self, 'model_') and
+ self.model_ and
+ os.path.isfile(self.model_))
+
@trace
def _invoke_inference_method(self, method, X, **params):
"""
Returns predictions. Can be predicted labels, probabilities
or else decision values.
"""
- check_is_fitted(self, ["X_", "y_"])
+ if not self._is_fitted:
+ raise ValueError("Model is not fitted. "
+ "fit() must be called before {}.".format(method))
# Check that the input is of the same shape as the one passed
# during
@@ -89,6 +97,10 @@ def _invoke_inference_method(self, method, X, **params):
data = getattr(pipeline, method)(X, **params)
return data
+ @trace
+ def get_feature_contributions(self, X, **params):
+ return self._invoke_inference_method('get_feature_contributions', X, **params)
+
@trace
def predict(self, X, **params):
"""
diff --git a/src/python/nimbusml/examples/PipelineWithFeatureContributions.py b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py
new file mode 100644
index 00000000..426241a2
--- /dev/null
+++ b/src/python/nimbusml/examples/PipelineWithFeatureContributions.py
@@ -0,0 +1,85 @@
+###############################################################################
+# Pipeline with observation level feature contributions
+
+# Scoring a dataset with a trained model produces a score, or prediction, for
+# each example. To understand and explain these predictions it can be useful to
+# inspect which features influenced them most significantly. This function
+# computes a model-specific list of per-feature contributions to the score for
+# each example. These contributions can be positive (they make the score
+# higher) or negative (they make the score lower).
+
+from nimbusml import Pipeline, FileDataStream
+from nimbusml.datasets import get_dataset
+from nimbusml.ensemble import FastTreesBinaryClassifier
+from nimbusml.linear_model import LogisticRegressionBinaryClassifier
+
+# data input (as a FileDataStream)
+path = get_dataset('uciadult_train').as_filepath()
+
+data = FileDataStream.read_csv(path)
+print(data.head())
+# label workclass education ... capital-loss hours-per-week
+# 0 0 Private 11th ... 0 40
+# 1 0 Private HS-grad ... 0 50
+# 2 1 Local-gov Assoc-acdm ... 0 40
+# 3 1 Private Some-college ... 0 40
+# 4 0 ? Some-college ... 0 30
+
+# define the training pipeline with a linear model
+lr_pipeline = Pipeline([LogisticRegressionBinaryClassifier(
+ feature=['age', 'education-num', 'hours-per-week'], label='label')])
+
+# train the model
+lr_model = lr_pipeline.fit(data)
+
+# For linear models, the contribution of a given feature is equal to the
+# product of feature value times the corresponding weight. Similarly, for
+# Generalized Additive Models (GAM), the contribution of a feature is equal to
+# the shape function for the given feature evaluated at the feature value.
+lr_feature_contributions = lr_model.get_feature_contributions(data)
+
+# Print predictions with feature contributions, which give a relative measure
+# of how much each feature impacted the Score.
+print("========== Feature Contributions for Linear Model ==========")
+print(lr_feature_contributions.head())
+# label ... PredictedLabel Score ... FeatureContributions.hours-per-week
+# 0 0 ... 0 -2.010687 ... 0.833069
+# 1 0 ... 0 -1.216163 ... 0.809928
+# 2 1 ... 0 -1.248412 ... 0.485957
+# 3 1 ... 0 -1.132419 ... 0.583148
+# 4 0 ... 0 -1.969522 ... 0.437361
+
+# define the training pipeline with a tree model
+tree_pipeline = Pipeline([FastTreesBinaryClassifier(
+ feature=['age', 'education-num', 'hours-per-week'], label='label')])
+
+# train the model
+tree_model = tree_pipeline.fit(data)
+
+# For tree-based models, the calculation of feature contribution essentially
+# consists in determining which splits in the tree have the most impact on the
+# final score and assigning the value of the impact to the features determining
+# the split. More precisely, the contribution of a feature is equal to the
+# change in score produced by exploring the opposite sub-tree every time a
+# decision node for the given feature is encountered.
+#
+# Consider a simple case with a single decision tree that has a decision node
+# for the binary feature F1. Given an example that has feature F1 equal to
+# true, we can calculate the score it would have obtained if we chose the
+# subtree corresponding to the feature F1 being equal to false while keeping
+# the other features constant. The contribution of feature F1 for the given
+# example is the difference between the original score and the score obtained
+# by taking the opposite decision at the node corresponding to feature F1. This
+# algorithm extends naturally to models with many decision trees.
+tree_feature_contributions = tree_model.get_feature_contributions(data)
+
+# Print predictions with feature contributions, which give a relative measure
+# of how much each feature impacted the Score.
+print("========== Feature Contributions for Tree Model ==========")
+print(tree_feature_contributions.head())
+# label ... PredictedLabel Score ... FeatureContributions.hours-per-week
+# 0 0 ... 0 -16.717360 ... -0.608664
+# 1 0 ... 0 -7.688200 ... -0.541213
+# 2 1 ... 1 1.571164 ... 0.032862
+# 3 1 ... 1 2.115638 ... 0.537077
+# 4 0 ... 0 -23.038410 ... -0.682764
diff --git a/src/python/nimbusml/internal/core/base_pipeline_item.py b/src/python/nimbusml/internal/core/base_pipeline_item.py
index e45b3d0c..cfd1aee9 100644
--- a/src/python/nimbusml/internal/core/base_pipeline_item.py
+++ b/src/python/nimbusml/internal/core/base_pipeline_item.py
@@ -15,6 +15,7 @@
from abc import ABCMeta, abstractmethod
from collections import OrderedDict
from itertools import chain
+from shutil import copyfile
from textwrap import wrap
import six
@@ -375,6 +376,8 @@ def _get_node(self, **params):
def __getstate__(self):
"Selects what to pickle."
odict = self.__dict__.copy()
+ odict['export_version'] = 1
+
if hasattr(self, 'model_') and \
self.model_ is not None and os.path.isfile(self.model_):
with open(self.model_, "rb") as mfile:
@@ -387,8 +390,11 @@ def __getstate__(self):
def __setstate__(self, state):
"Restore a pickled object."
for k, v in state.items():
- if k not in {'modelbytes', 'type'}:
+ if k not in {'modelbytes', 'type', 'export_version'}:
setattr(self, k, v)
+
+ # Note: modelbytes and type were
+ # added before export_version 1
if 'modelbytes' in state:
(fd, modelfile) = tempfile.mkstemp()
fl = os.fdopen(fd, "wb")
@@ -442,6 +448,19 @@ def get_roles_params(self):
res["columns"] = pars
return res
+ @trace
+ def save_model(self, dst):
+ """
+ Save model to file. For more details, please refer to
+ `load/save model `_
+
+ :param dst: filename to be saved with
+
+ """
+ if self.model_ is not None:
+ if os.path.isfile(self.model_):
+ copyfile(self.model_, dst)
+
def __getitem__(self, cols):
"""
Returns a View on this element restricted to the selected column.
diff --git a/src/python/nimbusml/pipeline.py b/src/python/nimbusml/pipeline.py
index 5a15bac4..93384176 100644
--- a/src/python/nimbusml/pipeline.py
+++ b/src/python/nimbusml/pipeline.py
@@ -5,6 +5,7 @@
import inspect
import itertools
import os
+import tempfile
import time
import warnings
from collections import OrderedDict, namedtuple, defaultdict
@@ -40,6 +41,8 @@
transforms_datasetscorer
from .internal.entrypoints.transforms_featurecombiner import \
transforms_featurecombiner
+from .internal.entrypoints.transforms_featurecontributioncalculationtransformer import \
+ transforms_featurecontributioncalculationtransformer
from .internal.entrypoints.transforms_labelcolumnkeybooleanconverter \
import \
transforms_labelcolumnkeybooleanconverter
@@ -1693,6 +1696,120 @@ def getn(n):
"only fit(X) is allowed or the training becomes "
"ambiguous.")
+ @trace
+ def get_feature_contributions(self, X, top=10, bottom=10, verbose=0,
+ as_binary_data_stream=False, **params):
+ """
+ Calculates observation level feature contributions. Returns dataframe
+ with raw data, predictions, and feature contributiuons for each
+ prediction. Feature contributions are not supported for transforms, so
+ make sure that the last step in a pipeline is a model. Feature
+ contriutions are supported for the following models:
+
+ * Regression:
+
+ * OrdinaryLeastSquaresRegressor
+ * FastLinearRegressor
+ * OnlineGradientDescentRegressor
+ * PoissonRegressionRegressor
+ * GamRegressor
+ * LightGbmRegressor
+ * FastTreesRegressor
+ * FastForestRegressor
+ * FastTreesTweedieRegressor
+
+ * Binary Classification:
+
+ * AveragedPerceptronBinaryClassifier
+ * LinearSvmBinaryClassifier
+ * LogisticRegressionBinaryClassifier
+ * FastLinearBinaryClassifier
+ * SgdBinaryClassifier
+ * SymSgdBinaryClassifier
+ * GamBinaryClassifier
+ * FastForestBinaryClassifier
+ * FastTreesBinaryClassifier
+ * LightGbmBinaryClassifier
+
+ * Ranking:
+
+ * LightGbmRanker
+
+ :param X: {array-like [n_samples, n_features],
+ :py:class:`nimbusml.FileDataStream` }
+ :param top: the number of positive contributions with highest magnitude
+ to report.
+ :param bottom: The number of negative contributions with highest
+ magnitude to report.
+ :return: dataframe of containing the raw data, predicted label, score,
+ probabilities, and feature contributions.
+ """
+ self.verbose = verbose
+
+ if not self._is_fitted:
+ raise ValueError(
+ "Model is not fitted. Train or load a model before test().")
+
+ if len(self.steps) > 0:
+ last_node = self.last_node
+ if last_node.type == 'transform':
+ raise ValueError(
+ "Pipeline needs a trainer as last step for test()")
+
+ X, y_temp, columns_renamed, feature_columns, label_column, \
+ schema, weights, weight_column = self._preprocess_X_y(X)
+
+ all_nodes = []
+ inputs = dict([('data', ''), ('predictor_model', self.model)])
+ if isinstance(X, FileDataStream):
+ importtext_node = data_customtextloader(
+ input_file="$file",
+ data="$data",
+ custom_schema=schema.to_string(
+ add_sep=True))
+ all_nodes = [importtext_node]
+ inputs = dict([('file', ''), ('predictor_model', self.model)])
+
+ score_node = transforms_datasetscorer(
+ data="$data",
+ predictor_model="$predictor_model",
+ scored_data="$scoredvectordata")
+
+ fcc_node = transforms_featurecontributioncalculationtransformer(
+ data="$scoredvectordata",
+ predictor_model="$predictor_model",
+ output_data="$output_data",
+ top=top,
+ bottom=bottom,
+ normalize=True)
+
+ all_nodes.extend([score_node, fcc_node])
+
+ outputs = dict(output_data="")
+
+ graph = Graph(
+ inputs,
+ outputs,
+ as_binary_data_stream,
+ *all_nodes)
+
+ class_name = type(self).__name__
+ method_name = inspect.currentframe().f_code.co_name
+ telemetry_info = ".".join([class_name, method_name])
+
+ try:
+ (out_model, out_data, out_metrics) = graph.run(
+ X=X,
+ random_state=self.random_state,
+ model=self.model,
+ verbose=verbose,
+ telemetry_info=telemetry_info,
+ **params)
+ except RuntimeError as e:
+ raise e
+
+ return out_data
+
@trace
def _predict(self, X, y=None,
evaltype='auto', group_id=None,
@@ -1942,7 +2059,7 @@ def test(
otherwise None
in the returned tuple.
:return: tuple (dataframe of evaluation metrics, dataframe of
- scores). Is scores are
+ scores). If scores are
required, set `output_scores`=True, otherwise None is
returned by default.
"""
@@ -2265,6 +2382,38 @@ def load_model(self, src):
self.model = src
self.steps = []
+ def __getstate__(self):
+ odict = {'export_version': 1}
+
+ if hasattr(self, 'steps'):
+ odict['steps'] = self.steps
+
+ if (hasattr(self, 'model') and
+ self.model is not None and
+ os.path.isfile(self.model)):
+
+ with open(self.model, "rb") as f:
+ odict['modelbytes'] = f.read()
+
+ return odict
+
+ def __setstate__(self, state):
+ self.steps = []
+ self.model = None
+ self.random_state = None
+
+ for k, v in state.items():
+ if k not in {'modelbytes', 'export_version'}:
+ setattr(self, k, v)
+
+ if state.get('export_version', 0) == 1:
+ if 'modelbytes' in state:
+ (fd, modelfile) = tempfile.mkstemp()
+ fl = os.fdopen(fd, "wb")
+ fl.write(state['modelbytes'])
+ fl.close()
+ self.model = modelfile
+
@trace
def score(
self,
diff --git a/src/python/nimbusml/tests/pipeline/test_load_save.py b/src/python/nimbusml/tests/pipeline/test_load_save.py
index 309650b5..339c9664 100644
--- a/src/python/nimbusml/tests/pipeline/test_load_save.py
+++ b/src/python/nimbusml/tests/pipeline/test_load_save.py
@@ -3,6 +3,7 @@
# Licensed under the MIT License.
# --------------------------------------------------------------------------------------------
+import os
import pickle
import unittest
@@ -44,8 +45,14 @@ def test_model_dataframe(self):
model_nimbusml.fit(train, label)
# Save with pickle
- pickle.dump(model_nimbusml, open('nimbusml_model.p', 'wb'))
- model_nimbusml_pickle = pickle.load(open("nimbusml_model.p", "rb"))
+ pickle_filename = 'nimbusml_model.p'
+ with open(pickle_filename, 'wb') as f:
+ pickle.dump(model_nimbusml, f)
+
+ with open(pickle_filename, "rb") as f:
+ model_nimbusml_pickle = pickle.load(f)
+
+ os.remove(pickle_filename)
score1 = model_nimbusml.predict(test).head(5)
score2 = model_nimbusml_pickle.predict(test).head(5)
@@ -72,6 +79,8 @@ def test_model_dataframe(self):
model_nimbusml_load.sum().sum(),
decimal=2)
+ os.remove('model.nimbusml.m')
+
def test_model_datastream(self):
model_nimbusml = Pipeline(
steps=[
@@ -85,8 +94,14 @@ def test_model_datastream(self):
model_nimbusml.fit(train, label)
# Save with pickle
- pickle.dump(model_nimbusml, open('nimbusml_model.p', 'wb'))
- model_nimbusml_pickle = pickle.load(open("nimbusml_model.p", "rb"))
+ pickle_filename = 'nimbusml_model.p'
+ with open(pickle_filename, 'wb') as f:
+ pickle.dump(model_nimbusml, f)
+
+ with open(pickle_filename, "rb") as f:
+ model_nimbusml_pickle = pickle.load(f)
+
+ os.remove(pickle_filename)
score1 = model_nimbusml.predict(test).head(5)
score2 = model_nimbusml_pickle.predict(test).head(5)
@@ -119,6 +134,198 @@ def test_model_datastream(self):
model_nimbusml_load.sum().sum(),
decimal=2)
+ os.remove('model.nimbusml.m')
+
+ def test_pipeline_saves_complete_model_file_when_pickled(self):
+ model_nimbusml = Pipeline(
+ steps=[
+ ('cat',
+ OneHotVectorizer() << categorical_columns),
+ ('linear',
+ FastLinearBinaryClassifier(
+ shuffle=False,
+ number_of_threads=1))])
+
+ model_nimbusml.fit(train, label)
+ metrics, score = model_nimbusml.test(test, test_label, output_scores=True)
+
+ pickle_filename = 'nimbusml_model.p'
+
+ # Save with pickle
+ with open(pickle_filename, 'wb') as f:
+ pickle.dump(model_nimbusml, f)
+
+ # Remove the pipeline model from disk so
+ # that the unpickled pipeline is forced
+ # to get its model from the pickled file.
+ os.remove(model_nimbusml.model)
+
+ with open(pickle_filename, "rb") as f:
+ model_nimbusml_pickle = pickle.load(f)
+
+ os.remove(pickle_filename)
+
+ metrics_pickle, score_pickle = model_nimbusml_pickle.test(
+ test, test_label, output_scores=True)
+
+ assert_almost_equal(score.sum().sum(),
+ score_pickle.sum().sum(),
+ decimal=2)
+
+ assert_almost_equal(metrics.sum().sum(),
+ metrics_pickle.sum().sum(),
+ decimal=2)
+
+ def test_unfitted_pickled_pipeline_can_be_fit(self):
+ pipeline = Pipeline(
+ steps=[
+ ('cat',
+ OneHotVectorizer() << categorical_columns),
+ ('linear',
+ FastLinearBinaryClassifier(
+ shuffle=False,
+ number_of_threads=1))])
+
+ pipeline.fit(train, label)
+ metrics, score = pipeline.test(test, test_label, output_scores=True)
+
+ # Create a new unfitted pipeline
+ pipeline = Pipeline(
+ steps=[
+ ('cat',
+ OneHotVectorizer() << categorical_columns),
+ ('linear',
+ FastLinearBinaryClassifier(
+ shuffle=False,
+ number_of_threads=1))])
+
+ pickle_filename = 'nimbusml_model.p'
+
+ # Save with pickle
+ with open(pickle_filename, 'wb') as f:
+ pickle.dump(pipeline, f)
+
+ with open(pickle_filename, "rb") as f:
+ pipeline_pickle = pickle.load(f)
+
+ os.remove(pickle_filename)
+
+ pipeline_pickle.fit(train, label)
+ metrics_pickle, score_pickle = pipeline_pickle.test(
+ test, test_label, output_scores=True)
+
+ assert_almost_equal(score.sum().sum(),
+ score_pickle.sum().sum(),
+ decimal=2)
+
+ assert_almost_equal(metrics.sum().sum(),
+ metrics_pickle.sum().sum(),
+ decimal=2)
+
+ def test_unpickled_pipeline_has_feature_contributions(self):
+ features = ['age', 'education-num', 'hours-per-week']
+
+ model_nimbusml = Pipeline(
+ steps=[FastLinearBinaryClassifier(feature=features)])
+ model_nimbusml.fit(train, label)
+ fc = model_nimbusml.get_feature_contributions(test)
+
+ # Save with pickle
+ pickle_filename = 'nimbusml_model.p'
+ with open(pickle_filename, 'wb') as f:
+ pickle.dump(model_nimbusml, f)
+ # Unpickle model
+ with open(pickle_filename, "rb") as f:
+ model_nimbusml_pickle = pickle.load(f)
+
+ fc_pickle = model_nimbusml_pickle.get_feature_contributions(test)
+
+ assert ['FeatureContributions.' + feature in fc_pickle.columns
+ for feature in features]
+
+ assert [fc['FeatureContributions.' + feature].equals(
+ fc_pickle['FeatureContributions.' + feature])
+ for feature in features]
+
+ os.remove(pickle_filename)
+
+ def test_unpickled_predictor_has_feature_contributions(self):
+ features = ['age', 'education-num', 'hours-per-week']
+
+ model_nimbusml = FastLinearBinaryClassifier(feature=features)
+ model_nimbusml.fit(train, label)
+ fc = model_nimbusml.get_feature_contributions(test)
+
+ # Save with pickle
+ pickle_filename = 'nimbusml_model.p'
+ with open(pickle_filename, 'wb') as f:
+ pickle.dump(model_nimbusml, f)
+ # Unpickle model
+ with open(pickle_filename, "rb") as f:
+ model_nimbusml_pickle = pickle.load(f)
+
+ fc_pickle = model_nimbusml_pickle.get_feature_contributions(test)
+
+ assert ['FeatureContributions.' + feature in fc_pickle.columns
+ for feature in features]
+
+ assert [fc['FeatureContributions.' + feature].equals(
+ fc_pickle['FeatureContributions.' + feature])
+ for feature in features]
+
+ os.remove(pickle_filename)
+
+ def test_pipeline_loaded_from_zip_has_feature_contributions(self):
+ features = ['age', 'education-num', 'hours-per-week']
+
+ model_nimbusml = Pipeline(
+ steps=[FastLinearBinaryClassifier(feature=features)])
+ model_nimbusml.fit(train, label)
+ fc = model_nimbusml.get_feature_contributions(test)
+
+ # Save the model to zip
+ model_filename = 'nimbusml_model.zip'
+ model_nimbusml.save_model(model_filename)
+ # Load the model from zip
+ model_nimbusml_zip = Pipeline()
+ model_nimbusml_zip.load_model(model_filename)
+
+ fc_zip = model_nimbusml_zip.get_feature_contributions(test)
+
+ assert ['FeatureContributions.' + feature in fc_zip.columns
+ for feature in features]
+
+ assert [fc['FeatureContributions.' + feature].equals(
+ fc_zip['FeatureContributions.' + feature])
+ for feature in features]
+
+ os.remove(model_filename)
+
+ def test_predictor_loaded_from_zip_has_feature_contributions(self):
+ features = ['age', 'education-num', 'hours-per-week']
+
+ model_nimbusml = FastLinearBinaryClassifier(feature=features)
+ model_nimbusml.fit(train, label)
+ fc = model_nimbusml.get_feature_contributions(test)
+
+ # Save the model to zip
+ model_filename = 'nimbusml_model.zip'
+ model_nimbusml.save_model(model_filename)
+ # Load the model from zip
+ model_nimbusml_zip = Pipeline()
+ model_nimbusml_zip.load_model(model_filename)
+
+ fc_zip = model_nimbusml_zip.get_feature_contributions(test)
+
+ assert ['FeatureContributions.' + feature in fc_zip.columns
+ for feature in features]
+
+ assert [fc['FeatureContributions.' + feature].equals(
+ fc_zip['FeatureContributions.' + feature])
+ for feature in features]
+
+ os.remove(model_filename)
+
if __name__ == '__main__':
unittest.main()
diff --git a/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py b/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py
index 503c21a6..a08ce3b9 100644
--- a/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py
+++ b/src/python/nimbusml/tests/scikit/test_uci_adult_scikit.py
@@ -3,6 +3,7 @@
# Licensed under the MIT License.
# --------------------------------------------------------------------------------------------
+import os
import pickle
import unittest
@@ -111,6 +112,7 @@ def test_pickle_predictor(self):
# Unpickle model and score. We should get the exact same accuracy as
# above
s = pickle.dumps(ftree)
+ os.remove(ftree.model_)
ftree2 = pickle.loads(s)
scores2 = ftree2.predict(X_test)
accu2 = np.mean(y_test.values.ravel() == scores2.values)
@@ -130,6 +132,7 @@ def test_pickle_transform(self):
# Unpickle transform and generate output.
# We should get the exact same output as above
s = pickle.dumps(cat)
+ os.remove(cat.model_)
cat2 = pickle.loads(s)
out2 = cat2.transform(X_train)
assert_equal(
@@ -158,7 +161,10 @@ def test_pickle_pipeline(self):
# Unpickle model and score. We should get the exact same accuracy as
# above
s = pickle.dumps(pipe)
+ os.remove(cat.model_)
+ os.remove(ftree.model_)
pipe2 = pickle.loads(s)
+
scores2 = pipe2.predict(X_test)
accu2 = np.mean(y_test.values.ravel() == scores2.values)
assert_equal(