diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index 5daea049..85d00fda 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -122,6 +122,7 @@ + @@ -175,6 +176,8 @@ + + @@ -232,6 +235,7 @@ + @@ -279,6 +283,7 @@ + @@ -296,6 +301,7 @@ + @@ -626,6 +632,7 @@ + @@ -667,6 +674,7 @@ + diff --git a/src/python/nimbusml/examples/LpScaler.py b/src/python/nimbusml/examples/LpScaler.py new file mode 100644 index 00000000..7b80d69a --- /dev/null +++ b/src/python/nimbusml/examples/LpScaler.py @@ -0,0 +1,35 @@ +############################################################################### +# MinMaxScaler +import numpy +from nimbusml import FileDataStream +from nimbusml.datasets import get_dataset +from nimbusml.preprocessing.normalization import LpScaler + +# data input (as a FileDataStream) +path = get_dataset('infert').as_filepath() +data = FileDataStream.read_csv( + path, + sep=',', + numeric_dtype=numpy.float32) # Error with integer input +print(data.head()) +# age case education induced parity pooled.stratum row_num ... +# 0 26.0 1.0 0-5yrs 1.0 6.0 3.0 1.0 ... +# 1 42.0 1.0 0-5yrs 1.0 1.0 1.0 2.0 ... +# 2 39.0 1.0 0-5yrs 2.0 6.0 4.0 3.0 ... +# 3 34.0 1.0 0-5yrs 2.0 4.0 2.0 4.0 ... +# 4 35.0 1.0 6-11yrs 1.0 3.0 32.0 5.0 ... + +# transform usage +xf = LpScaler(columns={'in': 'induced', 'sp': 'spontaneous'}) + +# fit and transform +features = xf.fit_transform(data) + +# print features +print(features.head()) +# age case education in ... pooled.stratum row_num sp ... +# 0 26.0 1.0 0-5yrs 0.5 ... 3.0 1.0 1.0 ... +# 1 42.0 1.0 0-5yrs 0.5 ... 1.0 2.0 0.0 ... +# 2 39.0 1.0 0-5yrs 1.0 ... 4.0 3.0 0.0 ... +# 3 34.0 1.0 0-5yrs 1.0 ... 2.0 4.0 0.0 ... +# 4 35.0 1.0 6-11yrs 0.5 ... 32.0 5.0 0.5 ... diff --git a/src/python/nimbusml/examples/NGramExtractor.py b/src/python/nimbusml/examples/NGramExtractor.py new file mode 100644 index 00000000..9df254d3 --- /dev/null +++ b/src/python/nimbusml/examples/NGramExtractor.py @@ -0,0 +1,39 @@ +############################################################################### +# NGramFeaturizer +from nimbusml import FileDataStream, Pipeline +from nimbusml.datasets import get_dataset +from nimbusml.preprocessing.schema import ColumnDropper +from nimbusml.preprocessing.text import CharTokenizer +from nimbusml.feature_extraction.text import NGramExtractor +from nimbusml.feature_extraction.text.extractor import Ngram + +# data input (as a FileDataStream) +path = get_dataset("wiki_detox_train").as_filepath() + +data = FileDataStream.read_csv(path, sep='\t') +print(data.head()) +# Sentiment SentimentText +# 0 1 ==RUDE== Dude, you are rude upload that carl p... +# 1 1 == OK! == IM GOING TO VANDALIZE WILD ONES WIK... +# 2 1 Stop trolling, zapatancas, calling me a liar m... +# 3 1 ==You're cool== You seem like a really cool g... +# 4 1 ::::: Why are you threatening me? I'm not bein... + +# transform usage +pipe = Pipeline([ + CharTokenizer(columns={'SentimentText_Transform': 'SentimentText'}), + NGramExtractor(ngram_length=1, all_lengths=False, columns={'Ngrams': 'SentimentText_Transform'}), + ColumnDropper(columns=['SentimentText_Transform', 'SentimentText', 'Sentiment']) + ]) + +# fit and transform +features = pipe.fit_transform(data) + +# print features +print(features.head()) +# Sentiment ... features.douchiest features.award. +# 0 1 ... 0.0 0.0 +# 1 1 ... 0.0 0.0 +# 2 1 ... 0.0 0.0 +# 3 1 ... 0.0 0.0 +# 4 1 ... 0.0 0.0 diff --git a/src/python/nimbusml/examples/examples_from_dataframe/LpScaler_df.py b/src/python/nimbusml/examples/examples_from_dataframe/LpScaler_df.py new file mode 100644 index 00000000..aaaa576a --- /dev/null +++ b/src/python/nimbusml/examples/examples_from_dataframe/LpScaler_df.py @@ -0,0 +1,20 @@ +############################################################################### +# MinMaxScaler +import pandas as pd +from nimbusml.preprocessing.normalization import LpScaler + +in_df = pd.DataFrame( + data=dict( + Sepal_Length=[ + 2.5, 1, 2.1, 1.0], Sepal_Width=[ + .75, .9, .8, .76], Petal_Length=[ + 0, 2.5, 2.6, 2.4], Species=[ + "setosa", "viginica", "setosa", 'versicolor'])) + +# generate two new Columns - Petal_Normed and Sepal_Normed +normed = LpScaler() << { + 'Petal_Normed': 'Petal_Length', + 'Sepal_Normed': 'Sepal_Width'} +out_df = normed.fit_transform(in_df) + +print('LpScaler\n', (out_df)) diff --git a/src/python/nimbusml/feature_extraction/text/__init__.py b/src/python/nimbusml/feature_extraction/text/__init__.py index 7dbd24cf..f70aae97 100644 --- a/src/python/nimbusml/feature_extraction/text/__init__.py +++ b/src/python/nimbusml/feature_extraction/text/__init__.py @@ -1,4 +1,5 @@ from .lightlda import LightLda +from .ngramextractor import NGramExtractor from .ngramfeaturizer import NGramFeaturizer from .sentiment import Sentiment from .wordembedding import WordEmbedding @@ -6,6 +7,7 @@ __all__ = [ 'LightLda', 'NGramFeaturizer', + 'NGramExtractor', 'Sentiment', 'WordEmbedding' ] diff --git a/src/python/nimbusml/feature_extraction/text/ngramextractor.py b/src/python/nimbusml/feature_extraction/text/ngramextractor.py new file mode 100644 index 00000000..f27b7004 --- /dev/null +++ b/src/python/nimbusml/feature_extraction/text/ngramextractor.py @@ -0,0 +1,72 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +# - Generated by tools/entrypoint_compiler.py: do not edit by hand +""" +NGramExtractor +""" + +__all__ = ["NGramExtractor"] + + +from sklearn.base import TransformerMixin + +from ...base_transform import BaseTransform +from ...internal.core.feature_extraction.text.ngramextractor import \ + NGramExtractor as core +from ...internal.utils.utils import trace + + +class NGramExtractor(core, BaseTransform, TransformerMixin): + """ + **Description** + Produces a bag of counts of n-grams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of n-grams and using the id in the dictionary as the index in the bag. + + :param columns: see `Columns `_. + + :param ngram_length: Maximum n-gram length. + + :param all_lengths: Whether to store all n-gram lengths up to ngramLength, + or only ngramLength. + + :param skip_length: Maximum number of tokens to skip when constructing an + n-gram. + + :param max_num_terms: Maximum number of n-grams to store in the dictionary. + + :param weighting: The weighting criteria. + + :param params: Additional arguments sent to compute engine. + + """ + + @trace + def __init__( + self, + ngram_length=2, + all_lengths=True, + skip_length=0, + max_num_terms=[10000000], + weighting='Tf', + columns=None, + **params): + + if columns: + params['columns'] = columns + BaseTransform.__init__(self, **params) + core.__init__( + self, + ngram_length=ngram_length, + all_lengths=all_lengths, + skip_length=skip_length, + max_num_terms=max_num_terms, + weighting=weighting, + **params) + self._columns = columns + + def get_params(self, deep=False): + """ + Get the parameters for this operator. + """ + return core.get_params(self) diff --git a/src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py b/src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py new file mode 100644 index 00000000..c627addd --- /dev/null +++ b/src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py @@ -0,0 +1,111 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +# - Generated by tools/entrypoint_compiler.py: do not edit by hand +""" +NGramExtractor +""" + +__all__ = ["NGramExtractor"] + + +from ....entrypoints.transforms_ngramtranslator import \ + transforms_ngramtranslator +from ....utils.utils import trace +from ...base_pipeline_item import BasePipelineItem, DefaultSignature + + +class NGramExtractor(BasePipelineItem, DefaultSignature): + """ + **Description** + Produces a bag of counts of n-grams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of n-grams and using the id in the dictionary as the index in the bag. + + :param ngram_length: Maximum n-gram length. + + :param all_lengths: Whether to store all n-gram lengths up to ngramLength, + or only ngramLength. + + :param skip_length: Maximum number of tokens to skip when constructing an + n-gram. + + :param max_num_terms: Maximum number of n-grams to store in the dictionary. + + :param weighting: The weighting criteria. + + :param params: Additional arguments sent to compute engine. + + """ + + @trace + def __init__( + self, + ngram_length=2, + all_lengths=True, + skip_length=0, + max_num_terms=[10000000], + weighting='Tf', + **params): + BasePipelineItem.__init__( + self, type='transform', **params) + + self.ngram_length = ngram_length + self.all_lengths = all_lengths + self.skip_length = skip_length + self.max_num_terms = max_num_terms + self.weighting = weighting + + @property + def _entrypoint(self): + return transforms_ngramtranslator + + @trace + def _get_node(self, **all_args): + + input_columns = self.input + if input_columns is None and 'input' in all_args: + input_columns = all_args['input'] + if 'input' in all_args: + all_args.pop('input') + + output_columns = self.output + if output_columns is None and 'output' in all_args: + output_columns = all_args['output'] + if 'output' in all_args: + all_args.pop('output') + + # validate input + if input_columns is None: + raise ValueError( + "'None' input passed when it cannot be none.") + + if not isinstance(input_columns, list): + raise ValueError( + "input has to be a list of strings, instead got %s" % + type(input_columns)) + + # validate output + if output_columns is None: + output_columns = input_columns + + if not isinstance(output_columns, list): + raise ValueError( + "output has to be a list of strings, instead got %s" % + type(output_columns)) + + algo_args = dict( + column=[ + dict( + Source=i, + Name=o) for i, + o in zip( + input_columns, + output_columns)] if input_columns else None, + ngram_length=self.ngram_length, + all_lengths=self.all_lengths, + skip_length=self.skip_length, + max_num_terms=self.max_num_terms, + weighting=self.weighting) + + all_args.update(algo_args) + return self._entrypoint(**all_args) diff --git a/src/python/nimbusml/internal/core/preprocessing/normalization/lpscaler.py b/src/python/nimbusml/internal/core/preprocessing/normalization/lpscaler.py new file mode 100644 index 00000000..3dce5d56 --- /dev/null +++ b/src/python/nimbusml/internal/core/preprocessing/normalization/lpscaler.py @@ -0,0 +1,93 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +# - Generated by tools/entrypoint_compiler.py: do not edit by hand +""" +LpScaler +""" + +__all__ = ["LpScaler"] + + +from ....entrypoints.transforms_lpnormalizer import transforms_lpnormalizer +from ....utils.utils import trace +from ...base_pipeline_item import BasePipelineItem, DefaultSignature + + +class LpScaler(BasePipelineItem, DefaultSignature): + """ + **Description** + Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf). Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D is either L2 norm, L1 norm or LInf norm. + + :param norm: The norm to use to normalize each sample. + + :param sub_mean: Subtract mean from each value before normalizing. + + :param params: Additional arguments sent to compute engine. + + """ + + @trace + def __init__( + self, + norm='L2', + sub_mean=False, + **params): + BasePipelineItem.__init__( + self, type='transform', **params) + + self.norm = norm + self.sub_mean = sub_mean + + @property + def _entrypoint(self): + return transforms_lpnormalizer + + @trace + def _get_node(self, **all_args): + + input_columns = self.input + if input_columns is None and 'input' in all_args: + input_columns = all_args['input'] + if 'input' in all_args: + all_args.pop('input') + + output_columns = self.output + if output_columns is None and 'output' in all_args: + output_columns = all_args['output'] + if 'output' in all_args: + all_args.pop('output') + + # validate input + if input_columns is None: + raise ValueError( + "'None' input passed when it cannot be none.") + + if not isinstance(input_columns, list): + raise ValueError( + "input has to be a list of strings, instead got %s" % + type(input_columns)) + + # validate output + if output_columns is None: + output_columns = input_columns + + if not isinstance(output_columns, list): + raise ValueError( + "output has to be a list of strings, instead got %s" % + type(output_columns)) + + algo_args = dict( + column=[ + dict( + Source=i, + Name=o) for i, + o in zip( + input_columns, + output_columns)] if input_columns else None, + norm=self.norm, + sub_mean=self.sub_mean) + + all_args.update(algo_args) + return self._entrypoint(**all_args) diff --git a/src/python/nimbusml/preprocessing/normalization/__init__.py b/src/python/nimbusml/preprocessing/normalization/__init__.py index 5036a49a..788ebf67 100644 --- a/src/python/nimbusml/preprocessing/normalization/__init__.py +++ b/src/python/nimbusml/preprocessing/normalization/__init__.py @@ -1,6 +1,7 @@ from .binner import Binner from .globalcontrastrowscaler import GlobalContrastRowScaler from .logmeanvariancescaler import LogMeanVarianceScaler +from .lpscaler import LpScaler from .meanvariancescaler import MeanVarianceScaler from .minmaxscaler import MinMaxScaler @@ -8,6 +9,7 @@ 'Binner', 'GlobalContrastRowScaler', 'LogMeanVarianceScaler', + 'LpScaler', 'MeanVarianceScaler', 'MinMaxScaler', ] diff --git a/src/python/nimbusml/preprocessing/normalization/lpscaler.py b/src/python/nimbusml/preprocessing/normalization/lpscaler.py new file mode 100644 index 00000000..f47a8cdf --- /dev/null +++ b/src/python/nimbusml/preprocessing/normalization/lpscaler.py @@ -0,0 +1,58 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +# - Generated by tools/entrypoint_compiler.py: do not edit by hand +""" +LpScaler +""" + +__all__ = ["LpScaler"] + + +from sklearn.base import TransformerMixin + +from ...base_transform import BaseTransform +from ...internal.core.preprocessing.normalization.lpscaler import \ + LpScaler as core +from ...internal.utils.utils import trace + + +class LpScaler(core, BaseTransform, TransformerMixin): + """ + **Description** + Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf). Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D is either L2 norm, L1 norm or LInf norm. + + :param columns: see `Columns `_. + + :param norm: The norm to use to normalize each sample. + + :param sub_mean: Subtract mean from each value before normalizing. + + :param params: Additional arguments sent to compute engine. + + """ + + @trace + def __init__( + self, + norm='L2', + sub_mean=False, + columns=None, + **params): + + if columns: + params['columns'] = columns + BaseTransform.__init__(self, **params) + core.__init__( + self, + norm=norm, + sub_mean=sub_mean, + **params) + self._columns = columns + + def get_params(self, deep=False): + """ + Get the parameters for this operator. + """ + return core.get_params(self) diff --git a/src/python/nimbusml/tests/preprocessing/normalization/test_lpscaler.py b/src/python/nimbusml/tests/preprocessing/normalization/test_lpscaler.py new file mode 100644 index 00000000..aa1ee020 --- /dev/null +++ b/src/python/nimbusml/tests/preprocessing/normalization/test_lpscaler.py @@ -0,0 +1,52 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +import unittest +from collections import OrderedDict + +import pandas +from nimbusml import Pipeline +from nimbusml.preprocessing.normalization import LpScaler +from sklearn.utils.testing import assert_almost_equal, assert_equal + + +class TestLpScaler(unittest.TestCase): + + def test_transform_float(self): + in_df = pandas.DataFrame(data=dict(xpetal=[-1.1, -2.2, -3.3], + ipetal=[1, 2, 3])) + + normed = LpScaler() << ['xpetal', 'ipetal'] + pipeline = Pipeline([normed]) + out_df = pipeline.fit_transform(in_df, verbose=0) + assert_equal(out_df.shape, (3, 2)) + assert_almost_equal(out_df.loc[2, 'xpetal'], -1.3887302, decimal=3) + assert_almost_equal(out_df.loc[2, 'ipetal'], 1.38873, decimal=3) + + def test_transform_int(self): + in_df = pandas.DataFrame(data=dict(xpetal=[-1, -2, -3], + ipetal=[1, 2, 3])) + + normed = LpScaler() << ['xpetal', 'ipetal'] + pipeline = Pipeline([normed]) + out_df = pipeline.fit_transform(in_df, verbose=0) + assert_equal(out_df.shape, (3, 2)) + assert_almost_equal(out_df.loc[2, 'xpetal'], -1.3887302, decimal=3) + assert_almost_equal(out_df.loc[2, 'ipetal'], 1.38873, decimal=3) + + def test_transform_int_rename(self): + in_df = pandas.DataFrame(data=dict(xpetal=[-1, -2, -3], + ipetal=[1, 2, 3])) + + normed = LpScaler() << OrderedDict( + [('ii', 'xpetal'), ('jj', 'ipetal')]) + pipeline = Pipeline([normed]) + out_df = pipeline.fit_transform(in_df, verbose=0) + assert_equal(out_df.shape, (3, 4)) + assert_almost_equal(out_df.loc[2, 'ii'], -1.3887302, decimal=3) + assert_almost_equal(out_df.loc[2, 'jj'], 1.38873, decimal=3) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/python/tools/manifest_diff.json b/src/python/tools/manifest_diff.json index d8a64d82..92526a44 100644 --- a/src/python/tools/manifest_diff.json +++ b/src/python/tools/manifest_diff.json @@ -536,6 +536,12 @@ "Module": "preprocessing.normalization", "Type": "Transform" }, + { + "Name": "Transforms.LpNormalizer", + "NewName": "LpScaler", + "Module": "preprocessing.normalization", + "Type": "Transform" + }, { "Name": "Trainers.OrdinaryLeastSquaresRegressor", "NewName": "OrdinaryLeastSquaresRegressor", @@ -732,6 +738,12 @@ "Module": "feature_extraction.text", "Type": "Transform" }, + { + "Name": "Transforms.NGramTranslator", + "NewName": "NGramExtractor", + "Module": "feature_extraction.text", + "Type": "Transform" + }, { "Name": "Transforms.WordEmbeddings", "NewName": "WordEmbedding",