[WIP] ProduceNgrams

src/python/nimbusml.pyproj

@@ -122,6 +122,7 @@
     <Compile Include="nimbusml\examples\examples_from_dataframe\Handler_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\IidChangePointDetector_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\LinearSvmBinaryClassifier_df.py" />
+    <Compile Include="nimbusml\examples\examples_from_dataframe\LpScaler_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\SsaForecaster_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\SsaChangePointDetector_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\SsaSpikeDetector_df.py" />
@@ -175,6 +176,8 @@
     <Compile Include="nimbusml\examples\Hinge.py" />
     <Compile Include="nimbusml\examples\IidChangePointDetector.py" />
     <Compile Include="nimbusml\examples\LinearSvmBinaryClassifier.py" />
+    <Compile Include="nimbusml\examples\LpScaler.py" />
+    <Compile Include="nimbusml\examples\NGramExtractor.py" />
     <Compile Include="nimbusml\examples\SsaForecaster.py" />
     <Compile Include="nimbusml\examples\SsaChangePointDetector.py" />
     <Compile Include="nimbusml\examples\SsaSpikeDetector.py" />
@@ -232,6 +235,7 @@
     <Compile Include="nimbusml\feature_extraction\text\extractor\ngramhash.py" />
     <Compile Include="nimbusml\feature_extraction\text\extractor\__init__.py" />
     <Compile Include="nimbusml\feature_extraction\text\lightlda.py" />
+    <Compile Include="nimbusml\feature_extraction\text\ngramextractor.py" />
     <Compile Include="nimbusml\feature_extraction\text\stopwords\customstopwordsremover.py" />
     <Compile Include="nimbusml\feature_extraction\text\stopwords\predefinedstopwordsremover.py" />
     <Compile Include="nimbusml\feature_extraction\text\stopwords\__init__.py" />
@@ -279,6 +283,7 @@
     <Compile Include="nimbusml\internal\core\feature_extraction\text\extractor\ngramhash.py" />
     <Compile Include="nimbusml\internal\core\feature_extraction\text\extractor\__init__.py" />
     <Compile Include="nimbusml\internal\core\feature_extraction\text\lightlda.py" />
+    <Compile Include="nimbusml\internal\core\feature_extraction\text\ngramextractor.py" />
     <Compile Include="nimbusml\internal\core\feature_extraction\text\stopwords\customstopwordsremover.py" />
     <Compile Include="nimbusml\internal\core\feature_extraction\text\stopwords\predefinedstopwordsremover.py" />
     <Compile Include="nimbusml\internal\core\feature_extraction\text\stopwords\__init__.py" />
@@ -296,6 +301,7 @@
     <Compile Include="nimbusml\internal\core\preprocessing\datasettransformer.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\filter\skipfilter.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\filter\takefilter.py" />
+    <Compile Include="nimbusml\internal\core\preprocessing\normalization\lpscaler.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\schema\columnduplicator.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\schema\columndropper.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\tensorflowscorer.py" />
@@ -626,6 +632,7 @@
     <Compile Include="nimbusml\preprocessing\normalization\binner.py" />
     <Compile Include="nimbusml\preprocessing\normalization\globalcontrastrowscaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\logmeanvariancescaler.py" />
+    <Compile Include="nimbusml\preprocessing\normalization\lpscaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\meanvariancescaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\minmaxscaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\__init__.py" />
@@ -667,6 +674,7 @@
     <Compile Include="nimbusml\tests\linear_model\test_linearsvmbinaryclassifier.py" />
     <Compile Include="nimbusml\tests\pipeline\test_pipeline_combining.py" />
     <Compile Include="nimbusml\tests\pipeline\test_pipeline_subclassing.py" />
+    <Compile Include="nimbusml\tests\preprocessing\normalization\test_lpscaler.py" />
     <Compile Include="nimbusml\tests\preprocessing\normalization\test_meanvariancescaler.py" />
     <Compile Include="nimbusml\tests\preprocessing\test_datasettransformer.py" />
     <Compile Include="nimbusml\tests\timeseries\test_iidchangepointdetector.py" />

src/python/nimbusml/examples/LpScaler.py

@@ -0,0 +1,35 @@
+###############################################################################
+# MinMaxScaler
+import numpy
+from nimbusml import FileDataStream
+from nimbusml.datasets import get_dataset
+from nimbusml.preprocessing.normalization import LpScaler
+
+# data input (as a FileDataStream)
+path = get_dataset('infert').as_filepath()
+data = FileDataStream.read_csv(
+    path,
+    sep=',',
+    numeric_dtype=numpy.float32)  # Error with integer input
+print(data.head())
+#    age  case education  induced  parity  pooled.stratum  row_num  ...
+# 0  26.0   1.0    0-5yrs      1.0     6.0             3.0      1.0  ...
+# 1  42.0   1.0    0-5yrs      1.0     1.0             1.0      2.0  ...
+# 2  39.0   1.0    0-5yrs      2.0     6.0             4.0      3.0  ...
+# 3  34.0   1.0    0-5yrs      2.0     4.0             2.0      4.0  ...
+# 4  35.0   1.0   6-11yrs      1.0     3.0            32.0      5.0  ...
+
+# transform usage
+xf = LpScaler(columns={'in': 'induced', 'sp': 'spontaneous'})
+
+# fit and transform
+features = xf.fit_transform(data)
+
+# print features
+print(features.head())
+#    age  case education   in  ... pooled.stratum  row_num   sp  ...
+# 0  26.0   1.0    0-5yrs  0.5 ...            3.0      1.0  1.0  ...
+# 1  42.0   1.0    0-5yrs  0.5 ...            1.0      2.0  0.0  ...
+# 2  39.0   1.0    0-5yrs  1.0 ...            4.0      3.0  0.0  ...
+# 3  34.0   1.0    0-5yrs  1.0 ...            2.0      4.0  0.0  ...
+# 4  35.0   1.0   6-11yrs  0.5 ...           32.0      5.0  0.5  ...

src/python/nimbusml/examples/NGramExtractor.py

@@ -0,0 +1,39 @@
+###############################################################################
+# NGramFeaturizer
+from nimbusml import FileDataStream, Pipeline
+from nimbusml.datasets import get_dataset
+from nimbusml.preprocessing.schema import ColumnDropper
+from nimbusml.preprocessing.text import CharTokenizer
+from nimbusml.feature_extraction.text import NGramExtractor
+from nimbusml.feature_extraction.text.extractor import Ngram
+
+# data input (as a FileDataStream)
+path = get_dataset("wiki_detox_train").as_filepath()
+
+data = FileDataStream.read_csv(path, sep='\t')
+print(data.head())
+#   Sentiment                                      SentimentText
+# 0          1  ==RUDE== Dude, you are rude upload that carl p...
+# 1          1  == OK! ==  IM GOING TO VANDALIZE WILD ONES WIK...
+# 2          1  Stop trolling, zapatancas, calling me a liar m...
+# 3          1  ==You're cool==  You seem like a really cool g...
+# 4          1  ::::: Why are you threatening me? I'm not bein...
+
+# transform usage
+pipe = Pipeline([
+        CharTokenizer(columns={'SentimentText_Transform': 'SentimentText'}),
+        NGramExtractor(ngram_length=1, all_lengths=False, columns={'Ngrams': 'SentimentText_Transform'}),
+        ColumnDropper(columns=['SentimentText_Transform', 'SentimentText', 'Sentiment'])
+        ])
+
+# fit and transform
+features = pipe.fit_transform(data)
+
+# print features
+print(features.head())
+#   Sentiment   ...         features.douchiest  features.award.
+# 0          1  ...                        0.0              0.0
+# 1          1  ...                        0.0              0.0
+# 2          1  ...                        0.0              0.0
+# 3          1  ...                        0.0              0.0
+# 4          1  ...                        0.0              0.0

src/python/nimbusml/examples/examples_from_dataframe/LpScaler_df.py

@@ -0,0 +1,20 @@
+###############################################################################
+# MinMaxScaler
+import pandas as pd
+from nimbusml.preprocessing.normalization import LpScaler
+
+in_df = pd.DataFrame(
+    data=dict(
+        Sepal_Length=[
+            2.5, 1, 2.1, 1.0], Sepal_Width=[
+            .75, .9, .8, .76], Petal_Length=[
+            0, 2.5, 2.6, 2.4], Species=[
+            "setosa", "viginica", "setosa", 'versicolor']))
+
+# generate two new Columns - Petal_Normed and Sepal_Normed
+normed = LpScaler() << {
+    'Petal_Normed': 'Petal_Length',
+    'Sepal_Normed': 'Sepal_Width'}
+out_df = normed.fit_transform(in_df)
+
+print('LpScaler\n', (out_df))

src/python/nimbusml/feature_extraction/text/__init__.py

@@ -1,11 +1,13 @@
 from .lightlda import LightLda
+from .ngramextractor import NGramExtractor
 from .ngramfeaturizer import NGramFeaturizer
 from .sentiment import Sentiment
 from .wordembedding import WordEmbedding
 
 __all__ = [
     'LightLda',
     'NGramFeaturizer',
+    'NGramExtractor',
     'Sentiment',
     'WordEmbedding'
 ]

src/python/nimbusml/feature_extraction/text/ngramextractor.py

@@ -0,0 +1,72 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------------------------
+# - Generated by tools/entrypoint_compiler.py: do not edit by hand
+"""
+NGramExtractor
+"""
+
+__all__ = ["NGramExtractor"]
+
+
+from sklearn.base import TransformerMixin
+
+from ...base_transform import BaseTransform
+from ...internal.core.feature_extraction.text.ngramextractor import \
+    NGramExtractor as core
+from ...internal.utils.utils import trace
+
+
+class NGramExtractor(core, BaseTransform, TransformerMixin):
+    """
+    **Description**
+        Produces a bag of counts of n-grams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of n-grams and using the id in the dictionary as the index in the bag.
+
+    :param columns: see `Columns </nimbusml/concepts/columns>`_.
+
+    :param ngram_length: Maximum n-gram length.
+
+    :param all_lengths: Whether to store all n-gram lengths up to ngramLength,
+        or only ngramLength.
+
+    :param skip_length: Maximum number of tokens to skip when constructing an
+        n-gram.
+
+    :param max_num_terms: Maximum number of n-grams to store in the dictionary.
+
+    :param weighting: The weighting criteria.
+
+    :param params: Additional arguments sent to compute engine.
+
+    """
+
+    @trace
+    def __init__(
+            self,
+            ngram_length=2,
+            all_lengths=True,
+            skip_length=0,
+            max_num_terms=[10000000],
+            weighting='Tf',
+            columns=None,
+            **params):
+
+        if columns:
+            params['columns'] = columns
+        BaseTransform.__init__(self, **params)
+        core.__init__(
+            self,
+            ngram_length=ngram_length,
+            all_lengths=all_lengths,
+            skip_length=skip_length,
+            max_num_terms=max_num_terms,
+            weighting=weighting,
+            **params)
+        self._columns = columns
+
+    def get_params(self, deep=False):
+        """
+        Get the parameters for this operator.
+        """
+        return core.get_params(self)

src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py

@@ -0,0 +1,111 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------------------------
+# - Generated by tools/entrypoint_compiler.py: do not edit by hand
+"""
+NGramExtractor
+"""
+
+__all__ = ["NGramExtractor"]
+
+
+from ....entrypoints.transforms_ngramtranslator import \
+    transforms_ngramtranslator
+from ....utils.utils import trace
+from ...base_pipeline_item import BasePipelineItem, DefaultSignature
+
+
+class NGramExtractor(BasePipelineItem, DefaultSignature):
+    """
+    **Description**
+        Produces a bag of counts of n-grams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of n-grams and using the id in the dictionary as the index in the bag.
+
+    :param ngram_length: Maximum n-gram length.
+
+    :param all_lengths: Whether to store all n-gram lengths up to ngramLength,
+        or only ngramLength.
+
+    :param skip_length: Maximum number of tokens to skip when constructing an
+        n-gram.
+
+    :param max_num_terms: Maximum number of n-grams to store in the dictionary.
+
+    :param weighting: The weighting criteria.
+
+    :param params: Additional arguments sent to compute engine.
+
+    """
+
+    @trace
+    def __init__(
+            self,
+            ngram_length=2,
+            all_lengths=True,
+            skip_length=0,
+            max_num_terms=[10000000],
+            weighting='Tf',
+            **params):
+        BasePipelineItem.__init__(
+            self, type='transform', **params)
+
+        self.ngram_length = ngram_length
+        self.all_lengths = all_lengths
+        self.skip_length = skip_length
+        self.max_num_terms = max_num_terms
+        self.weighting = weighting
+
+    @property
+    def _entrypoint(self):
+        return transforms_ngramtranslator
+
+    @trace
+    def _get_node(self, **all_args):
+
+        input_columns = self.input
+        if input_columns is None and 'input' in all_args:
+            input_columns = all_args['input']
+        if 'input' in all_args:
+            all_args.pop('input')
+
+        output_columns = self.output
+        if output_columns is None and 'output' in all_args:
+            output_columns = all_args['output']
+        if 'output' in all_args:
+            all_args.pop('output')
+
+        # validate input
+        if input_columns is None:
+            raise ValueError(
+                "'None' input passed when it cannot be none.")
+
+        if not isinstance(input_columns, list):
+            raise ValueError(
+                "input has to be a list of strings, instead got %s" %
+                type(input_columns))
+
+        # validate output
+        if output_columns is None:
+            output_columns = input_columns
+
+        if not isinstance(output_columns, list):
+            raise ValueError(
+                "output has to be a list of strings, instead got %s" %
+                type(output_columns))
+
+        algo_args = dict(
+            column=[
+                dict(
+                    Source=i,
+                    Name=o) for i,
+                o in zip(
+                    input_columns,
+                    output_columns)] if input_columns else None,
+            ngram_length=self.ngram_length,
+            all_lengths=self.all_lengths,
+            skip_length=self.skip_length,
+            max_num_terms=self.max_num_terms,
+            weighting=self.weighting)
+
+        all_args.update(algo_args)
+        return self._entrypoint(**all_args)