[REVIEW] Add nvtext::porter_stemmer_measure API

CHANGELOG.md

@@ -56,6 +56,7 @@
 - PR #5645 Enforce pd.NA and Pandas nullable dtype parity
 - PR #5729 Create nvtext normalize_characters API from the subword_tokenize internal function
 - PR #5572 Add `cudf::encode` API.
+- PR #5767 Add `nvtext::porter_stemmer_measure` and `nvtext::is_letter` APIs
 - PR #5753 Add `cudf::lists::extract_list_element` API
 - PR #5568 Add support for `Series.keys()` and `DataFrame.keys()`
 - PR #5782 Add Kafka support to custreamz

cpp/CMakeLists.txt

@@ -566,6 +566,7 @@ add_library(cudf
             src/text/detokenize.cu
             src/text/generate_ngrams.cu
             src/text/normalize.cu
+            src/text/stemmer.cu
             src/text/tokenize.cu
             src/text/ngrams_tokenize.cu
             src/text/replace.cu

cpp/include/doxygen_groups.h

@@ -126,6 +126,7 @@
  * @{
  *   @defgroup nvtext_ngrams NGrams
  *   @defgroup nvtext_normalize Normalizing
+ *   @defgroup nvtext_stemmer Stemming
  *   @defgroup nvtext_tokenize Tokenizing
  *   @defgroup nvtext_replace Replacing
  * @}

cpp/include/nvtext/stemmer.hpp

@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <cudf/column/column.hpp>
+#include <cudf/scalar/scalar.hpp>
+#include <cudf/strings/strings_column_view.hpp>
+
+namespace nvtext {
+/**
+ * @addtogroup nvtext_stemmer
+ * @{
+ */
+
+/**
+ * @brief Used for specifying letter type to check.
+ */
+enum class letter_type {
+  CONSONANT,  ///< Letter is a consonant
+  VOWEL       ///< Letter is not a consonant
+};
+
+/**
+ * @brief Returns boolean column indicating if `character_index` of the input strings
+ * is a consonant or vowel.
+ *
+ * Determining consonants and vowels is described in the following
+ * paper: https://tartarus.org/martin/PorterStemmer/def.txt
+ *
+ * Each string in the input column is expected to contain a single, lower-cased
+ * word (or subword) with no punctuation and no whitespace otherwise the
+ * measure value for that string is undefined.
+ *
+ * Also, the algorithm only works with English words.
+ *
+ * @code{.pseudo}
+ * Example:
+ * st = ["trouble", "toy", "sygyzy"]
+ * b1 = is_letter(st, VOWEL, 1)
+ * b1 is now [false, true, true]
+ * @endcode
+ *
+ * A negative index value will check the character starting from the end
+ * of each string. That is, for `character_index < 0` the letter checked for string
+ * `strings[i]` is at position `strings[i].length + index`.
+ *
+ * @code{.pseudo}
+ * Example:
+ * st = ["trouble", "toy", "sygyzy"]
+ * b2 = is_letter(st, CONSONANT, -1) // last letter checked in each string
+ * b2 is now [false, true, false]
+ * @endcode
+ *
+ * A null input element at row `i` produces a corresponding null entry
+ * for row `i` in the output column.
+ *
+ * @param strings Strings column of words to measure.
+ * @param ltype Specify letter type to check.
+ * @param character_index The character position to check in each string.
+ * @param mr Device memory resource used to allocate the returned column's device memory.
+ * @return New BOOL column.
+ */
+std::unique_ptr<cudf::column> is_letter(
+  cudf::strings_column_view const& strings,
+  letter_type ltype,
+  cudf::size_type character_index,
+  rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource());
+
+/**
+ * @brief Returns boolean column indicating if character at `indices[i]` of `strings[i]`
+ * is a consonant or vowel.
+ *
+ * Determining consonants and vowels is described in the following
+ * paper: https://tartarus.org/martin/PorterStemmer/def.txt
+ *
+ * Each string in the input column is expected to contain a single, lower-cased
+ * word (or subword) with no punctuation and no whitespace otherwise the
+ * measure value for that string is undefined.
+ *
+ * Also, the algorithm only works with English words.
+ *
+ * @code{.pseudo}
+ * Example:
+ * st = ["trouble", "toy", "sygyzy"]
+ * ix = [3, 1, 4]
+ * b1 = is_letter(st, VOWEL, ix)
+ * b1 is now [true, true, false]
+ * @endcode
+ *
+ * A negative index value will check the character starting from the end
+ * of each string. That is, for `character_index < 0` the letter checked for string
+ * `strings[i]` is at position `strings[i].length + indices[i]`.
+ *
+ * @code{.pseudo}
+ * Example:
+ * st = ["trouble", "toy", "sygyzy"]
+ * ix = [3, -2, 4] // 2nd to last character in st[1] is checked
+ * b2 = is_letter(st, CONSONANT, ix)
+ * b2 is now [false, false, true]
+ * @endcode
+ *
+ * A null input element at row `i` produces a corresponding null entry
+ * for row `i` in the output column.
+ *
+ * @throw cudf::logic_error if `indices.size() != strings.size()`
+ * @throw cudf::logic_error if `indices` contain nulls.
+ *
+ * @param strings Strings column of words to measure.
+ * @param ltype Specify letter type to check.
+ * @param indices The character positions to check in each string.
+ * @param mr Device memory resource used to allocate the returned column's device memory.
+ * @return New BOOL column.
+ */
+std::unique_ptr<cudf::column> is_letter(
+  cudf::strings_column_view const& strings,
+  letter_type ltype,
+  cudf::column_view const& indices,
+  rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource());
+
+/**
+ * @brief Returns the Porter Stemmer measurements of a strings column.
+ *
+ * Porter stemming is used to normalize words by removing plural and tense endings
+ * from words in English. The stemming measurement involves counting consonant/vowel
+ * patterns within a string.
+ * Reference paper: https://tartarus.org/martin/PorterStemmer/def.txt
+ *
+ * Each string in the input column is expected to contain a single, lower-cased
+ * word (or subword) with no punctuation and no whitespace otherwise the
+ * measure value for that string is undefined.
+ *
+ * Also, the algorithm only works with English words.
+ *
+ * @code{.pseudo}
+ * Example:
+ * st = ["tr", "troubles", "trouble"]
+ * m = porter_stemmer_measure(st)
+ * m is now [0,2,1]
+ * @endcode
+ *
+ * A null input element at row `i` produces a corresponding null entry
+ * for row `i` in the output column.
+ *
+ * @param strings Strings column of words to measure.
+ * @param mr Device memory resource used to allocate the returned column's device memory.
+ * @return New INT32 column of measure values.
+ */
+std::unique_ptr<cudf::column> porter_stemmer_measure(
+  cudf::strings_column_view const& strings,
+  rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource());
+
+/** @} */  // end of group
+}  // namespace nvtext