diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b90dd67148f..d99fc8395ffe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ - PR #5645 Enforce pd.NA and Pandas nullable dtype parity - PR #5729 Create nvtext normalize_characters API from the subword_tokenize internal function - PR #5572 Add `cudf::encode` API. +- PR #5767 Add `nvtext::porter_stemmer_measure` and `nvtext::is_letter` APIs - PR #5753 Add `cudf::lists::extract_list_element` API - PR #5568 Add support for `Series.keys()` and `DataFrame.keys()` - PR #5782 Add Kafka support to custreamz diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0483e2c4cc54..57d46d0cbac8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -566,6 +566,7 @@ add_library(cudf src/text/detokenize.cu src/text/generate_ngrams.cu src/text/normalize.cu + src/text/stemmer.cu src/text/tokenize.cu src/text/ngrams_tokenize.cu src/text/replace.cu diff --git a/cpp/include/doxygen_groups.h b/cpp/include/doxygen_groups.h index a6238369afb3..7c741e4821a6 100644 --- a/cpp/include/doxygen_groups.h +++ b/cpp/include/doxygen_groups.h @@ -126,6 +126,7 @@ * @{ * @defgroup nvtext_ngrams NGrams * @defgroup nvtext_normalize Normalizing + * @defgroup nvtext_stemmer Stemming * @defgroup nvtext_tokenize Tokenizing * @defgroup nvtext_replace Replacing * @} diff --git a/cpp/include/nvtext/stemmer.hpp b/cpp/include/nvtext/stemmer.hpp new file mode 100644 index 000000000000..9c9e50848a55 --- /dev/null +++ b/cpp/include/nvtext/stemmer.hpp @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +namespace nvtext { +/** + * @addtogroup nvtext_stemmer + * @{ + */ + +/** + * @brief Used for specifying letter type to check. + */ +enum class letter_type { + CONSONANT, ///< Letter is a consonant + VOWEL ///< Letter is not a consonant +}; + +/** + * @brief Returns boolean column indicating if `character_index` of the input strings + * is a consonant or vowel. + * + * Determining consonants and vowels is described in the following + * paper: https://tartarus.org/martin/PorterStemmer/def.txt + * + * Each string in the input column is expected to contain a single, lower-cased + * word (or subword) with no punctuation and no whitespace otherwise the + * measure value for that string is undefined. + * + * Also, the algorithm only works with English words. + * + * @code{.pseudo} + * Example: + * st = ["trouble", "toy", "sygyzy"] + * b1 = is_letter(st, VOWEL, 1) + * b1 is now [false, true, true] + * @endcode + * + * A negative index value will check the character starting from the end + * of each string. That is, for `character_index < 0` the letter checked for string + * `strings[i]` is at position `strings[i].length + index`. + * + * @code{.pseudo} + * Example: + * st = ["trouble", "toy", "sygyzy"] + * b2 = is_letter(st, CONSONANT, -1) // last letter checked in each string + * b2 is now [false, true, false] + * @endcode + * + * A null input element at row `i` produces a corresponding null entry + * for row `i` in the output column. + * + * @param strings Strings column of words to measure. + * @param ltype Specify letter type to check. + * @param character_index The character position to check in each string. + * @param mr Device memory resource used to allocate the returned column's device memory. + * @return New BOOL column. + */ +std::unique_ptr is_letter( + cudf::strings_column_view const& strings, + letter_type ltype, + cudf::size_type character_index, + rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource()); + +/** + * @brief Returns boolean column indicating if character at `indices[i]` of `strings[i]` + * is a consonant or vowel. + * + * Determining consonants and vowels is described in the following + * paper: https://tartarus.org/martin/PorterStemmer/def.txt + * + * Each string in the input column is expected to contain a single, lower-cased + * word (or subword) with no punctuation and no whitespace otherwise the + * measure value for that string is undefined. + * + * Also, the algorithm only works with English words. + * + * @code{.pseudo} + * Example: + * st = ["trouble", "toy", "sygyzy"] + * ix = [3, 1, 4] + * b1 = is_letter(st, VOWEL, ix) + * b1 is now [true, true, false] + * @endcode + * + * A negative index value will check the character starting from the end + * of each string. That is, for `character_index < 0` the letter checked for string + * `strings[i]` is at position `strings[i].length + indices[i]`. + * + * @code{.pseudo} + * Example: + * st = ["trouble", "toy", "sygyzy"] + * ix = [3, -2, 4] // 2nd to last character in st[1] is checked + * b2 = is_letter(st, CONSONANT, ix) + * b2 is now [false, false, true] + * @endcode + * + * A null input element at row `i` produces a corresponding null entry + * for row `i` in the output column. + * + * @throw cudf::logic_error if `indices.size() != strings.size()` + * @throw cudf::logic_error if `indices` contain nulls. + * + * @param strings Strings column of words to measure. + * @param ltype Specify letter type to check. + * @param indices The character positions to check in each string. + * @param mr Device memory resource used to allocate the returned column's device memory. + * @return New BOOL column. + */ +std::unique_ptr is_letter( + cudf::strings_column_view const& strings, + letter_type ltype, + cudf::column_view const& indices, + rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource()); + +/** + * @brief Returns the Porter Stemmer measurements of a strings column. + * + * Porter stemming is used to normalize words by removing plural and tense endings + * from words in English. The stemming measurement involves counting consonant/vowel + * patterns within a string. + * Reference paper: https://tartarus.org/martin/PorterStemmer/def.txt + * + * Each string in the input column is expected to contain a single, lower-cased + * word (or subword) with no punctuation and no whitespace otherwise the + * measure value for that string is undefined. + * + * Also, the algorithm only works with English words. + * + * @code{.pseudo} + * Example: + * st = ["tr", "troubles", "trouble"] + * m = porter_stemmer_measure(st) + * m is now [0,2,1] + * @endcode + * + * A null input element at row `i` produces a corresponding null entry + * for row `i` in the output column. + * + * @param strings Strings column of words to measure. + * @param mr Device memory resource used to allocate the returned column's device memory. + * @return New INT32 column of measure values. + */ +std::unique_ptr porter_stemmer_measure( + cudf::strings_column_view const& strings, + rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource()); + +/** @} */ // end of group +} // namespace nvtext diff --git a/cpp/src/text/stemmer.cu b/cpp/src/text/stemmer.cu new file mode 100644 index 000000000000..ec059de54aa5 --- /dev/null +++ b/cpp/src/text/stemmer.cu @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace nvtext { +namespace detail { +namespace { + +/** + * @brief Return true if character at current iterator position + * is a consonant. + * + * A consonant is a letter other than a, e, i, o or u, and other + * than y preceded by a consonant. + * + * For `toy` the consonants are `t` and `y`, and in `syzygy` they + * are `s`, `z` and `g`. + * + * A _vowel_ is defined as _not a consonant_. + * + * @param string_iterator Iterator positioned to the character to check. + * @return True if the character at the iterator is a consonant. + */ +__device__ bool is_consonant(cudf::string_view::const_iterator string_iterator) +{ + auto ch = *string_iterator; + cudf::string_view const d_vowels("aeiou", 5); + if (d_vowels.find(ch) >= 0) return false; + if ((ch != 'y') || (string_iterator.position() == 0)) return true; + // for 'y' case, check previous character is a consonant + --string_iterator; + return d_vowels.find(*string_iterator) >= 0; +} + +/** + * @brief Functor for the detail::is_letter_fn called to return true/false + * indicating the specified character is a consonant or a vowel. + */ +template +struct is_letter_fn { + cudf::column_device_view const d_strings; + letter_type ltype; + PositionIterator position_itr; + + __device__ bool operator()(cudf::size_type idx) + { + if (d_strings.is_null(idx)) return false; + auto const d_str = d_strings.element(idx); + if (d_str.empty()) return false; + auto const position = position_itr[idx]; + auto const length = d_str.length(); + if ((position >= length) || (position < -length)) return false; + return is_consonant(d_str.begin() + ((position + length) % length)) + ? ltype == letter_type::CONSONANT + : ltype == letter_type::VOWEL; + } +}; + +} // namespace + +// details API + +template +std::unique_ptr is_letter(cudf::strings_column_view const& strings, + letter_type ltype, + PositionIterator position_itr, + cudaStream_t stream, + rmm::mr::device_memory_resource* mr) +{ + if (strings.size() == 0) return cudf::make_empty_column(cudf::data_type{cudf::type_id::BOOL8}); + + // create empty output column + auto results = cudf::make_fixed_width_column(cudf::data_type{cudf::type_id::BOOL8}, + strings.size(), + copy_bitmask(strings.parent(), stream, mr), + strings.null_count(), + stream, + mr); + // set values into output column + auto strings_column = cudf::column_device_view::create(strings.parent(), stream); + thrust::transform(rmm::exec_policy(stream)->on(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings.size()), + results->mutable_view().data(), + is_letter_fn{*strings_column, ltype, position_itr}); + return results; +} + +namespace { + +/** + * @brief For dispatching index-type of indices parameter in the nvtext::is_letter API. + */ +struct dispatch_is_letter_fn { + template ()>* = nullptr> + std::unique_ptr operator()(cudf::strings_column_view const& strings, + letter_type ltype, + cudf::column_view const& indices, + cudaStream_t stream, + rmm::mr::device_memory_resource* mr) const + { + CUDF_EXPECTS(strings.size() == indices.size(), + "strings column and indices column must be the same size"); + CUDF_EXPECTS(!indices.has_nulls(), "indices column must not contain nulls"); + // resolve and pass an iterator for the indices column to the detail function + return is_letter(strings, ltype, indices.begin(), stream, mr); + } + template ()>* = nullptr> + std::unique_ptr operator()(Args&&... args) const + { + CUDF_FAIL("The is_letter indices parameter must be an integer type."); + } +}; + +/** + * @brief Returns the measure for each string. + * + * Text description here is from https://tartarus.org/martin/PorterStemmer/def.txt + * + * A consonant will be denoted by `c`, a vowel by `v`. A list `ccc...` of length + * greater than 0 will be denoted by `C`, and a list `vvv...` of length greater + * than 0 will be denoted by `V`. Any word, or part of a word, therefore has one + * of the four forms: + * + * @code{.pseudo} + * CVCV ... C + * CVCV ... V + * VCVC ... C + * VCVC ... V + * @endcode + * + * These may all be represented by the single form `[C]VCVC ... [V]` + * where the square brackets denote arbitrary presence of their contents. + * Using `(VC){m}` to denote `VC` repeated `m` times, this may again be written as + * `[C](VC){m}[V]`. + * + * And `m` will be called the _measure_ of any word or word part when represented in + * this form. The case `m = 0` covers the null or empty string. + * + * Examples: + * @code{.pseudo} + * m=0: TR, EE, TREE, Y, BY. + * m=1: TROUBLE, OATS, TREES, IVY. + * m=2: TROUBLES, PRIVATE, OATEN, ORRERY. + * @endcode + */ +struct porter_stemmer_measure_fn { + cudf::column_device_view const d_strings; // strings to measure + + __device__ int32_t operator()(cudf::size_type idx) const + { + if (d_strings.is_null(idx)) return 0; + cudf::string_view d_str = d_strings.element(idx); + if (d_str.empty()) return 0; + int32_t measure = 0; + auto itr = d_str.begin(); + bool vowel_run = !is_consonant(itr); + while (itr != d_str.end()) { + if (is_consonant(itr)) { + if (vowel_run) measure++; + vowel_run = false; + } else { + vowel_run = true; + } + ++itr; + } + return measure; + } +}; + +} // namespace + +std::unique_ptr porter_stemmer_measure(cudf::strings_column_view const& strings, + cudaStream_t stream, + rmm::mr::device_memory_resource* mr) +{ + if (strings.size() == 0) return cudf::make_empty_column(cudf::data_type{cudf::type_id::INT32}); + + // create empty output column + auto results = cudf::make_fixed_width_column(cudf::data_type{cudf::type_id::INT32}, + strings.size(), + copy_bitmask(strings.parent(), stream, mr), + strings.null_count(), + stream, + mr); + // compute measures into output column + auto strings_column = cudf::column_device_view::create(strings.parent(), stream); + thrust::transform(rmm::exec_policy(stream)->on(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings.size()), + results->mutable_view().data(), + porter_stemmer_measure_fn{*strings_column}); + return results; +} + +std::unique_ptr is_letter(cudf::strings_column_view const& strings, + letter_type ltype, + cudf::column_view const& indices, + cudaStream_t stream, + rmm::mr::device_memory_resource* mr) +{ + return cudf::type_dispatcher( + indices.type(), dispatch_is_letter_fn{}, strings, ltype, indices, stream, mr); +} + +} // namespace detail + +// external APIs + +std::unique_ptr is_letter(cudf::strings_column_view const& strings, + letter_type ltype, + cudf::size_type character_index, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::is_letter( + strings, ltype, thrust::make_constant_iterator(character_index), 0, mr); +} + +std::unique_ptr is_letter(cudf::strings_column_view const& strings, + letter_type ltype, + cudf::column_view const& indices, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::is_letter(strings, ltype, indices, 0, mr); +} + +/** + * @copydoc nvtext::porter_stemmer_measure + */ +std::unique_ptr porter_stemmer_measure(cudf::strings_column_view const& strings, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::porter_stemmer_measure(strings, 0, mr); +} + +} // namespace nvtext diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index f3cdf2c66ebd..cfce1d66a192 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -541,8 +541,9 @@ set(TEXT_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/text/ngrams_tokenize_tests.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/text/normalize_tests.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/text/replace_tests.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/text/tokenize_tests.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/text/subword_tests.cpp") + "${CMAKE_CURRENT_SOURCE_DIR}/text/stemmer_tests.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/text/subword_tests.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/text/tokenize_tests.cpp") ConfigureTest(TEXT_TEST "${TEXT_TEST_SRC}") diff --git a/cpp/tests/text/stemmer_tests.cpp b/cpp/tests/text/stemmer_tests.cpp new file mode 100644 index 000000000000..149254e130b6 --- /dev/null +++ b/cpp/tests/text/stemmer_tests.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +#include + +#include + +struct TextStemmerTest : public cudf::test::BaseFixture { +}; + +TEST_F(TextStemmerTest, PorterStemmer) +{ + std::vector h_strings{"abandon", + nullptr, + "abbey", + "cleans", + "trouble", + "", + "yearly", + "tree", + "y", + "by", + "oats", + "ivy", + "private", + "orrery"}; + auto validity = + thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; }); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end(), validity); + + cudf::test::fixed_width_column_wrapper expected( + {3, 0, 2, 1, 1, 0, 1, 0, 0, 0, 1, 1, 2, 2}, validity); + auto const results = nvtext::porter_stemmer_measure(cudf::strings_column_view(strings)); + cudf::test::expect_columns_equal(*results, expected); +} + +TEST_F(TextStemmerTest, IsLetterIndex) +{ + std::vector h_strings{"abandon", + nullptr, + "abbey", + "cleans", + "trouble", + "", + "yearly", + "tree", + "y", + "by", + "oats", + "ivy", + "private", + "orrery"}; + auto validity = + thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; }); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end(), validity); + + cudf::strings_column_view sv(strings); + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::VOWEL, 0); + cudf::test::fixed_width_column_wrapper expected( + {1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1}, validity); + cudf::test::expect_columns_equal(*results, expected); + } + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::CONSONANT, 0); + cudf::test::fixed_width_column_wrapper expected( + {0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0}, validity); + cudf::test::expect_columns_equal(*results, expected); + } + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::VOWEL, 5); + cudf::test::fixed_width_column_wrapper expected( + {1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1}, validity); + cudf::test::expect_columns_equal(*results, expected); + } + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::CONSONANT, 5); + cudf::test::fixed_width_column_wrapper expected( + {0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0}, validity); + cudf::test::expect_columns_equal(*results, expected); + } + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::VOWEL, -2); + cudf::test::fixed_width_column_wrapper expected( + {1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0}, validity); + cudf::test::expect_columns_equal(*results, expected); + } + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::CONSONANT, -2); + cudf::test::fixed_width_column_wrapper expected( + {0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1}, validity); + cudf::test::expect_columns_equal(*results, expected); + } +} + +TEST_F(TextStemmerTest, IsLetterIndices) +{ + std::vector h_strings{"abandon", + nullptr, + "abbey", + "cleans", + "trouble", + "", + "yearly", + "tree", + "y", + "by", + "oats", + "ivy", + "private", + "orrery"}; + auto validity = + thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; }); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end(), validity); + cudf::test::fixed_width_column_wrapper indices( + {0, 1, 2, 3, 4, 5, 4, 3, 2, 1, -1, -2, -3, -4}); + + cudf::strings_column_view sv(strings); + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::VOWEL, indices); + cudf::test::fixed_width_column_wrapper expected( + {1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0}, validity); + cudf::test::expect_columns_equal(*results, expected); + } + { + auto const results = nvtext::is_letter(sv, nvtext::letter_type::CONSONANT, indices); + cudf::test::fixed_width_column_wrapper expected( + {0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1}, validity); + cudf::test::expect_columns_equal(*results, expected); + } +} + +TEST_F(TextStemmerTest, EmptyTest) +{ + auto strings = cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); + cudf::strings_column_view sv(strings->view()); + auto results = nvtext::porter_stemmer_measure(sv); + EXPECT_EQ(results->size(), 0); + results = nvtext::is_letter(sv, nvtext::letter_type::CONSONANT, 0); + EXPECT_EQ(results->size(), 0); + auto indices = cudf::make_empty_column(cudf::data_type{cudf::type_id::INT32}); + results = nvtext::is_letter(sv, nvtext::letter_type::VOWEL, indices->view()); + EXPECT_EQ(results->size(), 0); +} + +TEST_F(TextStemmerTest, ErrorTest) +{ + auto empty = cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); + cudf::test::fixed_width_column_wrapper indices({0}, {0}); + EXPECT_THROW(nvtext::is_letter( + cudf::strings_column_view(empty->view()), nvtext::letter_type::VOWEL, indices), + cudf::logic_error); + cudf::test::strings_column_wrapper strings({"abc"}); + EXPECT_THROW( + nvtext::is_letter(cudf::strings_column_view(strings), nvtext::letter_type::VOWEL, indices), + cudf::logic_error); +} diff --git a/python/cudf/cudf/_lib/cpp/nvtext/stemmer.pxd b/python/cudf/cudf/_lib/cpp/nvtext/stemmer.pxd new file mode 100644 index 000000000000..b8b816c212e1 --- /dev/null +++ b/python/cudf/cudf/_lib/cpp/nvtext/stemmer.pxd @@ -0,0 +1,29 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. + +from libc.stdint cimport int32_t +from libcpp.memory cimport unique_ptr + +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.column.column_view cimport column_view +from cudf._lib.cpp.types cimport size_type + +cdef extern from "nvtext/stemmer.hpp" namespace "nvtext" nogil: + ctypedef enum letter_type: + CONSONANT 'nvtext::letter_type::CONSONANT' + VOWEL 'nvtext::letter_type::VOWEL' + + cdef unique_ptr[column] porter_stemmer_measure( + const column_view & strings + ) except + + + cdef unique_ptr[column] is_letter( + column_view source_strings, + letter_type ltype, + size_type character_index) except + + + cdef unique_ptr[column] is_letter( + column_view source_strings, + letter_type ltype, + column_view indices) except + + +ctypedef int32_t underlying_type_t_letter_type diff --git a/python/cudf/cudf/_lib/nvtext/stemmer.pyx b/python/cudf/cudf/_lib/nvtext/stemmer.pyx new file mode 100644 index 000000000000..d3e7c98331bc --- /dev/null +++ b/python/cudf/cudf/_lib/nvtext/stemmer.pyx @@ -0,0 +1,63 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from enum import IntEnum + +from cudf._lib.move cimport move +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.column.column_view cimport column_view +from cudf._lib.cpp.types cimport size_type +from cudf._lib.column cimport Column + +from cudf._lib.cpp.nvtext.stemmer cimport ( + porter_stemmer_measure as cpp_porter_stemmer_measure, + is_letter as cpp_is_letter, + letter_type as letter_type +) +from cudf._lib.cpp.nvtext.stemmer cimport underlying_type_t_letter_type + + +class LetterType(IntEnum): + CONSONANT = letter_type.CONSONANT + VOWEL = letter_type.VOWEL + + +def porter_stemmer_measure(Column strings): + cdef column_view c_strings = strings.view() + cdef unique_ptr[column] c_result + + with nogil: + c_result = move(cpp_porter_stemmer_measure(c_strings)) + + return Column.from_unique_ptr(move(c_result)) + + +def is_letter(Column strings, + object ltype, + size_type index): + cdef column_view c_strings = strings.view() + cdef letter_type c_ltype = ( + ltype + ) + cdef unique_ptr[column] c_result + + with nogil: + c_result = move(cpp_is_letter(c_strings, c_ltype, index)) + + return Column.from_unique_ptr(move(c_result)) + + +def is_letter_multi(Column strings, + object ltype, + Column indices): + cdef column_view c_strings = strings.view() + cdef column_view c_indices = indices.view() + cdef letter_type c_ltype = ( + ltype + ) + cdef unique_ptr[column] c_result + + with nogil: + c_result = move(cpp_is_letter(c_strings, c_ltype, c_indices)) + + return Column.from_unique_ptr(move(c_result)) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 7796b01d9bf8..3394db21d661 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -26,6 +26,12 @@ filter_tokens as cpp_filter_tokens, replace_tokens as cpp_replace_tokens, ) +from cudf._lib.nvtext.stemmer import ( + LetterType, + is_letter as cpp_is_letter, + is_letter_multi as cpp_is_letter_multi, + porter_stemmer_measure as cpp_porter_stemmer_measure, +) from cudf._lib.nvtext.subword_tokenize import ( subword_tokenize as cpp_subword_tokenize, ) @@ -4174,6 +4180,119 @@ def subword_tokenize( cupy.asarray(metadata), ) + def porter_stemmer_measure(self, **kwargs): + """ + Compute the Porter Stemmer measure for each string. + The Porter Stemmer algorithm is described `here + `_. + + Returns + ------- + Series or Index of object. + + Examples + -------- + >>> import cudf + >>> ser = cudf.Series(["hello", "super"]) + >>> ser.str.porter_stemmer_measure() + 0 1 + 1 2 + dtype: int32 + """ + return self._return_or_inplace( + cpp_porter_stemmer_measure(self._column), **kwargs + ) + + def is_consonant(self, position, **kwargs): + """ + Return true for strings where the character at ``position`` is a + consonant. The ``position`` parameter may also be a list of integers + to check different characters per string. + If the ``position`` is larger than the string length, False is + returned for that string. + + Parameters + ---------- + position: int or list-like + The character position to check within each string. + + Returns + ------- + Series or Index of bool dtype. + + Examples + -------- + >>> import cudf + >>> ser = cudf.Series(["toy", "trouble"]) + >>> ser.str.is_consonant(1) + 0 False + 1 True + dtype: bool + >>> positions = cudf.Series([2, 3]) + >>> ser.str.is_consonant(positions) + 0 True + 1 False + dtype: bool + """ + ltype = LetterType.CONSONANT + + if can_convert_to_column(position): + return self._return_or_inplace( + cpp_is_letter_multi( + self._column, ltype, column.as_column(position) + ), + **kwargs, + ) + + return self._return_or_inplace( + cpp_is_letter(self._column, ltype, position), **kwargs + ) + + def is_vowel(self, position, **kwargs): + """ + Return true for strings where the character at ``position`` is a + vowel -- not a consonant. The ``position`` parameter may also be + a list of integers to check different characters per string. + If the ``position`` is larger than the string length, False is + returned for that string. + + Parameters + ---------- + position: int or list-like + The character position to check within each string. + + Returns + ------- + Series or Index of bool dtype. + + Examples + -------- + >>> import cudf + >>> ser = cudf.Series(["toy", "trouble"]) + >>> ser.str.is_vowel(1) + 0 True + 1 False + dtype: bool + >>> positions = cudf.Series([2, 3]) + >>> ser.str.is_vowel(positions) + 0 False + 1 True + dtype: bool + """ + ltype = LetterType.VOWEL + + if can_convert_to_column(position): + return self._return_or_inplace( + cpp_is_letter_multi( + self._column, ltype, column.as_column(position) + ), + **kwargs, + ) + + return self._return_or_inplace( + cpp_is_letter(self._column, ltype, position), **kwargs + ) + def _massage_string_arg(value, name, allow_col=False): if isinstance(value, str): diff --git a/python/cudf/cudf/tests/test_text.py b/python/cudf/cudf/tests/test_text.py index fd9c39d27581..af9867b5b408 100644 --- a/python/cudf/cudf/tests/test_text.py +++ b/python/cudf/cudf/tests/test_text.py @@ -772,3 +772,67 @@ def test_text_subword_tokenize(tmpdir): [0, 0, 3, 1, 0, 3, 2, 0, 3, 3, 0, 1, 4, 0, 1], dtype=np.uint32 ) assert_eq(expected_metadata, metadata) + + +def test_porter_stemmer_measure(): + strings = cudf.Series( + [ + "tr", + "ee", + "tree", + "y", + "by", + "trouble", + "oats", + "trees", + "ivy", + "troubles", + "private", + "oaten", + "orrery", + None, + "", + ] + ) + expected = cudf.Series( + [0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, None, 0], dtype=np.int32 + ) + + actual = strings.str.porter_stemmer_measure() + + assert type(expected) == type(actual) + assert_eq(expected, actual) + + +def test_is_vowel_consonant(): + strings = cudf.Series( + ["tr", "ee", "tree", "y", "by", "oats", "ivy", "orrery", None, ""] + ) + expected = cudf.Series( + [False, False, True, False, False, False, True, False, None, False] + ) + actual = strings.str.is_vowel(2) + assert type(expected) == type(actual) + assert_eq(expected, actual) + + expected = cudf.Series( + [True, False, True, False, False, False, True, True, None, False] + ) + actual = strings.str.is_consonant(1) + assert type(expected) == type(actual) + assert_eq(expected, actual) + + indices = cudf.Series([2, 1, 0, 0, 1, 2, 0, 3, 0, 0]) + expected = cudf.Series( + [False, True, False, False, True, False, True, True, None, False] + ) + actual = strings.str.is_vowel(indices) + assert type(expected) == type(actual) + assert_eq(expected, actual) + + expected = cudf.Series( + [False, False, True, True, False, True, False, False, None, False] + ) + actual = strings.str.is_consonant(indices) + assert type(expected) == type(actual) + assert_eq(expected, actual)