diff --git a/CHANGELOG.md b/CHANGELOG.md index 159bc3d317ff..ed44485d856a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - PR #5568 Add support for `Series.keys()` and `DataFrame.keys()` - PR #5782 Add Kafka support to custreamz - PR #5642 Add `GroupBy.groups()` +- PR #5811 Add `nvtext::edit_distance` API - PR #5789 Add groupby support for duration types - PR #5810 Make Cython subdirs packages and simplify package_data - PR #5817 Enable more `fixed_point` unit tests by introducing "scale-less" constructor diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9da4c533cbac..3edee6aed7a3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -565,6 +565,7 @@ add_library(cudf src/lists/copying/concatenate.cu src/lists/copying/gather.cu src/text/detokenize.cu + src/text/edit_distance.cu src/text/generate_ngrams.cu src/text/normalize.cu src/text/stemmer.cu diff --git a/cpp/include/doxygen_groups.h b/cpp/include/doxygen_groups.h index 7c741e4821a6..23c07d94b55b 100644 --- a/cpp/include/doxygen_groups.h +++ b/cpp/include/doxygen_groups.h @@ -127,6 +127,7 @@ * @defgroup nvtext_ngrams NGrams * @defgroup nvtext_normalize Normalizing * @defgroup nvtext_stemmer Stemming + * @defgroup nvtext_edit_distance Edit Distance * @defgroup nvtext_tokenize Tokenizing * @defgroup nvtext_replace Replacing * @} diff --git a/cpp/include/nvtext/edit_distance.hpp b/cpp/include/nvtext/edit_distance.hpp new file mode 100644 index 000000000000..861a385fcd42 --- /dev/null +++ b/cpp/include/nvtext/edit_distance.hpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +//! NVText APIs +namespace nvtext { +/** + * @addtogroup nvtext_edit_distance + * @{ + */ + +/** + * @brief Compute the edit distance between individual strings in two strings columns. + * + * The `output[i]` is the edit distance between `strings[i]` and `targets[i]`. + * This edit distance calculation uses the Levenshtein algorithm as documented here: + * https://www.cuelogic.com/blog/the-levenshtein-algorithm + * + * @code{.pseudo} + * Example: + * s = ["hello", "", "world"] + * t = ["hallo", "goodbye", "world"] + * d = edit_distance(s, t) + * d is now [1, 7, 0] + * @endcode + * + * Any null entries for either `strings` or `targets` is ignored and the edit distance + * is computed as though the null entry is an empty string. + * + * The `targets.size()` must equal `strings.size()` unless `targets.size()==1`. + * In this case, all `strings` will be computed against the single `targets[0]` string. + * + * @throw cudf::logic_error if `targets.size() != strings.size()` and + * if `targets.size() != 1` + * + * @param strings Strings column of input strings + * @param targets Strings to compute edit distance against `strings` + * @param mr Device memory resource used to allocate the returned column's device memory. + * @return New strings columns of with replaced strings. + */ +std::unique_ptr edit_distance( + cudf::strings_column_view const& strings, + cudf::strings_column_view const& targets, + rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource()); + +/** + * @brief Compute the edit distance between all the strings in the input column. + * + * This uses the Levenshtein algorithm to calculate the edit distance between + * two strings as documented here: https://www.cuelogic.com/blog/the-levenshtein-algorithm + * + * The output is essentially a `strings.size() x strings.size()` square matrix of integers. + * All values at diagonal `row == col` are 0 since the edit distance between two identical + * strings is zero. All values above the diagonal are reflected below since the edit distance + * calculation is also commutative. + * + * @code{.pseudo} + * Example: + * s = ["hello", "hallo", "hella"] + * d = edit_distance_matrix(s) + * d is now [[0, 1, 1], + * [1, 0, 2] + * [1, 2, 0]] + * @endcode + * + * Null entries for `strings` are ignored and the edit distance + * is computed as though the null entry is an empty string. + * + * The output is a lists column of size `strings.size()` and where each list item + * is `strings.size()` elements. + * + * @throw cudf::logic_error if `strings.size() == 1` + * + * @param strings Strings column of input strings + * @param mr Device memory resource used to allocate the returned column's device memory. + * @return New lists column of edit distance values. + */ +std::unique_ptr edit_distance_matrix( + cudf::strings_column_view const& strings, + rmm::mr::device_memory_resource* mr = rmm::mr::get_default_resource()); + +/** @} */ // end of group +} // namespace nvtext diff --git a/cpp/src/text/edit_distance.cu b/cpp/src/text/edit_distance.cu new file mode 100644 index 000000000000..8dd959c40dce --- /dev/null +++ b/cpp/src/text/edit_distance.cu @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace nvtext { +namespace detail { +namespace { + +/** + * @brief Compute the edit-distance between two strings + * + * The temporary buffer must be able to hold 3 int16 values for each character + * in the smaller of the two provided strings. + * + * @param d_str First string + * @param d_tgt Second string + * @param buffer Temporary memory buffer used for the calculation. + * @return Edit distance value + */ +__device__ int32_t compute_distance(cudf::string_view const& d_str, + cudf::string_view const& d_tgt, + int16_t* buffer) +{ + auto const str_length = d_str.length(); + auto const tgt_length = d_tgt.length(); + if (str_length == 0) return tgt_length; + if (tgt_length == 0) return str_length; + + auto itr_A = str_length < tgt_length ? d_str.begin() : d_tgt.begin(); + auto itr_B = str_length < tgt_length ? d_tgt.begin() : d_str.begin(); + // .first is min and .second is max + auto const lengths = std::minmax(str_length, tgt_length); + // setup compute buffer pointers + auto line2 = buffer; + auto line1 = line2 + lengths.first; + auto line0 = line1 + lengths.first; + // range is both lengths + auto const range = lengths.first + lengths.second - 1; + for (cudf::size_type i = 0; i < range; ++i) { + auto tmp = line2; + line2 = line1; + line1 = line0; + line0 = tmp; + // checking pairs of characters + for (int x = (i < lengths.second ? 0 : i - lengths.second + 1); + (x < lengths.first) && (x < i + 1); + ++x) { + int const y = i - x; + itr_A += (x - itr_A.position()); // point to next + itr_B += (y - itr_B.position()); // characters to check + int16_t const w = + (((x > 0) && (y > 0)) ? line2[x - 1] : static_cast(std::max(x, y))) + + static_cast(*itr_A != *itr_B); // add 1 if characters do not match + int16_t const u = (y > 0 ? line1[x] : x + 1) + 1; + int16_t const v = (x > 0 ? line1[x - 1] : y + 1) + 1; + // store min(u,v,w) + line0[x] = std::min(std::min(u, v), w); + } + } + return static_cast(line0[lengths.first - 1]); +} + +/** + * @brief Compute the Levenshtein distance for each string. + * + * Documentation here: https://www.cuelogic.com/blog/the-levenshtein-algorithm + * And here: https://en.wikipedia.org/wiki/Levenshtein_distances + */ +struct edit_distance_levenshtein_algorithm { + cudf::column_device_view d_strings; // computing these + cudf::column_device_view d_targets; // against these; + int16_t* d_buffer; // compute buffer for each string + int32_t* d_results; // input is buffer offset; output is edit distance + + __device__ void operator()(cudf::size_type idx) + { + auto d_str = + d_strings.is_null(idx) ? cudf::string_view{} : d_strings.element(idx); + auto d_tgt = [&] __device__ { // d_targets is also allowed to have only one entry + if (d_targets.is_null(idx)) return cudf::string_view{}; + return d_targets.size() == 1 ? d_targets.element(0) + : d_targets.element(idx); + }(); + d_results[idx] = compute_distance(d_str, d_tgt, d_buffer + d_results[idx]); + } +}; + +struct edit_distance_matrix_levenshtein_algorithm { + cudf::column_device_view d_strings; // computing these against itself + int16_t* d_buffer; // compute buffer for each string + int32_t const* d_offsets; // locate sub-buffer for each string + int32_t* d_results; // edit distance values + + __device__ void operator()(cudf::size_type idx) + { + auto const strings_count = d_strings.size(); + auto const row = idx / strings_count; + auto const col = idx % strings_count; + if (row > col) return; // bottom half is computed with the top half of matrix + cudf::string_view d_str1 = + d_strings.is_null(row) ? cudf::string_view{} : d_strings.element(row); + cudf::string_view d_str2 = + d_strings.is_null(col) ? cudf::string_view{} : d_strings.element(col); + auto work_buffer = d_buffer + d_offsets[idx - ((row + 1) * (row + 2)) / 2]; + int32_t const distance = (row == col) ? 0 : compute_distance(d_str1, d_str2, work_buffer); + d_results[idx] = distance; // top half of matrix + d_results[col * strings_count + row] = distance; // bottom half of matrix + } +}; + +} // namespace + +/** + * @copydoc nvtext::edit_distance + */ +std::unique_ptr edit_distance(cudf::strings_column_view const& strings, + cudf::strings_column_view const& targets, + cudaStream_t stream, + rmm::mr::device_memory_resource* mr) +{ + cudf::size_type strings_count = strings.size(); + if (strings_count == 0) return cudf::make_empty_column(cudf::data_type{cudf::type_id::INT32}); + if (targets.size() > 1) + CUDF_EXPECTS(strings_count == targets.size(), "targets.size() must equal strings.size()"); + + // create device columns from the input columns + auto strings_column = cudf::column_device_view::create(strings.parent(), stream); + auto d_strings = *strings_column; + auto targets_column = cudf::column_device_view::create(targets.parent(), stream); + auto d_targets = *targets_column; + + // calculate the size of the compute-buffer; + // we can use the output column buffer to hold the size/offset values temporarily + auto results = cudf::make_fixed_width_column(cudf::data_type{cudf::type_id::INT32}, + strings_count, + rmm::device_buffer{0, stream, mr}, + 0, + stream, + mr); + auto d_results = results->mutable_view().data(); + auto execpol = rmm::exec_policy(stream); + thrust::transform(execpol->on(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings_count), + d_results, + [d_strings, d_targets] __device__(auto idx) { + if (d_strings.is_null(idx) || d_targets.is_null(idx)) return int32_t{0}; + auto d_str = d_strings.element(idx); + auto d_tgt = d_targets.size() == 1 + ? d_targets.element(0) + : d_targets.element(idx); + // just need 3 int16's for each character of the shorter string + return static_cast(std::min(d_str.length(), d_tgt.length()) * 3); + }); + + // get the total size of the temporary compute buffer + size_t compute_size = + thrust::reduce(execpol->on(stream), d_results, d_results + strings_count, size_t{0}); + // convert sizes to offsets in-place + thrust::exclusive_scan(execpol->on(stream), d_results, d_results + strings_count, d_results); + // create the temporary compute buffer + rmm::device_uvector compute_buffer(compute_size, stream); + auto d_buffer = compute_buffer.data(); + + // compute the edit distance into the output column in-place + // - on input, d_results is the offset to the working section of d_buffer for each row + // - on output, d_results is the calculated edit distance for that row + thrust::for_each_n( + execpol->on(stream), + thrust::make_counting_iterator(0), + strings_count, + edit_distance_levenshtein_algorithm{d_strings, d_targets, d_buffer, d_results}); + return results; +} + +/** + * @copydoc nvtext::edit_distance_matrix + */ +std::unique_ptr edit_distance_matrix(cudf::strings_column_view const& strings, + cudaStream_t stream, + rmm::mr::device_memory_resource* mr) +{ + cudf::size_type strings_count = strings.size(); + if (strings_count == 0) return cudf::make_empty_column(cudf::data_type{cudf::type_id::INT32}); + CUDF_EXPECTS(strings_count > 1, "the input strings must include at least 2 strings"); + CUDF_EXPECTS(size_t{strings_count} * size_t{strings_count} < std::numeric_limits().max(), + "too many strings to create the output column"); + + // create device column of the input strings column + auto strings_column = cudf::column_device_view::create(strings.parent(), stream); + auto d_strings = *strings_column; + auto execpol = rmm::exec_policy(stream); + + // Calculate the size of the compute-buffer. + // We only need memory for half the size of the output matrix since the edit distance calculation + // is commutative -- `distance(strings[i],strings[j]) == distance(strings[j],strings[i])` + cudf::size_type n_upper = (strings_count * (strings_count - 1)) / 2; + rmm::device_uvector offsets(n_upper, stream); + auto d_offsets = offsets.data(); + CUDA_TRY(cudaMemsetAsync(d_offsets, 0, n_upper * sizeof(cudf::size_type), stream)); + thrust::for_each_n( + execpol->on(stream), + thrust::make_counting_iterator(0), + strings_count * strings_count, + [d_strings, d_offsets, strings_count] __device__(cudf::size_type idx) { + auto const row = idx / strings_count; + auto const col = idx % strings_count; + if (row >= col) return; // compute only the top half + cudf::string_view const d_str1 = + d_strings.is_null(row) ? cudf::string_view{} : d_strings.element(row); + cudf::string_view const d_str2 = + d_strings.is_null(col) ? cudf::string_view{} : d_strings.element(col); + if (d_str1.empty() || d_str2.empty()) return; + // the temp size needed is 3 int16s per character of the shorter string + d_offsets[idx - ((row + 1) * (row + 2)) / 2] = std::min(d_str1.length(), d_str2.length()) * 3; + }); + + // get the total size for the compute buffer + size_t compute_size = + thrust::reduce(execpol->on(stream), offsets.begin(), offsets.end(), size_t{0}); + // convert sizes to offsets in-place + thrust::exclusive_scan(execpol->on(stream), offsets.begin(), offsets.end(), offsets.begin()); + // create the compute buffer + rmm::device_uvector compute_buffer(compute_size, stream); + auto d_buffer = compute_buffer.data(); + + // compute the edit distance into the output column + auto results = cudf::make_fixed_width_column(cudf::data_type{cudf::type_id::INT32}, + strings_count * strings_count, + rmm::device_buffer{0, stream, mr}, + 0, + stream, + mr); + auto d_results = results->mutable_view().data(); + thrust::for_each_n( + execpol->on(stream), + thrust::make_counting_iterator(0), + strings_count * strings_count, + edit_distance_matrix_levenshtein_algorithm{d_strings, d_buffer, d_offsets, d_results}); + + // build a lists column of the results + auto offsets_column = cudf::make_fixed_width_column(cudf::data_type{cudf::type_id::INT32}, + strings_count + 1, + rmm::device_buffer{0, stream, mr}, + 0, + stream, + mr); + thrust::transform_exclusive_scan( + execpol->on(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings_count + 1), + offsets_column->mutable_view().data(), + [strings_count] __device__(auto idx) { return strings_count; }, + int32_t{0}, + thrust::plus()); + return cudf::make_lists_column(strings_count, + std::move(offsets_column), + std::move(results), + 0, // no nulls + rmm::device_buffer{0, stream, mr}, + stream, + mr); +} + +} // namespace detail + +// external APIs + +/** + * @copydoc nvtext::edit_distance + */ +std::unique_ptr edit_distance(cudf::strings_column_view const& strings, + cudf::strings_column_view const& targets, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::edit_distance(strings, targets, 0, mr); +} + +/** + * @copydoc nvtext::edit_distance_matrix + */ +std::unique_ptr edit_distance_matrix(cudf::strings_column_view const& strings, + rmm::mr::device_memory_resource* mr) +{ + CUDF_FUNC_RANGE(); + return detail::edit_distance_matrix(strings, 0, mr); +} + +} // namespace nvtext diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index b3c4c677bb1e..0d0e5e6786b5 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -538,6 +538,7 @@ ConfigureTest(STRINGS_TEST "${STRINGS_TEST_SRC}") # - nvtext test ---------------------------------------------------------------------------------- set(TEXT_TEST_SRC + "${CMAKE_CURRENT_SOURCE_DIR}/text/edit_distance_tests.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/text/ngrams_tests.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/text/ngrams_tokenize_tests.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/text/normalize_tests.cpp" diff --git a/cpp/tests/text/edit_distance_tests.cpp b/cpp/tests/text/edit_distance_tests.cpp new file mode 100644 index 000000000000..47a95cfd96b7 --- /dev/null +++ b/cpp/tests/text/edit_distance_tests.cpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +#include + +struct TextEditDistanceTest : public cudf::test::BaseFixture { +}; + +TEST_F(TextEditDistanceTest, EditDistance) +{ + std::vector h_strings{"dog", nullptr, "cat", "mouse", "pup", "", "puppy", "thé"}; + cudf::test::strings_column_wrapper strings( + h_strings.begin(), + h_strings.end(), + thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; })); + + std::vector h_targets{"hog", "not", "cake", "house", "fox", nullptr, "puppy", "the"}; + cudf::test::strings_column_wrapper targets( + h_targets.begin(), + h_targets.end(), + thrust::make_transform_iterator(h_targets.begin(), [](auto str) { return str != nullptr; })); + { + auto results = + nvtext::edit_distance(cudf::strings_column_view(strings), cudf::strings_column_view(targets)); + cudf::test::fixed_width_column_wrapper expected({1, 3, 2, 1, 3, 0, 0, 1}); + cudf::test::expect_columns_equal(*results, expected); + } + { + cudf::test::strings_column_wrapper single({"pup"}); + auto results = + nvtext::edit_distance(cudf::strings_column_view(strings), cudf::strings_column_view(single)); + cudf::test::fixed_width_column_wrapper expected({3, 3, 3, 4, 0, 3, 2, 3}); + cudf::test::expect_columns_equal(*results, expected); + } +} + +TEST_F(TextEditDistanceTest, EditDistanceMatrix) +{ + std::vector h_strings{"dog", nullptr, "hog", "frog", "cat", "", "hat", "clog"}; + cudf::test::strings_column_wrapper strings( + h_strings.begin(), + h_strings.end(), + thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; })); + + { + auto results = nvtext::edit_distance_matrix(cudf::strings_column_view(strings)); + + using LCW = cudf::test::lists_column_wrapper; + LCW expected({LCW{0, 3, 1, 2, 3, 3, 3, 2}, + LCW{3, 0, 3, 4, 3, 0, 3, 4}, + LCW{1, 3, 0, 2, 3, 3, 2, 2}, + LCW{2, 4, 2, 0, 4, 4, 4, 2}, + LCW{3, 3, 3, 4, 0, 3, 1, 3}, + LCW{3, 0, 3, 4, 3, 0, 3, 4}, + LCW{3, 3, 2, 4, 1, 3, 0, 4}, + LCW{2, 4, 2, 2, 3, 4, 4, 0}}); + cudf::test::expect_columns_equal(*results, expected); + } +} + +TEST_F(TextEditDistanceTest, EmptyTest) +{ + auto strings = cudf::make_empty_column(cudf::data_type{cudf::type_id::STRING}); + cudf::strings_column_view strings_view(strings->view()); + auto results = nvtext::edit_distance(strings_view, strings_view); + EXPECT_EQ(results->size(), 0); + results = nvtext::edit_distance_matrix(strings_view); + EXPECT_EQ(results->size(), 0); +} + +TEST_F(TextEditDistanceTest, ErrorsTest) +{ + cudf::test::strings_column_wrapper strings({"pup"}); + cudf::test::strings_column_wrapper targets({"pup", ""}); + EXPECT_THROW( + nvtext::edit_distance(cudf::strings_column_view(strings), cudf::strings_column_view(targets)), + cudf::logic_error); + EXPECT_THROW(nvtext::edit_distance_matrix(cudf::strings_column_view(strings)), cudf::logic_error); +} diff --git a/python/cudf/cudf/_lib/cpp/nvtext/edit_distance.pxd b/python/cudf/cudf/_lib/cpp/nvtext/edit_distance.pxd new file mode 100644 index 000000000000..2a27cd3c338f --- /dev/null +++ b/python/cudf/cudf/_lib/cpp/nvtext/edit_distance.pxd @@ -0,0 +1,14 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. + +from libcpp cimport bool +from libcpp.memory cimport unique_ptr + +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.column.column_view cimport column_view + +cdef extern from "nvtext/edit_distance.hpp" namespace "nvtext" nogil: + + cdef unique_ptr[column] edit_distance( + const column_view & strings, + const column_view & targets + ) except + diff --git a/python/cudf/cudf/_lib/nvtext/edit_distance.pyx b/python/cudf/cudf/_lib/nvtext/edit_distance.pyx new file mode 100644 index 000000000000..94ca35c54990 --- /dev/null +++ b/python/cudf/cudf/_lib/nvtext/edit_distance.pyx @@ -0,0 +1,23 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. + +from libcpp cimport bool +from libcpp.memory cimport unique_ptr +from cudf._lib.move cimport move + +from cudf._lib.cpp.column.column cimport column +from cudf._lib.cpp.column.column_view cimport column_view +from cudf._lib.cpp.nvtext.edit_distance cimport ( + edit_distance as cpp_edit_distance +) +from cudf._lib.column cimport Column + + +def edit_distance(Column strings, Column targets): + cdef column_view c_strings = strings.view() + cdef column_view c_targets = targets.view() + cdef unique_ptr[column] c_result + + with nogil: + c_result = move(cpp_edit_distance(c_strings, c_targets)) + + return Column.from_unique_ptr(move(c_result)) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 3394db21d661..75687791d1b7 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -11,6 +11,7 @@ from cudf import _lib as libcudf from cudf._lib import string_casting as str_cast from cudf._lib.column import Column +from cudf._lib.nvtext.edit_distance import edit_distance as cpp_edit_distance from cudf._lib.nvtext.generate_ngrams import ( generate_character_ngrams as cpp_generate_character_ngrams, generate_ngrams as cpp_generate_ngrams, @@ -4293,6 +4294,55 @@ def is_vowel(self, position, **kwargs): cpp_is_letter(self._column, ltype, position), **kwargs ) + def edit_distance(self, targets, **kwargs): + """ + The ``targets`` strings are measured against the strings in this + instance using the Levenshtein edit distance algorithm. + https://www.cuelogic.com/blog/the-levenshtein-algorithm + + The ``targets`` parameter may also be a single string in which + case the edit distance is computed for all the strings against + that single string. + + Parameters + ---------- + targets : array-like, Sequence or Series or str + The string(s) to measure against each string. + + Returns + ------- + Series or Index of int32. + + Examples + -------- + >>> import cudf + >>> sr = cudf.Series(["puppy", "doggy", "kitty"]) + >>> targets = cudf.Series(["pup", "dogie", "kitten"]) + >>> sr.str.edit_distance(targets=targets) + 0 2 + 1 2 + 2 2 + dtype: int32 + >>> sr.str.edit_distance("puppy") + 0 0 + 1 4 + 2 4 + dtype: int32 + """ + if is_scalar(targets): + targets_column = column.as_column([targets]) + elif can_convert_to_column(targets): + targets_column = column.as_column(targets) + else: + raise TypeError( + f"targets should be an str, array-like or Series object, " + f"found {type(targets)}" + ) + + return self._return_or_inplace( + cpp_edit_distance(self._column, targets_column), **kwargs + ) + def _massage_string_arg(value, name, allow_col=False): if isinstance(value, str): diff --git a/python/cudf/cudf/tests/test_text.py b/python/cudf/cudf/tests/test_text.py index af9867b5b408..74465c4a54dc 100644 --- a/python/cudf/cudf/tests/test_text.py +++ b/python/cudf/cudf/tests/test_text.py @@ -774,6 +774,19 @@ def test_text_subword_tokenize(tmpdir): assert_eq(expected_metadata, metadata) +def test_edit_distance(): + sr = cudf.Series(["kitten", "saturday", "address", "book"]) + tg = cudf.Series(["sitting", "sunday", "addressee", "back"]) + + expected = cudf.Series([3, 3, 2, 2], dtype=np.int32) + actual = sr.str.edit_distance(tg) + assert_eq(expected, actual) + + expected = cudf.Series([0, 7, 6, 6], dtype=np.int32) + actual = sr.str.edit_distance("kitten") + assert_eq(expected, actual) + + def test_porter_stemmer_measure(): strings = cudf.Series( [