Skip to content

Commit

Permalink
feature: add TypoOption field to control requiring non-english words
Browse files Browse the repository at this point in the history
  • Loading branch information
skeptrunedev committed Sep 2, 2024
1 parent 02427f1 commit 0c020cb
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 2 deletions.
2 changes: 2 additions & 0 deletions server/src/data/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5296,6 +5296,8 @@ pub struct TypoOptions {
pub two_typo_word_range: Option<TypoRange>,
/// Words that should not be corrected. If not specified, this defaults to an empty list.
pub disable_on_word: Option<Vec<String>>,
/// Auto-require non-english words present in the dataset to exist in each results chunk_html text. If not specified, this defaults to true.
pub auto_require_non_english_words: Option<bool>,
}

#[derive(Serialize, Deserialize, Debug, Clone, ToSchema, Default)]
Expand Down
15 changes: 13 additions & 2 deletions server/src/operators/typo_operator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,8 @@ fn correct_query_helper(
) -> CorrectedQuery {
let query_words: Vec<&str> = query.query.split_whitespace().collect();
let mut corrections = HashMap::new();
let mut new_quote_words = Vec::new();

let excluded_words: HashSet<_> = options
.disable_on_word
.clone()
Expand Down Expand Up @@ -608,7 +610,10 @@ fn correct_query_helper(
continue;
}

if !tree.find(word.to_string(), 0).is_empty() {
if options.auto_require_non_english_words.unwrap_or(true)
&& !tree.find(word.to_string(), 0).is_empty()
{
new_quote_words.push(word);
query.quote_words = match query.quote_words {
Some(mut existing_words) => {
existing_words.push(word.to_string());
Expand Down Expand Up @@ -659,16 +664,22 @@ fn correct_query_helper(
}
}

if corrections.is_empty() {
if corrections.is_empty() && new_quote_words.is_empty() {
CorrectedQuery {
query: Some(query),
corrected: false,
}
} else {
let mut corrected_query = query.query.clone();

for (original, correction) in corrections {
corrected_query = corrected_query.replace(original, &correction);
}

for word in new_quote_words {
corrected_query = corrected_query.replace(word, &format!("\"{}\"", word));
}

query.query = corrected_query;
CorrectedQuery {
query: Some(query),
Expand Down

0 comments on commit 0c020cb

Please sign in to comment.