add stop words for highlight

Endle · Dec 21, 2023 · 4623c53 · 4623c53
1 parent 5ab9ccc
commit 4623c53
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 6 deletions.
diff --git a/fire_seq_search_server/src/language_tools/mod.rs b/fire_seq_search_server/src/language_tools/mod.rs
@@ -37,6 +37,7 @@ pub fn generate_stopwords_list() -> HashSet<String> {
     nltk.insert("theorem");
     nltk.insert("-");
 
+    nltk.insert("view");
 
 
     let mut nltk: HashSet<String> = nltk.iter().map(|&s|s.into()).collect();
@@ -87,4 +88,4 @@ mod test_language_detect {
         assert!(!is_chinese("Это статья ."));
     }
 }
-// assert_eq!(detected_language, Some(English));
+// assert_eq!(detected_language, Some(English));
diff --git a/fire_seq_search_server/src/post_query/highlighter.rs b/fire_seq_search_server/src/post_query/highlighter.rs
@@ -14,15 +14,10 @@ pub fn highlight_keywords_in_body(body: &str, term_tokens: &Vec<String>,
                                   show_summary_single_line_chars_limit: usize) -> String {
 
     let blocks = split_body_to_blocks(body, show_summary_single_line_chars_limit);
-    // let nltk = generate_stopwords_list();
     let nltk = &STOPWORDS_LIST;
 
     let terms_selected: Vec<&str> = crate::language_tools::tokenizer::filter_out_stopwords(
         &term_tokens, nltk);
-    // let term_ref: Vec<&str> = term_tokens.iter().map(|s| &**s).collect();
-    // let terms_selected: Vec<&str> = term_ref.into_iter()
-    //     .filter(|&s| !nltk.contains(s))
-    //     .collect();
     info!("Highlight terms: {:?}", &terms_selected);