Add jupyter notebook tutorial for single node mulilingual dataset

tutorials/single_node_tutorial/config/heuristic_filter_non-en.yaml

@@ -0,0 +1,82 @@
+input_field: text
+filters:
+  # The filters below define a chain of heuristic filters to be applied to each document in a corpus.
+  # This particular cascade of filters is intended to filter generic non-English data that use spaces for separating words.
+  # The filter listed at the top will be applied first, and the following filters will be applied in
+  # the order they appear in this file. Each filter can be removed and re-ordered as desired.
+  - name: nemo_curator.filters.heuristic_filter.SymbolsToWordsFilter
+    log_score: True
+    params:
+      max_symbol_to_word_ratio: 0.1
+  - name: nemo_curator.filters.heuristic_filter.NumbersFilter
+    log_score: True
+    params:
+      max_number_to_text_ratio: 0.15
+  - name: nemo_curator.filters.heuristic_filter.UrlsFilter
+    log_score: True
+    params:
+      max_url_to_text_ratio: 0.2
+  - name: nemo_curator.filters.heuristic_filter.WhiteSpaceFilter
+    log_score: True
+    params:
+      max_white_space_ratio: 0.25
+  - name: nemo_curator.filters.heuristic_filter.ParenthesesFilter
+    log_score: True
+    params:
+      max_parentheses_ratio: 0.1
+  - name: nemo_curator.filters.heuristic_filter.BoilerPlateStringFilter
+    log_score: True
+    params:
+      remove_if_at_top_or_bottom: True
+      max_boilerplate_string_ratio: 0.4
+  - name: nemo_curator.filters.heuristic_filter.RepeatedLinesFilter
+    log_score: True
+    params:
+      max_repeated_line_fraction: 0.7
+  - name: nemo_curator.filters.heuristic_filter.RepeatedParagraphsFilter
+    log_score: True
+    params:
+      max_repeated_paragraphs_ratio: 0.7
+  - name: nemo_curator.filters.heuristic_filter.RepeatedLinesByCharFilter
+    params:
+      max_repeated_lines_char_ratio: 0.8
+  - name: nemo_curator.filters.heuristic_filter.RepeatedParagraphsByCharFilter
+    log_score: True
+    params:
+     max_repeated_paragraphs_char_ratio: 0.8
+  - name: nemo_curator.filters.heuristic_filter.WordCountFilter
+    log_score: True
+    params:
+      min_words: 50
+      max_words: 100000
+  # NOTE: This filter tends to remove many documents and will need to
+  # be tuned per language
+#   - name: nemo_curator.filters.heuristic_filter.PunctuationFilter
+#     params:
+#       max_num_sentences_without_endmark_ratio: 0.85
+#   - name: nemo_curator.filters.heuristic_filter.MeanWordLengthFilter
+#     params:
+#       max_mean_word_length: 10
+#       min_mean_word_length: 3
+#   - name: nemo_curator.filters.heuristic_filter.LongWordFilter
+#     params:
+#       max_word_length: 1000
+#   - name: nemo_curator.filters.heuristic_filter.EllipsisFilter
+#     params:
+#       max_num_lines_ending_with_ellipsis_ratio: 0.3
+  # Top N-Gram filters for N-grams 2, 3, and 4
+  - name: nemo_curator.filters.heuristic_filter.RepeatingTopNGramsFilter
+    log_score: True
+    params:
+      n: 2
+      max_repeating_ngram_ratio: 0.2
+  - name: nemo_curator.filters.heuristic_filter.RepeatingTopNGramsFilter
+    log_score: True
+    params:
+      n: 3
+      max_repeating_ngram_ratio: 0.18
+  - name: nemo_curator.filters.heuristic_filter.RepeatingTopNGramsFilter
+    log_score: True
+    params:
+      n: 4
+      max_repeating_ngram_ratio: 0.16