docs: update Fern docs for filter/modifier reorganization (PR #1472)

fern/versions/v26.04/pages/about/concepts/audio/text-integration.mdx

@@ -265,8 +265,8 @@ NeMo Curator provides these audio quality assessment capabilities:
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.audio.inference.asr_nemo import InferenceAsrNemoStage
 from nemo_curator.stages.audio.io.convert import AudioToDocumentStage
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
-from nemo_curator.filters import WordCountFilter  # Example filter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter  # Example filter
 
 # Define a text quality filter
 text_quality_filter = WordCountFilter(min_words=10)

fern/versions/v26.04/pages/about/concepts/text/data-processing-concepts.mdx

@@ -86,7 +86,7 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
+from nemo_curator.stages.text.filters import ScoreFilter
 from nemo_curator.stages.text.filters import (
     WordCountFilter,
     NonAlphaNumericFilter,
@@ -167,8 +167,8 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import Modify
-from nemo_curator.stages.text.modifiers import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
 
 # Start Ray client
 ray_client = RayClient()

fern/versions/v26.04/pages/about/release-notes/index.mdx

@@ -57,6 +57,16 @@ Upgraded Cosmos-Xenna from 0.1.2 to 0.2.0 with a simplified resource model and i
 - **Xenna-managed CUDA devices**: Xenna now manages CUDA device visibility directly, replacing the previous Ray-managed approach.
 - **Ray 2.54**: Updated Ray dependency to version 2.54 for compatibility with Cosmos-Xenna 0.2.0.
 
+### Filter and Modifier Directory Reorganization (PR #1472)
+
+Reorganized the `DocumentFilter` and `DocumentModifier` directory structures to avoid eagerly importing heavy dependencies:
+
+- **Lazy imports**: Importing `DocumentFilter` or `DocumentModifier` no longer pulls in heavyweight dependencies like HuggingFace Transformers, fastText, or histogram libraries.
+- **Grouped by dependency weight**: Filters are now organized into `heuristic/`, `token/`, `histogram/`, and `fasttext/` subdirectories.
+- **Modifiers reorganized**: Modifiers are now grouped into `string/`, `unicode/`, and `fasttext/` subdirectories.
+- **`ScoreFilter`/`Filter`/`Score` moved**: These stages moved from `stages.text.modules` to `stages.text.filters`.
+- **`Modify` moved**: Moved from `stages.text.modules` to `stages.text.modifiers`.
+
 ### Fused Document Iterate and Extract Stages (PR #1458)
 
 The data acquisition pipeline now uses a three-stage architecture instead of four, fusing the iterate and extract steps into a single `DocumentIterateExtractStage`. This reduces memory overhead and improves pipeline performance:

fern/versions/v26.04/pages/curate-text/load-data/read-existing.mdx

@@ -22,8 +22,8 @@ Use Curator's `JsonlReader` and `ParquetReader` to read existing datasets into a
 from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
 
 # Initialize Ray client
 ray_client = RayClient()
@@ -67,8 +67,8 @@ ray_client.stop()
 from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import ParquetReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
 
 # Initialize Ray client
 ray_client = RayClient()

fern/versions/v26.04/pages/curate-text/process-data/content-processing/index.mdx

@@ -57,8 +57,9 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
 
 # Initialize Ray client
 ray_client = RayClient()

fern/versions/v26.04/pages/curate-text/process-data/content-processing/text-cleaning.mdx

@@ -40,8 +40,9 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
 
 def main():
     # Initialize Ray client
@@ -85,7 +86,7 @@ You can create your own custom text cleaner by extending the `DocumentModifier`
 ```python
 import re
 
-from nemo_curator.stages.text.modifiers.doc_modifier import DocumentModifier
+from nemo_curator.stages.text.modifiers import DocumentModifier
 
 URL_REGEX = re.compile(r"https?://\S+|www\.\S+", flags=re.IGNORECASE)
 

fern/versions/v26.04/pages/curate-text/process-data/language-management/index.mdx

@@ -30,8 +30,8 @@ Language management in NeMo Curator typically follows this pattern using the Pip
 ```python
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import FastTextLangId
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
 
 # 1) Build the pipeline
 pipeline = Pipeline(name="language_management")

fern/versions/v26.04/pages/curate-text/process-data/language-management/language.mdx

@@ -47,9 +47,9 @@ The following example demonstrates how to create a language identification pipel
 """Language identification using Curator."""
 
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.filters import FastTextLangId
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
 from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
 
 def create_language_identification_pipeline(data_dir: str) -> Pipeline:
     """Create a pipeline for language identification."""

fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/classifier.mdx

@@ -83,8 +83,8 @@ results = pipeline.run()
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import FastTextQualityFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter
 
 # Create pipeline with FastText filter (requires pre-trained model)
 pipeline = Pipeline(name="fasttext_quality_pipeline")
@@ -119,7 +119,7 @@ You can configure quality classifiers and filters with different parameters:
 
 ```python
 from nemo_curator.stages.text.classifiers import QualityClassifier
-from nemo_curator.stages.text.filters import FastTextQualityFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter
 
 # DeBERTa quality classifier configurations
 basic_deberta_classifier = QualityClassifier(

fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/heuristic.mdx

@@ -35,12 +35,9 @@ For details on filter structure and the filtering process, refer to [Data Proces
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
-    WordCountFilter,
-    RepeatingTopNGramsFilter,
-    PunctuationFilter
-)
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter, PunctuationFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
 
 # Create pipeline
 pipeline = Pipeline(name="heuristic_filtering")
@@ -87,13 +84,13 @@ results = pipeline.run()
 <Tab title="Configuration">
 ```python
 # Example configuration for common heuristic filters
-from nemo_curator.stages.text.filters import (
+from nemo_curator.stages.text.filters.heuristic import (
     WordCountFilter,
     PunctuationFilter,
-    RepeatingTopNGramsFilter,
     SymbolsToWordsFilter,
     CommonEnglishWordsFilter
 )
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
 
 # Define filter configurations
 filters_config = [
@@ -197,17 +194,17 @@ stages:
     file_paths: ${input_path}
     fields: null
 
-  - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+  - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
     filter_obj:
-      _target_: nemo_curator.stages.text.filters.heuristic_filter.WordCountFilter
+      _target_: nemo_curator.stages.text.filters.heuristic.string.WordCountFilter
       min_words: 50
       max_words: 100000
     text_field: ${text_field}
     score_field: word_count
 
-  - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+  - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
     filter_obj:
-      _target_: nemo_curator.stages.text.filters.heuristic_filter.PunctuationFilter
+      _target_: nemo_curator.stages.text.filters.heuristic.string.PunctuationFilter
       max_num_sentences_without_endmark_ratio: 0.85
     text_field: ${text_field}
     score_field: null
@@ -232,8 +229,9 @@ When building filter chains, follow these best practices:
 ```python
 # Efficient ordering - place fast filters first
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter, UrlsFilter, RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter, UrlsFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
 
 pipeline = Pipeline(name="efficient_filtering")
 # Fast filters first
@@ -262,8 +260,8 @@ strict_filter = WordCountFilter(min_words=100, max_words=10000)
 <Tab title="Language Considerations">
 ```python
 # Chinese text filter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import SymbolsToWordsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import SymbolsToWordsFilter
 
 cn_filter = ScoreFilter(
     filter_obj=SymbolsToWordsFilter(max_symbol_to_word_ratio=0.15, lang="zh"),
@@ -276,13 +274,13 @@ cn_filter = ScoreFilter(
 ```python
 # Comprehensive quality filter pipeline
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
-    WordCountFilter, 
-    PunctuationFilter, 
-    CommonEnglishWordsFilter, 
-    RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import (
+    WordCountFilter,
+    PunctuationFilter,
+    CommonEnglishWordsFilter,
 )
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
 
 quality_pipeline = Pipeline(name="comprehensive_quality")
 
@@ -325,8 +323,9 @@ Use `Score` to add score columns to your data without removing any documents:
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import Score
-from nemo_curator.stages.text.filters import WordCountFilter, RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import Score
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
 
 # Create scoring pipeline (no filtering)
 pipeline = Pipeline(name="score_analysis")
@@ -403,8 +402,9 @@ After analyzing distributions, apply filters with your chosen thresholds:
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter, RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
 
 pipeline = Pipeline(name="filtering_pipeline")
 pipeline.add_stage(JsonlReader(file_paths="input_data/", fields=["text", "id"]))

fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/index.mdx

@@ -27,8 +27,8 @@ The `ScoreFilter` is at the center of filtering in NeMo Curator. It applies a fi
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
 
 # Create pipeline
 pipeline = Pipeline(name="quality_filtering")
@@ -78,7 +78,7 @@ For more specific use cases, NeMo Curator provides two specialized modules:
 
 ```python
 # Example: Score documents without filtering
-from nemo_curator.stages.text.modules import Score
+from nemo_curator.stages.text.filters import Score
 
 scoring_step = Score(
     WordCountFilter().score_document,  # Use just the scoring part
@@ -95,7 +95,7 @@ scored_dataset = scoring_step.process(dataset)
 
 ```python
 # Example: Filter using pre-computed scores
-from nemo_curator.stages.text.modules import Filter
+from nemo_curator.stages.text.filters import Filter
 
 filter_step = Filter(
     lambda score: score >= 100,  # Keep documents with score >= 100
@@ -108,7 +108,7 @@ You can combine these modules in pipelines:
 
 ```python
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import Score, Filter
+from nemo_curator.stages.text.filters import Score, Filter
 # Assume `word_counter` and `symbol_counter` are callables that return numeric scores
 pipeline = Pipeline(name="multi_stage_filtering")
 pipeline.add_stage(Score(word_counter, score_field="word_count"))
@@ -158,8 +158,8 @@ NeMo Curator provides programmatic interfaces for document filtering through the
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
 
 # Create and configure pipeline
 pipeline = Pipeline(name="document_filtering")

fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/code.mdx

@@ -27,8 +27,8 @@ Here's an example of applying code filters to a dataset:
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import (
     PythonCommentToCodeFilter,
     NumberOfLinesOfCodeFilter,
     AlphaFilter
@@ -197,8 +197,8 @@ When filtering code datasets, consider these best practices:
 1. **Language-specific configurations**: Adjust thresholds based on the programming language
 
    ```python
-   from nemo_curator.stages.text.modules import ScoreFilter
-   from nemo_curator.stages.text.filters import PythonCommentToCodeFilter, GeneralCommentToCodeFilter
+   from nemo_curator.stages.text.filters import ScoreFilter
+   from nemo_curator.stages.text.filters.heuristic.code import PythonCommentToCodeFilter, GeneralCommentToCodeFilter
 
    # Python tends to have more comments than C
    python_comment_filter = ScoreFilter(
@@ -214,8 +214,8 @@ When filtering code datasets, consider these best practices:
 2. **Preserve code structure**: Ensure filters don't inadvertently remove valid coding patterns
 
    ```python
-   from nemo_curator.stages.text.modules import ScoreFilter
-   from nemo_curator.stages.text.filters import GeneralCommentToCodeFilter
+   from nemo_curator.stages.text.filters import ScoreFilter
+   from nemo_curator.stages.text.filters.heuristic.code import GeneralCommentToCodeFilter
 
    # Some languages naturally have low comment ratios
    assembly_filter = ScoreFilter(
@@ -231,9 +231,9 @@ When filtering code datasets, consider these best practices:
 
    ```python
    # First check if the content is actually Python using FastText language ID
-   from nemo_curator.stages.text.filters import FastTextLangId
+   from nemo_curator.stages.text.filters.fasttext import FastTextLangId
    from nemo_curator.pipeline import Pipeline
-   from nemo_curator.stages.text.modules import ScoreFilter
+   from nemo_curator.stages.text.filters import ScoreFilter
 
    # Create pipeline for Python code filtering with language detection
    pipeline = Pipeline(name="python_code_filtering")
@@ -286,8 +286,8 @@ When filtering code datasets, consider these best practices:
 
 ```python
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import NumberOfLinesOfCodeFilter, XMLHeaderFilter, GeneralCommentToCodeFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import NumberOfLinesOfCodeFilter, XMLHeaderFilter, GeneralCommentToCodeFilter
 
 # Create pipeline to filter non-functional code snippets
 pipeline = Pipeline(name="code_cleaning")
@@ -317,8 +317,8 @@ pipeline.add_stage(ScoreFilter(
 
 ```python
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import AlphaFilter, TokenizerFertilityFilter, HTMLBoilerplateFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import AlphaFilter, TokenizerFertilityFilter, HTMLBoilerplateFilter
 
 # Create pipeline for training data preparation
 pipeline = Pipeline(name="training_data_prep")

fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/index.mdx

@@ -47,8 +47,8 @@ languages
 
 ```python
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter
 from nemo_curator.stages.text.io.reader import JsonlReader
 
 # Filter Python code based on quality metrics