docs: update semdedup docs for vLLM default backend (PR #1606)

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "nvidia",
-  "version": "4.42.1"
+  "version": "4.43.1"
 }

fern/versions/v26.04/pages/about/release-notes/index.mdx

@@ -12,6 +12,15 @@ modality: "universal"
 
 ## What's New in 26.04
 
+### vLLM Default for Semantic Deduplication Embeddings
+
+The default embedding backend for `TextSemanticDeduplicationWorkflow` switched from SentenceTransformers to vLLM:
+
+- **Default model**: `google/embeddinggemma-300m` (previously `sentence-transformers/all-MiniLM-L6-v2`)
+- **Backend**: `VLLMEmbeddingModelStage` replaces `EmbeddingCreatorStage` for embedding generation in the semantic dedup workflow
+- **New parameters**: `embedding_pretokenize`, `embedding_vllm_init_kwargs`, `model_cache_dir` replace the previous SentenceTransformers-specific parameters
+- **Bug fix**: Resolved CUDA fork error when running vLLM stages with `RayDataExecutor` by setting `VLLM_WORKER_MULTIPROC_METHOD=spawn` in Ray remote tasks
+
 ### Multi-User Metrics Isolation
 
 Improved Prometheus and Grafana monitoring support for shared clusters:
@@ -58,6 +67,8 @@ Fixed a race condition in `CaptionGenerationStage` and `CaptionEnhancementStage`
 
 ## Breaking Changes
 
+- **`TextSemanticDeduplicationWorkflow` parameter changes**: `embedding_max_seq_length`, `embedding_padding_side`, `embedding_pooling`, and `embedding_model_inference_batch_size` are removed. Use `embedding_pretokenize`, `embedding_vllm_init_kwargs`, and `model_cache_dir` instead.
+- **Default embedding model changed**: `TextSemanticDeduplicationWorkflow` now defaults to `google/embeddinggemma-300m` with vLLM instead of `sentence-transformers/all-MiniLM-L6-v2` with SentenceTransformers.
 - **`Resources` API**: The `nvdecs`, `nvencs`, and `entire_gpu` fields have been removed from `Resources`. Stages that previously used `entire_gpu=True` should use `gpus=1` instead. Stages that used `nvdecs` or `nvencs` should use `gpus` for GPU allocation.
 - **`ExactDeduplicationWorkflow.run()` and `FuzzyDeduplicationWorkflow.run()`** now return `WorkflowRunResult` instead of `None`
 - **`SemanticDeduplicationWorkflow.run()` and `TextSemanticDeduplicationWorkflow.run()`** now return `WorkflowRunResult` instead of `dict`

fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx

@@ -109,7 +109,6 @@ text_workflow = TextSemanticDeduplicationWorkflow(
     output_path="/path/to/output",
     cache_path="/path/to/cache",
     text_field="text",
-    model_identifier="sentence-transformers/all-MiniLM-L6-v2",
     n_clusters=100,
     eps=0.01,  # Similarity threshold
     perform_removal=True  # Complete deduplication
@@ -133,19 +132,20 @@ For fine-grained control, break semantic deduplication into separate stages:
 
 ```python
 from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor
-from nemo_curator.stages.text.embedders import EmbeddingCreatorStage
+from nemo_curator.stages.text.embedders.vllm import VLLMEmbeddingModelStage
 from nemo_curator.stages.deduplication.semantic import SemanticDeduplicationWorkflow
 
 # 1. Create ID generator
 create_id_generator_actor()
 
-# 2. Generate embeddings separately
+# 2. Generate embeddings separately (using vLLM)
 embedding_pipeline = Pipeline(
     stages=[
         ParquetReader(file_paths=input_path, _generate_ids=True),
-        EmbeddingCreatorStage(
-            model_identifier="sentence-transformers/all-MiniLM-L6-v2",
-            text_field="text"
+        VLLMEmbeddingModelStage(
+            model_identifier="google/embeddinggemma-300m",
+            text_field="text",
+            embedding_field="embeddings",
         ),
         ParquetWriter(path=embedding_output_path, fields=["_curator_dedup_id", "embeddings"])
     ]

fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx

@@ -53,11 +53,11 @@ Get started with semantic deduplication using the following example of identifyi
 from nemo_curator.stages.text.deduplication.semantic import TextSemanticDeduplicationWorkflow
 from nemo_curator.backends.experimental.ray_data import RayDataExecutor
 
+# Default: uses vLLM with google/embeddinggemma-300m
 workflow = TextSemanticDeduplicationWorkflow(
     input_path="input_data/",
-    output_path="./results", 
+    output_path="./results",
     cache_path="./sem_cache",
-    model_identifier="sentence-transformers/all-MiniLM-L6-v2",
     n_clusters=100,
     eps=0.07,  # Similarity threshold
     id_field="doc_id",
@@ -79,7 +79,7 @@ For fine-grained control, break semantic deduplication into separate stages:
 
 ```python
 from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor
-from nemo_curator.stages.text.embedders import EmbeddingCreatorStage
+from nemo_curator.stages.text.embedders.vllm import VLLMEmbeddingModelStage
 from nemo_curator.stages.deduplication.semantic import SemanticDeduplicationWorkflow, IdentifyDuplicatesStage
 from nemo_curator.stages.text.deduplication.removal_workflow import TextDuplicatesRemovalWorkflow
 from nemo_curator.pipeline import Pipeline
@@ -89,16 +89,15 @@ from nemo_curator.stages.text.io.writer import ParquetWriter
 # Step 1: Create ID generator
 create_id_generator_actor()
 
-# Step 2: Generate embeddings separately
+# Step 2: Generate embeddings separately (using vLLM)
 embedding_pipeline = Pipeline(
     name="embedding_pipeline",
     stages=[
         ParquetReader(file_paths=input_path, files_per_partition=1, fields=["text"], _generate_ids=True),
-        EmbeddingCreatorStage(
-            model_identifier="sentence-transformers/all-MiniLM-L6-v2",
+        VLLMEmbeddingModelStage(
+            model_identifier="google/embeddinggemma-300m",
             text_field="text",
-            embedding_pooling="mean_pooling",
-            model_inference_batch_size=256,
+            embedding_field="embeddings",
         ),
         ParquetWriter(path=embedding_output_path, fields=["_curator_dedup_id", "embeddings"]),
     ],
@@ -139,54 +138,60 @@ Compare semantic deduplication with other methods:
 
 | Parameter | Type | Default | Description |
 | --- | --- | --- | --- |
-| `model_identifier` | str | "sentence-transformers/all-MiniLM-L6-v2" | Pre-trained model for embedding generation |
-| `embedding_model_inference_batch_size` | int | 256 | Number of samples per embedding batch |
-| `n_clusters` | int | 100 | Number of clusters for k-means clustering |
-| `kmeans_max_iter` | int | 300 | Maximum iterations for clustering |
-| `eps` | float | 0.01 | Threshold for deduplication (higher = more aggressive) |
-| `which_to_keep` | str | "hard" | Strategy for keeping duplicates ("hard"/"easy"/"random") |
-| `pairwise_batch_size` | int | 1024 | Batch size for similarity computation |
-| `distance_metric` | str | "cosine" | Distance metric for similarity ("cosine" or "l2") |
-| `embedding_pooling` | str | "mean_pooling" | Pooling strategy ("mean_pooling" or "last_token") |
-| `perform_removal` | bool | true | Whether to perform duplicate removal |
-| `text_field` | str | "text" | Name of the text field in input data |
-| `id_field` | str | "_curator_dedup_id" | Name of the ID field in the data |
+| `model_identifier` | str | `"google/embeddinggemma-300m"` | Pre-trained model for embedding generation (uses vLLM backend) |
+| `embedding_pretokenize` | bool | `False` | Whether to pre-tokenize input before passing to vLLM |
+| `embedding_vllm_init_kwargs` | dict | `None` | Additional kwargs passed to vLLM's `LLM` initializer (for example, `{"enforce_eager": True, "max_model_len": 2048}`) |
+| `model_cache_dir` | str | `None` | Directory to cache model weights (uses default HuggingFace cache if not set) |
+| `n_clusters` | int | `100` | Number of clusters for k-means clustering |
+| `kmeans_max_iter` | int | `300` | Maximum iterations for clustering |
+| `eps` | float | `0.01` | Threshold for deduplication (higher = more aggressive) |
+| `which_to_keep` | str | `"hard"` | Strategy for keeping duplicates ("hard"/"easy"/"random") |
+| `pairwise_batch_size` | int | `1024` | Batch size for similarity computation |
+| `distance_metric` | str | `"cosine"` | Distance metric for similarity ("cosine" or "l2") |
+| `perform_removal` | bool | `True` | Whether to perform duplicate removal |
+| `text_field` | str | `"text"` | Name of the text field in input data |
+| `id_field` | str | `"_curator_dedup_id"` | Name of the ID field in the data |
 
 ### Similarity Threshold
 
 Control deduplication aggressiveness with `eps`:
 
-- **Lower values** (e.g., 0.001): More strict, less deduplication, higher confidence
-- **Higher values** (e.g., 0.1): Less strict, more aggressive deduplication
+- **Lower values** (such as 0.001): More strict, less deduplication, higher confidence
+- **Higher values** (such as 0.1): Less strict, more aggressive deduplication
 
 Experiment with different values to balance data reduction and dataset diversity.
 
 <Accordion title="Embedding Models">
 
-**Sentence Transformers** (recommended for text):
+The default embedding backend uses **vLLM** with `google/embeddinggemma-300m`. You can use any HuggingFace embedding model that vLLM supports.
+
+**Default (vLLM)**:
 
 ```python
 workflow = TextSemanticDeduplicationWorkflow(
-    model_identifier="sentence-transformers/all-MiniLM-L6-v2",
+    model_identifier="google/embeddinggemma-300m",
     # ... other parameters
 )
 ```
 
-**HuggingFace Models**:
+**Custom vLLM configuration**:
 
 ```python
 workflow = TextSemanticDeduplicationWorkflow(
-    model_identifier="facebook/opt-125m",
+    model_identifier="google/embeddinggemma-300m",
+    embedding_pretokenize=True,
+    embedding_vllm_init_kwargs={"enforce_eager": True, "max_model_len": 2048},
+    model_cache_dir="/path/to/model/cache",
     # ... other parameters
 )
 ```
 
 **When choosing a model**:
 
-- Ensure compatibility with your data type
-- Adjust `embedding_model_inference_batch_size` for memory requirements
+- Use models trained for sentence embeddings, such as EmbeddingGemma, E5, BGE, or SBERT
+- Avoid generic decoder-only LLMs such as OPT or GPT for embeddings
 - Choose models appropriate for your language or domain
-- Avoid generic decoder-only LLMs (e.g., OPT/GPT) for embeddings; prefer models trained for sentence embeddings (e.g., E5/BGE/SBERT)
+- Use `embedding_vllm_init_kwargs` to tune vLLM behavior (for example, `max_model_len`, `enforce_eager`)
 </Accordion>
 
 <Accordion title="Advanced Configuration">
@@ -197,25 +202,25 @@ workflow = TextSemanticDeduplicationWorkflow(
     input_path="input_data/",
     output_path="results/",
     cache_path="semdedup_cache",
-    
-    # Embedding generation
+
+    # Embedding generation (vLLM backend)
     text_field="text",
-    model_identifier="sentence-transformers/all-MiniLM-L6-v2",
-    embedding_max_seq_length=512,
-    embedding_pooling="mean_pooling",
-    embedding_model_inference_batch_size=256,
-    
+    model_identifier="google/embeddinggemma-300m",
+    embedding_pretokenize=False,
+    embedding_vllm_init_kwargs={"enforce_eager": True, "max_model_len": 2048},
+    model_cache_dir=None,
+
     # Deduplication
     n_clusters=100,
     eps=0.01,  # Similarity threshold
     distance_metric="cosine",
     which_to_keep="hard",
-    
+
     # K-means
     kmeans_max_iter=300,
     kmeans_tol=1e-4,
     pairwise_batch_size=1024,
-    
+
     perform_removal=True
 )
 ```