diff --git a/fern/fern.config.json b/fern/fern.config.json
index a6de8643c9..3dd6020ae7 100644
--- a/fern/fern.config.json
+++ b/fern/fern.config.json
@@ -1,4 +1,4 @@
{
"organization": "nvidia",
- "version": "4.42.1"
+ "version": "4.43.1"
}
diff --git a/fern/versions/v26.04/pages/about/release-notes/index.mdx b/fern/versions/v26.04/pages/about/release-notes/index.mdx
index b7df52f8d9..b89883dd2d 100644
--- a/fern/versions/v26.04/pages/about/release-notes/index.mdx
+++ b/fern/versions/v26.04/pages/about/release-notes/index.mdx
@@ -12,6 +12,15 @@ modality: "universal"
## What's New in 26.04
+### vLLM Default for Semantic Deduplication Embeddings
+
+The default embedding backend for `TextSemanticDeduplicationWorkflow` switched from SentenceTransformers to vLLM:
+
+- **Default model**: `google/embeddinggemma-300m` (previously `sentence-transformers/all-MiniLM-L6-v2`)
+- **Backend**: `VLLMEmbeddingModelStage` replaces `EmbeddingCreatorStage` for embedding generation in the semantic dedup workflow
+- **New parameters**: `embedding_pretokenize`, `embedding_vllm_init_kwargs`, `model_cache_dir` replace the previous SentenceTransformers-specific parameters
+- **Bug fix**: Resolved CUDA fork error when running vLLM stages with `RayDataExecutor` by setting `VLLM_WORKER_MULTIPROC_METHOD=spawn` in Ray remote tasks
+
### Multi-User Metrics Isolation
Improved Prometheus and Grafana monitoring support for shared clusters:
@@ -58,6 +67,8 @@ Fixed a race condition in `CaptionGenerationStage` and `CaptionEnhancementStage`
## Breaking Changes
+- **`TextSemanticDeduplicationWorkflow` parameter changes**: `embedding_max_seq_length`, `embedding_padding_side`, `embedding_pooling`, and `embedding_model_inference_batch_size` are removed. Use `embedding_pretokenize`, `embedding_vllm_init_kwargs`, and `model_cache_dir` instead.
+- **Default embedding model changed**: `TextSemanticDeduplicationWorkflow` now defaults to `google/embeddinggemma-300m` with vLLM instead of `sentence-transformers/all-MiniLM-L6-v2` with SentenceTransformers.
- **`Resources` API**: The `nvdecs`, `nvencs`, and `entire_gpu` fields have been removed from `Resources`. Stages that previously used `entire_gpu=True` should use `gpus=1` instead. Stages that used `nvdecs` or `nvencs` should use `gpus` for GPU allocation.
- **`ExactDeduplicationWorkflow.run()` and `FuzzyDeduplicationWorkflow.run()`** now return `WorkflowRunResult` instead of `None`
- **`SemanticDeduplicationWorkflow.run()` and `TextSemanticDeduplicationWorkflow.run()`** now return `WorkflowRunResult` instead of `dict`
diff --git a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx
index 016759addd..555ed81d2f 100644
--- a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx
+++ b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx
@@ -109,7 +109,6 @@ text_workflow = TextSemanticDeduplicationWorkflow(
output_path="/path/to/output",
cache_path="/path/to/cache",
text_field="text",
- model_identifier="sentence-transformers/all-MiniLM-L6-v2",
n_clusters=100,
eps=0.01, # Similarity threshold
perform_removal=True # Complete deduplication
@@ -133,19 +132,20 @@ For fine-grained control, break semantic deduplication into separate stages:
```python
from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor
-from nemo_curator.stages.text.embedders import EmbeddingCreatorStage
+from nemo_curator.stages.text.embedders.vllm import VLLMEmbeddingModelStage
from nemo_curator.stages.deduplication.semantic import SemanticDeduplicationWorkflow
# 1. Create ID generator
create_id_generator_actor()
-# 2. Generate embeddings separately
+# 2. Generate embeddings separately (using vLLM)
embedding_pipeline = Pipeline(
stages=[
ParquetReader(file_paths=input_path, _generate_ids=True),
- EmbeddingCreatorStage(
- model_identifier="sentence-transformers/all-MiniLM-L6-v2",
- text_field="text"
+ VLLMEmbeddingModelStage(
+ model_identifier="google/embeddinggemma-300m",
+ text_field="text",
+ embedding_field="embeddings",
),
ParquetWriter(path=embedding_output_path, fields=["_curator_dedup_id", "embeddings"])
]
diff --git a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx
index c68d344d6d..4885569a14 100644
--- a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx
+++ b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx
@@ -53,11 +53,11 @@ Get started with semantic deduplication using the following example of identifyi
from nemo_curator.stages.text.deduplication.semantic import TextSemanticDeduplicationWorkflow
from nemo_curator.backends.experimental.ray_data import RayDataExecutor
+# Default: uses vLLM with google/embeddinggemma-300m
workflow = TextSemanticDeduplicationWorkflow(
input_path="input_data/",
- output_path="./results",
+ output_path="./results",
cache_path="./sem_cache",
- model_identifier="sentence-transformers/all-MiniLM-L6-v2",
n_clusters=100,
eps=0.07, # Similarity threshold
id_field="doc_id",
@@ -79,7 +79,7 @@ For fine-grained control, break semantic deduplication into separate stages:
```python
from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor
-from nemo_curator.stages.text.embedders import EmbeddingCreatorStage
+from nemo_curator.stages.text.embedders.vllm import VLLMEmbeddingModelStage
from nemo_curator.stages.deduplication.semantic import SemanticDeduplicationWorkflow, IdentifyDuplicatesStage
from nemo_curator.stages.text.deduplication.removal_workflow import TextDuplicatesRemovalWorkflow
from nemo_curator.pipeline import Pipeline
@@ -89,16 +89,15 @@ from nemo_curator.stages.text.io.writer import ParquetWriter
# Step 1: Create ID generator
create_id_generator_actor()
-# Step 2: Generate embeddings separately
+# Step 2: Generate embeddings separately (using vLLM)
embedding_pipeline = Pipeline(
name="embedding_pipeline",
stages=[
ParquetReader(file_paths=input_path, files_per_partition=1, fields=["text"], _generate_ids=True),
- EmbeddingCreatorStage(
- model_identifier="sentence-transformers/all-MiniLM-L6-v2",
+ VLLMEmbeddingModelStage(
+ model_identifier="google/embeddinggemma-300m",
text_field="text",
- embedding_pooling="mean_pooling",
- model_inference_batch_size=256,
+ embedding_field="embeddings",
),
ParquetWriter(path=embedding_output_path, fields=["_curator_dedup_id", "embeddings"]),
],
@@ -139,54 +138,60 @@ Compare semantic deduplication with other methods:
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
-| `model_identifier` | str | "sentence-transformers/all-MiniLM-L6-v2" | Pre-trained model for embedding generation |
-| `embedding_model_inference_batch_size` | int | 256 | Number of samples per embedding batch |
-| `n_clusters` | int | 100 | Number of clusters for k-means clustering |
-| `kmeans_max_iter` | int | 300 | Maximum iterations for clustering |
-| `eps` | float | 0.01 | Threshold for deduplication (higher = more aggressive) |
-| `which_to_keep` | str | "hard" | Strategy for keeping duplicates ("hard"/"easy"/"random") |
-| `pairwise_batch_size` | int | 1024 | Batch size for similarity computation |
-| `distance_metric` | str | "cosine" | Distance metric for similarity ("cosine" or "l2") |
-| `embedding_pooling` | str | "mean_pooling" | Pooling strategy ("mean_pooling" or "last_token") |
-| `perform_removal` | bool | true | Whether to perform duplicate removal |
-| `text_field` | str | "text" | Name of the text field in input data |
-| `id_field` | str | "_curator_dedup_id" | Name of the ID field in the data |
+| `model_identifier` | str | `"google/embeddinggemma-300m"` | Pre-trained model for embedding generation (uses vLLM backend) |
+| `embedding_pretokenize` | bool | `False` | Whether to pre-tokenize input before passing to vLLM |
+| `embedding_vllm_init_kwargs` | dict | `None` | Additional kwargs passed to vLLM's `LLM` initializer (for example, `{"enforce_eager": True, "max_model_len": 2048}`) |
+| `model_cache_dir` | str | `None` | Directory to cache model weights (uses default HuggingFace cache if not set) |
+| `n_clusters` | int | `100` | Number of clusters for k-means clustering |
+| `kmeans_max_iter` | int | `300` | Maximum iterations for clustering |
+| `eps` | float | `0.01` | Threshold for deduplication (higher = more aggressive) |
+| `which_to_keep` | str | `"hard"` | Strategy for keeping duplicates ("hard"/"easy"/"random") |
+| `pairwise_batch_size` | int | `1024` | Batch size for similarity computation |
+| `distance_metric` | str | `"cosine"` | Distance metric for similarity ("cosine" or "l2") |
+| `perform_removal` | bool | `True` | Whether to perform duplicate removal |
+| `text_field` | str | `"text"` | Name of the text field in input data |
+| `id_field` | str | `"_curator_dedup_id"` | Name of the ID field in the data |
### Similarity Threshold
Control deduplication aggressiveness with `eps`:
-- **Lower values** (e.g., 0.001): More strict, less deduplication, higher confidence
-- **Higher values** (e.g., 0.1): Less strict, more aggressive deduplication
+- **Lower values** (such as 0.001): More strict, less deduplication, higher confidence
+- **Higher values** (such as 0.1): Less strict, more aggressive deduplication
Experiment with different values to balance data reduction and dataset diversity.
-**Sentence Transformers** (recommended for text):
+The default embedding backend uses **vLLM** with `google/embeddinggemma-300m`. You can use any HuggingFace embedding model that vLLM supports.
+
+**Default (vLLM)**:
```python
workflow = TextSemanticDeduplicationWorkflow(
- model_identifier="sentence-transformers/all-MiniLM-L6-v2",
+ model_identifier="google/embeddinggemma-300m",
# ... other parameters
)
```
-**HuggingFace Models**:
+**Custom vLLM configuration**:
```python
workflow = TextSemanticDeduplicationWorkflow(
- model_identifier="facebook/opt-125m",
+ model_identifier="google/embeddinggemma-300m",
+ embedding_pretokenize=True,
+ embedding_vllm_init_kwargs={"enforce_eager": True, "max_model_len": 2048},
+ model_cache_dir="/path/to/model/cache",
# ... other parameters
)
```
**When choosing a model**:
-- Ensure compatibility with your data type
-- Adjust `embedding_model_inference_batch_size` for memory requirements
+- Use models trained for sentence embeddings, such as EmbeddingGemma, E5, BGE, or SBERT
+- Avoid generic decoder-only LLMs such as OPT or GPT for embeddings
- Choose models appropriate for your language or domain
-- Avoid generic decoder-only LLMs (e.g., OPT/GPT) for embeddings; prefer models trained for sentence embeddings (e.g., E5/BGE/SBERT)
+- Use `embedding_vllm_init_kwargs` to tune vLLM behavior (for example, `max_model_len`, `enforce_eager`)
@@ -197,25 +202,25 @@ workflow = TextSemanticDeduplicationWorkflow(
input_path="input_data/",
output_path="results/",
cache_path="semdedup_cache",
-
- # Embedding generation
+
+ # Embedding generation (vLLM backend)
text_field="text",
- model_identifier="sentence-transformers/all-MiniLM-L6-v2",
- embedding_max_seq_length=512,
- embedding_pooling="mean_pooling",
- embedding_model_inference_batch_size=256,
-
+ model_identifier="google/embeddinggemma-300m",
+ embedding_pretokenize=False,
+ embedding_vllm_init_kwargs={"enforce_eager": True, "max_model_len": 2048},
+ model_cache_dir=None,
+
# Deduplication
n_clusters=100,
eps=0.01, # Similarity threshold
distance_metric="cosine",
which_to_keep="hard",
-
+
# K-means
kmeans_max_iter=300,
kmeans_tol=1e-4,
pairwise_batch_size=1024,
-
+
perform_removal=True
)
```