diff --git a/fern/fern.config.json b/fern/fern.config.json index a6de8643c9..3dd6020ae7 100644 --- a/fern/fern.config.json +++ b/fern/fern.config.json @@ -1,4 +1,4 @@ { "organization": "nvidia", - "version": "4.42.1" + "version": "4.43.1" } diff --git a/fern/versions/v26.04/pages/about/release-notes/index.mdx b/fern/versions/v26.04/pages/about/release-notes/index.mdx index b7df52f8d9..b89883dd2d 100644 --- a/fern/versions/v26.04/pages/about/release-notes/index.mdx +++ b/fern/versions/v26.04/pages/about/release-notes/index.mdx @@ -12,6 +12,15 @@ modality: "universal" ## What's New in 26.04 +### vLLM Default for Semantic Deduplication Embeddings + +The default embedding backend for `TextSemanticDeduplicationWorkflow` switched from SentenceTransformers to vLLM: + +- **Default model**: `google/embeddinggemma-300m` (previously `sentence-transformers/all-MiniLM-L6-v2`) +- **Backend**: `VLLMEmbeddingModelStage` replaces `EmbeddingCreatorStage` for embedding generation in the semantic dedup workflow +- **New parameters**: `embedding_pretokenize`, `embedding_vllm_init_kwargs`, `model_cache_dir` replace the previous SentenceTransformers-specific parameters +- **Bug fix**: Resolved CUDA fork error when running vLLM stages with `RayDataExecutor` by setting `VLLM_WORKER_MULTIPROC_METHOD=spawn` in Ray remote tasks + ### Multi-User Metrics Isolation Improved Prometheus and Grafana monitoring support for shared clusters: @@ -58,6 +67,8 @@ Fixed a race condition in `CaptionGenerationStage` and `CaptionEnhancementStage` ## Breaking Changes +- **`TextSemanticDeduplicationWorkflow` parameter changes**: `embedding_max_seq_length`, `embedding_padding_side`, `embedding_pooling`, and `embedding_model_inference_batch_size` are removed. Use `embedding_pretokenize`, `embedding_vllm_init_kwargs`, and `model_cache_dir` instead. +- **Default embedding model changed**: `TextSemanticDeduplicationWorkflow` now defaults to `google/embeddinggemma-300m` with vLLM instead of `sentence-transformers/all-MiniLM-L6-v2` with SentenceTransformers. - **`Resources` API**: The `nvdecs`, `nvencs`, and `entire_gpu` fields have been removed from `Resources`. Stages that previously used `entire_gpu=True` should use `gpus=1` instead. Stages that used `nvdecs` or `nvencs` should use `gpus` for GPU allocation. - **`ExactDeduplicationWorkflow.run()` and `FuzzyDeduplicationWorkflow.run()`** now return `WorkflowRunResult` instead of `None` - **`SemanticDeduplicationWorkflow.run()` and `TextSemanticDeduplicationWorkflow.run()`** now return `WorkflowRunResult` instead of `dict` diff --git a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx index 016759addd..555ed81d2f 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/index.mdx @@ -109,7 +109,6 @@ text_workflow = TextSemanticDeduplicationWorkflow( output_path="/path/to/output", cache_path="/path/to/cache", text_field="text", - model_identifier="sentence-transformers/all-MiniLM-L6-v2", n_clusters=100, eps=0.01, # Similarity threshold perform_removal=True # Complete deduplication @@ -133,19 +132,20 @@ For fine-grained control, break semantic deduplication into separate stages: ```python from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor -from nemo_curator.stages.text.embedders import EmbeddingCreatorStage +from nemo_curator.stages.text.embedders.vllm import VLLMEmbeddingModelStage from nemo_curator.stages.deduplication.semantic import SemanticDeduplicationWorkflow # 1. Create ID generator create_id_generator_actor() -# 2. Generate embeddings separately +# 2. Generate embeddings separately (using vLLM) embedding_pipeline = Pipeline( stages=[ ParquetReader(file_paths=input_path, _generate_ids=True), - EmbeddingCreatorStage( - model_identifier="sentence-transformers/all-MiniLM-L6-v2", - text_field="text" + VLLMEmbeddingModelStage( + model_identifier="google/embeddinggemma-300m", + text_field="text", + embedding_field="embeddings", ), ParquetWriter(path=embedding_output_path, fields=["_curator_dedup_id", "embeddings"]) ] diff --git a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx index c68d344d6d..4885569a14 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/deduplication/semdedup.mdx @@ -53,11 +53,11 @@ Get started with semantic deduplication using the following example of identifyi from nemo_curator.stages.text.deduplication.semantic import TextSemanticDeduplicationWorkflow from nemo_curator.backends.experimental.ray_data import RayDataExecutor +# Default: uses vLLM with google/embeddinggemma-300m workflow = TextSemanticDeduplicationWorkflow( input_path="input_data/", - output_path="./results", + output_path="./results", cache_path="./sem_cache", - model_identifier="sentence-transformers/all-MiniLM-L6-v2", n_clusters=100, eps=0.07, # Similarity threshold id_field="doc_id", @@ -79,7 +79,7 @@ For fine-grained control, break semantic deduplication into separate stages: ```python from nemo_curator.stages.deduplication.id_generator import create_id_generator_actor -from nemo_curator.stages.text.embedders import EmbeddingCreatorStage +from nemo_curator.stages.text.embedders.vllm import VLLMEmbeddingModelStage from nemo_curator.stages.deduplication.semantic import SemanticDeduplicationWorkflow, IdentifyDuplicatesStage from nemo_curator.stages.text.deduplication.removal_workflow import TextDuplicatesRemovalWorkflow from nemo_curator.pipeline import Pipeline @@ -89,16 +89,15 @@ from nemo_curator.stages.text.io.writer import ParquetWriter # Step 1: Create ID generator create_id_generator_actor() -# Step 2: Generate embeddings separately +# Step 2: Generate embeddings separately (using vLLM) embedding_pipeline = Pipeline( name="embedding_pipeline", stages=[ ParquetReader(file_paths=input_path, files_per_partition=1, fields=["text"], _generate_ids=True), - EmbeddingCreatorStage( - model_identifier="sentence-transformers/all-MiniLM-L6-v2", + VLLMEmbeddingModelStage( + model_identifier="google/embeddinggemma-300m", text_field="text", - embedding_pooling="mean_pooling", - model_inference_batch_size=256, + embedding_field="embeddings", ), ParquetWriter(path=embedding_output_path, fields=["_curator_dedup_id", "embeddings"]), ], @@ -139,54 +138,60 @@ Compare semantic deduplication with other methods: | Parameter | Type | Default | Description | | --- | --- | --- | --- | -| `model_identifier` | str | "sentence-transformers/all-MiniLM-L6-v2" | Pre-trained model for embedding generation | -| `embedding_model_inference_batch_size` | int | 256 | Number of samples per embedding batch | -| `n_clusters` | int | 100 | Number of clusters for k-means clustering | -| `kmeans_max_iter` | int | 300 | Maximum iterations for clustering | -| `eps` | float | 0.01 | Threshold for deduplication (higher = more aggressive) | -| `which_to_keep` | str | "hard" | Strategy for keeping duplicates ("hard"/"easy"/"random") | -| `pairwise_batch_size` | int | 1024 | Batch size for similarity computation | -| `distance_metric` | str | "cosine" | Distance metric for similarity ("cosine" or "l2") | -| `embedding_pooling` | str | "mean_pooling" | Pooling strategy ("mean_pooling" or "last_token") | -| `perform_removal` | bool | true | Whether to perform duplicate removal | -| `text_field` | str | "text" | Name of the text field in input data | -| `id_field` | str | "_curator_dedup_id" | Name of the ID field in the data | +| `model_identifier` | str | `"google/embeddinggemma-300m"` | Pre-trained model for embedding generation (uses vLLM backend) | +| `embedding_pretokenize` | bool | `False` | Whether to pre-tokenize input before passing to vLLM | +| `embedding_vllm_init_kwargs` | dict | `None` | Additional kwargs passed to vLLM's `LLM` initializer (for example, `{"enforce_eager": True, "max_model_len": 2048}`) | +| `model_cache_dir` | str | `None` | Directory to cache model weights (uses default HuggingFace cache if not set) | +| `n_clusters` | int | `100` | Number of clusters for k-means clustering | +| `kmeans_max_iter` | int | `300` | Maximum iterations for clustering | +| `eps` | float | `0.01` | Threshold for deduplication (higher = more aggressive) | +| `which_to_keep` | str | `"hard"` | Strategy for keeping duplicates ("hard"/"easy"/"random") | +| `pairwise_batch_size` | int | `1024` | Batch size for similarity computation | +| `distance_metric` | str | `"cosine"` | Distance metric for similarity ("cosine" or "l2") | +| `perform_removal` | bool | `True` | Whether to perform duplicate removal | +| `text_field` | str | `"text"` | Name of the text field in input data | +| `id_field` | str | `"_curator_dedup_id"` | Name of the ID field in the data | ### Similarity Threshold Control deduplication aggressiveness with `eps`: -- **Lower values** (e.g., 0.001): More strict, less deduplication, higher confidence -- **Higher values** (e.g., 0.1): Less strict, more aggressive deduplication +- **Lower values** (such as 0.001): More strict, less deduplication, higher confidence +- **Higher values** (such as 0.1): Less strict, more aggressive deduplication Experiment with different values to balance data reduction and dataset diversity. -**Sentence Transformers** (recommended for text): +The default embedding backend uses **vLLM** with `google/embeddinggemma-300m`. You can use any HuggingFace embedding model that vLLM supports. + +**Default (vLLM)**: ```python workflow = TextSemanticDeduplicationWorkflow( - model_identifier="sentence-transformers/all-MiniLM-L6-v2", + model_identifier="google/embeddinggemma-300m", # ... other parameters ) ``` -**HuggingFace Models**: +**Custom vLLM configuration**: ```python workflow = TextSemanticDeduplicationWorkflow( - model_identifier="facebook/opt-125m", + model_identifier="google/embeddinggemma-300m", + embedding_pretokenize=True, + embedding_vllm_init_kwargs={"enforce_eager": True, "max_model_len": 2048}, + model_cache_dir="/path/to/model/cache", # ... other parameters ) ``` **When choosing a model**: -- Ensure compatibility with your data type -- Adjust `embedding_model_inference_batch_size` for memory requirements +- Use models trained for sentence embeddings, such as EmbeddingGemma, E5, BGE, or SBERT +- Avoid generic decoder-only LLMs such as OPT or GPT for embeddings - Choose models appropriate for your language or domain -- Avoid generic decoder-only LLMs (e.g., OPT/GPT) for embeddings; prefer models trained for sentence embeddings (e.g., E5/BGE/SBERT) +- Use `embedding_vllm_init_kwargs` to tune vLLM behavior (for example, `max_model_len`, `enforce_eager`) @@ -197,25 +202,25 @@ workflow = TextSemanticDeduplicationWorkflow( input_path="input_data/", output_path="results/", cache_path="semdedup_cache", - - # Embedding generation + + # Embedding generation (vLLM backend) text_field="text", - model_identifier="sentence-transformers/all-MiniLM-L6-v2", - embedding_max_seq_length=512, - embedding_pooling="mean_pooling", - embedding_model_inference_batch_size=256, - + model_identifier="google/embeddinggemma-300m", + embedding_pretokenize=False, + embedding_vllm_init_kwargs={"enforce_eager": True, "max_model_len": 2048}, + model_cache_dir=None, + # Deduplication n_clusters=100, eps=0.01, # Similarity threshold distance_metric="cosine", which_to_keep="hard", - + # K-means kmeans_max_iter=300, kmeans_tol=1e-4, pairwise_batch_size=1024, - + perform_removal=True ) ```