diff --git a/fern/versions/v26.04/pages/about/concepts/image/data-loading-concepts.mdx b/fern/versions/v26.04/pages/about/concepts/image/data-loading-concepts.mdx index d640859f14..525e3cbb6e 100644 --- a/fern/versions/v26.04/pages/about/concepts/image/data-loading-concepts.mdx +++ b/fern/versions/v26.04/pages/about/concepts/image/data-loading-concepts.mdx @@ -63,7 +63,7 @@ pipeline.add_stage(FilePartitioningStage( # Load JPEG images from tar files using DALI pipeline.add_stage(ImageReaderStage( - batch_size=100, # Number of images per batch + dali_batch_size=100, # Number of images per batch verbose=True, num_threads=8, # Number of threads for I/O operations num_gpus_per_worker=0.25, # Allocate 1/4 GPU per worker diff --git a/fern/versions/v26.04/pages/about/release-notes/migration-guide.mdx b/fern/versions/v26.04/pages/about/release-notes/migration-guide.mdx index 0c1fc7d24f..1fe3afc1d5 100644 --- a/fern/versions/v26.04/pages/about/release-notes/migration-guide.mdx +++ b/fern/versions/v26.04/pages/about/release-notes/migration-guide.mdx @@ -251,7 +251,7 @@ In the new version, data loading is encapsulated in a dedicated pipeline stage ( ```python # New: Read images from webdataset tar files read_stage = ImageReaderStage( - batch_size=args.batch_size, + dali_batch_size=args.batch_size, num_threads=16, num_gpus_per_worker=0.25, ) diff --git a/fern/versions/v26.04/pages/curate-images/load-data/tar-archives.mdx b/fern/versions/v26.04/pages/curate-images/load-data/tar-archives.mdx index 77f6af8754..76f1b8a0ea 100644 --- a/fern/versions/v26.04/pages/curate-images/load-data/tar-archives.mdx +++ b/fern/versions/v26.04/pages/curate-images/load-data/tar-archives.mdx @@ -61,7 +61,7 @@ pipeline.add_stage(FilePartitioningStage( # Stage 2: Read JPEG images from tar files using DALI pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, verbose=True, num_threads=8, num_gpus_per_worker=0.25, @@ -79,7 +79,7 @@ results = pipeline.run() - `file_paths`: Path to directory containing tar files - `files_per_partition`: Number of tar files to process per partition (controls parallelism) -- `batch_size`: Number of images per ImageBatch for processing +- `dali_batch_size`: Number of images per ImageBatch for processing --- @@ -116,7 +116,7 @@ The `ImageReaderStage` is the core component that handles tar archive loading wi | Parameter | Type | Default | Description | | --- | --- | --- | --- | -| `batch_size` | int | 100 | Number of images per ImageBatch for processing | +| `dali_batch_size` | int | 100 | Number of images per ImageBatch for processing | | `verbose` | bool | True | Enable verbose logging for debugging | | `num_threads` | int | 8 | Number of threads for DALI operations | | `num_gpus_per_worker` | float | 0.25 | GPU allocation per worker (0.25 = 1/4 GPU) | diff --git a/fern/versions/v26.04/pages/curate-images/process-data/embeddings/clip-embedder.mdx b/fern/versions/v26.04/pages/curate-images/process-data/embeddings/clip-embedder.mdx index d832d8c1a4..24f2e9296a 100644 --- a/fern/versions/v26.04/pages/curate-images/process-data/embeddings/clip-embedder.mdx +++ b/fern/versions/v26.04/pages/curate-images/process-data/embeddings/clip-embedder.mdx @@ -63,7 +63,7 @@ pipeline.add_stage(FilePartitioningStage( # Stage 2: Read images pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, num_threads=8, num_gpus_per_worker=0.25, )) diff --git a/fern/versions/v26.04/pages/curate-images/process-data/filters/aesthetic.mdx b/fern/versions/v26.04/pages/curate-images/process-data/filters/aesthetic.mdx index 5f764d9aab..8ad6227e80 100644 --- a/fern/versions/v26.04/pages/curate-images/process-data/filters/aesthetic.mdx +++ b/fern/versions/v26.04/pages/curate-images/process-data/filters/aesthetic.mdx @@ -66,7 +66,7 @@ pipeline.add_stage(FilePartitioningStage( # Stage 2: Read images pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, num_gpus_per_worker=0.25, )) diff --git a/fern/versions/v26.04/pages/curate-images/process-data/filters/nsfw.mdx b/fern/versions/v26.04/pages/curate-images/process-data/filters/nsfw.mdx index ec46396321..202c868922 100644 --- a/fern/versions/v26.04/pages/curate-images/process-data/filters/nsfw.mdx +++ b/fern/versions/v26.04/pages/curate-images/process-data/filters/nsfw.mdx @@ -66,7 +66,7 @@ pipeline.add_stage(FilePartitioningStage( # Stage 2: Read images pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, num_gpus_per_worker=0.25, )) diff --git a/fern/versions/v26.04/pages/curate-images/tutorials/beginner.mdx b/fern/versions/v26.04/pages/curate-images/tutorials/beginner.mdx index 62ac2c08e0..d871b7ce9c 100644 --- a/fern/versions/v26.04/pages/curate-images/tutorials/beginner.mdx +++ b/fern/versions/v26.04/pages/curate-images/tutorials/beginner.mdx @@ -104,7 +104,7 @@ Load images from tar archives and extract metadata. ```python pipeline.add_stage(ImageReaderStage( - batch_size=100, # Images per batch + dali_batch_size=100, # Images per batch verbose=True, num_threads=16, # I/O threads num_gpus_per_worker=0.25, @@ -215,7 +215,7 @@ def create_image_curation_pipeline(): )) pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25, diff --git a/fern/versions/v26.04/pages/curate-images/tutorials/dedup-workflow.mdx b/fern/versions/v26.04/pages/curate-images/tutorials/dedup-workflow.mdx index 3df3af9b2a..50d8584615 100644 --- a/fern/versions/v26.04/pages/curate-images/tutorials/dedup-workflow.mdx +++ b/fern/versions/v26.04/pages/curate-images/tutorials/dedup-workflow.mdx @@ -56,7 +56,7 @@ def create_image_embedding_pipeline(input_dir, embeddings_dir, model_dir): # Read images from tar archives pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25, @@ -207,7 +207,7 @@ def create_image_removal_pipeline(input_dir, removal_dir, output_dir): # Read original images pipeline.add_stage(ImageReaderStage( - batch_size=100, + dali_batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25, @@ -324,7 +324,7 @@ def run_image_deduplication_workflow(): file_paths=INPUT_TAR_DIR, files_per_partition=1, file_extensions=[".tar"] )) embedding_pipeline.add_stage(ImageReaderStage( - batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25 + dali_batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25 )) embedding_pipeline.add_stage(ImageEmbeddingStage( model_dir=MODEL_DIR, num_gpus_per_worker=0.25, @@ -360,7 +360,7 @@ def run_image_deduplication_workflow(): file_paths=INPUT_TAR_DIR, files_per_partition=1, file_extensions=[".tar"] )) removal_pipeline.add_stage(ImageReaderStage( - batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25 + dali_batch_size=100, verbose=True, num_threads=16, num_gpus_per_worker=0.25 )) removal_pipeline.add_stage(ImageDuplicatesRemovalStage( removal_parquets_dir=REMOVAL_DIR + "/duplicates", diff --git a/fern/versions/v26.04/pages/get-started/image.mdx b/fern/versions/v26.04/pages/get-started/image.mdx index b26571d788..733b11e941 100644 --- a/fern/versions/v26.04/pages/get-started/image.mdx +++ b/fern/versions/v26.04/pages/get-started/image.mdx @@ -116,7 +116,7 @@ Here's a simple example to get started with NeMo Curator's image curation pipeli **CPU Memory Considerations** Image loading and decoding happens in CPU memory before GPU processing. If you encounter out-of-memory errors during the `ImageReaderStage`, reduce: -- `batch_size`: Number of images per batch (reduce to 32-50 for systems with limited RAM) +- `dali_batch_size`: Number of images per batch (reduce to 32-50 for systems with limited RAM) - `num_threads`: Parallel decoding threads (reduce to 4 for systems with limited RAM) - `num_cpus`: Ray Client CPU allocation (reduce to 8-16 for systems with limited RAM) @@ -152,7 +152,7 @@ pipeline.add_stage(FilePartitioningStage( # Stage 2: Read images from tar files using DALI pipeline.add_stage(ImageReaderStage( - batch_size=50, + dali_batch_size=50, verbose=True, num_threads=4, num_gpus_per_worker=0.25,