Pin sklearn to < 1.8.0 for cuml 25.10

benchmarking/nightly-benchmark.yaml

@@ -28,7 +28,14 @@ datasets:
     formats:
     - type: "parquet"
       path: "{datasets_path}/commoncrawl/IDs/parquet_data"
-
+  - name: "mscoco"
+    formats:
+    - type: "wds"
+      path: "{datasets_path}/mscoco/wds/truncated_100K_mscoco_benchmarking"
+  - name: "mscoco_model_weights"
+    formats:
+    - type: "files"
+      path: "{datasets_path}/mscoco/model_weights"
 
 default_timeout_s: 7200
 
@@ -56,65 +63,108 @@ entries:
     script: domain_classification_benchmark.py
     args: >-
       --benchmark-results-path={session_entry_dir}
+      --output-path={session_entry_dir}/scratch/output
       --executor=ray_data
       --input-path={dataset:tinystories,parquet}
       --dataset-size-gb=10
       --model-inference-batch-size=1024
-    timeout_s: 20000
+    timeout_s: 1000
     sink_data:
       - name: slack
         # Additional metrics to include in the Slack report.  These must be present in the metrics.json file generated by the script.
-        additional_metrics: ["num_documents_processed", "throughput_docs_per_sec"]
+        additional_metrics:
+          - throughput_docs_per_sec
+          - number_of_domains_predicted
     ray:
       num_cpus: 64
       num_gpus: 4
       enable_object_spilling: false
-    # Additional requirements for the benchmark to pass.  These will result in the benchmark being marked as failed if not met.
     requirements:
+      # Observed throughput of 2700 docs/sec so we allow a 5% buffer to account for variability
       - metric: throughput_docs_per_sec
-        min_value: 0.2
+        min_value: 2565
+      - metric: number_of_domains_predicted
+        exact_value: 26
+      # We choose two domains randomly and ensure that the counts are as expected
+      - metric: domain_label_games_count
+        exact_value: 149816
+      - metric: domain_label_news_count
+        exact_value: 2817
 
+    object_store_size_bytes: 214748364800
   - name: domain_classification_xenna
     enabled: true
     script: domain_classification_benchmark.py
     args: >-
       --benchmark-results-path={session_entry_dir}
+      --output-path={session_entry_dir}/scratch/output
       --executor=xenna
       --input-path={dataset:tinystories,parquet}
       --dataset-size-gb=10
       --model-inference-batch-size=1024
-    timeout_s: 20000
+    timeout_s: 1000
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - throughput_docs_per_sec
+          - number_of_domains_predicted
+    requirements:
+      # Observed throughput of 2900 docs/sec so we allow a 5% buffer to account for variability
+      - metric: throughput_docs_per_sec
+        min_value: 2755
+      - metric: number_of_domains_predicted
+        exact_value: 26
+      # We choose two domains randomly and ensure that the counts are as expected
+      - metric: domain_label_games_count
+        exact_value: 149816
+      - metric: domain_label_news_count
+        exact_value: 2817
 
   - name: embedding_generation_raydata
     enabled: true
     script: embedding_generation_benchmark.py
     args: >-
       --benchmark-results-path={session_entry_dir}
+      --output-path={session_entry_dir}/scratch/output
       --executor=ray_data
       --input-path={dataset:tinystories,parquet}
       --dataset-size-gb=10
       --model-identifier=sentence-transformers/all-MiniLM-L6-v2
       --model-inference-batch-size=1024
-    timeout_s: 20000
+    timeout_s: 350
     sink_data:
       - name: slack
-        additional_metrics: ["num_documents_processed", "throughput_docs_per_sec"]
+        additional_metrics:
+          - throughput_docs_per_sec
     ray:
       num_cpus: 64
       num_gpus: 4
       enable_object_spilling: false
-
+    requirements:
+      # observed throughput of 7200 docs/sec so we allow a 5% buffer to account for variability
+      - metric: throughput_docs_per_sec
+        min_value: 6840
   - name: embedding_generation_xenna
     enabled: true
     script: embedding_generation_benchmark.py
     args: >-
       --benchmark-results-path={session_entry_dir}
+      --output-path={session_entry_dir}/scratch/output
       --executor=xenna
       --input-path={dataset:tinystories,parquet}
       --dataset-size-gb=10
       --model-identifier=sentence-transformers/all-MiniLM-L6-v2
       --model-inference-batch-size=1024
-    timeout_s: 20000
+    timeout_s: 350
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - num_documents_processed
+          - throughput_docs_per_sec
+    requirements:
+      # Observed throughput of 8600 docs/sec so we allow a 5% buffer to account for variability
+      - metric: throughput_docs_per_sec
+        min_value: 8170
 
   - name: fuzzy_dedup_identification
     enabled: true
@@ -123,16 +173,29 @@ entries:
       --benchmark-results-path={session_entry_dir}
       --input-path={dataset:commoncrawl,jsonl}
       --cache-path={session_entry_dir}/scratch/cache
-      --output-path={session_entry_dir}/output
+      --output-path={session_entry_dir}/scratch/output
       --input-filetype=jsonl
       --bands-per-iteration=20
       --text-field=text
       --input-blocksize=1.5GiB
-    timeout_s: 20000
+    timeout_s: 700
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - num_duplicates
+          - minhash_percent_time
+          - lsh_percent_time
+          - connected_components_percent_time
+          - workflow_total_time
     ray:
       num_cpus: 64
       num_gpus: 4
       enable_object_spilling: false
+    requirements:
+      - metric: num_duplicates
+        # 59,226,133 is expected, ensure +/- 1%
+        min_value: 58633871
+        max_value: 59818394
 
   - name: dedup_removal_raydata
     enabled: true
@@ -149,11 +212,18 @@ entries:
       --id-field=_curator_dedup_id
       --duplicate-id-field=_curator_dedup_id
       --blocksize=1.5GiB
-    timeout_s: 20000
+    timeout_s: 1000
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - num_duplicates_removed
     ray:
       num_cpus: 64
-      num_gpus: 4
+      num_gpus: 0
       enable_object_spilling: false
+    requirements:
+      - metric: num_duplicates_removed
+        exact_value: 59226133
 
   - name: dedup_removal_xenna
     enabled: true
@@ -170,4 +240,109 @@ entries:
       --id-field=_curator_dedup_id
       --duplicate-id-field=_curator_dedup_id
       --blocksize=1.5GiB
+    timeout_s: 1000
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - num_duplicates_removed
+          - io_percentage
+    ray:
+      num_cpus: 64
+      num_gpus: 0
+      enable_object_spilling: false
+    requirements:
+      - metric: num_duplicates_removed
+        exact_value: 59226133
+      - metric: io_percentage
+        min_value: 97
+
+  - name: score_filter_raydata
+    enabled: true
+    script: score_filter_benchmark.py
+    args: >-
+      --benchmark-results-path={session_entry_dir}
+      --output-path={session_entry_dir}/scratch/output
+      --executor=ray_data
+      --input-path={dataset:tinystories,parquet}
+      --yaml-config={curator_repo_dir}/nemo_curator/config/text/heuristic_filter_english_pipeline.yaml
+      --overrides="stages.0._target_=nemo_curator.stages.text.io.reader.ParquetReader"
     timeout_s: 20000
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - num_kept_documents
+          - throughput_docs_per_sec
+    ray:
+      num_cpus: 64
+      num_gpus: 0
+      enable_object_spilling: false
+    requirements:
+      # ensure the total number of documents processed is correct
+      - metric: num_documents_processed
+        exact_value: 2119489
+    # account for stochastic filters
+      - metric: num_kept_documents
+        min_value: 2090470
+        max_value: 2090490
+      - metric: throughput_docs_per_sec
+        min_value: 19000
+
+  - name: score_filter_xenna
+    enabled: true
+    script: score_filter_benchmark.py
+    args: >-
+      --benchmark-results-path={session_entry_dir}
+      --output-path={session_entry_dir}/scratch/output
+      --executor=xenna
+      --input-path={dataset:tinystories,parquet}
+      --yaml-config={curator_repo_dir}/nemo_curator/config/text/heuristic_filter_english_pipeline.yaml
+      --overrides="stages.0._target_=nemo_curator.stages.text.io.reader.ParquetReader"
+    timeout_s: 20000
+    sink_data:
+      - name: slack
+        additional_metrics:
+          - num_kept_documents
+          - throughput_docs_per_sec
+    ray:
+      num_cpus: 64
+      num_gpus: 0
+      enable_object_spilling: false
+    requirements:
+      # ensure the total number of documents processed is correct
+      - metric: num_documents_processed
+        exact_value: 2119489
+    # account for stochastic filters
+      - metric: num_kept_documents
+        min_value: 2090470
+        max_value: 2090490
+      - metric: throughput_docs_per_sec
+        min_value: 8500
+
+  - name: image_curation
+    enabled: true
+    script: "{curator_repo_dir}/tutorials/image/getting-started/image_curation_example.py"
+    args: >-
+      --input-wds-dataset-dir {dataset:mscoco,wds}
+      --output-dataset-dir {session_entry_dir}/scratch/output
+      --model-dir {dataset:mscoco_model_weights,files}
+      --batch-size 100
+      --embedding-batch-size 100
+      --aesthetic-batch-size 100
+      --nsfw-batch-size 100
+      --tar-files-per-partition 10
+      --aesthetic-threshold 0.9
+      --nsfw-threshold 0.9
+      --skip-download
+      --verbose
+
+  - name: audio_fleurs
+    enabled: true
+    script: audio_fleurs_benchmark.py
+    args: >-
+      --benchmark-results-path={session_entry_dir}
+      --scratch-output-path={session_entry_dir}/scratch
+      --model-name=nvidia/stt_hy_fastconformer_hybrid_large_pc
+      --lang=hy_am
+      --split=dev
+      --wer-threshold=5.5
+      --gpus=1