From b65410d9c5c784004043401a41bf95c8aa03ff6c Mon Sep 17 00:00:00 2001 From: Onur Yilmaz Date: Tue, 7 Apr 2026 12:33:53 -0400 Subject: [PATCH] Fix merge conflicts Signed-off-by: Onur Yilmaz --- benchmarking/scripts/utils.py | 2 +- benchmarking/tools/generate_ci_tests.py | 4 +- docs/_extensions/rich_metadata/__init__.py | 4 +- .../rich_metadata/verify_metadata.py | 13 +- fern/_fix_broken_links.py | 5 +- fern/substitute_variables.py | 4 +- .../api-reference/executors/experimental.mdx | 2 +- .../infrastructure/execution-backends.mdx | 2 +- .../api-reference/executors/experimental.mdx | 8 +- .../ray_actor_pool/__init__.py | 0 .../ray_actor_pool/adapter.py | 2 +- .../ray_actor_pool/executor.py | 3 +- .../ray_actor_pool/raft_adapter.py | 2 +- .../ray_actor_pool/shuffle_adapter.py | 2 +- .../ray_actor_pool/utils.py | 2 +- nemo_curator/backends/ray_data/adapter.py | 2 +- nemo_curator/backends/ray_data/executor.py | 3 +- nemo_curator/backends/ray_data/utils.py | 2 +- nemo_curator/backends/utils.py | 124 +- nemo_curator/models/transnetv2.py | 1 + nemo_curator/models/vllm_model.py | 16 +- .../audio_data_filter/config.py | 11 +- .../fleurs/create_initial_manifest.py | 2 +- .../readspeech/create_initial_manifest.py | 2 +- .../stages/audio/inference/sortformer.py | 4 +- .../audio/preprocessing/concatenation.py | 3 +- .../audio/segmentation/speaker_separation.py | 2 +- .../audio/segmentation/vad_segmentation.py | 2 +- .../stages/deduplication/exact/workflow.py | 2 +- .../fuzzy/connected_components.py | 2 +- .../stages/deduplication/fuzzy/lsh/stage.py | 2 +- .../stages/deduplication/fuzzy/workflow.py | 2 +- .../deduplication/semantic/pairwise_io.py | 2 +- .../stages/deduplication/semantic/workflow.py | 2 +- .../deduplication/shuffle_utils/stage.py | 2 +- nemo_curator/stages/file_partitioning.py | 2 +- .../stages/image/deduplication/removal.py | 13 +- .../stages/image/embedders/clip_embedder.py | 5 +- .../stages/image/filters/aesthetic_filter.py | 5 +- nemo_curator/stages/image/filters/base.py | 1 + .../stages/image/filters/nsfw_filter.py | 11 +- nemo_curator/stages/image/io/convert.py | 1 + nemo_curator/stages/image/io/image_reader.py | 2 +- .../stages/math/modifiers/chunking.py | 4 +- .../stages/math/modifiers/llm_cleanup.py | 4 +- .../nemo_data_designer/data_designer.py | 1 + .../stages/synthetic/nemotron_cc/base.py | 9 +- .../nemotron_cc/nemo_data_designer/base.py | 3 +- .../nemo_data_designer/nemotron_cc.py | 2 - .../synthetic/nemotron_cc/nemotron_cc.py | 5 + .../stages/text/deduplication/semantic.py | 2 +- .../stages/text/download/base/iterator.py | 2 +- nemo_curator/stages/text/io/reader/base.py | 2 +- nemo_curator/stages/text/modules/joiner.py | 1 - nemo_curator/stages/text/modules/splitter.py | 1 - .../video/clipping/clip_extraction_stages.py | 2 +- nemo_curator/utils/column_utils.py | 1 + nemo_curator/utils/split_large_files.py | 16 +- tests/backends/experimental/__init__.py | 0 .../experimental/ray_actor_pool/__init__.py | 0 tests/backends/experimental/test_utils.py | 232 -- .../backends/ray_actor_pool}/__init__.py | 0 .../ray_actor_pool/test_executor.py | 2 +- tests/backends/ray_data/test_max_calls_pid.py | 2 +- tests/backends/ray_data/test_utils.py | 36 +- tests/backends/test_integration.py | 2 +- tests/backends/test_utils.py | 259 +- tests/backends/utils.py | 2 +- .../test_audio_data_filter.py | 45 +- .../test_fleurs_create_initial_manifest.py | 2 +- tests/stages/audio/filtering/test_sigmos.py | 20 +- .../segmentation/test_vad_segmentation.py | 2 +- .../exact/test_identification.py | 2 +- .../fuzzy/test_connected_components_stage.py | 2 +- .../deduplication/fuzzy/test_lsh_stage.py | 2 +- .../deduplication/semantic/test_kmeans.py | 2 +- .../shuffle_utils/test_shuffle_stage.py | 2 +- .../image/embedders/test_clip_embedder.py | 55 +- .../stages/image/filters/test_nsfw_filter.py | 49 +- tests/stages/image/io/test_image_reader.py | 12 +- .../download/test_lynx_extractor.py | 24 +- .../nemo_data_designer/test_base.py | 10 +- .../nemo_data_designer/test_nemotron_cc.py | 18 +- .../synthetic/nemotron_cc/test_nemotron_cc.py | 22 +- .../text/download/arxiv/test_download.py | 4 +- .../text/download/base/test_iterator.py | 2 +- .../clipping/test_clip_transcoding_stage.py | 2 +- .../clipping/test_transnetv2_extraction.py | 4 +- tests/utils/test_split_large_files.py | 12 +- .../audio/readspeech/extract_segments.py | 10 +- tutorials/audio/readspeech/pipeline.py | 4 +- tutorials/image/getting-started/helper.py | 14 +- .../getting-started/image_curation_example.py | 176 +- .../getting-started/image_dedup_example.py | 167 +- .../ndd_data_generation_example.ipynb | 3078 ++++++++--------- .../ndd_data_generation_example.py | 2 +- ...on_cc_sdg_high_quality_example_pipeline.py | 9 +- ...ron_cc_sdg_low_quality_example_pipeline.py | 9 +- 98 files changed, 2325 insertions(+), 2310 deletions(-) rename nemo_curator/backends/{experimental => }/ray_actor_pool/__init__.py (100%) rename nemo_curator/backends/{experimental => }/ray_actor_pool/adapter.py (96%) rename nemo_curator/backends/{experimental => }/ray_actor_pool/executor.py (99%) rename nemo_curator/backends/{experimental => }/ray_actor_pool/raft_adapter.py (98%) rename nemo_curator/backends/{experimental => }/ray_actor_pool/shuffle_adapter.py (98%) rename nemo_curator/backends/{experimental => }/ray_actor_pool/utils.py (98%) delete mode 100644 tests/backends/experimental/__init__.py delete mode 100644 tests/backends/experimental/ray_actor_pool/__init__.py delete mode 100644 tests/backends/experimental/test_utils.py rename {nemo_curator/backends/experimental => tests/backends/ray_actor_pool}/__init__.py (100%) rename tests/backends/{experimental => }/ray_actor_pool/test_executor.py (93%) diff --git a/benchmarking/scripts/utils.py b/benchmarking/scripts/utils.py index 23b17d8aa0..9691ac8ddb 100644 --- a/benchmarking/scripts/utils.py +++ b/benchmarking/scripts/utils.py @@ -25,7 +25,7 @@ import pyarrow.compute as pc import pyarrow.parquet as pq -from nemo_curator.backends.experimental.ray_actor_pool.executor import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool.executor import RayActorPoolExecutor from nemo_curator.backends.ray_data import RayDataExecutor from nemo_curator.backends.xenna import XennaExecutor from nemo_curator.utils.file_utils import get_all_file_paths_and_size_under diff --git a/benchmarking/tools/generate_ci_tests.py b/benchmarking/tools/generate_ci_tests.py index e79df61743..7eebaed561 100644 --- a/benchmarking/tools/generate_ci_tests.py +++ b/benchmarking/tools/generate_ci_tests.py @@ -158,9 +158,7 @@ def generate_pipeline(curator_dir: str, scope: str, test_paths: str) -> dict: def main() -> None: - parser = argparse.ArgumentParser( - description="Generate GitLab CI jobs for Curator benchmarks" - ) + parser = argparse.ArgumentParser(description="Generate GitLab CI jobs for Curator benchmarks") parser.add_argument( "--curator-dir", type=str, diff --git a/docs/_extensions/rich_metadata/__init__.py b/docs/_extensions/rich_metadata/__init__.py index 10dc81b730..322fa2494d 100644 --- a/docs/_extensions/rich_metadata/__init__.py +++ b/docs/_extensions/rich_metadata/__init__.py @@ -27,6 +27,7 @@ # Import YAML for frontmatter parsing try: import yaml + YAML_AVAILABLE = True except ImportError: YAML_AVAILABLE = False @@ -250,7 +251,7 @@ def build_meta_tags(metadata: dict[str, Any], context: dict[str, Any]) -> dict[s "basic": _add_basic_fields(metadata), "opengraph": _add_opengraph_fields(metadata, context), "twitter": _add_twitter_fields(metadata, context), - "custom": _add_custom_fields(metadata) + "custom": _add_custom_fields(metadata), } @@ -474,4 +475,3 @@ def setup(app: Sphinx) -> dict[str, Any]: "parallel_read_safe": True, "parallel_write_safe": True, } - diff --git a/docs/_extensions/rich_metadata/verify_metadata.py b/docs/_extensions/rich_metadata/verify_metadata.py index 94675b319a..faf052fe78 100755 --- a/docs/_extensions/rich_metadata/verify_metadata.py +++ b/docs/_extensions/rich_metadata/verify_metadata.py @@ -126,9 +126,9 @@ def verify_html_file(html_path: Path) -> bool: print(f"❌ File not found: {html_path}") return False - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print(f"Verifying: {html_path.name}") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") html_content = html_path.read_text(encoding="utf-8") @@ -154,9 +154,7 @@ def verify_html_file(html_path: Path) -> bool: def main() -> None: """Main entry point for the verification script.""" - parser = argparse.ArgumentParser( - description="Verify rich metadata injection in built HTML files" - ) + parser = argparse.ArgumentParser(description="Verify rich metadata injection in built HTML files") parser.add_argument( "html_files", nargs="+", @@ -177,16 +175,15 @@ def main() -> None: if not verify_html_file(html_file): all_passed = False - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") if all_passed: print("βœ… All files verified successfully!") else: print("⚠️ Some files are missing metadata") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") sys.exit(0 if all_passed else 1) if __name__ == "__main__": main() - diff --git a/fern/_fix_broken_links.py b/fern/_fix_broken_links.py index e7c6863f1c..dcbbedae86 100644 --- a/fern/_fix_broken_links.py +++ b/fern/_fix_broken_links.py @@ -125,7 +125,10 @@ ("../../curate-audio/process-data/audio-analysis/index.md", "/curate-audio/process-data/audio-analysis"), ("../../curate-audio/process-data/text-integration/index.md", "/curate-audio/process-data/text-integration"), ("../../curate-text/index.md", "/curate-text"), - ("../../curate-text/process-data/quality-assessment/distributed-classifier.md", "/curate-text/process-data/quality-assessment/distributed-classifier"), + ( + "../../curate-text/process-data/quality-assessment/distributed-classifier.md", + "/curate-text/process-data/quality-assessment/distributed-classifier", + ), ("../../curate-text/process-data/deduplication/semdedup.md", "/curate-text/process-data/deduplication/semdedup"), ("../../curate-images/index.md", "/curate-images"), ("../../curate-images/process-data/embeddings/index.md", "/curate-images/process-data/embeddings"), diff --git a/fern/substitute_variables.py b/fern/substitute_variables.py index 58034b5457..3412ac7444 100644 --- a/fern/substitute_variables.py +++ b/fern/substitute_variables.py @@ -74,7 +74,9 @@ def main() -> int: default="versions/v26.02", help="Path under fern/ containing MDX (e.g. versions/v25.09, versions/v26.02)", ) - parser.add_argument("--version", help="Version string for version/container_version/current_release (e.g. 25.09, 26.02)") + parser.add_argument( + "--version", help="Version string for version/container_version/current_release (e.g. 25.09, 26.02)" + ) parser.add_argument("--dry-run", action="store_true", help="Show what would be changed without modifying files") args = parser.parse_args() diff --git a/fern/versions/v25.09/pages/api-reference/executors/experimental.mdx b/fern/versions/v25.09/pages/api-reference/executors/experimental.mdx index bfbecb0585..3c76596bf6 100644 --- a/fern/versions/v25.09/pages/api-reference/executors/experimental.mdx +++ b/fern/versions/v25.09/pages/api-reference/executors/experimental.mdx @@ -46,7 +46,7 @@ Uses Ray Actor Pool for distributed execution. ### Import ```python -from nemo_curator.backends.experimental import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor ``` ### Usage diff --git a/fern/versions/v25.09/pages/reference/infrastructure/execution-backends.mdx b/fern/versions/v25.09/pages/reference/infrastructure/execution-backends.mdx index 9a16fa4432..0588354445 100644 --- a/fern/versions/v25.09/pages/reference/infrastructure/execution-backends.mdx +++ b/fern/versions/v25.09/pages/reference/infrastructure/execution-backends.mdx @@ -124,7 +124,7 @@ results = pipeline.run(executor) Executor using Ray Actor pools for custom distributed processing patterns such as deduplication. ```python -from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor executor = RayActorPoolExecutor() results = pipeline.run(executor) diff --git a/fern/versions/v26.02/pages/api-reference/executors/experimental.mdx b/fern/versions/v26.02/pages/api-reference/executors/experimental.mdx index bfbecb0585..4b79e3f77d 100644 --- a/fern/versions/v26.02/pages/api-reference/executors/experimental.mdx +++ b/fern/versions/v26.02/pages/api-reference/executors/experimental.mdx @@ -16,7 +16,7 @@ Uses Ray Data for distributed execution. ### Import ```python -from nemo_curator.backends.experimental import RayDataExecutor +from nemo_curator.backends.ray_data import RayDataExecutor ``` ### Usage @@ -46,7 +46,7 @@ Uses Ray Actor Pool for distributed execution. ### Import ```python -from nemo_curator.backends.experimental import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor ``` ### Usage @@ -149,8 +149,8 @@ class MyCustomExecutor(BaseExecutor): | Executor | Best For | Considerations | |----------|----------|----------------| | `XennaExecutor` | Production workloads | Default choice, most stable | -| `RayDataExecutor` | Ray-native environments | Experimental | -| `RayActorPoolExecutor` | Fine-grained actor control | Experimental | +| `RayDataExecutor` | Ray-native environments | Production | +| `RayActorPoolExecutor` | Fine-grained actor control | Production | ## Source Code diff --git a/nemo_curator/backends/experimental/ray_actor_pool/__init__.py b/nemo_curator/backends/ray_actor_pool/__init__.py similarity index 100% rename from nemo_curator/backends/experimental/ray_actor_pool/__init__.py rename to nemo_curator/backends/ray_actor_pool/__init__.py diff --git a/nemo_curator/backends/experimental/ray_actor_pool/adapter.py b/nemo_curator/backends/ray_actor_pool/adapter.py similarity index 96% rename from nemo_curator/backends/experimental/ray_actor_pool/adapter.py rename to nemo_curator/backends/ray_actor_pool/adapter.py index 37e53906c0..f68ac0f0c2 100644 --- a/nemo_curator/backends/experimental/ray_actor_pool/adapter.py +++ b/nemo_curator/backends/ray_actor_pool/adapter.py @@ -15,7 +15,7 @@ from loguru import logger from nemo_curator.backends.base import BaseStageAdapter -from nemo_curator.backends.experimental.utils import get_worker_metadata_and_node_id +from nemo_curator.backends.utils import get_worker_metadata_and_node_id from nemo_curator.stages.base import ProcessingStage diff --git a/nemo_curator/backends/experimental/ray_actor_pool/executor.py b/nemo_curator/backends/ray_actor_pool/executor.py similarity index 99% rename from nemo_curator/backends/experimental/ray_actor_pool/executor.py rename to nemo_curator/backends/ray_actor_pool/executor.py index f2e2273db7..c46217a748 100644 --- a/nemo_curator/backends/experimental/ray_actor_pool/executor.py +++ b/nemo_curator/backends/ray_actor_pool/executor.py @@ -23,8 +23,7 @@ from tqdm import tqdm from nemo_curator.backends.base import BaseExecutor -from nemo_curator.backends.experimental.utils import RayStageSpecKeys, execute_setup_on_node -from nemo_curator.backends.utils import register_loguru_serializer +from nemo_curator.backends.utils import RayStageSpecKeys, execute_setup_on_node, register_loguru_serializer from nemo_curator.tasks import EmptyTask, Task from .adapter import RayActorPoolStageAdapter diff --git a/nemo_curator/backends/experimental/ray_actor_pool/raft_adapter.py b/nemo_curator/backends/ray_actor_pool/raft_adapter.py similarity index 98% rename from nemo_curator/backends/experimental/ray_actor_pool/raft_adapter.py rename to nemo_curator/backends/ray_actor_pool/raft_adapter.py index 02ee38d98c..ff321951cb 100644 --- a/nemo_curator/backends/experimental/ray_actor_pool/raft_adapter.py +++ b/nemo_curator/backends/ray_actor_pool/raft_adapter.py @@ -18,7 +18,7 @@ from loguru import logger from nemo_curator.backends.base import BaseStageAdapter -from nemo_curator.backends.experimental.utils import get_worker_metadata_and_node_id +from nemo_curator.backends.utils import get_worker_metadata_and_node_id from nemo_curator.stages.base import ProcessingStage if TYPE_CHECKING: diff --git a/nemo_curator/backends/experimental/ray_actor_pool/shuffle_adapter.py b/nemo_curator/backends/ray_actor_pool/shuffle_adapter.py similarity index 98% rename from nemo_curator/backends/experimental/ray_actor_pool/shuffle_adapter.py rename to nemo_curator/backends/ray_actor_pool/shuffle_adapter.py index 69ec06633f..5f8d9baddd 100644 --- a/nemo_curator/backends/experimental/ray_actor_pool/shuffle_adapter.py +++ b/nemo_curator/backends/ray_actor_pool/shuffle_adapter.py @@ -19,7 +19,7 @@ from loguru import logger from nemo_curator.backends.base import BaseStageAdapter -from nemo_curator.backends.experimental.utils import RayStageSpecKeys, get_worker_metadata_and_node_id +from nemo_curator.backends.utils import RayStageSpecKeys, get_worker_metadata_and_node_id from nemo_curator.tasks import FileGroupTask if TYPE_CHECKING: diff --git a/nemo_curator/backends/experimental/ray_actor_pool/utils.py b/nemo_curator/backends/ray_actor_pool/utils.py similarity index 98% rename from nemo_curator/backends/experimental/ray_actor_pool/utils.py rename to nemo_curator/backends/ray_actor_pool/utils.py index d1c4f45dd0..b1fd19f84d 100644 --- a/nemo_curator/backends/experimental/ray_actor_pool/utils.py +++ b/nemo_curator/backends/ray_actor_pool/utils.py @@ -18,7 +18,7 @@ import ray from loguru import logger -from nemo_curator.backends.experimental.utils import get_available_cpu_gpu_resources +from nemo_curator.backends.utils import get_available_cpu_gpu_resources if TYPE_CHECKING: from ray.actor import ActorClass diff --git a/nemo_curator/backends/ray_data/adapter.py b/nemo_curator/backends/ray_data/adapter.py index e3abe2847d..47ca2e70af 100644 --- a/nemo_curator/backends/ray_data/adapter.py +++ b/nemo_curator/backends/ray_data/adapter.py @@ -20,7 +20,7 @@ from ray.data import Dataset from nemo_curator.backends.base import BaseStageAdapter -from nemo_curator.backends.experimental.utils import RayStageSpecKeys, get_worker_metadata_and_node_id +from nemo_curator.backends.utils import RayStageSpecKeys, get_worker_metadata_and_node_id from nemo_curator.stages.base import ProcessingStage from .utils import calculate_concurrency_for_actors_for_stage, is_actor_stage diff --git a/nemo_curator/backends/ray_data/executor.py b/nemo_curator/backends/ray_data/executor.py index dbf9870f9e..a9e33cb814 100644 --- a/nemo_curator/backends/ray_data/executor.py +++ b/nemo_curator/backends/ray_data/executor.py @@ -19,8 +19,7 @@ from ray.data import DataContext, Dataset from nemo_curator.backends.base import BaseExecutor -from nemo_curator.backends.experimental.utils import execute_setup_on_node -from nemo_curator.backends.utils import register_loguru_serializer +from nemo_curator.backends.utils import execute_setup_on_node, register_loguru_serializer from nemo_curator.tasks import EmptyTask, Task from .adapter import RayDataStageAdapter diff --git a/nemo_curator/backends/ray_data/utils.py b/nemo_curator/backends/ray_data/utils.py index e98d60cada..ed3d9a6954 100644 --- a/nemo_curator/backends/ray_data/utils.py +++ b/nemo_curator/backends/ray_data/utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nemo_curator.backends.experimental.utils import get_available_cpu_gpu_resources +from nemo_curator.backends.utils import get_available_cpu_gpu_resources from nemo_curator.stages.base import ProcessingStage diff --git a/nemo_curator/backends/utils.py b/nemo_curator/backends/utils.py index a974c1bbe9..ca7b793964 100644 --- a/nemo_curator/backends/utils.py +++ b/nemo_curator/backends/utils.py @@ -12,15 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import time from copy import deepcopy -from typing import TYPE_CHECKING +from enum import Enum +from typing import TYPE_CHECKING, Any import ray from loguru import logger +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy + +from nemo_curator.backends.base import NodeInfo, WorkerMetadata +from nemo_curator.stages.base import ProcessingStage if TYPE_CHECKING: import loguru +# Global variable to cache head node ID +_HEAD_NODE_ID_CACHE = None + def _logger_custom_serializer( _: "loguru.Logger", @@ -112,3 +122,115 @@ def warn_on_env_var_override(existing_config: dict | None, merged_config: dict | "Update the executor configuration before running if this is unintended.", overridden_keys, ) + + +def is_head_node(node: dict[str, Any]) -> bool: + """Check if a node is the head node.""" + return "node:__internal_head__" in node.get("Resources", {}) + + +def get_head_node_id() -> str | None: + """Get the head node ID from the Ray cluster, with lazy evaluation and caching. + + Returns: + The head node ID if a head node exists, otherwise None. + """ + global _HEAD_NODE_ID_CACHE # noqa: PLW0603 + + if _HEAD_NODE_ID_CACHE is not None: + return _HEAD_NODE_ID_CACHE + + # Compute head node ID + for node in ray.nodes(): + if is_head_node(node): + _HEAD_NODE_ID_CACHE = node["NodeID"] + return _HEAD_NODE_ID_CACHE + + return None + + +class RayStageSpecKeys(str, Enum): + """String enum of different flags that define keys inside ray_stage_spec.""" + + IS_ACTOR_STAGE = "is_actor_stage" + IS_FANOUT_STAGE = "is_fanout_stage" + IS_RAFT_ACTOR = "is_raft_actor" + IS_LSH_STAGE = "is_lsh_stage" + IS_SHUFFLE_STAGE = "is_shuffle_stage" + MAX_CALLS_PER_WORKER = "max_calls_per_worker" + + +def get_worker_metadata_and_node_id() -> tuple[NodeInfo, WorkerMetadata]: + """Get the worker metadata and node id from the runtime context.""" + ray_context = ray.get_runtime_context() + return NodeInfo(node_id=ray_context.get_node_id()), WorkerMetadata(worker_id=ray_context.get_worker_id()) + + +def get_available_cpu_gpu_resources( + init_and_shutdown: bool = False, ignore_head_node: bool = False +) -> tuple[int, int]: + """Get available CPU and GPU resources from Ray.""" + if init_and_shutdown: + ray.init(ignore_reinit_error=True) + time.sleep(0.2) # ray.available_resources() returns might have a lag + # available resources can be different from total resources, however curator assumes + # entire cluster is available for use and only one pipeline is being run at a time. + # therefore available resources should match total resources. + available_resources = ray.available_resources() + available_cpus = available_resources.get("CPU", 0) + available_gpus = available_resources.get("GPU", 0) + if ignore_head_node: + head_node_id = get_head_node_id() + if head_node_id is not None: + total_resources = ray.state.total_resources_per_node().get(head_node_id, {}) + head_node_cpus = total_resources.get("CPU", 0) + head_node_gpus = total_resources.get("GPU", 0) + logger.info( + f"Ignoring head node {head_node_id} with {head_node_cpus} CPUs and {head_node_gpus} GPUs for resource calculation" + ) + available_cpus = max(0, available_cpus - head_node_cpus) + available_gpus = max(0, available_gpus - head_node_gpus) + else: + logger.warning("ignore_head_node=True but no head node found in the cluster") + if init_and_shutdown: + ray.shutdown() + return (available_cpus, available_gpus) + + +@ray.remote +def _setup_stage_on_node(stage: ProcessingStage, node_info: NodeInfo, worker_metadata: WorkerMetadata) -> None: + """Ray remote function to execute setup_on_node for a stage. + + This runs as a Ray remote task (not an actor). + vLLM's auto-detection only forces the spawn multiprocessing method inside Ray actors, + not in Ray tasks. Without this override, vLLM defaults to fork in tasks and hits + RuntimeError: Cannot re-initialize CUDA in forked subprocess. + We explicitly set the environment variable to spawn to prevent this. + """ + os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + stage.setup_on_node(node_info, worker_metadata) + + +def execute_setup_on_node(stages: list[ProcessingStage], ignore_head_node: bool = False) -> None: + """Execute setup on node for a stage.""" + head_node_id = get_head_node_id() + ray_tasks = [] + for node in ray.nodes(): + node_id = node["NodeID"] + node_info = NodeInfo(node_id=node_id) + worker_metadata = WorkerMetadata(worker_id="", allocation=None) + if ignore_head_node and node_id == head_node_id: + logger.info(f"Ignoring setup on head node {node_id}") + continue + + logger.info(f"Executing setup on node {node_id} for {len(stages)} stages") + + for stage in stages: + ray_tasks.append( + _setup_stage_on_node.options( + num_cpus=stage.resources.cpus if stage.resources is not None else 1, + num_gpus=stage.resources.gpus if stage.resources is not None else 0, + scheduling_strategy=NodeAffinitySchedulingStrategy(node_id=node_id, soft=False), + ).remote(stage, node_info, worker_metadata) + ) + ray.get(ray_tasks) diff --git a/nemo_curator/models/transnetv2.py b/nemo_curator/models/transnetv2.py index 0f334aa114..e1ae590503 100644 --- a/nemo_curator/models/transnetv2.py +++ b/nemo_curator/models/transnetv2.py @@ -39,6 +39,7 @@ _TRANSNETV2_MODEL_WEIGHTS: Final = "transnetv2-pytorch-weights.pth" _TRANSNETV2_MODEL_REVISION: Final = "db6ceab" + class _TransNetV2(nn.Module): def __init__( # noqa: PLR0913 self, diff --git a/nemo_curator/models/vllm_model.py b/nemo_curator/models/vllm_model.py index c7186830d9..09e1e67ccf 100644 --- a/nemo_curator/models/vllm_model.py +++ b/nemo_curator/models/vllm_model.py @@ -90,10 +90,7 @@ def model_id_names(self) -> list[str]: def setup(self) -> None: """Set up the vLLM model and sampling parameters.""" if not VLLM_AVAILABLE: - msg = ( - "vLLM is required for VLLMModel. " - "Please install it: pip install vllm" - ) + msg = "vLLM is required for VLLMModel. Please install it: pip install vllm" raise ImportError(msg) # Fetch max_model_len from user param or auto-detect from HuggingFace AutoConfig @@ -132,11 +129,7 @@ def setup(self) -> None: self._llm = LLM(**llm_kwargs) self._final_max_model_len = final_max_model_len - max_gen_tokens = ( - self.max_tokens - if self.max_tokens is not None - else final_max_model_len - ) + max_gen_tokens = self.max_tokens if self.max_tokens is not None else final_max_model_len if max_gen_tokens is None: logger.warning( "max_tokens is None and max_model_len could not be auto-detected. " @@ -190,10 +183,7 @@ def generate( sampling_params=self._sampling_params, use_tqdm=False, ) - return [ - out.outputs[0].text if out.outputs else "" - for out in outputs - ] + return [out.outputs[0].text if out.outputs else "" for out in outputs] except (RuntimeError, ValueError, TypeError) as e: msg = f"Error generating text: {e}" raise RuntimeError(msg) from e diff --git a/nemo_curator/stages/audio/advanced_pipelines/audio_data_filter/config.py b/nemo_curator/stages/audio/advanced_pipelines/audio_data_filter/config.py index 17b5a6217c..c40f2ca59e 100644 --- a/nemo_curator/stages/audio/advanced_pipelines/audio_data_filter/config.py +++ b/nemo_curator/stages/audio/advanced_pipelines/audio_data_filter/config.py @@ -122,8 +122,15 @@ def _validate(cfg: dict[str, Any]) -> None: # noqa: C901 sigmos = cfg.get("sigmos", {}) if sigmos.get("enable", True): - for key in ("noise_threshold", "ovrl_threshold", "sig_threshold", - "col_threshold", "disc_threshold", "loud_threshold", "reverb_threshold"): + for key in ( + "noise_threshold", + "ovrl_threshold", + "sig_threshold", + "col_threshold", + "disc_threshold", + "loud_threshold", + "reverb_threshold", + ): val = sigmos.get(key) if val is not None and not 0.0 <= val <= _MOS_MAX: msg = f"sigmos.{key} must be in [0, {_MOS_MAX}] (MOS scale), got {val}" diff --git a/nemo_curator/stages/audio/datasets/fleurs/create_initial_manifest.py b/nemo_curator/stages/audio/datasets/fleurs/create_initial_manifest.py index 588eeaaef5..fe5099ea08 100644 --- a/nemo_curator/stages/audio/datasets/fleurs/create_initial_manifest.py +++ b/nemo_curator/stages/audio/datasets/fleurs/create_initial_manifest.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from typing import Any -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.audio.datasets.file_utils import download_file, extract_archive from nemo_curator.stages.base import ProcessingStage from nemo_curator.tasks import AudioTask, _EmptyTask diff --git a/nemo_curator/stages/audio/datasets/readspeech/create_initial_manifest.py b/nemo_curator/stages/audio/datasets/readspeech/create_initial_manifest.py index f3d8329ee2..533ada385e 100644 --- a/nemo_curator/stages/audio/datasets/readspeech/create_initial_manifest.py +++ b/nemo_curator/stages/audio/datasets/readspeech/create_initial_manifest.py @@ -20,7 +20,7 @@ from loguru import logger -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.audio.datasets.file_utils import download_file from nemo_curator.stages.base import ProcessingStage from nemo_curator.tasks import AudioTask, _EmptyTask diff --git a/nemo_curator/stages/audio/inference/sortformer.py b/nemo_curator/stages/audio/inference/sortformer.py index 7f33c5f31a..03f69b3607 100644 --- a/nemo_curator/stages/audio/inference/sortformer.py +++ b/nemo_curator/stages/audio/inference/sortformer.py @@ -189,9 +189,7 @@ def process(self, task: AudioTask) -> AudioTask: file_path = task.data[self.filepath_key] sess_name = task.data.get("session_name") - resolved_sess_name = ( - sess_name if sess_name is not None else os.path.splitext(os.path.basename(file_path))[0] - ) + resolved_sess_name = sess_name if sess_name is not None else os.path.splitext(os.path.basename(file_path))[0] all_segments = self.diarize([file_path]) segments = all_segments[0] diff --git a/nemo_curator/stages/audio/preprocessing/concatenation.py b/nemo_curator/stages/audio/preprocessing/concatenation.py index 36fcfaa53c..91453c7b60 100755 --- a/nemo_curator/stages/audio/preprocessing/concatenation.py +++ b/nemo_curator/stages/audio/preprocessing/concatenation.py @@ -166,8 +166,7 @@ def _concatenate( if parts and sr != sample_rate: logger.warning( - f"[SegmentConcat] Sample rate mismatch: " - f"expected {sample_rate}Hz, got {sr}Hz. Skipping segment." + f"[SegmentConcat] Sample rate mismatch: expected {sample_rate}Hz, got {sr}Hz. Skipping segment." ) continue sample_rate = sr diff --git a/nemo_curator/stages/audio/segmentation/speaker_separation.py b/nemo_curator/stages/audio/segmentation/speaker_separation.py index 6a9e916ba3..31641a2e4b 100755 --- a/nemo_curator/stages/audio/segmentation/speaker_separation.py +++ b/nemo_curator/stages/audio/segmentation/speaker_separation.py @@ -44,7 +44,7 @@ SortformerEncLabelModel = None from nemo_curator.backends.base import WorkerMetadata -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.audio.common import resolve_waveform_from_item from nemo_curator.stages.audio.segmentation.speaker_separation_module.speaker_sep import SpeakerSeparator from nemo_curator.stages.base import ProcessingStage diff --git a/nemo_curator/stages/audio/segmentation/vad_segmentation.py b/nemo_curator/stages/audio/segmentation/vad_segmentation.py index adc52a9a41..bdcc6c790c 100755 --- a/nemo_curator/stages/audio/segmentation/vad_segmentation.py +++ b/nemo_curator/stages/audio/segmentation/vad_segmentation.py @@ -47,7 +47,7 @@ from silero_vad import get_speech_timestamps, load_silero_vad from nemo_curator.backends.base import WorkerMetadata -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.audio.common import ensure_waveform_2d, load_audio_file from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.resources import Resources diff --git a/nemo_curator/stages/deduplication/exact/workflow.py b/nemo_curator/stages/deduplication/exact/workflow.py index 7f14ad02e2..6955913b27 100644 --- a/nemo_curator/stages/deduplication/exact/workflow.py +++ b/nemo_curator/stages/deduplication/exact/workflow.py @@ -18,7 +18,7 @@ from loguru import logger -from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.backends.utils import merge_executor_configs, warn_on_env_var_override from nemo_curator.pipeline import Pipeline from nemo_curator.pipeline.workflow import WorkflowBase, WorkflowRunResult diff --git a/nemo_curator/stages/deduplication/fuzzy/connected_components.py b/nemo_curator/stages/deduplication/fuzzy/connected_components.py index 495143203a..333fbad077 100644 --- a/nemo_curator/stages/deduplication/fuzzy/connected_components.py +++ b/nemo_curator/stages/deduplication/fuzzy/connected_components.py @@ -21,7 +21,7 @@ from pylibcugraph import weakly_connected_components as pylibcugraph_wcc from pylibcugraph.comms.comms_wrapper import init_subcomms as c_init_subcomms -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.deduplication.fuzzy.utils import CURATOR_FUZZY_DUPLICATE_GROUP_FIELD from nemo_curator.stages.deduplication.id_generator import CURATOR_DEDUP_ID_STR diff --git a/nemo_curator/stages/deduplication/fuzzy/lsh/stage.py b/nemo_curator/stages/deduplication/fuzzy/lsh/stage.py index 6abdfa9ce3..0d94aa0669 100644 --- a/nemo_curator/stages/deduplication/fuzzy/lsh/stage.py +++ b/nemo_curator/stages/deduplication/fuzzy/lsh/stage.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from typing import Any, Literal -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.deduplication.fuzzy.lsh.lsh import LSHActor from nemo_curator.stages.deduplication.fuzzy.utils import CURATOR_DEFAULT_MINHASH_FIELD diff --git a/nemo_curator/stages/deduplication/fuzzy/workflow.py b/nemo_curator/stages/deduplication/fuzzy/workflow.py index 6d3216a623..9acc86b272 100644 --- a/nemo_curator/stages/deduplication/fuzzy/workflow.py +++ b/nemo_curator/stages/deduplication/fuzzy/workflow.py @@ -17,7 +17,7 @@ from loguru import logger -from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.backends.utils import merge_executor_configs, warn_on_env_var_override from nemo_curator.pipeline import Pipeline from nemo_curator.pipeline.workflow import WorkflowBase, WorkflowRunResult diff --git a/nemo_curator/stages/deduplication/semantic/pairwise_io.py b/nemo_curator/stages/deduplication/semantic/pairwise_io.py index 5dd9dc34a3..a801d03e66 100644 --- a/nemo_curator/stages/deduplication/semantic/pairwise_io.py +++ b/nemo_curator/stages/deduplication/semantic/pairwise_io.py @@ -17,7 +17,7 @@ from loguru import logger from nemo_curator.backends.base import WorkerMetadata -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.resources import Resources from nemo_curator.tasks import FileGroupTask, _EmptyTask diff --git a/nemo_curator/stages/deduplication/semantic/workflow.py b/nemo_curator/stages/deduplication/semantic/workflow.py index 5fc27785f3..93da746b71 100644 --- a/nemo_curator/stages/deduplication/semantic/workflow.py +++ b/nemo_curator/stages/deduplication/semantic/workflow.py @@ -29,7 +29,7 @@ # Ray Curator imports from nemo_curator.backends.base import BaseExecutor -from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.backends.xenna import XennaExecutor from nemo_curator.pipeline import Pipeline from nemo_curator.pipeline.workflow import WorkflowBase, WorkflowRunResult diff --git a/nemo_curator/stages/deduplication/shuffle_utils/stage.py b/nemo_curator/stages/deduplication/shuffle_utils/stage.py index 4e24c29636..9465b735ae 100644 --- a/nemo_curator/stages/deduplication/shuffle_utils/stage.py +++ b/nemo_curator/stages/deduplication/shuffle_utils/stage.py @@ -14,7 +14,7 @@ from typing import Any, Literal -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.deduplication.shuffle_utils.rapidsmpf_shuffler import BulkRapidsMPFShuffler from nemo_curator.stages.resources import Resources diff --git a/nemo_curator/stages/file_partitioning.py b/nemo_curator/stages/file_partitioning.py index f5b577ef32..b8733bb4fe 100644 --- a/nemo_curator/stages/file_partitioning.py +++ b/nemo_curator/stages/file_partitioning.py @@ -17,7 +17,7 @@ from loguru import logger -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.resources import Resources from nemo_curator.tasks import FileGroupTask, _EmptyTask diff --git a/nemo_curator/stages/image/deduplication/removal.py b/nemo_curator/stages/image/deduplication/removal.py index d30f433b29..43f9ba095f 100644 --- a/nemo_curator/stages/image/deduplication/removal.py +++ b/nemo_curator/stages/image/deduplication/removal.py @@ -58,7 +58,11 @@ def outputs(self) -> tuple[list[str], list[str]]: return ["data"], [] def setup(self, _worker_metadata=None) -> None: # noqa: ANN001 - removal_parquets = [os.path.join(self.removal_parquets_dir, f) for f in os.listdir(self.removal_parquets_dir) if f.endswith(".parquet")] + removal_parquets = [ + os.path.join(self.removal_parquets_dir, f) + for f in os.listdir(self.removal_parquets_dir) + if f.endswith(".parquet") + ] if not removal_parquets: msg = f"No parquet files found in {self.removal_parquets_dir}" logger.error(msg) @@ -73,9 +77,7 @@ def setup(self, _worker_metadata=None) -> None: # noqa: ANN001 self._ids_to_remove.update(ids_array) if self.verbose: - logger.debug( - f"Loaded {len(self._ids_to_remove)} IDs to remove from '{self.removal_parquets_dir}'" - ) + logger.debug(f"Loaded {len(self._ids_to_remove)} IDs to remove from '{self.removal_parquets_dir}'") def process(self, task: ImageBatch) -> ImageBatch: original_count = len(task.data) @@ -85,8 +87,7 @@ def process(self, task: ImageBatch) -> ImageBatch: removed_count = original_count - len(filtered_images) if self.verbose: logger.debug( - f"Dedup filtering: kept {len(filtered_images)}/{original_count} images, " - f"removed {removed_count} by ID" + f"Dedup filtering: kept {len(filtered_images)}/{original_count} images, removed {removed_count} by ID" ) return ImageBatch( diff --git a/nemo_curator/stages/image/embedders/clip_embedder.py b/nemo_curator/stages/image/embedders/clip_embedder.py index 5f8500a69a..6c0592c725 100644 --- a/nemo_curator/stages/image/embedders/clip_embedder.py +++ b/nemo_curator/stages/image/embedders/clip_embedder.py @@ -33,6 +33,7 @@ class ImageEmbeddingStage(ProcessingStage[ImageBatch, ImageBatch]): embeddings for each image. It assumes image data is already loaded in ImageObject.image_data and stores embeddings in ImageObject.embedding. """ + model_dir: str = None num_gpus_per_worker: float = 0.25 model_inference_batch_size: int = 32 # Number of images to process through model at once @@ -110,8 +111,6 @@ def process(self, task: ImageBatch) -> ImageBatch: image_obj.image_data = None if self.verbose: - logger.info( - f"Generated embeddings for {len(batch)} images." - ) + logger.info(f"Generated embeddings for {len(batch)} images.") return task diff --git a/nemo_curator/stages/image/filters/aesthetic_filter.py b/nemo_curator/stages/image/filters/aesthetic_filter.py index a6e1a4432d..f993b052c8 100644 --- a/nemo_curator/stages/image/filters/aesthetic_filter.py +++ b/nemo_curator/stages/image/filters/aesthetic_filter.py @@ -31,6 +31,7 @@ class ImageAestheticFilterStage(BaseFilterStage): This class processes image batches through an aesthetic scoring model to generate aesthetic scores for each image. Images with scores below the threshold will be filtered out. """ + model_dir: str = None num_gpus_per_worker: float = 0.25 model_inference_batch_size: int = 32 # Number of images to process through model at once @@ -38,7 +39,9 @@ class ImageAestheticFilterStage(BaseFilterStage): verbose: bool = False name: str = "image_aesthetic_filter" - def setup_on_node(self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None) -> None: + def setup_on_node( + self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None + ) -> None: """Download aesthetic model weights from HF""" AestheticScorer.download_weights_on_node(self.model_dir) diff --git a/nemo_curator/stages/image/filters/base.py b/nemo_curator/stages/image/filters/base.py index 282d603e8e..20b4464f96 100644 --- a/nemo_curator/stages/image/filters/base.py +++ b/nemo_curator/stages/image/filters/base.py @@ -29,6 +29,7 @@ class BaseFilterStage(ProcessingStage[ImageBatch, ImageBatch]): This class provides a base class for image filtering stages. """ + model_dir: str = None num_gpus_per_worker: float = 0.25 model_inference_batch_size: int = 32 # Number of images to process through model at once diff --git a/nemo_curator/stages/image/filters/nsfw_filter.py b/nemo_curator/stages/image/filters/nsfw_filter.py index de9bfbf0c0..4c63be1f7f 100644 --- a/nemo_curator/stages/image/filters/nsfw_filter.py +++ b/nemo_curator/stages/image/filters/nsfw_filter.py @@ -32,10 +32,13 @@ class ImageNSFWFilterStage(BaseFilterStage): NSFW probability scores for each image. Images with scores above the threshold will be filtered out as NSFW content. """ + weights_path: str = None name: str = "image_nsfw_filter" - def setup_on_node(self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None) -> None: + def setup_on_node( + self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None + ) -> None: """Download NSFW model weights from LAION repository.""" NSFWScorer.download_weights_on_node(self.model_dir) @@ -73,8 +76,7 @@ def process(self, task: ImageBatch) -> ImageBatch: if self.verbose: logger.info( - f"Generated NSFW scores for {len(batch)} images " - f"in batch {i}-{i + self.model_inference_batch_size}" + f"Generated NSFW scores for {len(batch)} images in batch {i}-{i + self.model_inference_batch_size}" ) # Filter images based on NSFW score threshold @@ -94,8 +96,7 @@ def process(self, task: ImageBatch) -> ImageBatch: if self.verbose: logger.info( - f"NSFW filtering: {len(filtered_images)}/{len(task.data)} images passed, " - f"{filtered_count} filtered out" + f"NSFW filtering: {len(filtered_images)}/{len(task.data)} images passed, {filtered_count} filtered out" ) # Return new ImageBatch with filtered images diff --git a/nemo_curator/stages/image/io/convert.py b/nemo_curator/stages/image/io/convert.py index e257a95680..d8e1374782 100644 --- a/nemo_curator/stages/image/io/convert.py +++ b/nemo_curator/stages/image/io/convert.py @@ -28,6 +28,7 @@ class ConvertImageBatchToDocumentBatchStage(ProcessingStage[ImageBatch, Document Args: fields: list of fields of ImageObject to convert to DocumentBatch """ + fields: list[str] = field(default_factory=list) name: str = "convert_image_batch_to_document_batch" diff --git a/nemo_curator/stages/image/io/image_reader.py b/nemo_curator/stages/image/io/image_reader.py index b65576e6c4..9a08789e46 100644 --- a/nemo_curator/stages/image/io/image_reader.py +++ b/nemo_curator/stages/image/io/image_reader.py @@ -21,7 +21,7 @@ import torch from loguru import logger -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.resources import Resources from nemo_curator.tasks import FileGroupTask, ImageBatch, ImageObject diff --git a/nemo_curator/stages/math/modifiers/chunking.py b/nemo_curator/stages/math/modifiers/chunking.py index 618e3679f1..aee2c6bba0 100644 --- a/nemo_curator/stages/math/modifiers/chunking.py +++ b/nemo_curator/stages/math/modifiers/chunking.py @@ -45,7 +45,9 @@ def __init__( # noqa: PLR0913 self._tokenizer = None self.name = format_name_with_suffix(self.model_name, suffix="_token_splitter") - def setup_on_node(self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None) -> None: + def setup_on_node( + self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None + ) -> None: """Download model weights to local cache once per physical node.""" from huggingface_hub import snapshot_download diff --git a/nemo_curator/stages/math/modifiers/llm_cleanup.py b/nemo_curator/stages/math/modifiers/llm_cleanup.py index 00ca48f6c1..09c41d6bab 100644 --- a/nemo_curator/stages/math/modifiers/llm_cleanup.py +++ b/nemo_curator/stages/math/modifiers/llm_cleanup.py @@ -112,7 +112,9 @@ def _initialize_model(self) -> None: else: self._final_max_model_len = self.max_model_len - def setup_on_node(self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None) -> None: + def setup_on_node( + self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None + ) -> None: """Download weights and initialize vLLM once per node to avoid torch.compile race conditions.""" cache_dir = self._model_kwargs.get("cache_dir") if self._model is None else self._model.cache_dir diff --git a/nemo_curator/stages/synthetic/nemo_data_designer/data_designer.py b/nemo_curator/stages/synthetic/nemo_data_designer/data_designer.py index aff74fc177..cfbbd8e644 100644 --- a/nemo_curator/stages/synthetic/nemo_data_designer/data_designer.py +++ b/nemo_curator/stages/synthetic/nemo_data_designer/data_designer.py @@ -139,5 +139,6 @@ def process(self, batch: DocumentBatch) -> DocumentBatch: _stage_perf=batch._stage_perf, ) + # Explicitly export the class __all__ = ["DataDesignerStage"] diff --git a/nemo_curator/stages/synthetic/nemotron_cc/base.py b/nemo_curator/stages/synthetic/nemotron_cc/base.py index bc929d3254..ba07dd8f80 100644 --- a/nemo_curator/stages/synthetic/nemotron_cc/base.py +++ b/nemo_curator/stages/synthetic/nemotron_cc/base.py @@ -87,6 +87,7 @@ def _process_llm_response(self, response: list[str]) -> str: def _process_sync(self, df: pd.DataFrame) -> list[str]: """Process DataFrame using synchronous sequential processing.""" + def generate_response(row: pd.Series) -> str: prompt = self._process_llm_prompt(row) if self.system_prompt: @@ -95,9 +96,7 @@ def generate_response(row: pd.Series) -> str: {"role": "user", "content": prompt}, ] else: - messages = [ - {"role": "user", "content": prompt} - ] + messages = [{"role": "user", "content": prompt}] response = self.client.query_model( model=self.model_name, messages=messages, @@ -142,9 +141,7 @@ async def generate_response_async(row: pd.Series) -> str: {"role": "user", "content": prompt}, ] else: - messages = [ - {"role": "user", "content": prompt} - ] + messages = [{"role": "user", "content": prompt}] response = await self.client.query_model( model=self.model_name, messages=messages, diff --git a/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/base.py b/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/base.py index fd1689d2f9..f6e526c358 100644 --- a/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/base.py +++ b/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/base.py @@ -163,7 +163,8 @@ def process(self, batch: DocumentBatch) -> DocumentBatch: ) raise ValueError(msg) df[_FORMATTED_PROMPT_COL] = df.apply( - lambda row: self._process_llm_prompt(row.to_dict()), axis=1, + lambda row: self._process_llm_prompt(row.to_dict()), + axis=1, ) pre_batch = DocumentBatch( data=df, diff --git a/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc.py b/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc.py index 7f12201208..d92b5ed790 100644 --- a/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc.py +++ b/nemo_curator/stages/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc.py @@ -72,5 +72,3 @@ class KnowledgeListStage(NDDBaseSyntheticStage): prompt: str = KNOWLEDGE_LIST_PROMPT_TEMPLATE input_field: str = "text" output_field: str = "knowledge_list" - - diff --git a/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py b/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py index d8bc7bb690..6f23a2dc82 100644 --- a/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py +++ b/nemo_curator/stages/synthetic/nemotron_cc/nemotron_cc.py @@ -40,6 +40,7 @@ class WikipediaParaphrasingStage(BaseSyntheticStage): input_field: str = "text" output_field: str = "rephrased" + @dataclass class DiverseQAStage(BaseSyntheticStage): system_prompt: str = NEMOTRON_CC_SYSTEM_PROMPT @@ -117,6 +118,7 @@ def _format_row(row: pd.Series) -> str: _stage_perf=batch._stage_perf, ) + @dataclass class DistillStage(BaseSyntheticStage): system_prompt: str = NEMOTRON_CC_DISTILL_SYSTEM_PROMPT @@ -124,6 +126,7 @@ class DistillStage(BaseSyntheticStage): input_field: str = "text" output_field: str = "distill" + @dataclass class ExtractKnowledgeStage(BaseSyntheticStage): system_prompt: str = NEMOTRON_CC_SYSTEM_PROMPT @@ -131,6 +134,7 @@ class ExtractKnowledgeStage(BaseSyntheticStage): input_field: str = "text" output_field: str = "extract_knowledge" + @dataclass class KnowledgeListStage(BaseSyntheticStage): system_prompt: str = NEMOTRON_CC_SYSTEM_PROMPT @@ -138,6 +142,7 @@ class KnowledgeListStage(BaseSyntheticStage): input_field: str = "text" output_field: str = "knowledge_list" + @dataclass class KnowledgeListPostProcessingStage(ProcessingStage[DocumentBatch, DocumentBatch]): """ diff --git a/nemo_curator/stages/text/deduplication/semantic.py b/nemo_curator/stages/text/deduplication/semantic.py index 857b44fe0f..4af3f79010 100644 --- a/nemo_curator/stages/text/deduplication/semantic.py +++ b/nemo_curator/stages/text/deduplication/semantic.py @@ -431,7 +431,7 @@ def run( # noqa: C901, PLR0912, PLR0915 embedding_executor = pairwise_executor = removal_executor = streaming_executor if batch_executor is None: - from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor + from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor batch_executor = RayActorPoolExecutor() diff --git a/nemo_curator/stages/text/download/base/iterator.py b/nemo_curator/stages/text/download/base/iterator.py index 6cdd8d513e..b84ebb6922 100644 --- a/nemo_curator/stages/text/download/base/iterator.py +++ b/nemo_curator/stages/text/download/base/iterator.py @@ -21,7 +21,7 @@ import pandas as pd from loguru import logger -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.tasks import DocumentBatch, FileGroupTask from nemo_curator.utils.column_utils import resolve_filename_column diff --git a/nemo_curator/stages/text/io/reader/base.py b/nemo_curator/stages/text/io/reader/base.py index 6550066965..571c116eb5 100644 --- a/nemo_curator/stages/text/io/reader/base.py +++ b/nemo_curator/stages/text/io/reader/base.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from nemo_curator.backends.base import WorkerMetadata -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.tasks import DocumentBatch, FileGroupTask diff --git a/nemo_curator/stages/text/modules/joiner.py b/nemo_curator/stages/text/modules/joiner.py index d706a4b3b2..1c9ea98ca5 100644 --- a/nemo_curator/stages/text/modules/joiner.py +++ b/nemo_curator/stages/text/modules/joiner.py @@ -190,4 +190,3 @@ def process(self, batch: DocumentBatch) -> DocumentBatch: _metadata=batch._metadata, _stage_perf=batch._stage_perf, ) - diff --git a/nemo_curator/stages/text/modules/splitter.py b/nemo_curator/stages/text/modules/splitter.py index 174af34508..db816ba610 100644 --- a/nemo_curator/stages/text/modules/splitter.py +++ b/nemo_curator/stages/text/modules/splitter.py @@ -91,4 +91,3 @@ def process(self, batch: DocumentBatch) -> DocumentBatch: _metadata=batch._metadata, _stage_perf=batch._stage_perf, ) - diff --git a/nemo_curator/stages/video/clipping/clip_extraction_stages.py b/nemo_curator/stages/video/clipping/clip_extraction_stages.py index 83ff4ca6ad..6814f384b7 100644 --- a/nemo_curator/stages/video/clipping/clip_extraction_stages.py +++ b/nemo_curator/stages/video/clipping/clip_extraction_stages.py @@ -22,7 +22,7 @@ from loguru import logger from nemo_curator.backends.base import WorkerMetadata -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.resources import Resources from nemo_curator.tasks.video import Clip, Video, VideoTask diff --git a/nemo_curator/utils/column_utils.py b/nemo_curator/utils/column_utils.py index bae97ef275..065e3b52b7 100644 --- a/nemo_curator/utils/column_utils.py +++ b/nemo_curator/utils/column_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def resolve_filename_column(add_filename_column: bool | str) -> str | None: """Resolve the filename column name based on the input parameter. diff --git a/nemo_curator/utils/split_large_files.py b/nemo_curator/utils/split_large_files.py index 41b7e35045..0e9fd73f40 100644 --- a/nemo_curator/utils/split_large_files.py +++ b/nemo_curator/utils/split_large_files.py @@ -105,7 +105,11 @@ def split_parquet_file_by_size( if row_group.nbytes > target_size_bytes: # Flush any pending small row groups first to preserve order. if row_groups_to_write: - sub_table = row_groups_to_write[0] if len(row_groups_to_write) == 1 else pa.concat_tables(row_groups_to_write) + sub_table = ( + row_groups_to_write[0] + if len(row_groups_to_write) == 1 + else pa.concat_tables(row_groups_to_write) + ) out_file = _join_out_path(output_path, f"{outfile_prefix}_{file_idx}{ext}", so) _write_table_to_file(sub_table, out_file, so) file_idx += 1 @@ -129,7 +133,9 @@ def split_parquet_file_by_size( row_group_idx += 1 if row_groups_to_write: - sub_table = row_groups_to_write[0] if len(row_groups_to_write) == 1 else pa.concat_tables(row_groups_to_write) + sub_table = ( + row_groups_to_write[0] if len(row_groups_to_write) == 1 else pa.concat_tables(row_groups_to_write) + ) out_file = _join_out_path(output_path, f"{outfile_prefix}_{file_idx}{ext}", so) _write_table_to_file(sub_table, out_file, so) file_idx += 1 @@ -207,7 +213,9 @@ def parse_args(args: argparse.ArgumentParser | None = None) -> argparse.Namespac parser.add_argument( "--input-path", type=str, required=True, help="Path to input file, or directory of files, to split" ) - parser.add_argument("--file-type", type=str, required=True, help="Type of file to split", choices=["parquet", "jsonl"]) + parser.add_argument( + "--file-type", type=str, required=True, help="Type of file to split", choices=["parquet", "jsonl"] + ) parser.add_argument("--output-path", type=str, required=True, help="Output directory to store split files") parser.add_argument("--target-size-mb", type=int, default=128, help="Target size (in MB) of split output files") parser.add_argument( @@ -242,7 +250,7 @@ def main(args: argparse.ArgumentParser | None = None) -> None: target_size_mb=args.target_size_mb, storage_options=storage_options, ) - for f in files + for f in files ] ) diff --git a/tests/backends/experimental/__init__.py b/tests/backends/experimental/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/backends/experimental/ray_actor_pool/__init__.py b/tests/backends/experimental/ray_actor_pool/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/backends/experimental/test_utils.py b/tests/backends/experimental/test_utils.py deleted file mode 100644 index b56d1a883f..0000000000 --- a/tests/backends/experimental/test_utils.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid -from collections.abc import Iterator -from contextlib import contextmanager -from pathlib import Path -from typing import TYPE_CHECKING - -import pytest -import ray -from pytest import LogCaptureFixture - -from nemo_curator.backends.base import NodeInfo, WorkerMetadata -from nemo_curator.backends.experimental import utils -from nemo_curator.backends.experimental.utils import ( - RayStageSpecKeys, - execute_setup_on_node, - get_head_node_id, -) -from nemo_curator.stages.base import ProcessingStage -from nemo_curator.stages.resources import Resources - -if TYPE_CHECKING: - from nemo_curator.tasks import Task - - -@contextmanager -def _reset_head_node_cache_context() -> Iterator[None]: - original_value = utils._HEAD_NODE_ID_CACHE - utils._HEAD_NODE_ID_CACHE = None - try: - yield - finally: - utils._HEAD_NODE_ID_CACHE = original_value - - -@pytest.fixture -def reset_head_node_cache() -> Iterator[None]: - with _reset_head_node_cache_context(): - yield - - -class TestExecuteSetupOnNode: - """Test class for execute_setup_on_node function.""" - - def test_execute_setup_on_node_with_two_stages( - self, - shared_ray_client: None, - tmp_path: Path, - caplog: LogCaptureFixture, - ): - """Test execute_setup_on_node with two stages on the Ray cluster.""" - - class MockStage1(ProcessingStage): - name = "mock_stage_1" - resources = Resources(cpus=1.0, gpus=0.0) - - def process(self, task: "Task") -> "Task": - return task - - def setup_on_node( - self, node_info: NodeInfo | None = None, worker_metadata: WorkerMetadata | None = None - ) -> None: - # Write a file to record this call - node_id = node_info.node_id if node_info else "unknown" - worker_id = worker_metadata.worker_id if worker_metadata else "unknown" - filename = f"{self.name}_{uuid.uuid4()}.txt" - filepath = tmp_path / filename - with open(filepath, "w") as f: - f.write(f"{node_id},{worker_id}\n") - - stage1 = MockStage1() - stage2 = MockStage1().with_(name="mock_stage_2", resources=Resources(cpus=0.5, gpus=0.0)) - - # Test - execute_setup_on_node([stage1, stage2]) - - # Check the files written to the temp directory - # Verify that NodeInfo and WorkerMetadata were passed correctly - for stage_name in ["mock_stage_1", "mock_stage_2"]: - stage_files = list(tmp_path.glob(f"{stage_name}_*.txt")) - assert len(stage_files) == len(ray.nodes()), ( - f"Expected {len(ray.nodes())} calls to setup_on_node for {stage_name}, got {len(stage_files)}" - ) - node_ids = set() - for file_path in stage_files: - content = file_path.read_text().strip() - node_id, worker_id = content.split(",") - assert worker_id == "", f"{stage_name} Worker ID should be empty string, got '{worker_id}'" - node_ids.add(node_id) - assert len(node_ids) == len(ray.nodes()), ( - f"Expected {len(ray.nodes())} different node IDs for {stage_name}, got {node_ids}" - ) - assert node_ids == {node["NodeID"] for node in ray.nodes()}, ( - f"Expected node IDs to be the same as the Ray nodes, got {node_ids}" - ) - - # Check that there are exactly two log records that start with "Executing setup on node" and end with "for 2 stages" - matching_logs = [ - record.message - for record in caplog.records - if record.message.startswith("Executing setup on node") and record.message.endswith("for 2 stages") - ] - # TODO: When we add a cluster then we should check the value of len(ray.nodes()) too - assert len(matching_logs) == len(ray.nodes()), ( - f"Expected {len(ray.nodes())} logs for setup on node for 2 stages, got {len(matching_logs)}: {matching_logs}" - ) - - def test_execute_setup_on_node_ignore_head_node( - self, - shared_ray_client: None, - tmp_path: Path, - caplog: LogCaptureFixture, - reset_head_node_cache: None, - ): - """Test execute_setup_on_node with ignore_head_node=True to skip head node.""" - - class MockStage1(ProcessingStage): - name = "mock_stage_ignore_head" - resources = Resources(cpus=1.0, gpus=0.0) - - def process(self, task: "Task") -> "Task": - return task - - def setup_on_node( - self, node_info: NodeInfo | None = None, worker_metadata: WorkerMetadata | None = None - ) -> None: - # Write a file to record this call - node_id = node_info.node_id if node_info else "unknown" - worker_id = worker_metadata.worker_id if worker_metadata else "unknown" - filename = f"{self.name}_{uuid.uuid4()}.txt" - filepath = tmp_path / filename - with open(filepath, "w") as f: - f.write(f"{node_id},{worker_id}\n") - - stage = MockStage1() - - # Test with ignore_head_node=True - execute_setup_on_node([stage], ignore_head_node=True) - - # Verify the cache variable is set directly (not using the lazy function) - assert utils._HEAD_NODE_ID_CACHE is not None, "_HEAD_NODE_ID_CACHE should be set after execute_setup_on_node" - - # Verify it matches the actual head node in the cluster - expected_head_node_id = None - for node in ray.nodes(): - if "node:__internal_head__" in node.get("Resources", {}): - expected_head_node_id = node["NodeID"] - break - - assert expected_head_node_id is not None, "Expected head node ID should be set" - assert expected_head_node_id == utils._HEAD_NODE_ID_CACHE, ( - f"_HEAD_NODE_ID_CACHE should be {expected_head_node_id}, got {utils._HEAD_NODE_ID_CACHE}" - ) - - # Check the files written to the temp directory - stage_files = list(tmp_path.glob(f"{stage.name}_*.txt")) - expected_calls = len(ray.nodes()) - (1 if expected_head_node_id else 0) - assert len(stage_files) == expected_calls, ( - f"Expected {expected_calls} calls to setup_on_node (excluding head node), got {len(stage_files)}" - ) - - -class TestGetHeadNodeId: - def test_lazy_evaluation( - self, - shared_ray_client: None, - reset_head_node_cache: None, - ): - """Test that get_head_node_id uses lazy evaluation and caching.""" - - # Cache should start cleared by fixture - assert utils._HEAD_NODE_ID_CACHE is None, "Cache should be cleared before test" - - # First call should compute and cache - head_node_id_1 = get_head_node_id() - - # Cache should now be set - assert utils._HEAD_NODE_ID_CACHE is not None, "Cache should be set after first call" - - # Second call should return cached value - head_node_id_2 = get_head_node_id() - - # Both should be the same - assert head_node_id_1 == head_node_id_2, "Cached value should match" - - # Verify it's the actual head node - expected_head_node_id = None - for node in ray.nodes(): - if "node:__internal_head__" in node.get("Resources", {}): - expected_head_node_id = node["NodeID"] - break - - assert expected_head_node_id is not None, "Expected head node ID should be set" - assert head_node_id_1 == expected_head_node_id, ( - f"get_head_node_id() returned {head_node_id_1}, expected {expected_head_node_id}" - ) - - -class TestRayStageSpecKeys: - """Test class for RayStageSpecKeys enum compatibility.""" - - def test_enum_membership_compatibility(self): - """Test that the fixed pattern works across Python versions.""" - # Test data - valid_keys = ["is_actor_stage", "is_fanout_stage", "is_lsh_stage"] - invalid_keys = ["invalid_key", "another_bad_key"] - - # Test the fixed pattern - this is what's now used in the adapter - enum_values = {e.value for e in RayStageSpecKeys} - - # Testing valid keys - for key in valid_keys: - result = key not in enum_values - assert result is False, f"Valid key '{key}' should be found in enum values" - - # Testing invalid keys - for key in invalid_keys: - result = key not in enum_values - assert result is True, f"Invalid key '{key}' should not be found in enum values" diff --git a/nemo_curator/backends/experimental/__init__.py b/tests/backends/ray_actor_pool/__init__.py similarity index 100% rename from nemo_curator/backends/experimental/__init__.py rename to tests/backends/ray_actor_pool/__init__.py diff --git a/tests/backends/experimental/ray_actor_pool/test_executor.py b/tests/backends/ray_actor_pool/test_executor.py similarity index 93% rename from tests/backends/experimental/ray_actor_pool/test_executor.py rename to tests/backends/ray_actor_pool/test_executor.py index 99243b28c6..295be1c045 100644 --- a/tests/backends/experimental/ray_actor_pool/test_executor.py +++ b/tests/backends/ray_actor_pool/test_executor.py @@ -14,7 +14,7 @@ from unittest import mock -from nemo_curator.backends.experimental.ray_actor_pool.executor import _parse_runtime_env +from nemo_curator.backends.ray_actor_pool.executor import _parse_runtime_env class TestRayActorPoolExecutor: diff --git a/tests/backends/ray_data/test_max_calls_pid.py b/tests/backends/ray_data/test_max_calls_pid.py index 55464d74c4..e53530c6d8 100644 --- a/tests/backends/ray_data/test_max_calls_pid.py +++ b/tests/backends/ray_data/test_max_calls_pid.py @@ -22,8 +22,8 @@ import ray from loguru import logger -from nemo_curator.backends.experimental.utils import RayStageSpecKeys from nemo_curator.backends.ray_data.executor import RayDataExecutor +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.core.client import RayClient from nemo_curator.stages.base import ProcessingStage, Resources from nemo_curator.tasks import DocumentBatch, EmptyTask diff --git a/tests/backends/ray_data/test_utils.py b/tests/backends/ray_data/test_utils.py index d4dc961cd9..ac5d1a279c 100644 --- a/tests/backends/ray_data/test_utils.py +++ b/tests/backends/ray_data/test_utils.py @@ -71,63 +71,49 @@ def test_calculate_concurrency_explicit_num_workers(self, mock_get_resources: Ma # Should not call get_resources if num_workers is set mock_get_resources.assert_not_called() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0)) def test_calculate_concurrency_explicit_num_workers_zero_or_negative(self, mock_get_resources: MagicMock): """Test calculate_concurrency when num_workers is explicitly set to 0 or negative.""" mock_stage = Mock(num_workers=lambda: 0, resources=Resources(cpus=2.0, gpus=0.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 4) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0)) def test_calculate_concurrency_cpu_only_constraint(self, mock_get_resources: MagicMock): """Test calculate_concurrency with CPU-only constraint.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=2.0, gpus=0.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 4) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 4.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 4.0)) def test_calculate_concurrency_gpu_only_constraint(self, mock_get_resources: MagicMock): """Test calculate_concurrency with GPU-only constraint.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=0.0, gpus=1.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 4) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 4.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 4.0)) def test_calculate_concurrency_both_cpu_gpu_constraints(self, mock_get_resources: MagicMock): """Test calculate_concurrency with both CPU and GPU constraints.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=2.0, gpus=1.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 4) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(4.0, 8.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(4.0, 8.0)) def test_calculate_concurrency_cpu_more_limiting(self, mock_get_resources: MagicMock): """Test calculate_concurrency when CPU is more limiting than GPU.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=2.0, gpus=1.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 2) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(16.0, 2.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(16.0, 2.0)) def test_calculate_concurrency_gpu_more_limiting(self, mock_get_resources: MagicMock): """Test calculate_concurrency when GPU is more limiting than CPU.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=2.0, gpus=1.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 2) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0)) def test_calculate_concurrency_no_resource_requirements(self, mock_get_resources: MagicMock): """Test calculate_concurrency when stage has no resource requirements.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=0.0, gpus=0.0)) @@ -136,18 +122,14 @@ def test_calculate_concurrency_no_resource_requirements(self, mock_get_resources mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(1.0, 0.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(1.0, 0.0)) def test_calculate_concurrency_insufficient_resources(self, mock_get_resources: MagicMock): """Test calculate_concurrency when there are insufficient resources.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=4.0, gpus=2.0)) assert calculate_concurrency_for_actors_for_stage(mock_stage) == (1, 0) mock_get_resources.assert_called_once() - @patch( - "nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0) - ) + @patch("nemo_curator.backends.ray_data.utils.get_available_cpu_gpu_resources", return_value=(8.0, 2.0)) def test_calculate_concurrency_fractional_resources(self, mock_get_resources: MagicMock): """Test calculate_concurrency with fractional resource requirements.""" mock_stage = Mock(num_workers=lambda: None, resources=Resources(cpus=0.5, gpus=0.25)) diff --git a/tests/backends/test_integration.py b/tests/backends/test_integration.py index 0393f1a884..fa5e117c0c 100644 --- a/tests/backends/test_integration.py +++ b/tests/backends/test_integration.py @@ -24,7 +24,7 @@ from loguru import logger from nemo_curator.backends.base import BaseExecutor -from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor +from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.backends.ray_data import RayDataExecutor from nemo_curator.backends.xenna import XennaExecutor from nemo_curator.tasks import FileGroupTask diff --git a/tests/backends/test_utils.py b/tests/backends/test_utils.py index 6a5f5aabdb..6565b04164 100644 --- a/tests/backends/test_utils.py +++ b/tests/backends/test_utils.py @@ -12,44 +12,221 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nemo_curator.backends.utils import merge_executor_configs - - -class TestMergeExecutorConfig: - """Test class for merge_executor_configs function.""" - - def test_merge_nested_dicts(self): - """Test merging nested dictionaries.""" - base = { - "runtime_env": { - "env_vars": {"A": "1", "B": "2"}, - "pip": ["package1"], - }, - "other_config": "value1", - } - - override = { - "runtime_env": { - "env_vars": {"B": "3", "C": "4"}, - "working_dir": ".", - }, - "some_other_top_key": "value2", - } - - result = merge_executor_configs(base, override) - - # Check that nested dicts are merged - assert result["runtime_env"]["env_vars"]["A"] == "1" - assert result["runtime_env"]["env_vars"]["B"] == "3" - assert result["runtime_env"]["env_vars"]["C"] == "4" - # Check that other keys are preserved - assert result["runtime_env"]["pip"] == ["package1"] - assert result["runtime_env"]["working_dir"] == "." - assert result["other_config"] == "value1" - assert result["some_other_top_key"] == "value2" - - def test_merge_with_none(self): - """Test merging when base config is None.""" - assert merge_executor_configs(None, {"key": "value"}) == {"key": "value"} - assert merge_executor_configs({"key": "value"}, None) == {"key": "value"} - assert merge_executor_configs(None, None) == {} +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import ray +from pytest import LogCaptureFixture + +from nemo_curator.backends import utils +from nemo_curator.backends.base import NodeInfo, WorkerMetadata +from nemo_curator.backends.utils import ( + RayStageSpecKeys, + execute_setup_on_node, + get_head_node_id, +) +from nemo_curator.stages.base import ProcessingStage +from nemo_curator.stages.resources import Resources + +if TYPE_CHECKING: + from nemo_curator.tasks import Task + + +@contextmanager +def _reset_head_node_cache_context() -> Iterator[None]: + original_value = utils._HEAD_NODE_ID_CACHE + utils._HEAD_NODE_ID_CACHE = None + try: + yield + finally: + utils._HEAD_NODE_ID_CACHE = original_value + + +@pytest.fixture +def reset_head_node_cache() -> Iterator[None]: + with _reset_head_node_cache_context(): + yield + + +class TestExecuteSetupOnNode: + """Test class for execute_setup_on_node function.""" + + def test_execute_setup_on_node_with_two_stages( + self, + shared_ray_client: None, + tmp_path: Path, + caplog: LogCaptureFixture, + ): + """Test execute_setup_on_node with two stages on the Ray cluster.""" + + class MockStage1(ProcessingStage): + name = "mock_stage_1" + resources = Resources(cpus=1.0, gpus=0.0) + + def process(self, task: "Task") -> "Task": + return task + + def setup_on_node( + self, node_info: NodeInfo | None = None, worker_metadata: WorkerMetadata | None = None + ) -> None: + # Write a file to record this call + node_id = node_info.node_id if node_info else "unknown" + worker_id = worker_metadata.worker_id if worker_metadata else "unknown" + filename = f"{self.name}_{uuid.uuid4()}.txt" + filepath = tmp_path / filename + with open(filepath, "w") as f: + f.write(f"{node_id},{worker_id}\n") + + stage1 = MockStage1() + stage2 = MockStage1().with_(name="mock_stage_2", resources=Resources(cpus=0.5, gpus=0.0)) + + # Test + execute_setup_on_node([stage1, stage2]) + + # Check the files written to the temp directory + # Verify that NodeInfo and WorkerMetadata were passed correctly + for stage_name in ["mock_stage_1", "mock_stage_2"]: + stage_files = list(tmp_path.glob(f"{stage_name}_*.txt")) + assert len(stage_files) == len(ray.nodes()), ( + f"Expected {len(ray.nodes())} calls to setup_on_node for {stage_name}, got {len(stage_files)}" + ) + node_ids = set() + for file_path in stage_files: + content = file_path.read_text().strip() + node_id, worker_id = content.split(",") + assert worker_id == "", f"{stage_name} Worker ID should be empty string, got '{worker_id}'" + node_ids.add(node_id) + assert len(node_ids) == len(ray.nodes()), ( + f"Expected {len(ray.nodes())} different node IDs for {stage_name}, got {node_ids}" + ) + assert node_ids == {node["NodeID"] for node in ray.nodes()}, ( + f"Expected node IDs to be the same as the Ray nodes, got {node_ids}" + ) + + # Check that there are exactly two log records that start with "Executing setup on node" and end with "for 2 stages" + matching_logs = [ + record.message + for record in caplog.records + if record.message.startswith("Executing setup on node") and record.message.endswith("for 2 stages") + ] + # TODO: When we add a cluster then we should check the value of len(ray.nodes()) too + assert len(matching_logs) == len(ray.nodes()), ( + f"Expected {len(ray.nodes())} logs for setup on node for 2 stages, got {len(matching_logs)}: {matching_logs}" + ) + + def test_execute_setup_on_node_ignore_head_node( + self, + shared_ray_client: None, + tmp_path: Path, + caplog: LogCaptureFixture, + reset_head_node_cache: None, + ): + """Test execute_setup_on_node with ignore_head_node=True to skip head node.""" + + class MockStage1(ProcessingStage): + name = "mock_stage_ignore_head" + resources = Resources(cpus=1.0, gpus=0.0) + + def process(self, task: "Task") -> "Task": + return task + + def setup_on_node( + self, node_info: NodeInfo | None = None, worker_metadata: WorkerMetadata | None = None + ) -> None: + # Write a file to record this call + node_id = node_info.node_id if node_info else "unknown" + worker_id = worker_metadata.worker_id if worker_metadata else "unknown" + filename = f"{self.name}_{uuid.uuid4()}.txt" + filepath = tmp_path / filename + with open(filepath, "w") as f: + f.write(f"{node_id},{worker_id}\n") + + stage = MockStage1() + + # Test with ignore_head_node=True + execute_setup_on_node([stage], ignore_head_node=True) + + # Verify the cache variable is set directly (not using the lazy function) + assert utils._HEAD_NODE_ID_CACHE is not None, "_HEAD_NODE_ID_CACHE should be set after execute_setup_on_node" + + # Verify it matches the actual head node in the cluster + expected_head_node_id = None + for node in ray.nodes(): + if "node:__internal_head__" in node.get("Resources", {}): + expected_head_node_id = node["NodeID"] + break + + assert expected_head_node_id is not None, "Expected head node ID should be set" + assert expected_head_node_id == utils._HEAD_NODE_ID_CACHE, ( + f"_HEAD_NODE_ID_CACHE should be {expected_head_node_id}, got {utils._HEAD_NODE_ID_CACHE}" + ) + + # Check the files written to the temp directory + stage_files = list(tmp_path.glob(f"{stage.name}_*.txt")) + expected_calls = len(ray.nodes()) - (1 if expected_head_node_id else 0) + assert len(stage_files) == expected_calls, ( + f"Expected {expected_calls} calls to setup_on_node (excluding head node), got {len(stage_files)}" + ) + + +class TestGetHeadNodeId: + def test_lazy_evaluation( + self, + shared_ray_client: None, + reset_head_node_cache: None, + ): + """Test that get_head_node_id uses lazy evaluation and caching.""" + + # Cache should start cleared by fixture + assert utils._HEAD_NODE_ID_CACHE is None, "Cache should be cleared before test" + + # First call should compute and cache + head_node_id_1 = get_head_node_id() + + # Cache should now be set + assert utils._HEAD_NODE_ID_CACHE is not None, "Cache should be set after first call" + + # Second call should return cached value + head_node_id_2 = get_head_node_id() + + # Both should be the same + assert head_node_id_1 == head_node_id_2, "Cached value should match" + + # Verify it's the actual head node + expected_head_node_id = None + for node in ray.nodes(): + if "node:__internal_head__" in node.get("Resources", {}): + expected_head_node_id = node["NodeID"] + break + + assert expected_head_node_id is not None, "Expected head node ID should be set" + assert head_node_id_1 == expected_head_node_id, ( + f"get_head_node_id() returned {head_node_id_1}, expected {expected_head_node_id}" + ) + + +class TestRayStageSpecKeys: + """Test class for RayStageSpecKeys enum compatibility.""" + + def test_enum_membership_compatibility(self): + """Test that the fixed pattern works across Python versions.""" + # Test data + valid_keys = ["is_actor_stage", "is_fanout_stage", "is_lsh_stage"] + invalid_keys = ["invalid_key", "another_bad_key"] + + # Test the fixed pattern - this is what's now used in the adapter + enum_values = {e.value for e in RayStageSpecKeys} + + # Testing valid keys + for key in valid_keys: + result = key not in enum_values + assert result is False, f"Valid key '{key}' should be found in enum values" + + # Testing invalid keys + for key in invalid_keys: + result = key not in enum_values + assert result is True, f"Invalid key '{key}' should not be found in enum values" diff --git a/tests/backends/utils.py b/tests/backends/utils.py index 07cc225cb1..c6aef281e8 100644 --- a/tests/backends/utils.py +++ b/tests/backends/utils.py @@ -32,7 +32,7 @@ from loguru import logger from nemo_curator.backends.base import NodeInfo, WorkerMetadata -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.pipeline import Pipeline from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.text.io.reader import JsonlReader diff --git a/tests/stages/audio/advanced_pipelines/test_audio_data_filter.py b/tests/stages/audio/advanced_pipelines/test_audio_data_filter.py index 556e0567d3..8c4ab98986 100644 --- a/tests/stages/audio/advanced_pipelines/test_audio_data_filter.py +++ b/tests/stages/audio/advanced_pipelines/test_audio_data_filter.py @@ -195,21 +195,25 @@ def test_decompose_all_enabled_stage_count(self) -> None: assert len(stages) == 12 def test_decompose_all_disabled_except_mono(self) -> None: - stage = AudioDataFilterStage(config={ - "vad": {"enable": False}, - "band_filter": {"enable": False}, - "utmos": {"enable": False}, - "sigmos": {"enable": False}, - "speaker_separation": {"enable": False}, - }) + stage = AudioDataFilterStage( + config={ + "vad": {"enable": False}, + "band_filter": {"enable": False}, + "utmos": {"enable": False}, + "sigmos": {"enable": False}, + "speaker_separation": {"enable": False}, + } + ) stages = stage.decompose() assert len(stages) == 1 assert isinstance(stages[0], MonoConversionStage) def test_decompose_no_speaker_no_second_pass(self) -> None: - stage = AudioDataFilterStage(config={ - "speaker_separation": {"enable": False}, - }) + stage = AudioDataFilterStage( + config={ + "speaker_separation": {"enable": False}, + } + ) stages = stage.decompose() assert len(stages) == 6 stage_types = [type(s) for s in stages] @@ -257,10 +261,12 @@ def test_decompose_custom_thresholds(self) -> None: class TestDecomposeEdgeCases: def test_decompose_speaker_without_vad_no_concat(self) -> None: - stage = AudioDataFilterStage(config={ - "vad": {"enable": False}, - "speaker_separation": {"enable": True}, - }) + stage = AudioDataFilterStage( + config={ + "vad": {"enable": False}, + "speaker_separation": {"enable": True}, + } + ) stages = stage.decompose() stage_types = [type(s) for s in stages] assert SegmentConcatenationStage not in stage_types @@ -297,8 +303,14 @@ class TestDefaultYAMLConsistency: def test_default_yaml_all_stages_have_resources(self) -> None: cfg = load_config(None) stages_with_cpus = [ - "mono_conversion", "vad", "band_filter", "utmos", - "sigmos", "concatenation", "speaker_separation", "timestamp_mapper", + "mono_conversion", + "vad", + "band_filter", + "utmos", + "sigmos", + "concatenation", + "speaker_separation", + "timestamp_mapper", ] for stage_name in stages_with_cpus: assert "cpus" in cfg[stage_name], f"{stage_name} missing 'cpus' in config" @@ -306,4 +318,3 @@ def test_default_yaml_all_stages_have_resources(self) -> None: stages_with_gpus = ["vad", "band_filter", "utmos", "sigmos", "speaker_separation"] for stage_name in stages_with_gpus: assert "gpus" in cfg[stage_name], f"{stage_name} missing 'gpus' in config" - diff --git a/tests/stages/audio/datasets/test_fleurs_create_initial_manifest.py b/tests/stages/audio/datasets/test_fleurs_create_initial_manifest.py index 68b045208f..6e4ab28206 100644 --- a/tests/stages/audio/datasets/test_fleurs_create_initial_manifest.py +++ b/tests/stages/audio/datasets/test_fleurs_create_initial_manifest.py @@ -34,7 +34,7 @@ def _import_stage_module() -> tuple[Any, Any]: def test_ray_stage_spec(tmp_path: Path) -> None: - from nemo_curator.backends.experimental.utils import RayStageSpecKeys + from nemo_curator.backends.utils import RayStageSpecKeys stage_cls, _ = _import_stage_module() stage = stage_cls(lang="hy_am", split="dev", raw_data_dir=str(tmp_path / "fleurs")) diff --git a/tests/stages/audio/filtering/test_sigmos.py b/tests/stages/audio/filtering/test_sigmos.py index 45da1651c1..dc886c05e7 100644 --- a/tests/stages/audio/filtering/test_sigmos.py +++ b/tests/stages/audio/filtering/test_sigmos.py @@ -96,15 +96,17 @@ def test_none_thresholds_disable_checks(self, mock_init: MagicMock) -> None: @patch.object(SIGMOSFilterStage, "_initialize_model") def test_partial_threshold_fail(self, mock_init: MagicMock) -> None: stage = SIGMOSFilterStage(noise_threshold=4.0, ovrl_threshold=None) - stage._model = _make_mock_model({ - "MOS_NOISE": 3.0, - "MOS_OVRL": 5.0, - "MOS_SIG": 5.0, - "MOS_COL": 5.0, - "MOS_DISC": 5.0, - "MOS_LOUD": 5.0, - "MOS_REVERB": 5.0, - }) + stage._model = _make_mock_model( + { + "MOS_NOISE": 3.0, + "MOS_OVRL": 5.0, + "MOS_SIG": 5.0, + "MOS_COL": 5.0, + "MOS_DISC": 5.0, + "MOS_LOUD": 5.0, + "MOS_REVERB": 5.0, + } + ) result = stage.process(_make_task()) diff --git a/tests/stages/audio/segmentation/test_vad_segmentation.py b/tests/stages/audio/segmentation/test_vad_segmentation.py index bd6610f79c..ff51f9a7c8 100644 --- a/tests/stages/audio/segmentation/test_vad_segmentation.py +++ b/tests/stages/audio/segmentation/test_vad_segmentation.py @@ -18,7 +18,7 @@ import pytest import torch -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.audio.segmentation.vad_segmentation import VADSegmentationStage from nemo_curator.tasks import AudioTask diff --git a/tests/stages/deduplication/exact/test_identification.py b/tests/stages/deduplication/exact/test_identification.py index d97102bbc1..5fa80faa58 100644 --- a/tests/stages/deduplication/exact/test_identification.py +++ b/tests/stages/deduplication/exact/test_identification.py @@ -30,7 +30,7 @@ # Suppress GPU-related import errors when running pytest -m "not gpu" with suppress(ImportError): - from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor + from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.pipeline import Pipeline from nemo_curator.stages.deduplication.exact.identification import ExactDuplicateIdentification from nemo_curator.stages.deduplication.id_generator import CURATOR_DEDUP_ID_STR, get_id_generator_actor diff --git a/tests/stages/deduplication/fuzzy/test_connected_components_stage.py b/tests/stages/deduplication/fuzzy/test_connected_components_stage.py index 9626cc19cc..d7f48b4996 100644 --- a/tests/stages/deduplication/fuzzy/test_connected_components_stage.py +++ b/tests/stages/deduplication/fuzzy/test_connected_components_stage.py @@ -23,7 +23,7 @@ # Suppress GPU-related import errors when running pytest -m "not gpu" with suppress(ImportError): - from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor + from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.pipeline import Pipeline from nemo_curator.stages.deduplication.fuzzy.connected_components import ConnectedComponentsStage from nemo_curator.stages.deduplication.id_generator import CURATOR_DEDUP_ID_STR diff --git a/tests/stages/deduplication/fuzzy/test_lsh_stage.py b/tests/stages/deduplication/fuzzy/test_lsh_stage.py index b1ea1998cf..33344ec8d7 100644 --- a/tests/stages/deduplication/fuzzy/test_lsh_stage.py +++ b/tests/stages/deduplication/fuzzy/test_lsh_stage.py @@ -26,7 +26,7 @@ # Suppress GPU-related import errors when running pytest -m "not gpu" with suppress(ImportError): - from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor + from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.pipeline import Pipeline from nemo_curator.stages.deduplication.fuzzy.lsh.stage import LSHStage from nemo_curator.stages.deduplication.id_generator import CURATOR_DEDUP_ID_STR diff --git a/tests/stages/deduplication/semantic/test_kmeans.py b/tests/stages/deduplication/semantic/test_kmeans.py index efadfce2fa..9c1442abd1 100644 --- a/tests/stages/deduplication/semantic/test_kmeans.py +++ b/tests/stages/deduplication/semantic/test_kmeans.py @@ -33,7 +33,7 @@ # Suppress GPU-related import errors when running pytest -m "not gpu" with suppress(ImportError): - from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor + from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.pipeline import Pipeline from nemo_curator.stages.deduplication.semantic.kmeans import KMeansReadFitWriteStage, KMeansStage from nemo_curator.stages.deduplication.semantic.utils import get_array_from_df diff --git a/tests/stages/deduplication/shuffle_utils/test_shuffle_stage.py b/tests/stages/deduplication/shuffle_utils/test_shuffle_stage.py index 73de4b0817..74bae750b3 100644 --- a/tests/stages/deduplication/shuffle_utils/test_shuffle_stage.py +++ b/tests/stages/deduplication/shuffle_utils/test_shuffle_stage.py @@ -26,7 +26,7 @@ # Suppress GPU-related import errors when running pytest -m "not gpu" with suppress(ImportError): - from nemo_curator.backends.experimental.ray_actor_pool import RayActorPoolExecutor + from nemo_curator.backends.ray_actor_pool import RayActorPoolExecutor from nemo_curator.pipeline import Pipeline from nemo_curator.stages.deduplication.shuffle_utils.stage import ShuffleStage diff --git a/tests/stages/image/embedders/test_clip_embedder.py b/tests/stages/image/embedders/test_clip_embedder.py index 47b24de062..83541d8e03 100644 --- a/tests/stages/image/embedders/test_clip_embedder.py +++ b/tests/stages/image/embedders/test_clip_embedder.py @@ -30,11 +30,7 @@ class TestImageEmbeddingStage: @pytest.fixture def stage(self) -> ImageEmbeddingStage: """Create a test stage instance.""" - return ImageEmbeddingStage( - model_dir="test_models/clip", - model_inference_batch_size=2, - verbose=True - ) + return ImageEmbeddingStage(model_dir="test_models/clip", model_inference_batch_size=2, verbose=True) @pytest.fixture def mock_model(self) -> Mock: @@ -53,23 +49,23 @@ def sample_image_objects(self) -> list[ImageObject]: ImageObject( image_id="img_001", image_path="/path/to/img1.jpg", - image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8) + image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8), ), ImageObject( image_id="img_002", image_path="/path/to/img2.jpg", - image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8) + image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8), ), ImageObject( image_id="img_003", image_path="/path/to/img3.jpg", - image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8) + image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8), ), ImageObject( image_id="img_004", image_path="/path/to/img4.jpg", - image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8) - ) + image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8), + ), ] @pytest.fixture @@ -80,7 +76,7 @@ def sample_image_batch(self, sample_image_objects: list[ImageObject]) -> ImageBa dataset_name="test_dataset", task_id="test_task_001", _metadata={"test": "metadata"}, - _stage_perf={} + _stage_perf={}, ) def test_stage_properties(self, stage: ImageEmbeddingStage) -> None: @@ -110,9 +106,8 @@ def test_setup(self, mock_clip_embeddings: Mock, stage: ImageEmbeddingStage) -> mock_clip_embeddings.assert_called_once() call_args, call_kwargs = mock_clip_embeddings.call_args - assert ( - (len(call_args) >= 1 and call_args[0] == "test_models/clip") - or (call_kwargs.get("model_dir") == "test_models/clip") + assert (len(call_args) >= 1 and call_args[0] == "test_models/clip") or ( + call_kwargs.get("model_dir") == "test_models/clip" ) mock_model.setup.assert_called_once() assert stage.model == mock_model @@ -183,11 +178,13 @@ def test_batch_processing( rng = np.random.default_rng(42) images = [] for i in range(5): - images.append(ImageObject( - image_id=f"img_{i:03d}", - image_path=f"/path/to/img{i}.jpg", - image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8) - )) + images.append( + ImageObject( + image_id=f"img_{i:03d}", + image_path=f"/path/to/img{i}.jpg", + image_data=rng.integers(0, 255, (224, 224, 3), dtype=np.uint8), + ) + ) batch = ImageBatch(data=images, task_id="test_batch", dataset_name="test_dataset") @@ -241,14 +238,17 @@ def test_verbose_logging( stage.process(sample_image_batch) # Should log embedding generation - embedding_calls = [call for call in mock_logger.info.call_args_list - if "Generated embeddings for" in str(call)] + embedding_calls = [call for call in mock_logger.info.call_args_list if "Generated embeddings for" in str(call)] assert len(embedding_calls) > 0 @patch("nemo_curator.stages.image.embedders.clip_embedder.CLIPImageEmbeddings") @patch("transformers.CLIPProcessor.from_pretrained") def test_preserves_other_image_attributes( - self, mock_processor: Mock, mock_clip_embeddings: Mock, stage: ImageEmbeddingStage, sample_image_batch: ImageBatch + self, + mock_processor: Mock, + mock_clip_embeddings: Mock, + stage: ImageEmbeddingStage, + sample_image_batch: ImageBatch, ) -> None: """Test that processing preserves other image attributes.""" mock_clip_embeddings.return_value = Mock() @@ -332,9 +332,8 @@ def test_processor_integration( # Verify the model was instantiated and setup was called mock_clip_embeddings.assert_called_once() call_args, call_kwargs = mock_clip_embeddings.call_args - assert ( - (len(call_args) >= 1 and call_args[0] == "test_models/clip") - or (call_kwargs.get("model_dir") == "test_models/clip") + assert (len(call_args) >= 1 and call_args[0] == "test_models/clip") or ( + call_kwargs.get("model_dir") == "test_models/clip" ) mock_model_instance.setup.assert_called_once() @@ -364,13 +363,13 @@ def test_embedding_shape_consistency(self, stage: ImageEmbeddingStage) -> None: ImageObject( image_id="small_img", image_path="/path/to/small.jpg", - image_data=rng.integers(0, 255, (100, 100, 3), dtype=np.uint8) + image_data=rng.integers(0, 255, (100, 100, 3), dtype=np.uint8), ), ImageObject( image_id="large_img", image_path="/path/to/large.jpg", - image_data=rng.integers(0, 255, (500, 500, 3), dtype=np.uint8) - ) + image_data=rng.integers(0, 255, (500, 500, 3), dtype=np.uint8), + ), ] batch = ImageBatch(data=different_sized_images, task_id="shape_test", dataset_name="test_dataset") diff --git a/tests/stages/image/filters/test_nsfw_filter.py b/tests/stages/image/filters/test_nsfw_filter.py index a99c3da364..5a75985bbe 100644 --- a/tests/stages/image/filters/test_nsfw_filter.py +++ b/tests/stages/image/filters/test_nsfw_filter.py @@ -30,11 +30,7 @@ class TestImageNSFWFilterStage: @pytest.fixture def stage(self) -> ImageNSFWFilterStage: """Create a test stage instance.""" - return ImageNSFWFilterStage( - model_dir="test_models/nsfw", - score_threshold=0.5, - model_inference_batch_size=2 - ) + return ImageNSFWFilterStage(model_dir="test_models/nsfw", score_threshold=0.5, model_inference_batch_size=2) @pytest.fixture def mock_model(self) -> Mock: @@ -84,7 +80,7 @@ def sample_image_batch(self, sample_image_objects: list[ImageObject]) -> ImageBa dataset_name="test_dataset", task_id="test_task_001", _metadata={"test": "metadata"}, - _stage_perf={} + _stage_perf={}, ) def test_stage_properties(self, stage: ImageNSFWFilterStage) -> None: @@ -126,7 +122,7 @@ def test_process_filtering( # So keep img1 (0.3), img3 (0.2), filter out img2 (0.7), img4 (0.8) mock_model.side_effect = [ torch.tensor([0.3, 0.7]), # First batch - torch.tensor([0.2, 0.8]) # Second batch + torch.tensor([0.2, 0.8]), # Second batch ] result = stage.process(sample_image_batch) @@ -167,7 +163,7 @@ def test_process_high_nsfw_filtering( # All images have high NSFW scores (above threshold) mock_model.side_effect = [ torch.tensor([0.8, 0.9]), # First batch - torch.tensor([0.7, 0.6]) # Second batch + torch.tensor([0.7, 0.6]), # Second batch ] result = stage.process(sample_image_batch) @@ -206,17 +202,13 @@ def test_threshold_boundary_cases( """Test boundary cases at threshold.""" mock_nsfw_scorer.return_value = mock_model - stage = ImageNSFWFilterStage( - model_dir="test_models/nsfw", - score_threshold=0.5, - model_inference_batch_size=2 - ) + stage = ImageNSFWFilterStage(model_dir="test_models/nsfw", score_threshold=0.5, model_inference_batch_size=2) stage.setup() # Test scores around threshold (0.5) mock_model.side_effect = [ - torch.tensor([0.5, 0.49]), # First batch: exactly at and just below - torch.tensor([0.51, 0.499]) # Second batch: just above and just below + torch.tensor([0.5, 0.49]), # First batch: exactly at and just below + torch.tensor([0.51, 0.499]), # Second batch: just above and just below ] result = stage.process(sample_image_batch) @@ -236,17 +228,13 @@ def test_all_images_filtered( """Test when all images are filtered out.""" mock_nsfw_scorer.return_value = mock_model - stage = ImageNSFWFilterStage( - model_dir="test_models/nsfw", - score_threshold=0.5, - model_inference_batch_size=2 - ) + stage = ImageNSFWFilterStage(model_dir="test_models/nsfw", score_threshold=0.5, model_inference_batch_size=2) stage.setup() # All high NSFW scores mock_model.side_effect = [ torch.tensor([0.9, 0.8]), # First batch - torch.tensor([0.7, 0.6]) # Second batch + torch.tensor([0.7, 0.6]), # Second batch ] result = stage.process(sample_image_batch) @@ -265,17 +253,13 @@ def test_no_images_filtered( """Test when no images are filtered out.""" mock_nsfw_scorer.return_value = mock_model - stage = ImageNSFWFilterStage( - model_dir="test_models/nsfw", - score_threshold=0.5, - model_inference_batch_size=2 - ) + stage = ImageNSFWFilterStage(model_dir="test_models/nsfw", score_threshold=0.5, model_inference_batch_size=2) stage.setup() # All low NSFW scores mock_model.side_effect = [ torch.tensor([0.1, 0.2]), # First batch - torch.tensor([0.3, 0.4]) # Second batch + torch.tensor([0.3, 0.4]), # Second batch ] result = stage.process(sample_image_batch) @@ -326,21 +310,20 @@ def test_verbose_logging( model_dir="test_models/nsfw", score_threshold=0.5, model_inference_batch_size=2, # Match the mock data structure - verbose=True + verbose=True, ) verbose_stage.setup() verbose_stage.model = mock_model mock_model.side_effect = [ torch.tensor([0.3, 0.7]), # First batch: one pass, one fail - torch.tensor([0.2, 0.8]) # Second batch: one pass, one fail + torch.tensor([0.2, 0.8]), # Second batch: one pass, one fail ] verbose_stage.process(sample_image_batch) # Should log filtering results - filtering_calls = [call for call in mock_logger.info.call_args_list - if "NSFW" in str(call)] + filtering_calls = [call for call in mock_logger.info.call_args_list if "NSFW" in str(call)] assert len(filtering_calls) > 0 @patch("nemo_curator.stages.image.filters.nsfw_filter.NSFWScorer") @@ -385,7 +368,9 @@ def __call__(self, embeddings_numpy: np.ndarray) -> torch.Tensor: tmp_dir = tempfile.gettempdir() images = [ - ImageObject(image_id=f"img_{i}", image_path=f"{tmp_dir}/{i}.jpg", embedding=rng.normal(size=(8,)).astype(np.float32)) + ImageObject( + image_id=f"img_{i}", image_path=f"{tmp_dir}/{i}.jpg", embedding=rng.normal(size=(8,)).astype(np.float32) + ) for i in range(6) ] batch = ImageBatch(data=images, dataset_name="ds", task_id="t0") diff --git a/tests/stages/image/io/test_image_reader.py b/tests/stages/image/io/test_image_reader.py index 251f24ccf9..d80e2dcf80 100644 --- a/tests/stages/image/io/test_image_reader.py +++ b/tests/stages/image/io/test_image_reader.py @@ -35,9 +35,7 @@ class _FakeTensorList: """Minimal stand-in for a DALI TensorList returned by Pipeline.run().""" def __init__(self, batch_size: int, height: int = 8, width: int = 8) -> None: - self._arrays: list[np.ndarray] = [ - np.zeros((height, width, 3), dtype=np.uint8) for _ in range(batch_size) - ] + self._arrays: list[np.ndarray] = [np.zeros((height, width, 3), dtype=np.uint8) for _ in range(batch_size)] def as_cpu(self) -> _FakeTensorList: return self @@ -121,8 +119,10 @@ class _Types: sys.modules["nvidia.dali"] = dali sys.modules["nvidia.dali.pipeline"] = pipeline + def test_inputs_outputs_and_name() -> None: from nemo_curator.stages.image.io.image_reader import ImageReaderStage + with patch("torch.cuda.is_available", return_value=True): stage = ImageReaderStage(dali_batch_size=3, verbose=False) assert stage.inputs() == ([], []) @@ -132,6 +132,7 @@ def test_inputs_outputs_and_name() -> None: def test_init_allows_cpu_when_no_cuda() -> None: from nemo_curator.stages.image.io.image_reader import ImageReaderStage + # When CUDA is unavailable, the stage should initialize and use CPU DALI with patch("torch.cuda.is_available", return_value=False): stage = ImageReaderStage(dali_batch_size=2, verbose=False) @@ -140,6 +141,7 @@ def test_init_allows_cpu_when_no_cuda() -> None: def test_process_streams_batches_from_dali() -> None: from nemo_curator.stages.image.io.image_reader import ImageReaderStage + # Two tar files; each has 5 total samples, emitted in batches of 2 (2,2,1) task = FileGroupTask( task_id="t1", @@ -168,6 +170,7 @@ def test_process_streams_batches_from_dali() -> None: def test_process_raises_on_empty_task() -> None: from nemo_curator.stages.image.io.image_reader import ImageReaderStage + empty = FileGroupTask(task_id="e1", dataset_name="ds", data=[]) with patch("torch.cuda.is_available", return_value=True): @@ -177,9 +180,9 @@ def test_process_raises_on_empty_task() -> None: stage.process(empty) - def test_resources_with_cuda_available() -> None: from nemo_curator.stages.image.io.image_reader import ImageReaderStage + # Instantiate with CUDA available so __post_init__ passes with patch("torch.cuda.is_available", return_value=True): stage = ImageReaderStage(dali_batch_size=2, verbose=False) @@ -191,6 +194,7 @@ def test_resources_with_cuda_available() -> None: def test_resources_without_cuda() -> None: from nemo_curator.stages.image.io.image_reader import ImageReaderStage + # Create the stage without CUDA available with patch("torch.cuda.is_available", return_value=False): stage = ImageReaderStage(dali_batch_size=2, verbose=False) diff --git a/tests/stages/math_stages/download/test_lynx_extractor.py b/tests/stages/math_stages/download/test_lynx_extractor.py index 8366d511ee..920b0931b1 100644 --- a/tests/stages/math_stages/download/test_lynx_extractor.py +++ b/tests/stages/math_stages/download/test_lynx_extractor.py @@ -23,7 +23,9 @@ class TestLynxExtractor: @mock.patch("shutil.which", return_value="/usr/bin/lynx") @mock.patch("subprocess.run") - def test_lynx_extractor_extract_text_success(self, mock_run: mock.Mock, mock_which: mock.Mock, html_with_content: str) -> None: + def test_lynx_extractor_extract_text_success( + self, mock_run: mock.Mock, mock_which: mock.Mock, html_with_content: str + ) -> None: """Test successful lynx text extraction.""" # Mock successful subprocess call mock_process = mock.Mock() @@ -57,7 +59,9 @@ def test_lynx_extractor_extract_text_success(self, mock_run: mock.Mock, mock_whi @mock.patch("shutil.which", return_value="/usr/bin/lynx") @mock.patch("subprocess.run") - def test_lynx_extractor_extract_text_timeout(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + def test_lynx_extractor_extract_text_timeout( + self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str + ) -> None: """Test LynxExtractor timeout handling.""" mock_run.side_effect = subprocess.TimeoutExpired(["lynx"], timeout=20) @@ -70,7 +74,9 @@ def test_lynx_extractor_extract_text_timeout(self, mock_run: mock.Mock, mock_whi @mock.patch("shutil.which", return_value="/usr/bin/lynx") @mock.patch("subprocess.run") - def test_lynx_extractor_extract_text_failure(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + def test_lynx_extractor_extract_text_failure( + self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str + ) -> None: """Test LynxExtractor when lynx returns non-zero exit code.""" mock_process = mock.Mock() mock_process.returncode = 1 @@ -96,7 +102,9 @@ def test_lynx_extractor_extract_text_empty_input(self, mock_run: mock.Mock, mock @mock.patch("shutil.which", return_value="/usr/bin/lynx") @mock.patch("subprocess.run") - def test_lynx_extractor_extract_text_decode_error(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + def test_lynx_extractor_extract_text_decode_error( + self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str + ) -> None: """Test LynxExtractor with decode error handling.""" mock_process = mock.Mock() mock_process.returncode = 0 @@ -114,7 +122,9 @@ def test_lynx_extractor_extract_text_decode_error(self, mock_run: mock.Mock, moc @mock.patch("shutil.which", return_value="/usr/bin/lynx") @mock.patch("subprocess.run") - def test_lynx_extractor_extract_text_with_math_content(self, mock_run: mock.Mock, mock_which: mock.Mock, math_html: str) -> None: + def test_lynx_extractor_extract_text_with_math_content( + self, mock_run: mock.Mock, mock_which: mock.Mock, math_html: str + ) -> None: """Test LynxExtractor with mathematical content.""" # Simulate lynx extracting LaTeX/math content mock_process = mock.Mock() @@ -132,7 +142,9 @@ def test_lynx_extractor_extract_text_with_math_content(self, mock_run: mock.Mock @mock.patch("shutil.which", return_value="/usr/bin/lynx") @mock.patch("subprocess.run") - def test_lynx_extractor_subprocess_error(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + def test_lynx_extractor_subprocess_error( + self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str + ) -> None: """Test LynxExtractor with subprocess error handling.""" mock_run.side_effect = subprocess.SubprocessError("Subprocess failed") diff --git a/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_base.py b/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_base.py index 16b6400661..c66ec50695 100644 --- a/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_base.py +++ b/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_base.py @@ -143,10 +143,12 @@ def test_process(self) -> None: stage = _make_stage() stage.setup() - output_df = pd.DataFrame([ - {"text": "a", _FORMATTED_PROMPT_COL: "Rephrase: a", "result": "out_a"}, - {"text": "b", _FORMATTED_PROMPT_COL: "Rephrase: b", "result": "out_b"}, - ]) + output_df = pd.DataFrame( + [ + {"text": "a", _FORMATTED_PROMPT_COL: "Rephrase: a", "result": "out_a"}, + {"text": "b", _FORMATTED_PROMPT_COL: "Rephrase: b", "result": "out_b"}, + ] + ) stage.data_designer.preview = MagicMock( return_value=PreviewResults(config_builder=stage.config_builder, dataset=output_df) ) diff --git a/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_nemotron_cc.py b/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_nemotron_cc.py index d143da0d90..9cecc4cdf5 100644 --- a/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_nemotron_cc.py +++ b/tests/stages/synthetic/nemotron_cc/nemo_data_designer/test_nemotron_cc.py @@ -94,11 +94,15 @@ def test_process_smoke(self, stage_cls: type, output_field: str) -> None: stage = _make_stage(stage_cls) stage.setup() - output_df = pd.DataFrame([{ - "text": "doc", - _FORMATTED_PROMPT_COL: "prompt", - output_field: "generated", - }]) + output_df = pd.DataFrame( + [ + { + "text": "doc", + _FORMATTED_PROMPT_COL: "prompt", + output_field: "generated", + } + ] + ) stage.data_designer.preview = MagicMock( return_value=PreviewResults(config_builder=stage.config_builder, dataset=output_df) ) @@ -205,9 +209,7 @@ def test_pipeline_run_end_to_end( assert _FORMATTED_PROMPT_COL not in out.data.columns assert len(out.data) == 1 - def test_pipeline_e2e_reader_ndd_writer( - self, httpserver: pytest_httpserver.HTTPServer, tmp_path: Path - ) -> None: + def test_pipeline_e2e_reader_ndd_writer(self, httpserver: pytest_httpserver.HTTPServer, tmp_path: Path) -> None: """JsonlReader -> WikipediaParaphrasingStage -> JsonlWriter. Verifies files, _metadata, _stage_perf.""" from nemo_curator.backends.xenna import XennaExecutor diff --git a/tests/stages/synthetic/nemotron_cc/test_nemotron_cc.py b/tests/stages/synthetic/nemotron_cc/test_nemotron_cc.py index ae41cd561e..5c36621b75 100644 --- a/tests/stages/synthetic/nemotron_cc/test_nemotron_cc.py +++ b/tests/stages/synthetic/nemotron_cc/test_nemotron_cc.py @@ -102,8 +102,9 @@ def test_diverseqa_post_processing_basic() -> None: df = pd.DataFrame([{"text": "DOC", "diverse_qa": generated_text}]) batch = DocumentBatch(data=df, dataset_name="ds", task_id="t0") # Deterministic behavior: no shuffle and pick 2 pairs - with patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.shuffle", lambda _: None), patch( - "nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.randint", return_value=2 + with ( + patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.shuffle", lambda _: None), + patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.randint", return_value=2), ): out_batch = pp.process(batch) out = out_batch.data["diverse_qa"].iloc[0] @@ -122,8 +123,9 @@ def test_diverseqa_sync_end_to_end() -> None: pp = DiverseQAPostProcessingStage() df = pd.DataFrame([{"text": "DOC"}]) batch = DocumentBatch(data=df, dataset_name="ds", task_id="t1") - with patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.shuffle", lambda _: None), patch( - "nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.randint", return_value=1 + with ( + patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.shuffle", lambda _: None), + patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.randint", return_value=1), ): raw_batch = stage.process(batch) out_batch = pp.process(raw_batch) @@ -144,8 +146,9 @@ def test_diverseqa_async_multiple_rows() -> None: pp = DiverseQAPostProcessingStage() df = pd.DataFrame([{"text": "D1"}, {"text": "D2"}, {"text": "D3"}]) batch = DocumentBatch(data=df, dataset_name="ds", task_id="t2") - with patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.shuffle", lambda _: None), patch( - "nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.randint", return_value=1 + with ( + patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.shuffle", lambda _: None), + patch("nemo_curator.stages.synthetic.nemotron_cc.nemotron_cc.random.randint", return_value=1), ): raw_batch = stage.process(batch) out_batch = pp.process(raw_batch) @@ -158,12 +161,7 @@ def test_diverseqa_async_multiple_rows() -> None: def test_knowledge_list_process_llm_response() -> None: pp = KnowledgeListPostProcessingStage() # First line not starting with "-" should be skipped - generated = ( - "Header line\n" - "- item one\n" - " continuation\n" - "- item two" - ) + generated = "Header line\n- item one\n continuation\n- item two" df = pd.DataFrame([{"knowledge_list": generated}]) batch = DocumentBatch(data=df, dataset_name="ds", task_id="tkl") out_batch = pp.process(batch) diff --git a/tests/stages/text/download/arxiv/test_download.py b/tests/stages/text/download/arxiv/test_download.py index 1e1ee69ee9..15a8993d53 100644 --- a/tests/stages/text/download/arxiv/test_download.py +++ b/tests/stages/text/download/arxiv/test_download.py @@ -38,7 +38,9 @@ class TestArxivDownloader: @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) @mock.patch("subprocess.run", return_value=mock.Mock(returncode=0)) @pytest.mark.parametrize("verbose", [True, False]) - def test_download_to_path(self, mock_run: mock.Mock, mock_s5cmd_check: mock.Mock, tmp_path: Path, verbose: bool) -> None: + def test_download_to_path( + self, mock_run: mock.Mock, mock_s5cmd_check: mock.Mock, tmp_path: Path, verbose: bool + ) -> None: """Test _download_to_path with s5cmd.""" downloader = ArxivDownloader(str(tmp_path), verbose=verbose) diff --git a/tests/stages/text/download/base/test_iterator.py b/tests/stages/text/download/base/test_iterator.py index ab9023ab4b..cfc8ca50c2 100644 --- a/tests/stages/text/download/base/test_iterator.py +++ b/tests/stages/text/download/base/test_iterator.py @@ -19,7 +19,7 @@ import pytest -from nemo_curator.backends.experimental.utils import RayStageSpecKeys +from nemo_curator.backends.utils import RayStageSpecKeys from nemo_curator.stages.resources import Resources from nemo_curator.stages.text.download.base.iterator import DocumentIterateExtractStage, DocumentIterator from nemo_curator.tasks import DocumentBatch, FileGroupTask diff --git a/tests/stages/video/clipping/test_clip_transcoding_stage.py b/tests/stages/video/clipping/test_clip_transcoding_stage.py index 9d8a773cd7..35a054adf5 100644 --- a/tests/stages/video/clipping/test_clip_transcoding_stage.py +++ b/tests/stages/video/clipping/test_clip_transcoding_stage.py @@ -107,7 +107,7 @@ def test_ray_stage_spec(self) -> None: spec = self.stage.ray_stage_spec() # Verify the expected keys and values based on the git diff - from nemo_curator.backends.experimental.utils import RayStageSpecKeys + from nemo_curator.backends.utils import RayStageSpecKeys assert RayStageSpecKeys.IS_FANOUT_STAGE in spec assert spec[RayStageSpecKeys.IS_FANOUT_STAGE] is True diff --git a/tests/stages/video/clipping/test_transnetv2_extraction.py b/tests/stages/video/clipping/test_transnetv2_extraction.py index 51b37d21e7..4fff953dc6 100644 --- a/tests/stages/video/clipping/test_transnetv2_extraction.py +++ b/tests/stages/video/clipping/test_transnetv2_extraction.py @@ -694,7 +694,9 @@ def test_complete_pipeline_integration(self, mock_transnetv2_class: Mock): # Setup and process stage.setup() - with patch("nemo_curator.stages.video.clipping.transnetv2_extraction._get_predictions") as mock_get_predictions: + with patch( + "nemo_curator.stages.video.clipping.transnetv2_extraction._get_predictions" + ) as mock_get_predictions: # Mock predictions to create some transitions mock_get_predictions.return_value = np.array([[0], [1], [0], [0], [1], [0]] * 25, dtype=np.uint8) diff --git a/tests/utils/test_split_large_files.py b/tests/utils/test_split_large_files.py index 8fb87c5c94..455f0dccd6 100644 --- a/tests/utils/test_split_large_files.py +++ b/tests/utils/test_split_large_files.py @@ -57,9 +57,7 @@ def _(num_row_groups: int = 1) -> pathlib.Path: def test_default_target_size(parquet_file_factory: Callable, tmp_path: pathlib.Path): parquet_file = parquet_file_factory() - args = parse_args( - ["--input-path", str(parquet_file), "--output-path", str(tmp_path), "--file-type", "parquet"] - ) + args = parse_args(["--input-path", str(parquet_file), "--output-path", str(tmp_path), "--file-type", "parquet"]) assert args.target_size_mb == 128 @@ -70,7 +68,9 @@ def test_split_parquet_file_by_size(parquet_file_factory: Callable, tmp_path: pa target_size_mb = size_original_mb / 3 output_path = tmp_path / "out" output_path.mkdir(exist_ok=True) - split_parquet_file_by_size._function(input_file=str(parquet_file), output_path=str(output_path), target_size_mb=target_size_mb) + split_parquet_file_by_size._function( + input_file=str(parquet_file), output_path=str(output_path), target_size_mb=target_size_mb + ) expected = pd.read_parquet(parquet_file) result = pd.read_parquet(output_path) @@ -96,7 +96,9 @@ def test_split_jsonl_file_by_size(tmp_path: pathlib.Path): target_size_mb = max(size_original_mb / 4, 1e-6) output_path = tmp_path / "out" output_path.mkdir(exist_ok=True) - split_jsonl_file_by_size._function(input_file=str(jsonl_file), output_path=str(output_path), target_size_mb=target_size_mb) + split_jsonl_file_by_size._function( + input_file=str(jsonl_file), output_path=str(output_path), target_size_mb=target_size_mb + ) files = sorted(output_path.glob("data_*.jsonl")) assert len(files) >= 2 diff --git a/tutorials/audio/readspeech/extract_segments.py b/tutorials/audio/readspeech/extract_segments.py index df666acdb4..4a1e32ccb4 100755 --- a/tutorials/audio/readspeech/extract_segments.py +++ b/tutorials/audio/readspeech/extract_segments.py @@ -56,7 +56,6 @@ } - def load_manifest(manifest_path: str) -> list: """Load a single manifest.jsonl file and return list of segment entries.""" segments = [] @@ -117,9 +116,7 @@ def load_manifests(input_path: str, output_dir: str) -> list: return all_segments -def _write_segment( - output_path: str, segment_audio: np.ndarray, sample_rate: int, output_format: str -) -> None: +def _write_segment(output_path: str, segment_audio: np.ndarray, sample_rate: int, output_format: str) -> None: """Write a single audio segment to disk.""" sf.write(output_path, segment_audio, sample_rate, subtype=SOUNDFILE_FORMATS[output_format]) @@ -177,7 +174,10 @@ def _process_file_segments( start_sample = int(start_ms * sample_rate / 1000) end_sample = int(end_ms * sample_rate / 1000) segment_audio, _ = sf.read( - original_file, start=start_sample, stop=end_sample, dtype="float32", + original_file, + start=start_sample, + stop=end_sample, + dtype="float32", ) _write_segment(output_path, segment_audio, sample_rate, output_format) extracted += 1 diff --git a/tutorials/audio/readspeech/pipeline.py b/tutorials/audio/readspeech/pipeline.py index 55dc6fb533..7adc4907d4 100644 --- a/tutorials/audio/readspeech/pipeline.py +++ b/tutorials/audio/readspeech/pipeline.py @@ -260,7 +260,9 @@ def main() -> None: logger.info("Starting pipeline execution...") try: - executor = RayDataExecutor() if args.backend == "ray_data" else XennaExecutor(config={"execution_mode": "streaming"}) + executor = ( + RayDataExecutor() if args.backend == "ray_data" else XennaExecutor(config={"execution_mode": "streaming"}) + ) pipeline.run(executor) logger.info(f"Results written to {args.output_dir}/*.jsonl") diff --git a/tutorials/image/getting-started/helper.py b/tutorials/image/getting-started/helper.py index 8c8e49cc26..cd1a675b23 100644 --- a/tutorials/image/getting-started/helper.py +++ b/tutorials/image/getting-started/helper.py @@ -138,12 +138,7 @@ def download_webdataset( func = partial(process_parquet_chunk, output_dir=output_dir) # Use tqdm to track progress of chunk processing - list(tqdm( - pool.imap(func, chunks), - total=len(chunks), - desc="Processing chunks", - unit="chunk" - )) + list(tqdm(pool.imap(func, chunks), total=len(chunks), desc="Processing chunks", unit="chunk")) # Best-effort cleanup of legacy tmp dir from previous versions tmp_dir = os.path.join(output_dir, "tmp") @@ -254,7 +249,7 @@ def save_imagebatch_to_webdataset( # Process images in shards shard_id = 0 for i in range(0, len(all_image_objects), samples_per_shard): - shard_images = all_image_objects[i:i + samples_per_shard] + shard_images = all_image_objects[i : i + samples_per_shard] # Create output file paths parquet_filename = _name_partition(shard_id, max_shards=max_shards) @@ -270,10 +265,7 @@ def save_imagebatch_to_webdataset( for sample_idx, image_obj in enumerate(shard_images): # Generate new ID combining shard and sample indices new_id = _combine_id( - shard_id, - sample_idx, - max_shards=max_shards, - max_samples_per_shard=max_samples_per_shard + shard_id, sample_idx, max_shards=max_shards, max_samples_per_shard=max_samples_per_shard ) # Prepare metadata record for parquet diff --git a/tutorials/image/getting-started/image_curation_example.py b/tutorials/image/getting-started/image_curation_example.py index 6d58740cac..890f41a24a 100644 --- a/tutorials/image/getting-started/image_curation_example.py +++ b/tutorials/image/getting-started/image_curation_example.py @@ -36,54 +36,66 @@ def create_image_curation_pipeline(args: argparse.Namespace) -> Pipeline: pipeline = Pipeline(name="image_curation", description="Curate images with embeddings and quality scoring") # Stage 0: Partition tar files for parallel processing - pipeline.add_stage(FilePartitioningStage( - file_paths=args.input_wds_dataset_dir, - files_per_partition=args.tar_files_per_partition, - file_extensions=[".tar"], - )) + pipeline.add_stage( + FilePartitioningStage( + file_paths=args.input_wds_dataset_dir, + files_per_partition=args.tar_files_per_partition, + file_extensions=[".tar"], + ) + ) # Stage 1: Read images from webdataset tar files (now runs in parallel) - pipeline.add_stage(ImageReaderStage( - dali_batch_size=args.batch_size, - verbose=args.verbose, # Force verbose to see debug info - num_threads=16, # More threads for I/O - num_gpus_per_worker=0.25, - )) + pipeline.add_stage( + ImageReaderStage( + dali_batch_size=args.batch_size, + verbose=args.verbose, # Force verbose to see debug info + num_threads=16, # More threads for I/O + num_gpus_per_worker=0.25, + ) + ) # Stage 2: Generate CLIP embeddings for images - pipeline.add_stage(ImageEmbeddingStage( - model_dir=args.model_dir, - num_gpus_per_worker=args.embedding_gpus_per_worker, - model_inference_batch_size=args.embedding_batch_size, - remove_image_data=False, - verbose=args.verbose, - )) + pipeline.add_stage( + ImageEmbeddingStage( + model_dir=args.model_dir, + num_gpus_per_worker=args.embedding_gpus_per_worker, + model_inference_batch_size=args.embedding_batch_size, + remove_image_data=False, + verbose=args.verbose, + ) + ) # Stage 3: Generate aesthetic quality scores and filter - pipeline.add_stage(ImageAestheticFilterStage( - model_dir=args.model_dir, - num_gpus_per_worker=args.aesthetic_gpus_per_worker, - model_inference_batch_size=args.aesthetic_batch_size, - score_threshold=args.aesthetic_threshold, - verbose=args.verbose, - )) + pipeline.add_stage( + ImageAestheticFilterStage( + model_dir=args.model_dir, + num_gpus_per_worker=args.aesthetic_gpus_per_worker, + model_inference_batch_size=args.aesthetic_batch_size, + score_threshold=args.aesthetic_threshold, + verbose=args.verbose, + ) + ) # Stage 4: Generate NSFW probability scores and filter - pipeline.add_stage(ImageNSFWFilterStage( - model_dir=args.model_dir, - num_gpus_per_worker=args.nsfw_gpus_per_worker, - model_inference_batch_size=args.nsfw_batch_size, - score_threshold=args.nsfw_threshold, - verbose=args.verbose, - )) + pipeline.add_stage( + ImageNSFWFilterStage( + model_dir=args.model_dir, + num_gpus_per_worker=args.nsfw_gpus_per_worker, + model_inference_batch_size=args.nsfw_batch_size, + score_threshold=args.nsfw_threshold, + verbose=args.verbose, + ) + ) # Stage 5: Write down to disk - pipeline.add_stage(ImageWriterStage( - output_dir=args.output_dataset_dir, - images_per_tar=args.images_per_tar, - remove_image_data=True, - verbose=args.verbose, - )) + pipeline.add_stage( + ImageWriterStage( + output_dir=args.output_dataset_dir, + images_per_tar=args.images_per_tar, + remove_image_data=True, + verbose=args.verbose, + ) + ) return pipeline @@ -169,37 +181,22 @@ def main(args: argparse.Namespace) -> None: type=str, required=False, default=None, - help="Path to input parquet file containing image URLs and metadata" + help="Path to input parquet file containing image URLs and metadata", ) parser.add_argument( - "--input-wds-dataset-dir", - type=str, - required=True, - help="Directory to save the downloaded webdataset" + "--input-wds-dataset-dir", type=str, required=True, help="Directory to save the downloaded webdataset" ) parser.add_argument( - "--output-dataset-dir", - type=str, - required=True, - help="Directory to save the resulting webdataset" + "--output-dataset-dir", type=str, required=True, help="Directory to save the resulting webdataset" ) parser.add_argument( - "--download-processes", - type=int, - default=8, - help="Number of parallel processes for downloading images" + "--download-processes", type=int, default=8, help="Number of parallel processes for downloading images" ) parser.add_argument( - "--entries-per-tar", - type=int, - default=1000, - help="Number of entries per tar shard during download" + "--entries-per-tar", type=int, default=1000, help="Number of entries per tar shard during download" ) parser.add_argument( - "--skip-download", - action="store_true", - default=False, - help="Skip dataset download and use existing webdataset" + "--skip-download", action="store_true", default=False, help="Skip dataset download and use existing webdataset" ) # Image reader arguments @@ -207,89 +204,54 @@ def main(args: argparse.Namespace) -> None: "--tar-files-per-partition", type=int, default=1, - help="Number of tar files to process per partition (controls parallelism) for FilePartitioningStage" + help="Number of tar files to process per partition (controls parallelism) for FilePartitioningStage", ) parser.add_argument( - "--batch-size", - type=int, - default=100, - help="Number of images per ImageBatch for the reader stage" + "--batch-size", type=int, default=100, help="Number of images per ImageBatch for the reader stage" ) # General arguments parser.add_argument( - "--model-dir", - type=str, - required=True, - help="Path to model directory containing all model weights" - ) - parser.add_argument( - "--verbose", - action="store_true", - default=False, - help="Enable verbose logging for all stages" + "--model-dir", type=str, required=True, help="Path to model directory containing all model weights" ) + parser.add_argument("--verbose", action="store_true", default=False, help="Enable verbose logging for all stages") # Embedding stage arguments - parser.add_argument( - "--embedding-batch-size", - type=int, - default=32, - help="Batch size for embedding generation" - ) + parser.add_argument("--embedding-batch-size", type=int, default=32, help="Batch size for embedding generation") parser.add_argument( "--embedding-gpus-per-worker", type=float, default=0.25, - help="GPU allocation per worker for embedding generation" + help="GPU allocation per worker for embedding generation", ) # Aesthetic scoring arguments + parser.add_argument("--aesthetic-batch-size", type=int, default=32, help="Batch size for aesthetic scoring") parser.add_argument( - "--aesthetic-batch-size", - type=int, - default=32, - help="Batch size for aesthetic scoring" - ) - parser.add_argument( - "--aesthetic-gpus-per-worker", - type=float, - default=0.25, - help="GPU allocation per worker for aesthetic scoring" + "--aesthetic-gpus-per-worker", type=float, default=0.25, help="GPU allocation per worker for aesthetic scoring" ) parser.add_argument( "--aesthetic-threshold", type=float, default=0.5, - help="Aesthetic score threshold for filtering (images below this score will be filtered out)" + help="Aesthetic score threshold for filtering (images below this score will be filtered out)", ) # NSFW scoring arguments + parser.add_argument("--nsfw-batch-size", type=int, default=32, help="Batch size for NSFW scoring") parser.add_argument( - "--nsfw-batch-size", - type=int, - default=32, - help="Batch size for NSFW scoring" - ) - parser.add_argument( - "--nsfw-gpus-per-worker", - type=float, - default=0.25, - help="GPU allocation per worker for NSFW scoring" + "--nsfw-gpus-per-worker", type=float, default=0.25, help="GPU allocation per worker for NSFW scoring" ) parser.add_argument( "--nsfw-threshold", type=float, default=0.5, - help="NSFW score threshold for filtering (images above this score will be filtered out as NSFW)" + help="NSFW score threshold for filtering (images above this score will be filtered out as NSFW)", ) # Output dataset arguments parser.add_argument( - "--images-per-tar", - type=int, - default=100, - help="Number of images per tar file in output dataset" + "--images-per-tar", type=int, default=100, help="Number of images per tar file in output dataset" ) args = parser.parse_args() diff --git a/tutorials/image/getting-started/image_dedup_example.py b/tutorials/image/getting-started/image_dedup_example.py index d8199cd7dd..b5fd212e5a 100644 --- a/tutorials/image/getting-started/image_dedup_example.py +++ b/tutorials/image/getting-started/image_dedup_example.py @@ -37,38 +37,47 @@ def create_image_embedding_pipeline(args: argparse.Namespace) -> Pipeline: pipeline = Pipeline(name="image_curation", description="Curate images with embeddings and quality scoring") # Stage 0: Partition tar files for parallel processing - pipeline.add_stage(FilePartitioningStage( - file_paths=args.input_wds_dataset_dir, - files_per_partition=args.tar_files_per_partition, - file_extensions=[".tar"], - )) + pipeline.add_stage( + FilePartitioningStage( + file_paths=args.input_wds_dataset_dir, + files_per_partition=args.tar_files_per_partition, + file_extensions=[".tar"], + ) + ) # Stage 1: Read images from webdataset tar files (now runs in parallel) - pipeline.add_stage(ImageReaderStage( - dali_batch_size=args.batch_size, - verbose=args.verbose, - num_threads=16, # More threads for I/O - num_gpus_per_worker=0.25, - )) + pipeline.add_stage( + ImageReaderStage( + dali_batch_size=args.batch_size, + verbose=args.verbose, + num_threads=16, # More threads for I/O + num_gpus_per_worker=0.25, + ) + ) # Stage 2: Generate CLIP embeddings for images - pipeline.add_stage(ImageEmbeddingStage( - model_dir=args.model_dir, - num_gpus_per_worker=args.embedding_gpus_per_worker, - model_inference_batch_size=args.embedding_batch_size, - verbose=args.verbose, - )) + pipeline.add_stage( + ImageEmbeddingStage( + model_dir=args.model_dir, + num_gpus_per_worker=args.embedding_gpus_per_worker, + model_inference_batch_size=args.embedding_batch_size, + verbose=args.verbose, + ) + ) # Stage 3: Convert embeddings to document batch pipeline.add_stage(ConvertImageBatchToDocumentBatchStage(fields=["image_id", "embedding"])) # Stage 4: Save embeddings to parquet file - pipeline.add_stage(ParquetWriter( - path=args.embeddings_dir, - )) + pipeline.add_stage( + ParquetWriter( + path=args.embeddings_dir, + ) + ) return pipeline + def create_embedding_deduplication_workflow(args: argparse.Namespace) -> Pipeline: """Create image deduplication pipeline with embedding deduplication.""" return SemanticDeduplicationWorkflow( @@ -83,39 +92,48 @@ def create_embedding_deduplication_workflow(args: argparse.Namespace) -> Pipelin verbose=args.verbose, ) + def create_image_deduplication_pipeline(args: argparse.Namespace) -> Pipeline: """Create image deduplication pipeline with image deduplication.""" # Define pipeline pipeline = Pipeline(name="image_deduplication", description="Deduplicate images with image deduplication") # Stage 0: Partition tar files for parallel processing - pipeline.add_stage(FilePartitioningStage( - file_paths=args.input_wds_dataset_dir, - files_per_partition=args.tar_files_per_partition, - file_extensions=[".tar"], - )) + pipeline.add_stage( + FilePartitioningStage( + file_paths=args.input_wds_dataset_dir, + files_per_partition=args.tar_files_per_partition, + file_extensions=[".tar"], + ) + ) # Stage 1: Read images from webdataset tar files (now runs in parallel) - pipeline.add_stage(ImageReaderStage( - dali_batch_size=args.batch_size, - verbose=args.verbose, - num_threads=16, # More threads for I/O - num_gpus_per_worker=0.25, - )) + pipeline.add_stage( + ImageReaderStage( + dali_batch_size=args.batch_size, + verbose=args.verbose, + num_threads=16, # More threads for I/O + num_gpus_per_worker=0.25, + ) + ) # Stage 2: Read removal list from parquet file and filter images - pipeline.add_stage(ImageDuplicatesRemovalStage( - removal_parquets_dir=args.removal_parquets_dir + "/duplicates", - duplicate_id_field="id", - verbose=args.verbose, - )) + pipeline.add_stage( + ImageDuplicatesRemovalStage( + removal_parquets_dir=args.removal_parquets_dir + "/duplicates", + duplicate_id_field="id", + verbose=args.verbose, + ) + ) # Stage 3: Write filtered images to disk - pipeline.add_stage(ImageWriterStage( - output_dir=args.output_dataset_dir, - remove_image_data=True, - verbose=args.verbose, - )) + pipeline.add_stage( + ImageWriterStage( + output_dir=args.output_dataset_dir, + remove_image_data=True, + verbose=args.verbose, + ) + ) return pipeline @@ -210,49 +228,26 @@ def main(args: argparse.Namespace) -> None: type=str, required=False, default=None, - help="Path to input parquet file containing image URLs and metadata" + help="Path to input parquet file containing image URLs and metadata", ) parser.add_argument( - "--input-wds-dataset-dir", - type=str, - required=True, - help="Directory to save the downloaded webdataset" + "--input-wds-dataset-dir", type=str, required=True, help="Directory to save the downloaded webdataset" ) parser.add_argument( - "--output-dataset-dir", - type=str, - required=True, - help="Directory to save the resulting webdataset" + "--output-dataset-dir", type=str, required=True, help="Directory to save the resulting webdataset" ) + parser.add_argument("--embeddings-dir", type=str, required=True, help="Directory to save the embeddings") parser.add_argument( - "--embeddings-dir", - type=str, - required=True, - help="Directory to save the embeddings" + "--removal-parquets-dir", type=str, required=True, help="Directory to save the remove parquets" ) parser.add_argument( - "--removal-parquets-dir", - type=str, - required=True, - help="Directory to save the remove parquets" + "--download-processes", type=int, default=8, help="Number of parallel processes for downloading images" ) parser.add_argument( - "--download-processes", - type=int, - default=8, - help="Number of parallel processes for downloading images" + "--entries-per-tar", type=int, default=1000, help="Number of entries per tar shard during download" ) parser.add_argument( - "--entries-per-tar", - type=int, - default=1000, - help="Number of entries per tar shard during download" - ) - parser.add_argument( - "--skip-download", - action="store_true", - default=False, - help="Skip dataset download and use existing webdataset" + "--skip-download", action="store_true", default=False, help="Skip dataset download and use existing webdataset" ) # Image reader arguments @@ -260,41 +255,25 @@ def main(args: argparse.Namespace) -> None: "--tar-files-per-partition", type=int, default=1, - help="Number of tar files to process per partition (controls parallelism) for FilePartitioningStage" + help="Number of tar files to process per partition (controls parallelism) for FilePartitioningStage", ) parser.add_argument( - "--batch-size", - type=int, - default=100, - help="Number of images per ImageBatch for the reader stage" + "--batch-size", type=int, default=100, help="Number of images per ImageBatch for the reader stage" ) # General arguments parser.add_argument( - "--model-dir", - type=str, - required=True, - help="Path to model directory containing all model weights" - ) - parser.add_argument( - "--verbose", - action="store_true", - default=False, - help="Enable verbose logging for all stages" + "--model-dir", type=str, required=True, help="Path to model directory containing all model weights" ) + parser.add_argument("--verbose", action="store_true", default=False, help="Enable verbose logging for all stages") # Embedding stage arguments - parser.add_argument( - "--embedding-batch-size", - type=int, - default=32, - help="Batch size for embedding generation" - ) + parser.add_argument("--embedding-batch-size", type=int, default=32, help="Batch size for embedding generation") parser.add_argument( "--embedding-gpus-per-worker", type=float, default=0.25, - help="GPU allocation per worker for embedding generation" + help="GPU allocation per worker for embedding generation", ) args = parser.parse_args() diff --git a/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.ipynb b/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.ipynb index 158749fdc3..8e8d8cdd2a 100644 --- a/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.ipynb +++ b/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.ipynb @@ -1,1588 +1,1588 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "9da91285", - "metadata": {}, - "source": [ - "# NeMo Data Designer – Synthetic Data Generation Example\n", - "\n", - "This notebook demonstrates how to use **NeMo Data Designer** (NDD) stage to generate synthetic medical-notes data from a small seed dataset.\n", - "\n", - "The pipeline:\n", - "1. Downloads a public symptom-to-diagnosis CSV and converts it to JSONL shards.\n", - "2. Define the configuration for the NDD stage.\n", - "3. Build and run a Curator pipeline, composing of three stages: `JsonlReader`, `DataDesignerStage`, `JsonlWriter`." - ] - }, - { - "cell_type": "markdown", - "id": "01d8a476", - "metadata": {}, - "source": [ - "### Imports libraries" - ] - }, + "cells": [ + { + "cell_type": "markdown", + "id": "9da91285", + "metadata": {}, + "source": [ + "# NeMo Data Designer – Synthetic Data Generation Example\n", + "\n", + "This notebook demonstrates how to use **NeMo Data Designer** (NDD) stage to generate synthetic medical-notes data from a small seed dataset.\n", + "\n", + "The pipeline:\n", + "1. Downloads a public symptom-to-diagnosis CSV and converts it to JSONL shards.\n", + "2. Define the configuration for the NDD stage.\n", + "3. Build and run a Curator pipeline, composing of three stages: `JsonlReader`, `DataDesignerStage`, `JsonlWriter`." + ] + }, + { + "cell_type": "markdown", + "id": "01d8a476", + "metadata": {}, + "source": [ + "### Imports libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "d4e882de", + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 1, - "id": "d4e882de", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/raid/huvu/venvs/curator_env_fast/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "2026-03-31 14:46:09,801\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n" - ] - } - ], - "source": [ - "import time\n", - "from pathlib import Path\n", - "\n", - "import data_designer.config as dd\n", - "import pandas as pd\n", - "\n", - "from nemo_curator.core.client import RayClient\n", - "from nemo_curator.pipeline import Pipeline\n", - "from nemo_curator.stages.synthetic.nemo_data_designer.data_designer import DataDesignerStage\n", - "from nemo_curator.stages.text.io.reader.jsonl import JsonlReader\n", - "from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter\n", - "from nemo_curator.utils.file_utils import get_all_file_paths_under" - ] - }, + "name": "stderr", + "output_type": "stream", + "text": [ + "/raid/huvu/venvs/curator_env_fast/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "2026-03-31 14:46:09,801\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n" + ] + } + ], + "source": [ + "import time\n", + "from pathlib import Path\n", + "\n", + "import data_designer.config as dd\n", + "import pandas as pd\n", + "\n", + "from nemo_curator.core.client import RayClient\n", + "from nemo_curator.pipeline import Pipeline\n", + "from nemo_curator.stages.synthetic.nemo_data_designer.data_designer import DataDesignerStage\n", + "from nemo_curator.stages.text.io.reader.jsonl import JsonlReader\n", + "from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter\n", + "from nemo_curator.utils.file_utils import get_all_file_paths_under" + ] + }, + { + "cell_type": "markdown", + "id": "bd44bd21", + "metadata": {}, + "source": [ + "### Download and Prepare Seed Data\n", + "\n", + "The seed dataset is a public symptom-to-diagnosis CSV hosted on GitHub. \n", + "We download it, split it into small JSONL shards (10 rows each), and save them locally so the pipeline can read them in parallel.\n", + "We sample 200 records from the original dataset for testing purposes." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "943dfb0b", + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "id": "bd44bd21", - "metadata": {}, - "source": [ - "### Download and Prepare Seed Data\n", - "\n", - "The seed dataset is a public symptom-to-diagnosis CSV hosted on GitHub. \n", - "We download it, split it into small JSONL shards (10 rows each), and save them locally so the pipeline can read them in parallel.\n", - "We sample 200 records from the original dataset for testing purposes." - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "Preparing seed data (download + CSVβ†’JSONL)...\n", + "Seed data ready: processed_seed_data\n" + ] + } + ], + "source": [ + "SEED_CSV_URL = (\n", + " \"https://raw.githubusercontent.com/NVIDIA/GenerativeAIExamples/refs/heads/main\"\n", + " \"/nemo/NeMo-Data-Designer/data/gretelai_symptom_to_diagnosis.csv\"\n", + ")\n", + "\n", + "\n", + "def download_and_convert_seed_data(\n", + " output_dir: str | Path | None = None,\n", + " records_per_file: int = 10,\n", + " number_of_records: int = 200,\n", + ") -> str:\n", + " \"\"\"Download seed CSV from URL, convert to JSONL (chunked), return output dir path.\"\"\"\n", + " if output_dir is None:\n", + " output_dir = Path(\"processed_seed_data\")\n", + " output_dir = Path(output_dir)\n", + " output_dir.mkdir(parents=True, exist_ok=True)\n", + " df = pd.read_csv(SEED_CSV_URL, sep=\",\", encoding=\"utf-8\")\n", + " df = df.head(number_of_records)\n", + " for i, start in enumerate(range(0, len(df), records_per_file)):\n", + " chunk = df.iloc[start : start + records_per_file]\n", + " chunk.to_json(\n", + " output_dir / f\"{i:06d}.jsonl\",\n", + " orient=\"records\",\n", + " lines=True,\n", + " force_ascii=False,\n", + " date_format=\"iso\",\n", + " )\n", + " return str(output_dir)\n", + "\n", + "\n", + "print(\"Preparing seed data (download + CSVβ†’JSONL)...\")\n", + "seed_dir = download_and_convert_seed_data()\n", + "print(f\"Seed data ready: {seed_dir}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ef568089", + "metadata": {}, + "source": [ + "### Configuration\n", + "\n", + "Set the output directory for generated data here. \n", + "You can also point `DATA_DESIGNER_CONFIG_FILE` at a YAML config file; leave it as `None` to use the programmatic config defined in this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1dd6ac44", + "metadata": {}, + "outputs": [], + "source": [ + "OUTPUT_PATH = \"./synthetic_output\"\n", + "DATA_DESIGNER_CONFIG_FILE = None # set to a file path string to load a YAML config" + ] + }, + { + "cell_type": "markdown", + "id": "b72c2f8e", + "metadata": {}, + "source": [ + "### Define the Data Designer Config\n", + "\n", + "The DataDesignerStage is intialized from a NDD's configuration builder (https://nvidia-nemo.github.io/DataDesigner/latest/code_reference/config_builder/). In this step, we manually create a configuration for the NDD module. This includes setting the LLM model provider, and the steps taken inside NDD module. Users can learn more about NDD's API from NDD official documentation: https://nvidia-nemo.github.io/DataDesigner/latest/notebooks/.\n", + "\n", + "In this tutorial, we replicate the official NDD's tutorials on synthetic data genertion with seed data (https://nvidia-nemo.github.io/DataDesigner/latest/notebooks/3-seeding-with-a-dataset/), but with Nemo-Curator.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "db719310", + "metadata": {}, + "outputs": [], + "source": [ + "def _build_config(model_id: str, provider_name: str, model_alias: str) -> dd.DataDesignerConfigBuilder:\n", + " \"\"\"Build the Data Designer config with medical notes generation.\"\"\"\n", + " model_configs = [\n", + " dd.ModelConfig(\n", + " alias=model_alias,\n", + " model=model_id,\n", + " provider=provider_name,\n", + " skip_health_check=True,\n", + " inference_parameters=dd.ChatCompletionInferenceParams(\n", + " temperature=1.0,\n", + " top_p=1.0,\n", + " max_tokens=2048,\n", + " ),\n", + " )\n", + " ]\n", + "\n", + " config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)\n", + "\n", + " config_builder.add_column(\n", + " dd.SamplerColumnConfig(\n", + " name=\"patient_sampler\",\n", + " sampler_type=dd.SamplerType.PERSON_FROM_FAKER,\n", + " params=dd.PersonFromFakerSamplerParams(),\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.SamplerColumnConfig(\n", + " name=\"doctor_sampler\",\n", + " sampler_type=dd.SamplerType.PERSON_FROM_FAKER,\n", + " params=dd.PersonFromFakerSamplerParams(),\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.SamplerColumnConfig(\n", + " name=\"patient_id\",\n", + " sampler_type=dd.SamplerType.UUID,\n", + " params=dd.UUIDSamplerParams(\n", + " prefix=\"PT-\",\n", + " short_form=True,\n", + " uppercase=True,\n", + " ),\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.ExpressionColumnConfig(\n", + " name=\"first_name\",\n", + " expr=\"{{ patient_sampler.first_name}}\",\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.ExpressionColumnConfig(\n", + " name=\"last_name\",\n", + " expr=\"{{ patient_sampler.last_name }}\",\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.ExpressionColumnConfig(\n", + " name=\"dob\",\n", + " expr=\"{{ patient_sampler.birth_date }}\",\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.SamplerColumnConfig(\n", + " name=\"symptom_onset_date\",\n", + " sampler_type=dd.SamplerType.DATETIME,\n", + " params=dd.DatetimeSamplerParams(start=\"2024-01-01\", end=\"2024-12-31\"),\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.SamplerColumnConfig(\n", + " name=\"date_of_visit\",\n", + " sampler_type=dd.SamplerType.TIMEDELTA,\n", + " params=dd.TimeDeltaSamplerParams(\n", + " dt_min=1,\n", + " dt_max=30,\n", + " reference_column_name=\"symptom_onset_date\",\n", + " ),\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.ExpressionColumnConfig(\n", + " name=\"physician\",\n", + " expr=\"Dr. {{ doctor_sampler.last_name }}\",\n", + " )\n", + " )\n", + "\n", + " config_builder.add_column(\n", + " dd.LLMTextColumnConfig(\n", + " name=\"physician_notes\",\n", + " prompt=\"\"\"\\\n", + " You are a primary-care physician who just had an appointment with {{ first_name }} {{ last_name }},\n", + " who has been struggling with symptoms from {{ diagnosis }} since {{ symptom_onset_date }}.\n", + " The date of today's visit is {{ date_of_visit }}.\n", + "\n", + " {{ patient_summary }}\n", + "\n", + " Write careful notes about your visit with {{ first_name }},\n", + " as Dr. {{ doctor_sampler.first_name }} {{ doctor_sampler.last_name }}.\n", + "\n", + " Format the notes as a busy doctor might.\n", + " Respond with only the notes, no other text.\n", + " \"\"\",\n", + " model_alias=model_alias,\n", + " )\n", + " )\n", + "\n", + " return config_builder" + ] + }, + { + "cell_type": "markdown", + "id": "6dcdd577", + "metadata": {}, + "source": [ + "### Start Ray client" + ] + }, + { + "cell_type": "markdown", + "id": "e4b5fd6a", + "metadata": {}, + "source": [ + "User can add `NVIDIA_API_KEY` by running `os.environ[\"NVIDIA_API_KEY\"] = \"YOUR_API_KEY\"` if needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a859dbbf", + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 2, - "id": "943dfb0b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Preparing seed data (download + CSVβ†’JSONL)...\n", - "Seed data ready: processed_seed_data\n" - ] - } - ], - "source": [ - "SEED_CSV_URL = (\n", - " \"https://raw.githubusercontent.com/NVIDIA/GenerativeAIExamples/refs/heads/main\"\n", - " \"/nemo/NeMo-Data-Designer/data/gretelai_symptom_to_diagnosis.csv\"\n", - ")\n", - "\n", - "\n", - "def download_and_convert_seed_data(\n", - " output_dir: str | Path | None = None,\n", - " records_per_file: int = 10,\n", - " number_of_records: int = 200,\n", - ") -> str:\n", - " \"\"\"Download seed CSV from URL, convert to JSONL (chunked), return output dir path.\"\"\"\n", - " if output_dir is None:\n", - " output_dir = Path(\"processed_seed_data\")\n", - " output_dir = Path(output_dir)\n", - " output_dir.mkdir(parents=True, exist_ok=True)\n", - " df = pd.read_csv(SEED_CSV_URL, sep=\",\", encoding=\"utf-8\")\n", - " df = df.head(number_of_records)\n", - " for i, start in enumerate(range(0, len(df), records_per_file)):\n", - " chunk = df.iloc[start : start + records_per_file]\n", - " chunk.to_json(\n", - " output_dir / f\"{i:06d}.jsonl\",\n", - " orient=\"records\",\n", - " lines=True,\n", - " force_ascii=False,\n", - " date_format=\"iso\",\n", - " )\n", - " return str(output_dir)\n", - "\n", - "\n", - "print(\"Preparing seed data (download + CSVβ†’JSONL)...\")\n", - "seed_dir = download_and_convert_seed_data()\n", - "print(f\"Seed data ready: {seed_dir}\")" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2026-03-31 14:46:53.670\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.core.utils\u001b[0m:\u001b[36minit_cluster\u001b[0m:\u001b[36m185\u001b[0m - \u001b[1mRay start command: ray start --head --node-ip-address 127.0.1.1 --port 6381 --metrics-export-port 8082 --dashboard-host 127.0.0.1 --dashboard-port 8267 --ray-client-server-port 20002 --temp-dir /tmp/ray_huvu --disable-usage-stats --num-gpus 1 --num-cpus 8 --block\u001b[0m\n" + ] }, { - "cell_type": "markdown", - "id": "ef568089", - "metadata": {}, - "source": [ - "### Configuration\n", - "\n", - "Set the output directory for generated data here. \n", - "You can also point `DATA_DESIGNER_CONFIG_FILE` at a YAML config file; leave it as `None` to use the programmatic config defined in this notebook." - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "2026-03-31 14:46:54,830\tINFO usage_lib.py:447 -- Usage stats collection is disabled.\n", + "2026-03-31 14:46:54,830\tINFO scripts.py:936 -- \u001b[37mLocal node IP\u001b[39m: \u001b[1m127.0.1.1\u001b[22m\n", + "2026-03-31 14:47:09,078\tSUCC scripts.py:975 -- \u001b[32m--------------------\u001b[39m\n", + "2026-03-31 14:47:09,079\tSUCC scripts.py:976 -- \u001b[32mRay runtime started.\u001b[39m\n", + "2026-03-31 14:47:09,079\tSUCC scripts.py:977 -- \u001b[32m--------------------\u001b[39m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:979 -- \u001b[36mNext steps\u001b[39m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:982 -- To add another node to this Ray cluster, run\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:985 -- \u001b[1m ray start --address='127.0.1.1:6381'\u001b[22m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:996 -- To connect to this Ray cluster:\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:998 -- \u001b[35mimport\u001b[39m\u001b[26m ray\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:999 -- ray\u001b[35m.\u001b[39m\u001b[26minit(_node_ip_address\u001b[35m=\u001b[39m\u001b[26m\u001b[33m'127.0.1.1'\u001b[39m\u001b[26m)\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1013 -- To submit a Ray job using the Ray Jobs CLI:\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1014 -- \u001b[1m RAY_API_SERVER_ADDRESS='http://127.0.0.1:8267' ray job submit --working-dir . -- python my_script.py\u001b[22m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1023 -- See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html \n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1027 -- for more information on submitting Ray jobs to the Ray cluster.\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1032 -- To terminate the Ray runtime, run\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1033 -- \u001b[1m ray stop\u001b[22m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1036 -- To view the status of the cluster, use\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1037 -- \u001b[1mray status\u001b[22m\u001b[26m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1041 -- To monitor and debug Ray, view the dashboard at \n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1042 -- \u001b[1m127.0.0.1:8267\u001b[22m\u001b[26m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1049 -- \u001b[4mIf connection to the dashboard fails, check your firewall settings and network configuration.\u001b[24m\n", + "2026-03-31 14:47:09,079\tINFO scripts.py:1155 -- \u001b[36m\u001b[1m--block\u001b[22m\u001b[39m\n", + "2026-03-31 14:47:09,080\tINFO scripts.py:1156 -- This command will now block forever until terminated by a signal.\n", + "2026-03-31 14:47:09,080\tINFO scripts.py:1159 -- Running subprocesses are monitored and a message will be printed if any of them terminate unexpectedly. Subprocesses exit with SIGTERM will be treated as graceful, thus NOT reported.\n", + "2026-03-31 14:47:09,080\tINFO scripts.py:1164 -- Process exit logs will be saved to: \u001b[1m/tmp/ray_huvu/session_2026-03-31_14-46-55_041316_1372026/logs/ray_process_exit.log\u001b[22m\u001b[26m\n" + ] + } + ], + "source": [ + "client = RayClient() # change as needed\n", + "client.start()" + ] + }, + { + "cell_type": "markdown", + "id": "fe724ad9", + "metadata": {}, + "source": [ + "### Build the NeMo Curator Pipeline\n", + "\n", + "After defining the NDD configuration which will be used to initialize `DataDesignerStage` stage, we build the pipeline chains three stages:\n", + "\n", + "| # | Stage | Role |\n", + "|---|-------|------|\n", + "| 1 | `JsonlReader` | Read seed JSONL shards from disk |\n", + "| 2 | `DataDesignerStage` | Enrich each record using the NDD config |\n", + "| 3 | `JsonlWriter` | Write enriched records back to JSONL |\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aed6d35d", + "metadata": {}, + "outputs": [], + "source": [ + "provider = None # when set to None we use local Ray inference, set to \"nvidia\" to use Nvidia endpoint\n", + "model_alias = \"local-llm\"\n", + "provider_name = provider or \"local\"\n", + "inference_server = None\n", + "model_providers = None\n", + "model = \"openai/gpt-oss-20b\" # when use Nvidia endpoint, set to \"meta/llama-3.3-70b-instruct\" as an example" + ] + }, + { + "cell_type": "markdown", + "id": "126448e3", + "metadata": {}, + "source": [ + "If provider is None, launch Ray inference through vLLM, which requires GPUs locally. Otherwise, using the default Nvidia's endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "6193471e", + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 3, - "id": "1dd6ac44", - "metadata": {}, - "outputs": [], - "source": [ - "OUTPUT_PATH = \"./synthetic_output\"\n", - "DATA_DESIGNER_CONFIG_FILE = None # set to a file path string to load a YAML config" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-03-31 14:51:26,294\tINFO worker.py:1669 -- Using address 127.0.1.1:6381 set in the environment variable RAY_ADDRESS\n", + "2026-03-31 14:51:26,302\tINFO worker.py:1810 -- Connecting to existing Ray cluster at address: 127.0.1.1:6381...\n", + "2026-03-31 14:51:26,435\tINFO worker.py:2004 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8267 \u001b[39m\u001b[22m\n" + ] }, { - "cell_type": "markdown", - "id": "b72c2f8e", - "metadata": {}, - "source": [ - "### Define the Data Designer Config\n", - "\n", - "The DataDesignerStage is intialized from a NDD's configuration builder (https://nvidia-nemo.github.io/DataDesigner/latest/code_reference/config_builder/). In this step, we manually create a configuration for the NDD module. This includes setting the LLM model provider, and the steps taken inside NDD module. Users can learn more about NDD's API from NDD official documentation: https://nvidia-nemo.github.io/DataDesigner/latest/notebooks/.\n", - "\n", - "In this tutorial, we replicate the official NDD's tutorials on synthetic data genertion with seed data (https://nvidia-nemo.github.io/DataDesigner/latest/notebooks/3-seeding-with-a-dataset/), but with Nemo-Curator.\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Detected 1 GPUs, using tensor_parallel_size=1\n" + ] }, { - "cell_type": "code", - "execution_count": 4, - "id": "db719310", - "metadata": {}, - "outputs": [], - "source": [ - "def _build_config(model_id: str, provider_name: str, model_alias: str) -> dd.DataDesignerConfigBuilder:\n", - " \"\"\"Build the Data Designer config with medical notes generation.\"\"\"\n", - " model_configs = [\n", - " dd.ModelConfig(\n", - " alias=model_alias,\n", - " model=model_id,\n", - " provider=provider_name,\n", - " skip_health_check=True,\n", - " inference_parameters=dd.ChatCompletionInferenceParams(\n", - " temperature=1.0,\n", - " top_p=1.0,\n", - " max_tokens=2048,\n", - " ),\n", - " )\n", - " ]\n", - "\n", - " config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)\n", - "\n", - " config_builder.add_column(\n", - " dd.SamplerColumnConfig(\n", - " name=\"patient_sampler\",\n", - " sampler_type=dd.SamplerType.PERSON_FROM_FAKER,\n", - " params=dd.PersonFromFakerSamplerParams(),\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.SamplerColumnConfig(\n", - " name=\"doctor_sampler\",\n", - " sampler_type=dd.SamplerType.PERSON_FROM_FAKER,\n", - " params=dd.PersonFromFakerSamplerParams(),\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.SamplerColumnConfig(\n", - " name=\"patient_id\",\n", - " sampler_type=dd.SamplerType.UUID,\n", - " params=dd.UUIDSamplerParams(\n", - " prefix=\"PT-\",\n", - " short_form=True,\n", - " uppercase=True,\n", - " ),\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.ExpressionColumnConfig(\n", - " name=\"first_name\",\n", - " expr=\"{{ patient_sampler.first_name}}\",\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.ExpressionColumnConfig(\n", - " name=\"last_name\",\n", - " expr=\"{{ patient_sampler.last_name }}\",\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.ExpressionColumnConfig(\n", - " name=\"dob\",\n", - " expr=\"{{ patient_sampler.birth_date }}\",\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.SamplerColumnConfig(\n", - " name=\"symptom_onset_date\",\n", - " sampler_type=dd.SamplerType.DATETIME,\n", - " params=dd.DatetimeSamplerParams(start=\"2024-01-01\", end=\"2024-12-31\"),\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.SamplerColumnConfig(\n", - " name=\"date_of_visit\",\n", - " sampler_type=dd.SamplerType.TIMEDELTA,\n", - " params=dd.TimeDeltaSamplerParams(\n", - " dt_min=1,\n", - " dt_max=30,\n", - " reference_column_name=\"symptom_onset_date\",\n", - " ),\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.ExpressionColumnConfig(\n", - " name=\"physician\",\n", - " expr=\"Dr. {{ doctor_sampler.last_name }}\",\n", - " )\n", - " )\n", - "\n", - " config_builder.add_column(\n", - " dd.LLMTextColumnConfig(\n", - " name=\"physician_notes\",\n", - " prompt=\"\"\"\\\n", - " You are a primary-care physician who just had an appointment with {{ first_name }} {{ last_name }},\n", - " who has been struggling with symptoms from {{ diagnosis }} since {{ symptom_onset_date }}.\n", - " The date of today's visit is {{ date_of_visit }}.\n", - "\n", - " {{ patient_summary }}\n", - "\n", - " Write careful notes about your visit with {{ first_name }},\n", - " as Dr. {{ doctor_sampler.first_name }} {{ doctor_sampler.last_name }}.\n", - "\n", - " Format the notes as a busy doctor might.\n", - " Respond with only the notes, no other text.\n", - " \"\"\",\n", - " model_alias=model_alias,\n", - " )\n", - " )\n", - "\n", - " return config_builder" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-03-31 14:51:27,211\tINFO worker.py:1669 -- Using address 127.0.1.1:6381 set in the environment variable RAY_ADDRESS\n", + "2026-03-31 14:51:27,219\tINFO worker.py:1810 -- Connecting to existing Ray cluster at address: 127.0.1.1:6381...\n", + "2026-03-31 14:51:27,342\tINFO worker.py:2004 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8267 \u001b[39m\u001b[22m\n", + "\u001b[32m2026-03-31 14:51:27.361\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.core.serve\u001b[0m:\u001b[36m_deploy\u001b[0m:\u001b[36m227\u001b[0m - \u001b[1mStarting Ray Serve with models: ['openai/gpt-oss-20b'] on port 8000\u001b[0m\n", + "INFO 2026-03-31 14:51:36,588 serve 1369491 -- ============== Deployment Options ==============\n", + "INFO 2026-03-31 14:51:36,590 serve 1369491 -- {'autoscaling_config': {'max_replicas': 1,\n", + " 'min_replicas': 1,\n", + " 'target_ongoing_requests': 1000000000},\n", + " 'health_check_period_s': 10,\n", + " 'health_check_timeout_s': 10,\n", + " 'max_ongoing_requests': 1000000000,\n", + " 'name': 'LLMServer:openai--gpt-oss-20b',\n", + " 'placement_group_bundles': [{'CPU': 1, 'GPU': 1}],\n", + " 'placement_group_strategy': 'PACK',\n", + " 'ray_actor_options': {'runtime_env': {'env_vars': {'RAY_SERVE_LOG_TO_STDERR': '0',\n", + " 'VLLM_LOGGING_LEVEL': 'WARNING'},\n", + " 'worker_process_setup_hook': 'ray.llm._internal.serve._worker_process_setup_hook'}}}\n", + "INFO 2026-03-31 14:51:36,681 serve 1369491 -- ============== Ingress Options ==============\n", + "INFO 2026-03-31 14:51:36,682 serve 1369491 -- {'autoscaling_config': {'target_ongoing_requests': 1000000000},\n", + " 'max_ongoing_requests': 1000000000,\n", + " 'ray_actor_options': {'runtime_env': {'env_vars': {'RAY_SERVE_LOG_TO_STDERR': '0',\n", + " 'VLLM_LOGGING_LEVEL': 'WARNING'}}}}\n", + "INFO 2026-03-31 14:51:38,482 serve 1369491 -- Started Serve in namespace \"serve\".\n", + "INFO 2026-03-31 14:51:38,534 serve 1369491 -- Connecting to existing Serve app in namespace \"serve\". New http options will not be applied.\n", + "\u001b[36m(ServeReplica:default:LLMServer:openai--gpt-oss-20b pid=1401871)\u001b[0m W0331 14:51:45.136000 1401871 torch/utils/cpp_extension.py:117] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'\n", + "\u001b[36m(initialize_remote_node pid=1402166)\u001b[0m W0331 14:51:52.051000 1402166 torch/utils/cpp_extension.py:117] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[36m(initialize_remote_node pid=1402166)\u001b[0m No cloud storage mirror configured\n", + "Parse safetensors files: 0%| | 0/3 [00:00:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", + "\u001b[36m(RayWorkerWrapper pid=1403177)\u001b[0m :1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n", + "Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", + "\u001b[36m(ServeReplica:default:LLMServer:openai--gpt-oss-20b pid=1401871)\u001b[0m \u001b[0;36m(EngineCore_DP0 pid=1402804)\u001b[0;0m \u001b[36m(RayWorkerWrapper pid=1403177)\u001b[0m :1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n", + "Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", - "\u001b[36m(RayWorkerWrapper pid=1403177)\u001b[0m :1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n", - "Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", - "\u001b[36m(ServeReplica:default:LLMServer:openai--gpt-oss-20b pid=1401871)\u001b[0m \u001b[0;36m(EngineCore_DP0 pid=1402804)\u001b[0;0m \u001b[36m(RayWorkerWrapper pid=1403177)\u001b[0m :1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n", - "Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, num_setup_attempts_python=1, num_run_attempts_python=1, max_setup_failure_percentage=None, ignore_failures=False, reset_workers_on_failure=False, slots_per_actor=2, enable_work_stealing=False, max_tasks_to_poll_per_chunk=8, worker_max_lifetime_m=0, worker_restart_interval_m=1, logging_interval_s=60, failures_return_nones=False, return_last_stage_outputs=True, actor_pool_verbosity_level=, monitoring_verbosity_level=, mode_specific=StreamingSpecificSpec(autoscale_interval_s=180, autoscale_speed_estimation_window_duration_s=180.0, autoscale_speed_estimation_min_data_points=5, max_queued_multiplier=1.0, max_queued_lower_bound=8, autoscaler_verbosity_level=, executor_verbosity_level=), log_worker_allocation_layout=True, cpu_allocation_percentage=0.95, clear_cuda_visible_devices_on_cpu_actors=True)\n", - " job_info: None\n", - " serving_queues: None\n", - " Stage 0:\n", - " class_name: FilePartitioningStage\n", - " required_resources: Resources(cpus=0.5, gpus=0, is_spmd=false)\n", - " num_workers: None\n", - " num_workers_per_node: 1\n", - " num_setup_attempts_python: 1\n", - " num_run_attempts_python: 1\n", - " ignore_failures: False\n", - " reset_workers_on_failure: False\n", - " slots_per_actor: 2\n", - " worker_max_lifetime_m: 0\n", - " worker_restart_interval_m: 1\n", - " max_setup_failure_percentage: None\n", - " over_provision_factor: None\n", - " Stage 1:\n", - " class_name: JsonlReaderStage\n", - " required_resources: Resources(cpus=1, gpus=0, is_spmd=false)\n", - " num_workers: None\n", - " num_workers_per_node: None\n", - " num_setup_attempts_python: 1\n", - " num_run_attempts_python: 1\n", - " ignore_failures: False\n", - " reset_workers_on_failure: False\n", - " slots_per_actor: 2\n", - " worker_max_lifetime_m: 0\n", - " worker_restart_interval_m: 1\n", - " max_setup_failure_percentage: None\n", - " over_provision_factor: None\n", - " Stage 2:\n", - " class_name: DataDesignerStage\n", - " required_resources: Resources(cpus=1, gpus=0, is_spmd=false)\n", - " num_workers: 2\n", - " num_workers_per_node: None\n", - " num_setup_attempts_python: 1\n", - " num_run_attempts_python: 1\n", - " ignore_failures: False\n", - " reset_workers_on_failure: False\n", - " slots_per_actor: 2\n", - " worker_max_lifetime_m: 0\n", - " worker_restart_interval_m: 1\n", - " max_setup_failure_percentage: None\n", - " over_provision_factor: None\n", - " Stage 3:\n", - " class_name: JsonlWriter\n", - " required_resources: Resources(cpus=1, gpus=0, is_spmd=false)\n", - " num_workers: None\n", - " num_workers_per_node: None\n", - " num_setup_attempts_python: 1\n", - " num_run_attempts_python: 1\n", - " ignore_failures: False\n", - " reset_workers_on_failure: False\n", - " slots_per_actor: 2\n", - " worker_max_lifetime_m: 0\n", - " worker_restart_interval_m: 1\n", - " max_setup_failure_percentage: None\n", - " over_provision_factor: None\n", - "\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:28.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m474\u001b[0m - \u001b[1mInitialized Ray cluster.\u001b[0m\n", - "2026-03-31 14:55:28,577\tINFO worker.py:1669 -- Using address 127.0.1.1:6381 set in the environment variable RAY_ADDRESS\n", - "2026-03-31 14:55:28,584\tINFO worker.py:1810 -- Connecting to existing Ray cluster at address: 127.0.1.1:6381...\n", - "2026-03-31 14:55:28,584\tINFO worker.py:1831 -- Calling ray.init() again after it has already been called.\n", - "\u001b[32m2026-03-31 14:55:28.584\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.ray_utils.cluster\u001b[0m:\u001b[36minit_or_connect_to_cluster\u001b[0m:\u001b[36m95\u001b[0m - \u001b[1mRay dashboard url: 127.0.0.1:8267\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting synthetic data generation pipeline...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2026-03-31 14:55:29.564\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.565\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.565\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.566\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.566\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m484\u001b[0m - \u001b[1mCluster resources: ClusterResources(nodes={'8cb2464e86bb476e004fe7c5364a1c29818ca7e58df1e1ec1d3afd6a': NodeResources(used_cpus=0.0, total_cpus=7, gpus=[GpuResources(index=1, uuid_=UUID('dd398c2c-c263-6d18-cfd6-1704e8b6dbb8'), used_fraction=0.0)], name='8cb2464e86bb476e004fe7c5364a1c29818ca7e58df1e1ec1d3afd6a')})\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.567\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m485\u001b[0m - \u001b[1mCreated/connected to cluster with resources: PoolOfResources(cpus=7, gpus=1)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.567\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.568\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.568\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.568\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.570\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.570\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.571\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.571\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.572\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.574\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.576\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:29.576\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:30.630\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", - "Pipeline Stats:\n", - "Pipeline duration: 0.017511641979217528 minutes\n", - "Number of initial input samples: 1\n", - "Number of input samples remaining: 1\n", - "Streaming pipeline main loop rate: 0\n", - "\n", - "Cluster Resources:\n", - "╒══════════════════════════╀═════════╀═════════════╕\n", - "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", - "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Resource Usage by Stage:\n", - "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", - "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Stage state:\n", - "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", - "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", - "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", - "\u001b[32m2026-03-31 14:55:30.633\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mapply_autoscale_result_if_ready\u001b[0m:\u001b[36m380\u001b[0m - \u001b[33m\u001b[1mApplying autoscale results took 1.045501947402954 seconds\u001b[0m\n", - "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.\n", - "\u001b[32m2026-03-31 14:55:36.156\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 0\u001b[0m\n", - "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m 2026-03-31 14:55:36.135 | INFO | nemo_curator.stages.file_partitioning:process:106 - Found 20 files\n", - "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m 2026-03-31 14:55:36.135 | INFO | nemo_curator.stages.file_partitioning:process:117 - No partitions specified, defaulting to one file per partition\n", - "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m 2026-03-31 14:55:36.135 | INFO | nemo_curator.stages.file_partitioning:process:143 - Created 20 file groups from 20 files\n", - "\u001b[36m(Stage 03 - JsonlWriter pid=1411406)\u001b[0m Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.\u001b[32m [repeated 5x across cluster]\u001b[0m\n", - "\u001b[32m2026-03-31 14:56:30.684\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", - "Pipeline Stats:\n", - "Pipeline duration: 1.0184145490328471 minutes\n", - "Number of initial input samples: 1\n", - "Number of input samples remaining: 0\n", - "Streaming pipeline main loop rate: 97.63829758704817\n", - "\n", - "Cluster Resources:\n", - "╒══════════════════════════╀═════════╀═════════════╕\n", - "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", - "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Resource Usage by Stage:\n", - "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", - "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 2.2 β”‚ 0.21 β”‚ 2 β”‚ 1.1 β”‚ 0.1 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 4 β”‚ 0.64 β”‚ 2 β”‚ 2 β”‚ 0.32 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 2.2 β”‚ 0.21 β”‚ 2 β”‚ 1.1 β”‚ 0.1 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Stage state:\n", - "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", - "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", - "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 18 β”‚ 0 β”‚ 0 β”‚ 8 β”‚ 0 β”‚ 4 β”‚ 14.5755 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.027902 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 4 β”‚ 6.21696 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", - "\u001b[32m2026-03-31 14:56:30.687\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", - " Auto Scaling Apply : 0.000001 seconds\n", - " Pool Update : 0.000339 seconds\n", - " Auto Scaling Submit: 0.000000 seconds\n", - " Monitor Update : 0.000009 seconds\n", - " Add Tasks : 0.000036 seconds\n", - " Sleep : 0.009468 seconds\n", - " Total : 0.009854 seconds\u001b[0m\n", - "\u001b[32m2026-03-31 14:56:40.528\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 1\u001b[0m\n", - "\u001b[32m2026-03-31 14:57:30.767\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", - "Pipeline Stats:\n", - "Pipeline duration: 2.0197851856549582 minutes\n", - "Number of initial input samples: 1\n", - "Number of input samples remaining: 0\n", - "Streaming pipeline main loop rate: 99.29700819229902\n", - "\n", - "Cluster Resources:\n", - "╒══════════════════════════╀═════════╀═════════════╕\n", - "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", - "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Resource Usage by Stage:\n", - "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", - "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 17.4 β”‚ 0.65 β”‚ 2 β”‚ 8.7 β”‚ 0.33 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 7.5 β”‚ 0.21 β”‚ 2 β”‚ 3.75 β”‚ 0.11 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Stage state:\n", - "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", - "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", - "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 6 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 6 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.0329424 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 6 β”‚ 0 β”‚ 0 β”‚ 6 β”‚ 0 β”‚ 4 β”‚ 5.77572 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", - "\u001b[32m2026-03-31 14:57:30.769\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", - " Auto Scaling Apply : 0.000005 seconds\n", - " Pool Update : 0.000510 seconds\n", - " Auto Scaling Submit: 0.000004 seconds\n", - " Monitor Update : 0.000025 seconds\n", - " Add Tasks : 0.000097 seconds\n", - " Sleep : 0.009003 seconds\n", - " Total : 0.009643 seconds\u001b[0m\n", - "\u001b[32m2026-03-31 14:58:30.890\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", - "Pipeline Stats:\n", - "Pipeline duration: 3.021822698911031 minutes\n", - "Number of initial input samples: 1\n", - "Number of input samples remaining: 0\n", - "Streaming pipeline main loop rate: 99.17518454324004\n", - "\n", - "Cluster Resources:\n", - "╒══════════════════════════╀═════════╀═════════════╕\n", - "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", - "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Resource Usage by Stage:\n", - "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", - "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 56.8 β”‚ 0.66 β”‚ 2 β”‚ 28.4 β”‚ 0.33 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 6.6 β”‚ 0.21 β”‚ 2 β”‚ 3.3 β”‚ 0.11 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Stage state:\n", - "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", - "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", - "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 10 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.0300601 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 3 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 10 β”‚ 0 β”‚ 0 β”‚ 10 β”‚ 0 β”‚ 4 β”‚ 5.57381 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", - "\u001b[32m2026-03-31 14:58:30.891\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", - " Auto Scaling Apply : 0.000005 seconds\n", - " Pool Update : 0.000620 seconds\n", - " Auto Scaling Submit: 0.000002 seconds\n", - " Monitor Update : 0.000023 seconds\n", - " Add Tasks : 0.000098 seconds\n", - " Sleep : 0.008827 seconds\n", - " Total : 0.009575 seconds\u001b[0m\n", - "\u001b[32m2026-03-31 14:59:30.967\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", - "Pipeline Stats:\n", - "Pipeline duration: 4.023102553685506 minutes\n", - "Number of initial input samples: 1\n", - "Number of input samples remaining: 0\n", - "Streaming pipeline main loop rate: 99.10276753765037\n", - "\n", - "Cluster Resources:\n", - "╒══════════════════════════╀═════════╀═════════════╕\n", - "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", - "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Resource Usage by Stage:\n", - "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", - "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 53.6 β”‚ 0.67 β”‚ 2 β”‚ 26.8 β”‚ 0.34 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 17.8 β”‚ 0.53 β”‚ 5 β”‚ 3.56 β”‚ 0.11 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Stage state:\n", - "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", - "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", - "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 13 β”‚ 0 β”‚ 3 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.0295169 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 5 β”‚ 0 β”‚ 5 β”‚ 13 β”‚ 0 β”‚ 0 β”‚ 13 β”‚ 0 β”‚ 10 β”‚ 5.58376 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", - "\u001b[32m2026-03-31 14:59:30.970\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", - " Auto Scaling Apply : 0.000005 seconds\n", - " Pool Update : 0.000594 seconds\n", - " Auto Scaling Submit: 0.000003 seconds\n", - " Monitor Update : 0.000033 seconds\n", - " Add Tasks : 0.000111 seconds\n", - " Sleep : 0.008767 seconds\n", - " Total : 0.009513 seconds\u001b[0m\n", - "\u001b[32m2026-03-31 15:00:31.037\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", - "Pipeline Stats:\n", - "Pipeline duration: 5.02426985502243 minutes\n", - "Number of initial input samples: 1\n", - "Number of input samples remaining: 0\n", - "Streaming pipeline main loop rate: 99.17144009891904\n", - "\n", - "Cluster Resources:\n", - "╒══════════════════════════╀═════════╀═════════════╕\n", - "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", - "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Resource Usage by Stage:\n", - "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", - "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 5.2 β”‚ 0.68 β”‚ 2 β”‚ 2.6 β”‚ 0.34 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 10 β”‚ 0.53 β”‚ 5 β”‚ 2 β”‚ 0.11 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", - "\n", - "Stage state:\n", - "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", - "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", - "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", - "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", - "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 16 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.02932 β”‚\n", - "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", - "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 5 β”‚ 0 β”‚ 5 β”‚ 16 β”‚ 0 β”‚ 0 β”‚ 16 β”‚ 0 β”‚ 10 β”‚ 5.38209 β”‚\n", - "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", - "\u001b[32m2026-03-31 15:00:31.039\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", - " Auto Scaling Apply : 0.000005 seconds\n", - " Pool Update : 0.000534 seconds\n", - " Auto Scaling Submit: 0.000001 seconds\n", - " Monitor Update : 0.000022 seconds\n", - " Add Tasks : 0.000089 seconds\n", - " Sleep : 0.008995 seconds\n", - " Total : 0.009645 seconds\u001b[0m\n", - "\u001b[32m2026-03-31 15:01:10.019\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 2\u001b[0m\n", - "\u001b[32m2026-03-31 15:01:10.251\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 3\u001b[0m\n", - "\u001b[32m2026-03-31 15:01:10.258\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m818\u001b[0m - \u001b[1mAll stages are done. Finishing pipeline.\u001b[0m\n", - "\u001b[32m2026-03-31 15:01:20.303\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.executor\u001b[0m:\u001b[36mexecute\u001b[0m:\u001b[36m151\u001b[0m - \u001b[1mPipeline completed successfully with 20 output tasks\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Pipeline completed!\n", - "Total execution time: 352.16 seconds (5.87 minutes)\n" - ] - } - ], - "source": [ - "print(\"Starting synthetic data generation pipeline...\")\n", - "start_time = time.time()\n", - "pipeline.run()\n", - "end_time = time.time()\n", - "\n", - "elapsed_time = end_time - start_time\n", - "print(\"\\nPipeline completed!\")\n", - "print(f\"Total execution time: {elapsed_time:.2f} seconds ({elapsed_time / 60:.2f} minutes)\")" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "Pipeline: ndd_data_generation\n", + "Description: Generate synthetic text data using Nemo Data Designer\n", + "Stages: 3\n", + "\n", + "Stage 1: jsonl_reader\n", + " Resources: 1.0 CPUs\n", + " Batch size: 1\n", + " Outputs:\n", + " Output attributes: data\n", + " Output columns: diagnosis, patient_summary\n", + "Stage 2: DataDesignerStage\n", + " Resources: 1.0 CPUs\n", + " Batch size: 1\n", + " Inputs:\n", + " Required attributes: data\n", + " Outputs:\n", + " Output attributes: data\n", + "Stage 3: jsonl_writer\n", + " Resources: 1.0 CPUs\n", + " Batch size: 1\n", + " Inputs:\n", + " Required attributes: data\n", + " Outputs:\n", + " Output attributes: data\n", + "\n" + ] + } + ], + "source": [ + "pipeline = Pipeline(\n", + " name=\"ndd_data_generation\",\n", + " description=\"Generate synthetic text data using Nemo Data Designer\",\n", + ")\n", + "\n", + "pipeline.add_stage(\n", + " JsonlReader(\n", + " file_paths=seed_dir + \"/*.jsonl\",\n", + " fields=[\"diagnosis\", \"patient_summary\"],\n", + " )\n", + ")\n", + "\n", + "# Add the Nemo Data Designer stage (limit actors to stay within API rate limits)\n", + "ndd_stage = DataDesignerStage(config_builder=config_builder, model_providers=model_providers)\n", + "ndd_stage.xenna_stage_spec = lambda: {\"num_workers\": 2}\n", + "pipeline.add_stage(ndd_stage)\n", + "\n", + "pipeline.add_stage(JsonlWriter(path=OUTPUT_PATH))\n", + "\n", + "print(pipeline.describe())" + ] + }, + { + "cell_type": "markdown", + "id": "0f61f363", + "metadata": {}, + "source": [ + "### Run the Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "57f9236a", + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "id": "8fa5c637", - "metadata": {}, - "source": [ - "### Inspect the Output" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2026-03-31 14:55:28.401\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.pipeline.pipeline\u001b[0m:\u001b[36mbuild\u001b[0m:\u001b[36m70\u001b[0m - \u001b[1mPlanning pipeline: ndd_data_generation\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.402\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.pipeline.pipeline\u001b[0m:\u001b[36m_decompose_stages\u001b[0m:\u001b[36m106\u001b[0m - \u001b[1mDecomposing composite stage: jsonl_reader\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.402\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.pipeline.pipeline\u001b[0m:\u001b[36m_decompose_stages\u001b[0m:\u001b[36m120\u001b[0m - \u001b[1mExpanded 'jsonl_reader' into 2 execution stages\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.403\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.executor\u001b[0m:\u001b[36mexecute\u001b[0m:\u001b[36m135\u001b[0m - \u001b[1mExecution mode: STREAMING\u001b[0m\n", + "2026-03-31 14:55:28,404\tINFO worker.py:1669 -- Using address 127.0.1.1:6381 set in the environment variable RAY_ADDRESS\n", + "2026-03-31 14:55:28,411\tINFO worker.py:1810 -- Connecting to existing Ray cluster at address: 127.0.1.1:6381...\n", + "2026-03-31 14:55:28,548\tINFO worker.py:2004 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8267 \u001b[39m\u001b[22m\n", + "\u001b[32m2026-03-31 14:55:28.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.574\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.572\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m472\u001b[0m - \u001b[1mPipelineSpec:\n", + " config: PipelineConfig(execution_mode=, num_setup_attempts_python=1, num_run_attempts_python=1, max_setup_failure_percentage=None, ignore_failures=False, reset_workers_on_failure=False, slots_per_actor=2, enable_work_stealing=False, max_tasks_to_poll_per_chunk=8, worker_max_lifetime_m=0, worker_restart_interval_m=1, logging_interval_s=60, failures_return_nones=False, return_last_stage_outputs=True, actor_pool_verbosity_level=, monitoring_verbosity_level=, mode_specific=StreamingSpecificSpec(autoscale_interval_s=180, autoscale_speed_estimation_window_duration_s=180.0, autoscale_speed_estimation_min_data_points=5, max_queued_multiplier=1.0, max_queued_lower_bound=8, autoscaler_verbosity_level=, executor_verbosity_level=), log_worker_allocation_layout=True, cpu_allocation_percentage=0.95, clear_cuda_visible_devices_on_cpu_actors=True)\n", + " job_info: None\n", + " serving_queues: None\n", + " Stage 0:\n", + " class_name: FilePartitioningStage\n", + " required_resources: Resources(cpus=0.5, gpus=0, is_spmd=false)\n", + " num_workers: None\n", + " num_workers_per_node: 1\n", + " num_setup_attempts_python: 1\n", + " num_run_attempts_python: 1\n", + " ignore_failures: False\n", + " reset_workers_on_failure: False\n", + " slots_per_actor: 2\n", + " worker_max_lifetime_m: 0\n", + " worker_restart_interval_m: 1\n", + " max_setup_failure_percentage: None\n", + " over_provision_factor: None\n", + " Stage 1:\n", + " class_name: JsonlReaderStage\n", + " required_resources: Resources(cpus=1, gpus=0, is_spmd=false)\n", + " num_workers: None\n", + " num_workers_per_node: None\n", + " num_setup_attempts_python: 1\n", + " num_run_attempts_python: 1\n", + " ignore_failures: False\n", + " reset_workers_on_failure: False\n", + " slots_per_actor: 2\n", + " worker_max_lifetime_m: 0\n", + " worker_restart_interval_m: 1\n", + " max_setup_failure_percentage: None\n", + " over_provision_factor: None\n", + " Stage 2:\n", + " class_name: DataDesignerStage\n", + " required_resources: Resources(cpus=1, gpus=0, is_spmd=false)\n", + " num_workers: 2\n", + " num_workers_per_node: None\n", + " num_setup_attempts_python: 1\n", + " num_run_attempts_python: 1\n", + " ignore_failures: False\n", + " reset_workers_on_failure: False\n", + " slots_per_actor: 2\n", + " worker_max_lifetime_m: 0\n", + " worker_restart_interval_m: 1\n", + " max_setup_failure_percentage: None\n", + " over_provision_factor: None\n", + " Stage 3:\n", + " class_name: JsonlWriter\n", + " required_resources: Resources(cpus=1, gpus=0, is_spmd=false)\n", + " num_workers: None\n", + " num_workers_per_node: None\n", + " num_setup_attempts_python: 1\n", + " num_run_attempts_python: 1\n", + " ignore_failures: False\n", + " reset_workers_on_failure: False\n", + " slots_per_actor: 2\n", + " worker_max_lifetime_m: 0\n", + " worker_restart_interval_m: 1\n", + " max_setup_failure_percentage: None\n", + " over_provision_factor: None\n", + "\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:28.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m474\u001b[0m - \u001b[1mInitialized Ray cluster.\u001b[0m\n", + "2026-03-31 14:55:28,577\tINFO worker.py:1669 -- Using address 127.0.1.1:6381 set in the environment variable RAY_ADDRESS\n", + "2026-03-31 14:55:28,584\tINFO worker.py:1810 -- Connecting to existing Ray cluster at address: 127.0.1.1:6381...\n", + "2026-03-31 14:55:28,584\tINFO worker.py:1831 -- Calling ray.init() again after it has already been called.\n", + "\u001b[32m2026-03-31 14:55:28.584\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.ray_utils.cluster\u001b[0m:\u001b[36minit_or_connect_to_cluster\u001b[0m:\u001b[36m95\u001b[0m - \u001b[1mRay dashboard url: 127.0.0.1:8267\u001b[0m\n" + ] }, { - "cell_type": "markdown", - "id": "5398c620", - "metadata": {}, - "source": [ - "We can find the output results stored in `./synthetic_output` directory." - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting synthetic data generation pipeline...\n" + ] }, { - "cell_type": "code", - "execution_count": 19, - "id": "60ef2409", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generated data saved to: ./synthetic_output\n", - " - 2a32c234df7d.jsonl\n", - " - 383a455f5a8d.jsonl\n", - " - 44767c13dcd9.jsonl\n", - " - 572222cbb959.jsonl\n", - " - 5ed1794c3d4a.jsonl\n", - " - 642cda8b7e5b.jsonl\n", - " - 6af70db0176e.jsonl\n", - " - 6d23d6c30b3e.jsonl\n", - " - 6eedcc8ded3c.jsonl\n", - " - 7080f45e14c9.jsonl\n", - " - 7b44d21ee99f.jsonl\n", - " - 7d220206227d.jsonl\n", - " - 7ec98cc83a2c.jsonl\n", - " - 884f8629a7fd.jsonl\n", - " - 8928c29fad4e.jsonl\n", - " - c7fde8d4971e.jsonl\n", - " - cc45e91592c4.jsonl\n", - " - ead3a316b626.jsonl\n", - " - f7f60d292b3e.jsonl\n", - " - fb1bb122478d.jsonl\n" - ] - } - ], - "source": [ - "output_files = get_all_file_paths_under(OUTPUT_PATH, recurse_subdirectories=True, keep_extensions=\".jsonl\")\n", - "\n", - "print(f\"Generated data saved to: {OUTPUT_PATH}\")\n", - "for file_path in output_files:\n", - " print(f\" - {Path(file_path).name}\")\n", - "\n", - "all_data_frames = [pd.read_json(f, lines=True) for f in output_files]" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2026-03-31 14:55:29.564\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.565\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.565\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.566\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.566\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m484\u001b[0m - \u001b[1mCluster resources: ClusterResources(nodes={'8cb2464e86bb476e004fe7c5364a1c29818ca7e58df1e1ec1d3afd6a': NodeResources(used_cpus=0.0, total_cpus=7, gpus=[GpuResources(index=1, uuid_=UUID('dd398c2c-c263-6d18-cfd6-1704e8b6dbb8'), used_fraction=0.0)], name='8cb2464e86bb476e004fe7c5364a1c29818ca7e58df1e1ec1d3afd6a')})\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.567\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.pipelines\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m485\u001b[0m - \u001b[1mCreated/connected to cluster with resources: PoolOfResources(cpus=7, gpus=1)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.567\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.568\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.568\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.568\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.570\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.570\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.571\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.571\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.572\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.573\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.574\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=0.5, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.575\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.576\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:29.576\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.adapter\u001b[0m:\u001b[36mrequired_resources\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mResources: Resources(cpus=1.0, gpu_memory_gb=0.0, gpus=0.0)\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:30.630\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", + "Pipeline Stats:\n", + "Pipeline duration: 0.017511641979217528 minutes\n", + "Number of initial input samples: 1\n", + "Number of input samples remaining: 1\n", + "Streaming pipeline main loop rate: 0\n", + "\n", + "Cluster Resources:\n", + "╒══════════════════════════╀═════════╀═════════════╕\n", + "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", + "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Resource Usage by Stage:\n", + "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", + "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Stage state:\n", + "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", + "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", + "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", + "\u001b[32m2026-03-31 14:55:30.633\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mapply_autoscale_result_if_ready\u001b[0m:\u001b[36m380\u001b[0m - \u001b[33m\u001b[1mApplying autoscale results took 1.045501947402954 seconds\u001b[0m\n", + "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.\n", + "\u001b[32m2026-03-31 14:55:36.156\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 0\u001b[0m\n", + "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m 2026-03-31 14:55:36.135 | INFO | nemo_curator.stages.file_partitioning:process:106 - Found 20 files\n", + "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m 2026-03-31 14:55:36.135 | INFO | nemo_curator.stages.file_partitioning:process:117 - No partitions specified, defaulting to one file per partition\n", + "\u001b[36m(Stage 00 - FilePartitioningStage pid=1411403)\u001b[0m 2026-03-31 14:55:36.135 | INFO | nemo_curator.stages.file_partitioning:process:143 - Created 20 file groups from 20 files\n", + "\u001b[36m(Stage 03 - JsonlWriter pid=1411406)\u001b[0m Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.\u001b[32m [repeated 5x across cluster]\u001b[0m\n", + "\u001b[32m2026-03-31 14:56:30.684\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", + "Pipeline Stats:\n", + "Pipeline duration: 1.0184145490328471 minutes\n", + "Number of initial input samples: 1\n", + "Number of input samples remaining: 0\n", + "Streaming pipeline main loop rate: 97.63829758704817\n", + "\n", + "Cluster Resources:\n", + "╒══════════════════════════╀═════════╀═════════════╕\n", + "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", + "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Resource Usage by Stage:\n", + "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", + "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 2.2 β”‚ 0.21 β”‚ 2 β”‚ 1.1 β”‚ 0.1 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 4 β”‚ 0.64 β”‚ 2 β”‚ 2 β”‚ 0.32 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 2.2 β”‚ 0.21 β”‚ 2 β”‚ 1.1 β”‚ 0.1 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Stage state:\n", + "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", + "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", + "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 18 β”‚ 0 β”‚ 0 β”‚ 8 β”‚ 0 β”‚ 4 β”‚ 14.5755 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.027902 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 4 β”‚ 6.21696 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", + "\u001b[32m2026-03-31 14:56:30.687\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", + " Auto Scaling Apply : 0.000001 seconds\n", + " Pool Update : 0.000339 seconds\n", + " Auto Scaling Submit: 0.000000 seconds\n", + " Monitor Update : 0.000009 seconds\n", + " Add Tasks : 0.000036 seconds\n", + " Sleep : 0.009468 seconds\n", + " Total : 0.009854 seconds\u001b[0m\n", + "\u001b[32m2026-03-31 14:56:40.528\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 1\u001b[0m\n", + "\u001b[32m2026-03-31 14:57:30.767\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", + "Pipeline Stats:\n", + "Pipeline duration: 2.0197851856549582 minutes\n", + "Number of initial input samples: 1\n", + "Number of input samples remaining: 0\n", + "Streaming pipeline main loop rate: 99.29700819229902\n", + "\n", + "Cluster Resources:\n", + "╒══════════════════════════╀═════════╀═════════════╕\n", + "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", + "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Resource Usage by Stage:\n", + "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", + "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 17.4 β”‚ 0.65 β”‚ 2 β”‚ 8.7 β”‚ 0.33 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 7.5 β”‚ 0.21 β”‚ 2 β”‚ 3.75 β”‚ 0.11 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Stage state:\n", + "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", + "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", + "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 6 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 6 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.0329424 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 6 β”‚ 0 β”‚ 0 β”‚ 6 β”‚ 0 β”‚ 4 β”‚ 5.77572 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", + "\u001b[32m2026-03-31 14:57:30.769\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", + " Auto Scaling Apply : 0.000005 seconds\n", + " Pool Update : 0.000510 seconds\n", + " Auto Scaling Submit: 0.000004 seconds\n", + " Monitor Update : 0.000025 seconds\n", + " Add Tasks : 0.000097 seconds\n", + " Sleep : 0.009003 seconds\n", + " Total : 0.009643 seconds\u001b[0m\n", + "\u001b[32m2026-03-31 14:58:30.890\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", + "Pipeline Stats:\n", + "Pipeline duration: 3.021822698911031 minutes\n", + "Number of initial input samples: 1\n", + "Number of input samples remaining: 0\n", + "Streaming pipeline main loop rate: 99.17518454324004\n", + "\n", + "Cluster Resources:\n", + "╒══════════════════════════╀═════════╀═════════════╕\n", + "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", + "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Resource Usage by Stage:\n", + "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", + "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 56.8 β”‚ 0.66 β”‚ 2 β”‚ 28.4 β”‚ 0.33 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 6.6 β”‚ 0.21 β”‚ 2 β”‚ 3.3 β”‚ 0.11 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Stage state:\n", + "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", + "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", + "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 10 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.0300601 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 3 β”‚ 2 β”‚ 0 β”‚ 2 β”‚ 10 β”‚ 0 β”‚ 0 β”‚ 10 β”‚ 0 β”‚ 4 β”‚ 5.57381 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", + "\u001b[32m2026-03-31 14:58:30.891\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", + " Auto Scaling Apply : 0.000005 seconds\n", + " Pool Update : 0.000620 seconds\n", + " Auto Scaling Submit: 0.000002 seconds\n", + " Monitor Update : 0.000023 seconds\n", + " Add Tasks : 0.000098 seconds\n", + " Sleep : 0.008827 seconds\n", + " Total : 0.009575 seconds\u001b[0m\n", + "\u001b[32m2026-03-31 14:59:30.967\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", + "Pipeline Stats:\n", + "Pipeline duration: 4.023102553685506 minutes\n", + "Number of initial input samples: 1\n", + "Number of input samples remaining: 0\n", + "Streaming pipeline main loop rate: 99.10276753765037\n", + "\n", + "Cluster Resources:\n", + "╒══════════════════════════╀═════════╀═════════════╕\n", + "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", + "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Resource Usage by Stage:\n", + "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", + "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 53.6 β”‚ 0.67 β”‚ 2 β”‚ 26.8 β”‚ 0.34 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 17.8 β”‚ 0.53 β”‚ 5 β”‚ 3.56 β”‚ 0.11 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Stage state:\n", + "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", + "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", + "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 13 β”‚ 0 β”‚ 3 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.0295169 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 5 β”‚ 0 β”‚ 5 β”‚ 13 β”‚ 0 β”‚ 0 β”‚ 13 β”‚ 0 β”‚ 10 β”‚ 5.58376 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", + "\u001b[32m2026-03-31 14:59:30.970\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", + " Auto Scaling Apply : 0.000005 seconds\n", + " Pool Update : 0.000594 seconds\n", + " Auto Scaling Submit: 0.000003 seconds\n", + " Monitor Update : 0.000033 seconds\n", + " Add Tasks : 0.000111 seconds\n", + " Sleep : 0.008767 seconds\n", + " Total : 0.009513 seconds\u001b[0m\n", + "\u001b[32m2026-03-31 15:00:31.037\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.monitoring\u001b[0m:\u001b[36mupdate\u001b[0m:\u001b[36m339\u001b[0m - \u001b[1mPipeline stats:\n", + "Pipeline Stats:\n", + "Pipeline duration: 5.02426985502243 minutes\n", + "Number of initial input samples: 1\n", + "Number of input samples remaining: 0\n", + "Streaming pipeline main loop rate: 99.17144009891904\n", + "\n", + "Cluster Resources:\n", + "╒══════════════════════════╀═════════╀═════════════╕\n", + "β”‚ Resource β”‚ Total β”‚ Available β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═════════════║\n", + "β”‚ CPUs β”‚ 8 β”‚ 5 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ GPUs β”‚ 1 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Memory (GB) β”‚ 1898.18 β”‚ 1898.18 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Object Store Memory (GB) β”‚ 200 β”‚ 200 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Resource Usage by Stage:\n", + "╒══════════════════════════════════╀═════════╀═══════════════╀═══════════════╀════════════════════╀══════════════════════════╕\n", + "β”‚ Stage β”‚ CPU % β”‚ Memory (GB) β”‚ Actor Count β”‚ CPU % per worker β”‚ Memory (GB) per worker β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═════════β•ͺ═══════════════β•ͺ═══════════════β•ͺ════════════════════β•ͺ══════════════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 5.2 β”‚ 0.68 β”‚ 2 β”‚ 2.6 β”‚ 0.34 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 10 β”‚ 0.53 β”‚ 5 β”‚ 2 β”‚ 0.11 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\n", + "\n", + "Stage state:\n", + "╒══════════════════════════════════╀═══════════╀═══════════╀═══════════╀═══════════╀═══════════╀═════════════╀═════════════════╀══════════════╀═══════════════╀════════════╀═════════════╀═════════════════╕\n", + "β”‚ Stage β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Actors: β”‚ Tasks: β”‚ Tasks: β”‚ Queue: β”‚ Queue: β”‚ Slots: β”‚ Slots: β”‚ Speed: β”‚\n", + "β”‚ β”‚ Target β”‚ Pending β”‚ Ready β”‚ Running β”‚ Idle β”‚ Completed β”‚ Returned None β”‚ Input Size β”‚ Output Size β”‚ Num Used β”‚ Num Empty β”‚ Tasks/actor/s β”‚\n", + "β•žβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═══════════β•ͺ═════════════β•ͺ═════════════════β•ͺ══════════════β•ͺ═══════════════β•ͺ════════════β•ͺ═════════════β•ͺ═════════════════║\n", + "β”‚ Stage 00 - FilePartitioningStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 1 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 01 - JsonlReaderStage β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 20 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 02 - DataDesignerStage β”‚ 0 β”‚ 0 β”‚ 2 β”‚ 2 β”‚ 0 β”‚ 16 β”‚ 0 β”‚ 0 β”‚ 0 β”‚ 4 β”‚ 0 β”‚ 0.02932 β”‚\n", + "β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€\n", + "β”‚ Stage 03 - JsonlWriter β”‚ 0 β”‚ 0 β”‚ 5 β”‚ 0 β”‚ 5 β”‚ 16 β”‚ 0 β”‚ 0 β”‚ 16 β”‚ 0 β”‚ 10 β”‚ 5.38209 β”‚\n", + "β•˜β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•§β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•›\u001b[0m\n", + "\u001b[32m2026-03-31 15:00:31.039\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m688\u001b[0m - \u001b[1mStreamingExecutor Timing Summary:\n", + " Auto Scaling Apply : 0.000005 seconds\n", + " Pool Update : 0.000534 seconds\n", + " Auto Scaling Submit: 0.000001 seconds\n", + " Monitor Update : 0.000022 seconds\n", + " Add Tasks : 0.000089 seconds\n", + " Sleep : 0.008995 seconds\n", + " Total : 0.009645 seconds\u001b[0m\n", + "\u001b[32m2026-03-31 15:01:10.019\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 2\u001b[0m\n", + "\u001b[32m2026-03-31 15:01:10.251\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m811\u001b[0m - \u001b[1mStopping stages 3\u001b[0m\n", + "\u001b[32m2026-03-31 15:01:10.258\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcosmos_xenna.pipelines.private.streaming\u001b[0m:\u001b[36mrun_pipeline\u001b[0m:\u001b[36m818\u001b[0m - \u001b[1mAll stages are done. Finishing pipeline.\u001b[0m\n", + "\u001b[32m2026-03-31 15:01:20.303\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnemo_curator.backends.xenna.executor\u001b[0m:\u001b[36mexecute\u001b[0m:\u001b[36m151\u001b[0m - \u001b[1mPipeline completed successfully with 20 output tasks\u001b[0m\n" + ] }, { - "cell_type": "code", - "execution_count": 20, - "id": "ff38a005", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total records generated: 200\n" - ] - } - ], - "source": [ - "if all_data_frames:\n", - " combined_df = pd.concat(all_data_frames, ignore_index=True)\n", - " print(f\"Total records generated: {len(combined_df)}\")" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Pipeline completed!\n", + "Total execution time: 352.16 seconds (5.87 minutes)\n" + ] + } + ], + "source": [ + "print(\"Starting synthetic data generation pipeline...\")\n", + "start_time = time.time()\n", + "pipeline.run()\n", + "end_time = time.time()\n", + "\n", + "elapsed_time = end_time - start_time\n", + "print(\"\\nPipeline completed!\")\n", + "print(f\"Total execution time: {elapsed_time:.2f} seconds ({elapsed_time / 60:.2f} minutes)\")" + ] + }, + { + "cell_type": "markdown", + "id": "8fa5c637", + "metadata": {}, + "source": [ + "### Inspect the Output" + ] + }, + { + "cell_type": "markdown", + "id": "5398c620", + "metadata": {}, + "source": [ + "We can find the output results stored in `./synthetic_output` directory." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "60ef2409", + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "id": "d512fed3", - "metadata": {}, - "source": [ - "Inspecting output .jsonl files content, we can find the generated text from Nvidia's LLM endpoint corresponding to the input prompt." - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated data saved to: ./synthetic_output\n", + " - 2a32c234df7d.jsonl\n", + " - 383a455f5a8d.jsonl\n", + " - 44767c13dcd9.jsonl\n", + " - 572222cbb959.jsonl\n", + " - 5ed1794c3d4a.jsonl\n", + " - 642cda8b7e5b.jsonl\n", + " - 6af70db0176e.jsonl\n", + " - 6d23d6c30b3e.jsonl\n", + " - 6eedcc8ded3c.jsonl\n", + " - 7080f45e14c9.jsonl\n", + " - 7b44d21ee99f.jsonl\n", + " - 7d220206227d.jsonl\n", + " - 7ec98cc83a2c.jsonl\n", + " - 884f8629a7fd.jsonl\n", + " - 8928c29fad4e.jsonl\n", + " - c7fde8d4971e.jsonl\n", + " - cc45e91592c4.jsonl\n", + " - ead3a316b626.jsonl\n", + " - f7f60d292b3e.jsonl\n", + " - fb1bb122478d.jsonl\n" + ] + } + ], + "source": [ + "output_files = get_all_file_paths_under(OUTPUT_PATH, recurse_subdirectories=True, keep_extensions=\".jsonl\")\n", + "\n", + "print(f\"Generated data saved to: {OUTPUT_PATH}\")\n", + "for file_path in output_files:\n", + " print(f\" - {Path(file_path).name}\")\n", + "\n", + "all_data_frames = [pd.read_json(f, lines=True) for f in output_files]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "ff38a005", + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 21, - "id": "1fbfc212", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==================================================\n", - "Sample of generated documents:\n", - "==================================================\n", - "\n", - "File 1: 2a32c234df7d.jsonl\n", - "Number of documents: 10\n", - "\n", - "Generated text (showing first record):\n", - "Document 1:\n", - "[diagnosis]:\n", - "malaria\n", - "[patient_summary]:\n", - "I have a high fever, chills, and severe itching. I also have a headache, excessive sweating, nausea, and muscle aches.\n", - "[patient_sampler]:\n", - "{'uuid': '13a36ba1-5f8f-431c-86d1-372975b6c220', 'locale': 'en_US', 'first_name': 'Jeffrey', 'last_name': 'Watts', 'middle_name': None, 'sex': 'Male', 'street_number': '293', 'street_name': 'Harris Oval', 'city': 'New Jessica', 'state': 'Texas', 'postcode': '20375', 'age': 49, 'birth_date': '1976-05-09', 'country': 'American Samoa', 'marital_status': 'separated', 'education_level': 'secondary_education', 'unit': '', 'occupation': 'Educational psychologist', 'phone_number': '(484)575-5193', 'bachelors_field': 'no_degree'}\n", - "[doctor_sampler]:\n", - "{'uuid': '8b79d082-0a4e-47b0-b890-ba69ee9c2c6d', 'locale': 'en_US', 'first_name': 'Rachael', 'last_name': 'Cortez', 'middle_name': None, 'sex': 'Female', 'street_number': '8232', 'street_name': 'Pugh Manor', 'city': 'New Tracyborough', 'state': 'Wisconsin', 'postcode': '24123', 'age': 29, 'birth_date': '1996-12-18', 'country': 'New Zealand', 'marital_status': 'never_married', 'education_level': 'graduate', 'unit': '', 'occupation': 'Health visitor', 'phone_number': '795-875-2718', 'bachelors_field': 'stem'}\n", - "[patient_id]:\n", - "PT-E599ACBE\n", - "[symptom_onset_date]:\n", - "2024-03-17\n", - "[date_of_visit]:\n", - "2024-04-08\n", - "[first_name]:\n", - "Jeffrey\n", - "[last_name]:\n", - "Watts\n", - "[dob]:\n", - "1976-05-09\n", - "[physician]:\n", - "Dr. Cortez\n", - "[physician_notes]:\n", - "**Chief Complaint:** Fever, chills, itching, headache, nausea, muscle aches. \n", - "\n", - "**Date/Time:** 2024‑04‑08 10:12β€―am \n", - "\n", - "**Patient:** Jeffrey Watts (M30, known malaria case 3/17/24)\n", - "\n", - "**S:** \n", - "*c/o* high fever (39.2β€―Β°C/102.6β€―Β°F reported; temp noted 39β€―Β°C/102.2β€―Β°F), intense chills, generalized pruritus, severe frontal headache, profuse sweating, nausea (no vomiting), diffuse myalgias. \n", - "*No* dyspnea, chest pain, abdominal pain, rash except itching, or new joint pain. No recent travel deviation from last trip. \n", - "\n", - "**O:** \n", - "- Vitals: Tβ€―39.0β€―Β°C (102.2β€―Β°F), HRβ€―108β€―bpm, BPβ€―118/72β€―mmHg, RRβ€―18β€―bpm, SpOβ‚‚β€―98% RA. \n", - "- General: Alert, afebrile after 0.5β€―h after start of antimalarial therapy. \n", - "- HEENT: Pale conjunctivae, normal pupils, no meningeal signs. \n", - "- Cardiovascular: Regular rhythm, no murmurs. \n", - "- Respiratory: Clear to auscultation bilaterally. \n", - "- Abdomen: Soft, nontender, no hepatosplenomegaly. \n", - "- Skin: Diffuse pruritus, no maculopapular rash, no petechiae. \n", - "- Neurologic: GCS 15, cranial nerves intact, mild agitation from discomfort. \n", - "- Musculoskeletal: Diffuse muscle tenderness, no focal neuro deficits. \n", - "\n", - "**Labs:** \n", - "- CBC: WBCβ€―4.8 K/Β΅L, Hgbβ€―13.1β€―g/dL, Hctβ€―39%, Pltβ€―190 K/Β΅L. \n", - "- CMP: Normal electrolytes, ASTβ€―45 IU/L, ALTβ€―39 IU/L, BUNβ€―12 mg/dL, Crβ€―0.9β€―mg/dL. \n", - "- Malaria smear: 0.36% parasitemia (expired 2–3 days). \n", - "- Malaria PCR: Positive for *Plasmodium falciparum* 3‑day post‑therapy.\n", - "\n", - "**Assessment:** \n", - "1. *Plasmodium falciparum* malaria relapse – moderate parasite density, resolving after 3β€―days of artemether‑lumefantrine. \n", - "2. Febrile reaction to treatmentβ€”likely due to hot‐blood syndrome. \n", - "3. Symptomatic a‑septic itchingβ€”potential eosinophil release reaction. \n", - "\n", - "**Plan:** \n", - "- Continue artemether‑lumefantrine (total 3 doses, last dose 11:00β€―am). \n", - "- Add dapsone 100β€―mg PO BID for itching; counsel on rash risk. \n", - "- Continue acetaminophen 650β€―mg PO every 6β€―h PRN for fever/headache. \n", - "- Encourage oral hydration; advise oral rehydration salts. \n", - "- Re-check CBC, electrolytes, parasite smear in 48β€―h. \n", - "- Counsel on mosquito avoidance and prophylaxis if returning to endemic zone. \n", - "- SFA (safety follow‑up appointment) 2024‑04‑16; if any worsening (high fever >39.5β€―Β°C, seizures, AMS), call ED. \n", - "--- \n", - "\n", - "**Physician:** Dr. Rachael Cortez, MD \n", - "**Signature:** _____________________ Date: 2024‑04‑08\n", - "----------------------------------------\n", - "\n", - "File 2: 383a455f5a8d.jsonl\n", - "Number of documents: 10\n", - "\n", - "Generated text (showing first record):\n", - "Document 1:\n", - "[diagnosis]:\n", - "psoriasis\n", - "[patient_summary]:\n", - "I have a skin condition that causes dry, flaky patches on my skin. I also have joint pain and the skin on my knees and elbows is peeling off.\n", - "[patient_sampler]:\n", - "{'uuid': '9008b8fa-27cd-49e4-8a63-2a6bc2acbace', 'locale': 'en_US', 'first_name': 'William', 'last_name': 'Johnson', 'middle_name': None, 'sex': 'Male', 'street_number': '12058', 'street_name': 'John Corner', 'city': 'Calderonport', 'state': 'Alabama', 'postcode': '79215', 'age': 28, 'birth_date': '1997-09-13', 'country': 'Djibouti', 'marital_status': 'married_present', 'education_level': 'secondary_education', 'unit': '', 'occupation': 'Field trials officer', 'phone_number': '450-594-0263x9360', 'bachelors_field': 'no_degree'}\n", - "[doctor_sampler]:\n", - "{'uuid': 'f04c32f2-5064-47a0-baa2-47a3b9e03c14', 'locale': 'en_US', 'first_name': 'Latasha', 'last_name': 'Anderson', 'middle_name': None, 'sex': 'Female', 'street_number': '25467', 'street_name': 'Petersen Ferry', 'city': 'Solomonfurt', 'state': 'Massachusetts', 'postcode': '99698', 'age': 87, 'birth_date': '1939-02-10', 'country': 'Jordan', 'marital_status': 'widowed', 'education_level': 'bachelors', 'unit': '', 'occupation': 'Writer', 'phone_number': '001-659-997-5484', 'bachelors_field': 'business'}\n", - "[patient_id]:\n", - "PT-B50655DA\n", - "[symptom_onset_date]:\n", - "2024-02-07\n", - "[date_of_visit]:\n", - "2024-02-26\n", - "[first_name]:\n", - "William\n", - "[last_name]:\n", - "Johnson\n", - "[dob]:\n", - "1997-09-13\n", - "[physician]:\n", - "Dr. Anderson\n", - "[physician_notes]:\n", - "**Dr. Latasha Anderson, MD** \n", - "**Clinic Visit Notes – 2024‑02‑26** \n", - "Patient: William Johnson, MRN 0045321 \n", - "\n", - "---\n", - "\n", - "### S – Subjective \n", - "- **Chief Complaint:** β€œDry, flaky patches on skin, peeling at knees/elbows, joint pain.” \n", - "- **History of Present Illness:** \n", - " - Psoriasis first noted 2024‑02‑07: erythematous plaques, scaling on scalp, elbows, knees. \n", - " - Current flare: increased scaling, desquamation. \n", - " - Joint pain: bilateral knees, mild swelling, stiffness >30β€―min morning. \n", - " - No systemic symptoms (fever, weight loss). \n", - "- **Past Meds:** Topical steroid (clobetasol 0.05%) once daily, OTC ibuprofen PRN. \n", - "- **Allergies:** NKDA. \n", - "- **Social:** Non‑smoker, 3-4 drinks/week, works as accountant (desk job). \n", - "- **Family hx:** Mother has psoriasis, mother diagnosed with rheumatoid arthritis. \n", - "- **Review of Systems:** \n", - " - **Skin:** Paronychia, no nail pitting noted now. \n", - " - **Musculoskeletal:** Pain in knees, no swelling elsewhere. \n", - " - **GI, Cardiopulm, Neuro:** Negative. \n", - "\n", - "### O – Objective \n", - "- **Vital Signs:** BP 122/78, HR 78, Temp 98.6β€―Β°F, RR 16, SpOβ‚‚ 98% RA. \n", - "- **Physical Exam:**\n", - " - **Skin:** Extensive plaques on scalp (secondary alopecia), elbows, knees – approx. 15% BSA. Scaling >30β€―mm shagreen, fissuring in cubital fossa, mild erythema around lesions. No active ulcers. \n", - " - **Nails:** Mild pitting on thumb nails, onycholysis not obvious now. \n", - " - **Joints:** Bilateral knee tenderness, 1‑2β€―mm effusion. Range of motion preserved, mild swelling. No redness or warmth. \n", - "- **Other systems:** Normal.\n", - "\n", - "### A – Assessment \n", - "1. **Plaque psoriasis – moderate severity (BSA 10‑20%)** – worsening flare. \n", - "2. **Psoriatic arthritis – oligo‑arthralgia (knees)** – likely early psoriatic arthritis (based on joint pain, swelling, and psoriasis). \n", - "3. **No signs of infection or systemic disease.**\n", - "\n", - "### P – Plan \n", - "- **Skin:** \n", - " - Continue clobetasol 0.05% twice daily on erosive lesions. \n", - " - Add calcipotriene 0.005% ointment nightly. \n", - " - Start low‑dose oral prednisone 5β€―mg daily for 1β€―week, tapering over 2β€―weeks. \n", - " - Educate on moisturizers (ceramide‑based) 4Γ—/day. \n", - "- **Joint pain:** \n", - " - NSAID trial: naproxen 250β€―mg BID for 2β€―weeks. \n", - " - If inadequate relief or intolerant β†’ refer to rheumatology for DAS28 & possible DMARDs. \n", - "- **Monitoring:** \n", - " - Dermatology follow‑up in 4β€―weeks; if flare worsens, consider systemic therapy (methotrexate) sooner. \n", - " - Rheuma consult by end of March. \n", - "- **Lifestyle:** \n", - " - Advise avoiding triggers: stress, alcohol, smoking cessation if desired. \n", - " - Safe sun exposure, sunscreen SPF β‰₯30. \n", - "- **Lab:** CBC, CMP, ESR, CRP baseline for future DMARD therapy. \n", - "- **Patient education:** Hand hygiene, avoid scratching, use lukewarm baths. \n", - "- **Follow‑up:** Return in 2β€―weeks if no improvement, or sooner if worsening pain/swelling.\n", - "\n", - "*Signature:* \n", - "Dr. Latasha Anderson, MD \n", - "Primary Care Physician\n", - "----------------------------------------\n", - "\n", - "File 3: 44767c13dcd9.jsonl\n", - "Number of documents: 10\n", - "\n", - "Generated text (showing first record):\n", - "Document 1:\n", - "[diagnosis]:\n", - "urinary tract infection\n", - "[patient_summary]:\n", - "I have to urinate a lot, but when I do, it hurts and I only get a little bit out. The urine is cloudy and bloody and sometimes smells bad. I also feel nauseous.\n", - "[patient_sampler]:\n", - "{'uuid': 'fc07c371-d23d-47be-843c-ebd959b97a03', 'locale': 'en_US', 'first_name': 'Ebony', 'last_name': 'Gay', 'middle_name': None, 'sex': 'Female', 'street_number': '3916', 'street_name': 'Yang Via', 'city': 'West Jaredshire', 'state': 'California', 'postcode': '60171', 'age': 32, 'birth_date': '1993-11-01', 'country': 'Uzbekistan', 'marital_status': 'divorced', 'education_level': 'associates', 'unit': '', 'occupation': 'Acupuncturist', 'phone_number': '+1-631-401-5078x77445', 'bachelors_field': 'no_degree'}\n", - "[doctor_sampler]:\n", - "{'uuid': '17fb8e0b-ed61-479b-b4a2-7a7c16ec045f', 'locale': 'en_US', 'first_name': 'Felicia', 'last_name': 'Rodriguez', 'middle_name': None, 'sex': 'Female', 'street_number': '407', 'street_name': 'Murphy Pass', 'city': 'East David', 'state': 'Washington', 'postcode': '92562', 'age': 106, 'birth_date': '1919-11-02', 'country': 'Belize', 'marital_status': 'separated', 'education_level': 'associates', 'unit': '', 'occupation': 'Teacher, early years/pre', 'phone_number': '(749)928-7601x637', 'bachelors_field': 'no_degree'}\n", - "[patient_id]:\n", - "PT-A908AC1C\n", - "[symptom_onset_date]:\n", - "2024-10-06\n", - "[date_of_visit]:\n", - "2024-11-01\n", - "[first_name]:\n", - "Ebony\n", - "[last_name]:\n", - "Gay\n", - "[dob]:\n", - "1993-11-01\n", - "[physician]:\n", - "Dr. Rodriguez\n", - "[physician_notes]:\n", - "**V/1 – Office Visit** \n", - "**Date:** 2024‑11‑01 \n", - "**Provider:** Dr. Felicia Rodriguez, MD \n", - "**Patient:** Ebony Gay \n", - "**MRN:** 0034789 \n", - "\n", - "**Subjective** \n", - "- **CC:** β€œFrequent urination, painful, only a little out, cloudy and bloody urine, odor off, nauseous.” \n", - "- **HPI:** Onset 2024‑10‑06. Urinary frequency ~8–10 times per day. Dysuria; painful stream; urine scanty. Cloudy, occasionally gross hematuria (visible streaks). Foul odor. Nausea with occasional light‑headedness, no vomiting. No fever, chills, flank pain, abdominal cramps, or vaginal discharge. No recent sexual activity; denies new partners. No history of kidney stones or prior UTIs; last UTI 4 years ago. \n", - "- **Medications:** None currently; no known drug allergies. \n", - "- **Past MDHX:** Hypertension, controlled; no diabetes or immuno‑suppression. \n", - "- **Social History:** Non‑smoker, occasional alcohol, no illicit drug use. \n", - "- **Review of Systems:** \n", - " - GU: Positive for dysuria, hematuria, frequency; negative for incontinence. \n", - " - HEENT: No headache. \n", - " - Cardiovascular: No chest pain. \n", - " - Pulmonary: No dyspnea. \n", - " - GI: Nausea, no vomiting, melena. \n", - " - Musculoskeletal: No pain. \n", - "\n", - "**Objective** \n", - "- **Vitals:** BP 128/76, HR 84, RR 16, Temp 98.2Β°F, SpOβ‚‚ 98% RA. \n", - "- **General:** Alert, oriented, mild discomfort. \n", - "- **Abdominal/Pelvic:** Soft, non‑tender, normal bladder palpation. No costovertebral angle tenderness. No masses. \n", - "- **Genitourinary:** External genitalia normal. Vaginal exam: mucous membranes intact, no lesions. No discharge. \n", - "- **Labs:** Pending urinalysis (UA) and urine culture. \n", - "\n", - "**Assessment** \n", - "- Acute cystitis/UTI (likely lower urinary tract).\n", - "- Differential: Urinary bladder infection, urethritis, early pyelonephritis, bladder stone, hematuria from other sources.\n", - "- No signs of systemic infection or obstruction.\n", - "\n", - "**Plan** \n", - "1. **Diagnostics:** \n", - " - Urinalysis with microscopy, urine dipstick for nitrites, leukocyte esterase, blood.\n", - " - Urine culture & sensitivity (inoculation, 24‑h incubation).\n", - " - Consider urine microscopy for RBC morphology.\n", - "2. **Therapeutics:** \n", - " - Empiric antibiotic: Nitrofurantoin 100β€―mg PO BID for 7β€―days (assuming no contraindications).\n", - " - If creatinine >1.5 or allergy, switch to trimethoprim‑sulfamethoxazole (TMP/SMX) 160/800β€―mg PO BID for 7β€―days.\n", - " - Analgesic: Ibuprofen 400β€―mg PO PRN for dysuria.\n", - "3. **Symptoms:** \n", - " - Drink >2β€―L water daily. Avoid bladder irritants (caffeine, alcohol). \n", - " - Sore precautions: Sitz baths, no tight pants, adequate perineal hygiene.\n", - " - Hydration as tolerated; consider anti‑emetics (ondansetron 4β€―mg PO PRN) if nausea worsens.\n", - "4. **Follow‑up:** \n", - " - Return in 48β€―h if symptoms worsen or persist; otherwise, POCT for UA results in office on next visit (2024‑11‑07). \n", - " - If pyelonephritis signs develop (fever >100.4Β°F, flank pain), immediate ED and IV antibiotics. \n", - "5. **Patient Education:** \n", - " - Discussed typical UTI presentation, importance of completing antibiotics, red‑flag symptoms (fever, flank pain, worsening hematuria). \n", - " - Reviewed contraception; no recommendations needed. \n", - "6. **Coordination:** \n", - " - EHR note entered, results to be flagged; pending UA/culture results to trigger alert. \n", - "\n", - "**Disposition:** Outpatient; instructed to call office if symptoms not improving by 2β€―days. \n", - "\n", - "**Signature:** \n", - "Dr. Felicia Rodriguez, MD, Family Medicine \n", - "(Phone: 555‑123‑4567)\n", - "----------------------------------------\n" - ] - } - ], - "source": [ - "print(\"=\" * 50)\n", - "print(\"Sample of generated documents:\")\n", - "print(\"=\" * 50)\n", - "\n", - "for i, df in enumerate(all_data_frames[:3]):\n", - " print(f\"\\nFile {i + 1}: {Path(output_files[i]).name}\")\n", - " print(f\"Number of documents: {len(df)}\")\n", - " print(\"\\nGenerated text (showing first record):\")\n", - " for j, row in enumerate(df.head(1).to_dict(orient=\"records\")):\n", - " print(f\"Document {j + 1}:\")\n", - " for key, value in row.items():\n", - " print(f\"[{key}]:\")\n", - " print(f\"{value}\")\n", - " print(\"-\" * 40)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Total records generated: 200\n" + ] } - ], - "metadata": { - "kernelspec": { - "display_name": "curator_env_fast", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" + ], + "source": [ + "if all_data_frames:\n", + " combined_df = pd.concat(all_data_frames, ignore_index=True)\n", + " print(f\"Total records generated: {len(combined_df)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "d512fed3", + "metadata": {}, + "source": [ + "Inspecting output .jsonl files content, we can find the generated text from Nvidia's LLM endpoint corresponding to the input prompt." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "1fbfc212", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "==================================================\n", + "Sample of generated documents:\n", + "==================================================\n", + "\n", + "File 1: 2a32c234df7d.jsonl\n", + "Number of documents: 10\n", + "\n", + "Generated text (showing first record):\n", + "Document 1:\n", + "[diagnosis]:\n", + "malaria\n", + "[patient_summary]:\n", + "I have a high fever, chills, and severe itching. I also have a headache, excessive sweating, nausea, and muscle aches.\n", + "[patient_sampler]:\n", + "{'uuid': '13a36ba1-5f8f-431c-86d1-372975b6c220', 'locale': 'en_US', 'first_name': 'Jeffrey', 'last_name': 'Watts', 'middle_name': None, 'sex': 'Male', 'street_number': '293', 'street_name': 'Harris Oval', 'city': 'New Jessica', 'state': 'Texas', 'postcode': '20375', 'age': 49, 'birth_date': '1976-05-09', 'country': 'American Samoa', 'marital_status': 'separated', 'education_level': 'secondary_education', 'unit': '', 'occupation': 'Educational psychologist', 'phone_number': '(484)575-5193', 'bachelors_field': 'no_degree'}\n", + "[doctor_sampler]:\n", + "{'uuid': '8b79d082-0a4e-47b0-b890-ba69ee9c2c6d', 'locale': 'en_US', 'first_name': 'Rachael', 'last_name': 'Cortez', 'middle_name': None, 'sex': 'Female', 'street_number': '8232', 'street_name': 'Pugh Manor', 'city': 'New Tracyborough', 'state': 'Wisconsin', 'postcode': '24123', 'age': 29, 'birth_date': '1996-12-18', 'country': 'New Zealand', 'marital_status': 'never_married', 'education_level': 'graduate', 'unit': '', 'occupation': 'Health visitor', 'phone_number': '795-875-2718', 'bachelors_field': 'stem'}\n", + "[patient_id]:\n", + "PT-E599ACBE\n", + "[symptom_onset_date]:\n", + "2024-03-17\n", + "[date_of_visit]:\n", + "2024-04-08\n", + "[first_name]:\n", + "Jeffrey\n", + "[last_name]:\n", + "Watts\n", + "[dob]:\n", + "1976-05-09\n", + "[physician]:\n", + "Dr. Cortez\n", + "[physician_notes]:\n", + "**Chief Complaint:** Fever, chills, itching, headache, nausea, muscle aches. \n", + "\n", + "**Date/Time:** 2024‑04‑08 10:12β€―am \n", + "\n", + "**Patient:** Jeffrey Watts (M30, known malaria case 3/17/24)\n", + "\n", + "**S:** \n", + "*c/o* high fever (39.2β€―Β°C/102.6β€―Β°F reported; temp noted 39β€―Β°C/102.2β€―Β°F), intense chills, generalized pruritus, severe frontal headache, profuse sweating, nausea (no vomiting), diffuse myalgias. \n", + "*No* dyspnea, chest pain, abdominal pain, rash except itching, or new joint pain. No recent travel deviation from last trip. \n", + "\n", + "**O:** \n", + "- Vitals: Tβ€―39.0β€―Β°C (102.2β€―Β°F), HRβ€―108β€―bpm, BPβ€―118/72β€―mmHg, RRβ€―18β€―bpm, SpOβ‚‚β€―98% RA. \n", + "- General: Alert, afebrile after 0.5β€―h after start of antimalarial therapy. \n", + "- HEENT: Pale conjunctivae, normal pupils, no meningeal signs. \n", + "- Cardiovascular: Regular rhythm, no murmurs. \n", + "- Respiratory: Clear to auscultation bilaterally. \n", + "- Abdomen: Soft, nontender, no hepatosplenomegaly. \n", + "- Skin: Diffuse pruritus, no maculopapular rash, no petechiae. \n", + "- Neurologic: GCS 15, cranial nerves intact, mild agitation from discomfort. \n", + "- Musculoskeletal: Diffuse muscle tenderness, no focal neuro deficits. \n", + "\n", + "**Labs:** \n", + "- CBC: WBCβ€―4.8 K/Β΅L, Hgbβ€―13.1β€―g/dL, Hctβ€―39%, Pltβ€―190 K/Β΅L. \n", + "- CMP: Normal electrolytes, ASTβ€―45 IU/L, ALTβ€―39 IU/L, BUNβ€―12 mg/dL, Crβ€―0.9β€―mg/dL. \n", + "- Malaria smear: 0.36% parasitemia (expired 2–3 days). \n", + "- Malaria PCR: Positive for *Plasmodium falciparum* 3‑day post‑therapy.\n", + "\n", + "**Assessment:** \n", + "1. *Plasmodium falciparum* malaria relapse – moderate parasite density, resolving after 3β€―days of artemether‑lumefantrine. \n", + "2. Febrile reaction to treatmentβ€”likely due to hot‐blood syndrome. \n", + "3. Symptomatic a‑septic itchingβ€”potential eosinophil release reaction. \n", + "\n", + "**Plan:** \n", + "- Continue artemether‑lumefantrine (total 3 doses, last dose 11:00β€―am). \n", + "- Add dapsone 100β€―mg PO BID for itching; counsel on rash risk. \n", + "- Continue acetaminophen 650β€―mg PO every 6β€―h PRN for fever/headache. \n", + "- Encourage oral hydration; advise oral rehydration salts. \n", + "- Re-check CBC, electrolytes, parasite smear in 48β€―h. \n", + "- Counsel on mosquito avoidance and prophylaxis if returning to endemic zone. \n", + "- SFA (safety follow‑up appointment) 2024‑04‑16; if any worsening (high fever >39.5β€―Β°C, seizures, AMS), call ED. \n", + "--- \n", + "\n", + "**Physician:** Dr. Rachael Cortez, MD \n", + "**Signature:** _____________________ Date: 2024‑04‑08\n", + "----------------------------------------\n", + "\n", + "File 2: 383a455f5a8d.jsonl\n", + "Number of documents: 10\n", + "\n", + "Generated text (showing first record):\n", + "Document 1:\n", + "[diagnosis]:\n", + "psoriasis\n", + "[patient_summary]:\n", + "I have a skin condition that causes dry, flaky patches on my skin. I also have joint pain and the skin on my knees and elbows is peeling off.\n", + "[patient_sampler]:\n", + "{'uuid': '9008b8fa-27cd-49e4-8a63-2a6bc2acbace', 'locale': 'en_US', 'first_name': 'William', 'last_name': 'Johnson', 'middle_name': None, 'sex': 'Male', 'street_number': '12058', 'street_name': 'John Corner', 'city': 'Calderonport', 'state': 'Alabama', 'postcode': '79215', 'age': 28, 'birth_date': '1997-09-13', 'country': 'Djibouti', 'marital_status': 'married_present', 'education_level': 'secondary_education', 'unit': '', 'occupation': 'Field trials officer', 'phone_number': '450-594-0263x9360', 'bachelors_field': 'no_degree'}\n", + "[doctor_sampler]:\n", + "{'uuid': 'f04c32f2-5064-47a0-baa2-47a3b9e03c14', 'locale': 'en_US', 'first_name': 'Latasha', 'last_name': 'Anderson', 'middle_name': None, 'sex': 'Female', 'street_number': '25467', 'street_name': 'Petersen Ferry', 'city': 'Solomonfurt', 'state': 'Massachusetts', 'postcode': '99698', 'age': 87, 'birth_date': '1939-02-10', 'country': 'Jordan', 'marital_status': 'widowed', 'education_level': 'bachelors', 'unit': '', 'occupation': 'Writer', 'phone_number': '001-659-997-5484', 'bachelors_field': 'business'}\n", + "[patient_id]:\n", + "PT-B50655DA\n", + "[symptom_onset_date]:\n", + "2024-02-07\n", + "[date_of_visit]:\n", + "2024-02-26\n", + "[first_name]:\n", + "William\n", + "[last_name]:\n", + "Johnson\n", + "[dob]:\n", + "1997-09-13\n", + "[physician]:\n", + "Dr. Anderson\n", + "[physician_notes]:\n", + "**Dr. Latasha Anderson, MD** \n", + "**Clinic Visit Notes – 2024‑02‑26** \n", + "Patient: William Johnson, MRN 0045321 \n", + "\n", + "---\n", + "\n", + "### S – Subjective \n", + "- **Chief Complaint:** β€œDry, flaky patches on skin, peeling at knees/elbows, joint pain.” \n", + "- **History of Present Illness:** \n", + " - Psoriasis first noted 2024‑02‑07: erythematous plaques, scaling on scalp, elbows, knees. \n", + " - Current flare: increased scaling, desquamation. \n", + " - Joint pain: bilateral knees, mild swelling, stiffness >30β€―min morning. \n", + " - No systemic symptoms (fever, weight loss). \n", + "- **Past Meds:** Topical steroid (clobetasol 0.05%) once daily, OTC ibuprofen PRN. \n", + "- **Allergies:** NKDA. \n", + "- **Social:** Non‑smoker, 3-4 drinks/week, works as accountant (desk job). \n", + "- **Family hx:** Mother has psoriasis, mother diagnosed with rheumatoid arthritis. \n", + "- **Review of Systems:** \n", + " - **Skin:** Paronychia, no nail pitting noted now. \n", + " - **Musculoskeletal:** Pain in knees, no swelling elsewhere. \n", + " - **GI, Cardiopulm, Neuro:** Negative. \n", + "\n", + "### O – Objective \n", + "- **Vital Signs:** BP 122/78, HR 78, Temp 98.6β€―Β°F, RR 16, SpOβ‚‚ 98% RA. \n", + "- **Physical Exam:**\n", + " - **Skin:** Extensive plaques on scalp (secondary alopecia), elbows, knees – approx. 15% BSA. Scaling >30β€―mm shagreen, fissuring in cubital fossa, mild erythema around lesions. No active ulcers. \n", + " - **Nails:** Mild pitting on thumb nails, onycholysis not obvious now. \n", + " - **Joints:** Bilateral knee tenderness, 1‑2β€―mm effusion. Range of motion preserved, mild swelling. No redness or warmth. \n", + "- **Other systems:** Normal.\n", + "\n", + "### A – Assessment \n", + "1. **Plaque psoriasis – moderate severity (BSA 10‑20%)** – worsening flare. \n", + "2. **Psoriatic arthritis – oligo‑arthralgia (knees)** – likely early psoriatic arthritis (based on joint pain, swelling, and psoriasis). \n", + "3. **No signs of infection or systemic disease.**\n", + "\n", + "### P – Plan \n", + "- **Skin:** \n", + " - Continue clobetasol 0.05% twice daily on erosive lesions. \n", + " - Add calcipotriene 0.005% ointment nightly. \n", + " - Start low‑dose oral prednisone 5β€―mg daily for 1β€―week, tapering over 2β€―weeks. \n", + " - Educate on moisturizers (ceramide‑based) 4Γ—/day. \n", + "- **Joint pain:** \n", + " - NSAID trial: naproxen 250β€―mg BID for 2β€―weeks. \n", + " - If inadequate relief or intolerant β†’ refer to rheumatology for DAS28 & possible DMARDs. \n", + "- **Monitoring:** \n", + " - Dermatology follow‑up in 4β€―weeks; if flare worsens, consider systemic therapy (methotrexate) sooner. \n", + " - Rheuma consult by end of March. \n", + "- **Lifestyle:** \n", + " - Advise avoiding triggers: stress, alcohol, smoking cessation if desired. \n", + " - Safe sun exposure, sunscreen SPF β‰₯30. \n", + "- **Lab:** CBC, CMP, ESR, CRP baseline for future DMARD therapy. \n", + "- **Patient education:** Hand hygiene, avoid scratching, use lukewarm baths. \n", + "- **Follow‑up:** Return in 2β€―weeks if no improvement, or sooner if worsening pain/swelling.\n", + "\n", + "*Signature:* \n", + "Dr. Latasha Anderson, MD \n", + "Primary Care Physician\n", + "----------------------------------------\n", + "\n", + "File 3: 44767c13dcd9.jsonl\n", + "Number of documents: 10\n", + "\n", + "Generated text (showing first record):\n", + "Document 1:\n", + "[diagnosis]:\n", + "urinary tract infection\n", + "[patient_summary]:\n", + "I have to urinate a lot, but when I do, it hurts and I only get a little bit out. The urine is cloudy and bloody and sometimes smells bad. I also feel nauseous.\n", + "[patient_sampler]:\n", + "{'uuid': 'fc07c371-d23d-47be-843c-ebd959b97a03', 'locale': 'en_US', 'first_name': 'Ebony', 'last_name': 'Gay', 'middle_name': None, 'sex': 'Female', 'street_number': '3916', 'street_name': 'Yang Via', 'city': 'West Jaredshire', 'state': 'California', 'postcode': '60171', 'age': 32, 'birth_date': '1993-11-01', 'country': 'Uzbekistan', 'marital_status': 'divorced', 'education_level': 'associates', 'unit': '', 'occupation': 'Acupuncturist', 'phone_number': '+1-631-401-5078x77445', 'bachelors_field': 'no_degree'}\n", + "[doctor_sampler]:\n", + "{'uuid': '17fb8e0b-ed61-479b-b4a2-7a7c16ec045f', 'locale': 'en_US', 'first_name': 'Felicia', 'last_name': 'Rodriguez', 'middle_name': None, 'sex': 'Female', 'street_number': '407', 'street_name': 'Murphy Pass', 'city': 'East David', 'state': 'Washington', 'postcode': '92562', 'age': 106, 'birth_date': '1919-11-02', 'country': 'Belize', 'marital_status': 'separated', 'education_level': 'associates', 'unit': '', 'occupation': 'Teacher, early years/pre', 'phone_number': '(749)928-7601x637', 'bachelors_field': 'no_degree'}\n", + "[patient_id]:\n", + "PT-A908AC1C\n", + "[symptom_onset_date]:\n", + "2024-10-06\n", + "[date_of_visit]:\n", + "2024-11-01\n", + "[first_name]:\n", + "Ebony\n", + "[last_name]:\n", + "Gay\n", + "[dob]:\n", + "1993-11-01\n", + "[physician]:\n", + "Dr. Rodriguez\n", + "[physician_notes]:\n", + "**V/1 – Office Visit** \n", + "**Date:** 2024‑11‑01 \n", + "**Provider:** Dr. Felicia Rodriguez, MD \n", + "**Patient:** Ebony Gay \n", + "**MRN:** 0034789 \n", + "\n", + "**Subjective** \n", + "- **CC:** β€œFrequent urination, painful, only a little out, cloudy and bloody urine, odor off, nauseous.” \n", + "- **HPI:** Onset 2024‑10‑06. Urinary frequency ~8–10 times per day. Dysuria; painful stream; urine scanty. Cloudy, occasionally gross hematuria (visible streaks). Foul odor. Nausea with occasional light‑headedness, no vomiting. No fever, chills, flank pain, abdominal cramps, or vaginal discharge. No recent sexual activity; denies new partners. No history of kidney stones or prior UTIs; last UTI 4 years ago. \n", + "- **Medications:** None currently; no known drug allergies. \n", + "- **Past MDHX:** Hypertension, controlled; no diabetes or immuno‑suppression. \n", + "- **Social History:** Non‑smoker, occasional alcohol, no illicit drug use. \n", + "- **Review of Systems:** \n", + " - GU: Positive for dysuria, hematuria, frequency; negative for incontinence. \n", + " - HEENT: No headache. \n", + " - Cardiovascular: No chest pain. \n", + " - Pulmonary: No dyspnea. \n", + " - GI: Nausea, no vomiting, melena. \n", + " - Musculoskeletal: No pain. \n", + "\n", + "**Objective** \n", + "- **Vitals:** BP 128/76, HR 84, RR 16, Temp 98.2Β°F, SpOβ‚‚ 98% RA. \n", + "- **General:** Alert, oriented, mild discomfort. \n", + "- **Abdominal/Pelvic:** Soft, non‑tender, normal bladder palpation. No costovertebral angle tenderness. No masses. \n", + "- **Genitourinary:** External genitalia normal. Vaginal exam: mucous membranes intact, no lesions. No discharge. \n", + "- **Labs:** Pending urinalysis (UA) and urine culture. \n", + "\n", + "**Assessment** \n", + "- Acute cystitis/UTI (likely lower urinary tract).\n", + "- Differential: Urinary bladder infection, urethritis, early pyelonephritis, bladder stone, hematuria from other sources.\n", + "- No signs of systemic infection or obstruction.\n", + "\n", + "**Plan** \n", + "1. **Diagnostics:** \n", + " - Urinalysis with microscopy, urine dipstick for nitrites, leukocyte esterase, blood.\n", + " - Urine culture & sensitivity (inoculation, 24‑h incubation).\n", + " - Consider urine microscopy for RBC morphology.\n", + "2. **Therapeutics:** \n", + " - Empiric antibiotic: Nitrofurantoin 100β€―mg PO BID for 7β€―days (assuming no contraindications).\n", + " - If creatinine >1.5 or allergy, switch to trimethoprim‑sulfamethoxazole (TMP/SMX) 160/800β€―mg PO BID for 7β€―days.\n", + " - Analgesic: Ibuprofen 400β€―mg PO PRN for dysuria.\n", + "3. **Symptoms:** \n", + " - Drink >2β€―L water daily. Avoid bladder irritants (caffeine, alcohol). \n", + " - Sore precautions: Sitz baths, no tight pants, adequate perineal hygiene.\n", + " - Hydration as tolerated; consider anti‑emetics (ondansetron 4β€―mg PO PRN) if nausea worsens.\n", + "4. **Follow‑up:** \n", + " - Return in 48β€―h if symptoms worsen or persist; otherwise, POCT for UA results in office on next visit (2024‑11‑07). \n", + " - If pyelonephritis signs develop (fever >100.4Β°F, flank pain), immediate ED and IV antibiotics. \n", + "5. **Patient Education:** \n", + " - Discussed typical UTI presentation, importance of completing antibiotics, red‑flag symptoms (fever, flank pain, worsening hematuria). \n", + " - Reviewed contraception; no recommendations needed. \n", + "6. **Coordination:** \n", + " - EHR note entered, results to be flagged; pending UA/culture results to trigger alert. \n", + "\n", + "**Disposition:** Outpatient; instructed to call office if symptoms not improving by 2β€―days. \n", + "\n", + "**Signature:** \n", + "Dr. Felicia Rodriguez, MD, Family Medicine \n", + "(Phone: 555‑123‑4567)\n", + "----------------------------------------\n" + ] } + ], + "source": [ + "print(\"=\" * 50)\n", + "print(\"Sample of generated documents:\")\n", + "print(\"=\" * 50)\n", + "\n", + "for i, df in enumerate(all_data_frames[:3]):\n", + " print(f\"\\nFile {i + 1}: {Path(output_files[i]).name}\")\n", + " print(f\"Number of documents: {len(df)}\")\n", + " print(\"\\nGenerated text (showing first record):\")\n", + " for j, row in enumerate(df.head(1).to_dict(orient=\"records\")):\n", + " print(f\"Document {j + 1}:\")\n", + " for key, value in row.items():\n", + " print(f\"[{key}]:\")\n", + " print(f\"{value}\")\n", + " print(\"-\" * 40)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "curator_env_fast", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 5 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.py b/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.py index 71131275f5..92dc010a32 100644 --- a/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.py +++ b/tutorials/synthetic/nemo_data_designer/ndd_data_generation_example.py @@ -294,7 +294,7 @@ def main() -> None: # noqa: PLR0915 # If no remote provider specified, start a local InferenceServer if args.provider is None: - from nemo_curator.backends.experimental.utils import get_available_cpu_gpu_resources + from nemo_curator.backends.utils import get_available_cpu_gpu_resources from nemo_curator.core.serve import InferenceModelConfig, InferenceServer _, num_gpus = get_available_cpu_gpu_resources() diff --git a/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_high_quality_example_pipeline.py b/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_high_quality_example_pipeline.py index 7f7a6e49be..ac95944418 100644 --- a/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_high_quality_example_pipeline.py +++ b/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_high_quality_example_pipeline.py @@ -229,10 +229,7 @@ def main() -> None: # noqa: C901, PLR0912, PLR0915 try: from transformers import AutoTokenizer except ImportError as e: - msg = ( - "The 'transformers' package is required for tokenizer support. " - "Install it with: pip install transformers" - ) + msg = "The 'transformers' package is required for tokenizer support. Install it with: pip install transformers" raise ImportError(msg) from e args.tokenizer = AutoTokenizer.from_pretrained(args.tokenizer) args.hf_token = os.environ.get("HF_TOKEN", "") @@ -398,7 +395,9 @@ def main() -> None: # noqa: C901, PLR0912, PLR0915 ### Extract high quality data pipeline.add_stage( Filter( - filter_fn=lambda x: x is not None and not (isinstance(x, float) and math.isnan(x)) and int(x) > BUCKETED_RESULTS_THRESHOLD, + filter_fn=lambda x: x is not None + and not (isinstance(x, float) and math.isnan(x)) + and int(x) > BUCKETED_RESULTS_THRESHOLD, filter_field="bucketed_results", ), ) diff --git a/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_low_quality_example_pipeline.py b/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_low_quality_example_pipeline.py index 6e55edea70..c964e8d335 100644 --- a/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_low_quality_example_pipeline.py +++ b/tutorials/synthetic/nemotron_cc/nemo_data_designer/nemotron_cc_sdg_low_quality_example_pipeline.py @@ -182,10 +182,7 @@ def main() -> None: # noqa: C901, PLR0912, PLR0915 try: from transformers import AutoTokenizer except ImportError as e: - msg = ( - "The 'transformers' package is required for tokenizer support. " - "Install it with: pip install transformers" - ) + msg = "The 'transformers' package is required for tokenizer support. Install it with: pip install transformers" raise ImportError(msg) from e args.tokenizer = AutoTokenizer.from_pretrained(args.tokenizer) args.hf_token = os.environ.get("HF_TOKEN", "") @@ -314,7 +311,9 @@ def main() -> None: # noqa: C901, PLR0912, PLR0915 ### Filter low quality data pipeline.add_stage( Filter( - filter_fn=lambda x: x is not None and not (isinstance(x, float) and math.isnan(x)) and int(x) <= BUCKETED_RESULTS_THRESHOLD, + filter_fn=lambda x: x is not None + and not (isinstance(x, float) and math.isnan(x)) + and int(x) <= BUCKETED_RESULTS_THRESHOLD, filter_field="bucketed_results", ), )