Add stable retrieval SDG recipe APIs

plugins/data-designer-retrieval-sdg/README.md

@@ -107,11 +107,13 @@ data-designer-retrieval-sdg generate \
     --dataset-name my_retrieval_run \
     --buffer-size 200 \
     --resume if_possible \
-    --num-pairs 7
+    --num-pairs 10
 ```
 
 Generation writes DataDesigner artifacts under `--artifact-path` and exports a
-single JSONL file to `--output-dir`.
+single JSONL file to `--output-dir`. The default profile uses
+`nvidia/nemotron-3-ultra-550b-a55b` for generation and
+`nvidia/nemotron-3-embed-1b` for embedding deduplication.
 
 ### Convert to training format
 
@@ -120,8 +122,10 @@ data-designer-retrieval-sdg convert ./generated_output/my_retrieval_run.jsonl \
     --corpus-id my_corpus
 ```
 
-Legacy `generated_batch*.json` directories remain supported by `convert`, but
-`generate` no longer writes per-batch JSON files. The old manual restart flags
+Legacy `generated_batch*.json` directories remain supported by `convert`, but a
+directory containing more than one generated-data format class is rejected as
+ambiguous. Pass the exact JSONL, JSON, or parquet file in that case. `generate`
+no longer writes per-batch JSON files. The old manual restart flags
 `--batch-size`, `--start-batch-index`, and `--end-batch-index` were removed
 because DataDesigner now owns checkpointing through `--buffer-size` and
 `--resume`. For very large corpora, keep input partitions sized for
@@ -131,18 +135,48 @@ create/export path.
 ### Use as a library
 
 ```python
+from pathlib import Path
+
 from data_designer_retrieval_sdg import (
     DocumentChunkerSeedSource,
-    build_qa_generation_pipeline,
+    GenerationPipelineConfig,
+    GenerationRunConfig,
+    run_conversion,
+    run_generation,
 )
 
 seed_source = DocumentChunkerSeedSource(
     path="./docs",
     file_extensions=[".txt", ".md"],
 )
-config_builder = build_qa_generation_pipeline(seed_source)
+generation = run_generation(
+    GenerationRunConfig(
+        seed_source=seed_source,
+        output_dir=Path("./generated"),
+        artifact_path=Path("./artifacts"),
+        dataset_name="my_retrieval_run",
+        pipeline=GenerationPipelineConfig(
+            num_pairs=10,
+            min_hops=1,
+            max_hops=3,
+        ),
+    )
+)
+conversion = run_conversion(
+    input_path=str(generation.output_path),
+    corpus_id="my_corpus",
+)
+assert conversion.train_file is not None
 ```
 
+The generation result contains the exact exported JSONL path, resolved Data
+Designer dataset path and name, record count, and producer version. Conversion
+returns the generated train, validation, corpus, and evaluation paths plus
+example counts. `GenerationRunConfig` and `GenerationPipelineConfig` reject
+unknown fields so recipe adapters cannot silently pass misspelled settings.
+`GenerationRunConfig.to_redacted_dict()` returns every effective Python API
+setting while replacing provider credentials and authorization headers.
+
 ## Plugin configuration examples
 
 ### `embedding-dedup` column
@@ -178,5 +212,10 @@ seed_source = DocumentChunkerSeedSource(
 )
 ```
 
+Every emitted row includes a normalized corpus-relative `source_id`. Conversion
+uses it for chunk lookup so same-basename documents in different directories do
+not overwrite one another. Legacy records without `source_id` retain their full
+normalized `file_name` as the lookup key.
+
 Output schema (one record per row): `file_name`, `text`, `chunks`,
 `sections_structured`, `bundle_id`, `bundle_members`, `is_multi_doc`.

plugins/data-designer-retrieval-sdg/src/data_designer_retrieval_sdg/__init__.py

@@ -15,8 +15,11 @@
 helpers.
 """
 
+from importlib import import_module
+from typing import Any
+
 from data_designer_retrieval_sdg.config import EmbeddingDedupColumnConfig
-from data_designer_retrieval_sdg.pipeline import build_qa_generation_pipeline
+from data_designer_retrieval_sdg.pipeline import build_model_providers, build_qa_generation_pipeline
 from data_designer_retrieval_sdg.postprocess import (
     filter_qa_pairs_by_quality,
     load_positive_docs_with_modality,
@@ -27,8 +30,39 @@
 __all__ = [
     "DocumentChunkerSeedSource",
     "EmbeddingDedupColumnConfig",
+    "ConversionResult",
+    "GenerationResult",
+    "GenerationPipelineConfig",
+    "GenerationRunConfig",
+    "GenerationPreviewResult",
+    "build_model_providers",
     "build_qa_generation_pipeline",
     "filter_qa_pairs_by_quality",
     "load_positive_docs_with_modality",
     "postprocess_retriever_data",
+    "preview_generation",
+    "run_conversion",
+    "run_generation",
 ]
+
+_LAZY_EXPORTS = {
+    "ConversionResult": ("data_designer_retrieval_sdg.convert", "ConversionResult"),
+    "GenerationPreviewResult": ("data_designer_retrieval_sdg.generation", "GenerationPreviewResult"),
+    "GenerationResult": ("data_designer_retrieval_sdg.generation", "GenerationResult"),
+    "GenerationPipelineConfig": ("data_designer_retrieval_sdg.run_config", "GenerationPipelineConfig"),
+    "GenerationRunConfig": ("data_designer_retrieval_sdg.run_config", "GenerationRunConfig"),
+    "preview_generation": ("data_designer_retrieval_sdg.generation", "preview_generation"),
+    "run_conversion": ("data_designer_retrieval_sdg.convert", "run_conversion"),
+    "run_generation": ("data_designer_retrieval_sdg.generation", "run_generation"),
+}
+
+
+def __getattr__(name: str) -> Any:
+    """Load orchestration APIs lazily so Data Designer can discover plugins safely."""
+    try:
+        module_name, attribute_name = _LAZY_EXPORTS[name]
+    except KeyError as exc:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from exc
+    value = getattr(import_module(module_name), attribute_name)
+    globals()[name] = value
+    return value

plugins/data-designer-retrieval-sdg/src/data_designer_retrieval_sdg/chunking.py

@@ -15,6 +15,7 @@
 import json
 import logging
 import math
+import posixpath
 import re
 from collections import defaultdict, deque
 from pathlib import Path
@@ -27,6 +28,21 @@
 logger = logging.getLogger(__name__)
 
 
+def normalize_source_id(source_id: str) -> str:
+    """Normalize a source identifier without discarding path components."""
+    return posixpath.normpath(source_id.replace("\\", "/"))
+
+
+def build_source_id(source_ids: list[str]) -> str:
+    """Build a stable identifier for one source or a multi-source bundle."""
+    normalized = sorted(normalize_source_id(source_id) for source_id in source_ids)
+    if not normalized:
+        return ""
+    if len(normalized) == 1:
+        return normalized[0]
+    return hashlib.md5("||".join(normalized).encode()).hexdigest()[:16]
+
+
 def load_multi_doc_manifest(manifest_path: Path | None) -> list[list[str]]:
     """Load a multi-doc manifest file.
 
@@ -92,7 +108,7 @@ def build_bundle_id(bundle_members: list[str]) -> str:
     """
     if not bundle_members:
         return ""
-    normalized = "||".join(sorted(str(member) for member in bundle_members))
+    normalized = "||".join(sorted(normalize_source_id(str(member)) for member in bundle_members))
     return hashlib.md5(normalized.encode()).hexdigest()