feat(sdk): stage extension packages at wheel build time

packages/data_designer_nemo/src/data_designer_nemo/fileset_filesystem_provider.py

@@ -13,9 +13,9 @@
     SeedReaderFileSystemContext,
 )
 from data_designer_nemo.filesystem import make_filesystem
+from filesets import FilesetFileSystem, FilesetPathError, build_fileset_ref, parse_fileset_ref
 from fsspec.implementations.dirfs import DirFileSystem
 from nemo_platform import AsyncNeMoPlatform, NeMoPlatform
-from nemo_platform.filesets import FilesetFileSystem, FilesetPathError, build_fileset_ref, parse_fileset_ref
 from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient
 
 

packages/data_designer_nemo/src/data_designer_nemo/filesystem.py

@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from data_designer_nemo.sdk_translation import async_to_sync_sdk
+from filesets import FilesetFileSystem
 from nemo_platform import AsyncNeMoPlatform, NeMoPlatform
-from nemo_platform.filesets import FilesetFileSystem
 from nemo_platform_plugin.client.adapter import client_from_platform
 from nemo_platform_plugin.files.client import FilesClient
 

packages/data_designer_nemo/src/data_designer_nemo/seed.py

@@ -10,8 +10,8 @@
 from data_designer_nemo.fileset_file_seed_source import FilesetFileSeedSource
 from data_designer_nemo.fileset_filesystem_provider import is_local_directory
 from data_designer_nemo.secret_resolver import validate_secret
+from filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref
 from nemo_platform import AsyncNeMoPlatform
-from nemo_platform.filesets import FilesetPathError, build_fileset_ref, parse_fileset_ref
 from nemo_platform_plugin.client.adapter import client_from_platform
 from nemo_platform_plugin.client.errors import NotFoundError, PermissionDeniedError
 from nemo_platform_plugin.files.client import AsyncFilesClient

packages/filesets/pyproject.toml

@@ -31,6 +31,7 @@ packages = ["src/filesets"]
 [tool.vendor-package]
 package = "filesets"
 package_root = "packages/filesets"
+sdk_include_mode = "source-package"
 target_sdk_module = "filesets"
 included_paths = [
     "**/*.py",

packages/filesets/src/filesets/resources.py

@@ -12,10 +12,16 @@
 from dataclasses import dataclass
 from functools import cached_property
 from pathlib import PurePath
-from typing import Protocol, runtime_checkable
+from typing import Any, Protocol, runtime_checkable
 
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
 from fsspec.core import has_magic
+from nemo_platform.resources.files.files import (
+    AsyncFilesResource as GeneratedAsyncFilesResource,
+)
+from nemo_platform.resources.files.files import (
+    FilesResource as GeneratedFilesResource,
+)
 from nemo_platform.resources.files.filesets import AsyncFilesetsResource, FilesetsResource
 from nemo_platform.resources.files.otlp.otlp import AsyncOtlpResource, OtlpResource
 from nemo_platform_plugin.files.client import AsyncFilesClient, FilesClient
@@ -150,6 +156,7 @@ def __init__(
         # Retain the platform client so the generated fileset/otlp sub-resources
         # (which speak to the platform client, not the FilesClient) can be exposed.
         self._platform_client = client
+        self._generated_files = GeneratedFilesResource(client)
         self._async_client = async_files_client
         if files_client is not None:
             self._client = files_client
@@ -163,6 +170,9 @@ def client(self) -> FilesClient:
         """Access the underlying FilesClient for direct API calls."""
         return self._client
 
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._generated_files, name)
+
     @cached_property
     def filesets(self) -> FilesetsResource:
         """Fileset entity CRUD (create/list/get/update/delete) via the generated SDK resource."""
@@ -693,6 +703,7 @@ def __init__(self, client, *, files_client: AsyncFilesClient | None = None) -> N
         # Retain the platform client so the generated fileset/otlp sub-resources
         # (which speak to the platform client, not the FilesClient) can be exposed.
         self._platform_client = client
+        self._generated_files = GeneratedAsyncFilesResource(client)
         if files_client is not None:
             self._client = files_client
         else:
@@ -705,6 +716,9 @@ def client(self) -> AsyncFilesClient:
         """Access the underlying AsyncFilesClient for direct API calls."""
         return self._client
 
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._generated_files, name)
+
     @cached_property
     def filesets(self) -> AsyncFilesetsResource:
         """Fileset entity CRUD (create/list/get/update/delete) via the generated SDK resource."""

packages/models/pyproject.toml

@@ -27,6 +27,7 @@ dev = []
 [tool.vendor-package]
 package = "models"
 package_root = "packages/models"
+sdk_include_mode = "source-package"
 target_sdk_module = "models"
 included_paths = [
   "**/*.py",

packages/models/tests/test_client.py

@@ -449,8 +449,8 @@ def test_wait_for_openai_model_bounds_sleep_to_remaining_timeout(sdk):
             "get",
             side_effect=_not_found_error(),
         ),
-        patch("nemo_platform.models.resources.time.time", side_effect=clock.time),
-        patch("nemo_platform.models.resources.time.sleep", side_effect=clock.sleep),
+        patch("models.resources.time.time", side_effect=clock.time),
+        patch("models.resources.time.sleep", side_effect=clock.sleep),
     ):
         with pytest.raises(TimeoutError, match="OpenAI model ws/model-a not available"):
             sdk.models.wait_for_openai_model("model-a", workspace="ws", timeout=1.25, poll_interval=5)
@@ -631,8 +631,8 @@ async def test_async_wait_for_openai_model_bounds_sleep_to_remaining_timeout(asy
 
     with (
         patch.object(async_sdk.inference.gateway.openai.v1.models, "get", mock_get),
-        patch("nemo_platform.models.resources.time.time", side_effect=clock.time),
-        patch("nemo_platform.models.resources.asyncio.sleep", side_effect=clock.async_sleep),
+        patch("models.resources.time.time", side_effect=clock.time),
+        patch("models.resources.asyncio.sleep", side_effect=clock.async_sleep),
     ):
         with pytest.raises(TimeoutError, match="OpenAI model ws/model-a not available"):
             await async_sdk.models.wait_for_openai_model("model-a", workspace="ws", timeout=1.25, poll_interval=5)

packages/nemo_evaluator_sdk/pyproject.toml

@@ -116,6 +116,7 @@ addopts = ["--import-mode=importlib"]
 [tool.vendor-package]
 package = "nemo_evaluator_sdk"
 package_root = "packages/nemo_evaluator_sdk"
+sdk_include_mode = "source-package"
 source_module = "nemo_evaluator_sdk"
 target_sdk_module = "beta.evaluator"
 sdk_optional_dependencies_name = "nemo-evaluator-sdk"

packages/nemo_evaluator_sdk/tests/agent_eval/test_evidence.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import asyncio
+import importlib
 import json
 import os
 import time
@@ -372,9 +373,10 @@ async def test_trace_handle_exposes_typed_tool_evidence_and_retains_modeled_fiel
 def test_observation_models_are_exported_from_source_and_vendored_values_packages() -> None:
     from nemo_evaluator_sdk.values import Observation as SourceObservation
     from nemo_evaluator_sdk.values import ObservationResult as SourceObservationResult
-    from nemo_platform.beta.evaluator.values import Observation as VendoredObservation
-    from nemo_platform.beta.evaluator.values import ObservationResult as VendoredObservationResult
 
+    vendored_values = importlib.import_module("nemo_platform.beta.evaluator.values")
+    VendoredObservation = vendored_values.Observation
+    VendoredObservationResult = vendored_values.ObservationResult
     assert SourceObservation.__name__ == VendoredObservation.__name__ == "Observation"
     assert SourceObservationResult.__name__ == VendoredObservationResult.__name__ == "ObservationResult"
 

packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_vendored_import.py

@@ -8,20 +8,16 @@
 import importlib
 from pathlib import Path
 
-from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.providers.compose import (
-    ComposeCleanupError,
-    ComposeCommandResult,
-    ComposeServiceTopology,
-    ComposeTeardownContext,
-    DockerComposeSandboxProvider,
-    ProgressCallback,
-    PullPolicy,
-    TeardownHook,
-)
-
 
 def test_vendored_compose_public_imports_are_constructible_without_docker(tmp_path: Path) -> None:
     """The vendored public Compose façade remains importable without Docker."""
+    compose = importlib.import_module("nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.providers.compose")
+    ComposeCleanupError = compose.ComposeCleanupError
+    ComposeCommandResult = compose.ComposeCommandResult
+    ComposeServiceTopology = compose.ComposeServiceTopology
+    ComposeTeardownContext = compose.ComposeTeardownContext
+    DockerComposeSandboxProvider = compose.DockerComposeSandboxProvider
+
     for cls in (
         ComposeCleanupError,
         ComposeCommandResult,
@@ -31,9 +27,9 @@ def test_vendored_compose_public_imports_are_constructible_without_docker(tmp_pa
     ):
         assert getattr(importlib.import_module(cls.__module__), cls.__name__) is cls
 
-    assert ProgressCallback is not None
-    assert PullPolicy is not None
-    assert TeardownHook is not None
+    assert compose.ProgressCallback is not None
+    assert compose.PullPolicy is not None
+    assert compose.TeardownHook is not None
 
     topology = ComposeServiceTopology("agent", frozenset({"agent"}))
     command_result = ComposeCommandResult(("docker", "compose", "ps"), 0, "", "")

packages/nemo_platform/pyproject.toml

@@ -126,6 +126,7 @@ files-service = [
   "ngcsdk>=4.9.10",
   "duckdb>=1.1.3",
   "pandas>=1.5.3",
+  "filesets",
   "opentelemetry-proto>=1.28.2",
   "aioboto3>=15.5.0",
   "types-aioboto3[s3]>=15.5.0",
@@ -348,24 +349,24 @@ nemo-platform-plugin = [
 
 # Generated from [tool.bundle-package]; do not edit by hand.
 nemo-platform-sdk = [
-  "httpx>=0.23.0, <1",
-  "pydantic>=2.0.0,<3",
-  "typing-extensions>=4.14, <5",
-  "anyio>=4.0.0,<5",
-  "distro>=1.7.0, <2",
-  "sniffio",
   "nemo-platform-plugin",
   "typer>=0.20.0",
   "rich>=13.7.1",
   "prompt_toolkit>=3.0.0",
   "requests>=2.31.0",
+  "pydantic>=2.0.0,<3",
   "pyyaml>=6.0.0",
   "docker>=7.0.0",
   "ngcsdk>=4.8.2",
   "nvidia-ml-py>=13.0.0",
   "psutil>=5.9.0",
+  "httpx>=0.23.0,<1",
   "openai",
+  "anyio>=4.0.0,<5",
   "fsspec>=2023.1.0",
+  "typing-extensions>=4.14, <5",
+  "distro>=1.7.0, <2",
+  "sniffio",
 ]
 
 # Generated from [tool.bundle-package]; do not edit by hand.
@@ -459,9 +460,9 @@ services = [
   "nmp-common",
   "pyleak>=0.1.0",
   "rich>=14.1.0",
-  "nemo-platform[platform-seed-service]",
   "nemo-platform[core-service]",
   "nemo-platform[studio-service]",
+  "nemo-platform[platform-seed-service]",
   "nemo-platform[intake-service]",
   "nemo-platform[hello-world-service]",
   "nemo-platform[guardrails-service]",
@@ -592,6 +593,8 @@ safe-synthesizer = "nemo_safe_synthesizer_plugin.skills:get_skills_path"
 nemo-platform-sdk = { workspace = true }
 nemo-platform-ext = { workspace = true }
 nemo-evaluator-sdk = { workspace = true }
+models = { workspace = true }
+filesets = { workspace = true }
 nmp-build-tools = { workspace = true }
 
 [tool.hatch.version]
@@ -628,6 +631,9 @@ only-include = ["_empty"]
 # Library packages
 nmp-common = { source = "../../packages/nmp_common/src/nmp/common", module = "nmp/common", inherit = { "entry-points" = ["nemo.*"] } }
 nemo-platform-plugin = { source = "../../packages/nemo_platform_plugin/src/nemo_platform_plugin", module = "nemo_platform_plugin" }
+nemo-platform-ext = { source = "../../packages/nemo_platform_ext/src/nemo_platform_ext", module = "nemo_platform_ext", deps_group = "nemo-platform-sdk", force_include = { "../../../../docs" = "nemo_platform_ext/cli/docs" } }
+models = { source = "../../packages/models/src/models", module = "models", deps_group = "nemo-platform-sdk" }
+filesets = { source = "../../packages/filesets/src/filesets", module = "filesets", deps_group = "nemo-platform-sdk" }
 nemo-evaluator-sdk = { source = "../../packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk", module = "nemo_evaluator_sdk" }
 data-designer-nemo = { source = "../../packages/data_designer_nemo/src/data_designer_nemo", module = "data_designer_nemo" }
 nmp-platform-runner = { source = "../../packages/nmp_platform_runner/src/nmp/platform_runner", module = "nmp/platform_runner", deps_group = "services" }
@@ -666,7 +672,6 @@ module = "nemo_platform"
 inherit."optional-dependencies" = true
 inherit.scripts = true
 inherit."entry-points" = ["nemo.*"]
-force_include."../../../../../docs" = "nemo_platform/cli/docs"
 
 [tool.bundle-package.nemo-agents-example-calculator]
 source = "../../plugins/nemo-agents/examples/calculator-agent/src/calculator_agent"

packages/nemo_platform_ext/pyproject.toml

@@ -88,6 +88,7 @@ packages = ["src/nemo_platform_ext"]
 [tool.vendor-package]
 package = "nemo_platform_ext"
 package_root = "packages/nemo_platform_ext"
+sdk_include_mode = "source-package"
 # Globs are evaluated per top-level module (e.g. `skills/`, `cli/`, `quickstart/`).
 # `**/*.md` recursively includes SKILL.md plus any companion markdown files
 # skills ship under `resources/` (e.g. notes, sub-docs, prompts). Add further