feat: onboard dynamo-sdk basic and kv-router examples

Earthfile

@@ -26,17 +26,17 @@ golang-base:
 
 ############### ALL TARGETS ##############################
 all-test:
-    BUILD ./deploy/compoundai/operator+test
-#    BUILD ./deploy/compoundai/api-server+test  #TODO: mkhadkevich earthly tests fail https://gitlab-master.nvidia.com/aire/microservices/compoundai/-/jobs/144475821
+    BUILD ./deploy/dynemo/operator+test
+#    BUILD ./deploy/dynemo/api-server+test  #TODO: mkhadkevich earthly tests fail https://gitlab-master.nvidia.com/aire/microservices/compoundai/-/jobs/144475821
 
 all-docker:
     ARG CI_REGISTRY_IMAGE=my-registry
     ARG CI_COMMIT_SHA=latest
-    BUILD ./deploy/compoundai/operator+docker --CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE --CI_COMMIT_SHA=$CI_COMMIT_SHA
-    BUILD ./deploy/compoundai/api-server+docker --CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE --CI_COMMIT_SHA=$CI_COMMIT_SHA
+    BUILD ./deploy/dynemo/operator+docker --CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE --CI_COMMIT_SHA=$CI_COMMIT_SHA
+    BUILD ./deploy/dynemo/api-server+docker --CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE --CI_COMMIT_SHA=$CI_COMMIT_SHA
 
 all-lint:
-    BUILD ./deploy/compoundai/operator+lint
+    BUILD ./deploy/dynemo/operator+lint
 
 all:
     BUILD +all-test

container/Dockerfile

@@ -154,6 +154,9 @@ RUN cd examples/rust && \
     cp target/release/http /usr/local/bin/ && \
     cp target/release/llmctl /usr/local/bin/
 
+COPY deploy/dynemo/sdk /workspace/deploy/dynemo/sdk
+
+
 # Generate C bindings. Note that this is required for TRTLLM backend re-build
 COPY lib/bindings /workspace/lib/bindings
 RUN cd lib/bindings/c/ && \
@@ -165,7 +168,10 @@ RUN mkdir /opt/dynemo && \
     uv venv /opt/dynemo/venv --python 3.12 && \
     source /opt/dynemo/venv/bin/activate && \
     uv build --wheel --out-dir /workspace/dist && \
-    uv pip install /workspace/dist/dynemo*cp312*.whl
+    uv pip install /workspace/dist/dynemo*cp312*.whl && \
+    cd /workspace/deploy/dynemo/sdk && \
+    uv build --wheel --out-dir /workspace/dist && \
+    uv pip install /workspace/dist/dynemo_sdk*any.whl
 
 # Package the bindings
 RUN mkdir -p /opt/dynemo/bindings/wheels && \
@@ -177,7 +183,9 @@ RUN mkdir -p /opt/dynemo/bindings/wheels && \
 # Install dynemo.runtime and dynemo.llm wheels globally in container for tests that
 # currently run without virtual environment activated.
 # TODO: In future, we may use a virtualenv for everything and remove this.
-RUN pip install /opt/dynemo/bindings/wheels/dynemo*cp312*.whl
+RUN cd /opt/dynemo/bindings/wheels && \
+    pip install dynemo*cp312*.whl  && \
+    pip install /workspace/dist/dynemo_sdk*any.whl
 
 # Copy everything in after ginstall steps to avoid re-running build/install
 # commands on unrelated changes in other dirs.

container/Dockerfile.vllm

@@ -100,10 +100,14 @@ COPY lib/bindings /workspace/lib/bindings
 RUN cd lib/bindings/c && \
 cargo build --release --locked && cargo doc --no-deps
 
+COPY deploy/dynemo/sdk /workspace/deploy/dynemo/sdk
 # Build dynemo wheel
 RUN source /opt/dynemo/venv/bin/activate && \
     uv build --wheel --out-dir /workspace/dist && \
-    uv pip install /workspace/dist/dynemo*cp312*.whl
+    uv pip install /workspace/dist/dynemo*cp312*.whl && \
+    cd /workspace/deploy/dynemo/sdk && \
+    uv build --wheel --out-dir /workspace/dist && \
+    uv pip install /workspace/dist/dynemo_sdk*any.whl
 
 # Package the bindings
 RUN mkdir -p /opt/dynemo/bindings/wheels && \

container/Dockerfile.vllm_nixl

@@ -230,10 +230,14 @@ COPY lib/bindings /workspace/lib/bindings
 RUN cd lib/bindings/c && \
     cargo build --release --locked && cargo doc --no-deps
 
+COPY deploy/dynemo/sdk /workspace/deploy/dynemo/sdk
 # Build dynemo wheel
 RUN source /opt/dynemo/venv/bin/activate && \
     uv build --wheel --out-dir /workspace/dist && \
-    uv pip install /workspace/dist/dynemo*cp312*.whl
+    uv pip install /workspace/dist/dynemo*cp312*.whl && \
+    cd /workspace/deploy/dynemo/sdk && \
+    uv build --wheel --out-dir /workspace/dist && \
+    uv pip install /workspace/dist/dynemo_sdk*any.whl
 
 # Package the bindings
 RUN mkdir -p /opt/dynemo/bindings/wheels && \
@@ -248,6 +252,7 @@ ENV VLLM_KV_CAPI_PATH="/opt/dynemo/bindings/lib/libdynemo_llm_capi.so"
 # FIXME: Copy more specific folders in for dev/debug after directory restructure
 COPY . /workspace
 
+
 # FIXME: May want a modification with dynemo-distributed banner on entry
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 

container/deps/requirements.test.txt

@@ -21,3 +21,5 @@ pytest-cov
 pytest-md-report
 pytest-mypy
 pytest-timeout
+# add types library stub for PyYAML
+types-PyYAML

deploy/compoundai/operator/internal/nim/nim_test.go → deploy/dynemo/operator/internal/nim/nim_test.go

@@ -133,7 +133,7 @@ func TestGenerateCompoundAINIMDeployments(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name: "Test GenerateCompoundAINIMDeployments nova dependency",
+			name: "Test GenerateCompoundAINIMDeployments dynemo dependency",
 			args: args{
 				parentCompoundAIDeployment: &v1alpha1.CompoundAIDeployment{
 					ObjectMeta: metav1.ObjectMeta{

deploy/compoundai/sdk/pyproject.toml → deploy/dynemo/sdk/pyproject.toml

@@ -14,22 +14,31 @@
 #  limitations under the License.
 
 [project]
-name = "compoundai"
+name = "dynemo-sdk"
 version = "0.1.0"
-description = "Add your description here"
+description = "Distributed Inference Framework - SDK"
 readme = "README.md"
 authors = [
-    { name = "mabdulwahhab", email = "mabdulwahhab@nvidia.com" }
+    { name = "NVIDIA Inc.", email = "sw-dl-dynemo@nvidia.com" },
 ]
-requires-python = "==3.10.*"
 dependencies = [
-    "bentoml>=1.4.1",
+    # TODO: currently compatible with bentoml==1.4.1,
+    # v1.4.2 has removed normalize_identifier which is used in compoundai
+    # "dynemo=={dynemo_version}",
+    "bentoml==1.4.1",
     "types-psutil==7.0.0.20250218",
 ]
 
 [project.scripts]
-compoundai = "compoundai.cli.cli:cli"
+dynemo-sdk = "dynemo.sdk.cli.cli:cli"
+
+[tool.setuptools]
+namespace-packages = ["dynemo"]
+package-dir = {"dynemo.sdk" = "src/dynemo/sdk"}
 
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dynemo"]

deploy/compoundai/sdk/src/compoundai/__init__.py → deploy/dynemo/sdk/src/dynemo/sdk/__init__.py

@@ -12,3 +12,27 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+
+from typing import Any
+
+from bentoml import api  # type: ignore
+from bentoml._internal.context import server_context  # type: ignore
+
+from dynemo.sdk.lib.decorators import async_onstart, dynemo_api, dynemo_endpoint
+from dynemo.sdk.lib.dependency import depends
+from dynemo.sdk.lib.image import DYNEMO_IMAGE
+from dynemo.sdk.lib.service import service
+
+dynemo_context: dict[str, Any] = {}
+
+__all__ = [
+    "api",
+    "server_context",
+    "async_onstart",
+    "dynemo_api",
+    "dynemo_endpoint",
+    "depends",
+    "DYNEMO_IMAGE",
+    "service",
+    "dynemo_context",
+]

deploy/compoundai/sdk/src/compoundai/cli/cli.py → deploy/dynemo/sdk/src/dynemo/sdk/cli/cli.py

@@ -34,8 +34,9 @@ def create_bentoml_cli() -> click.Command:
     from bentoml_cli.models import model_command
     from bentoml_cli.secret import secret_command
     from bentoml_cli.utils import BentoMLCommandGroup, get_entry_points
-    from compoundai.cli.serve import serve_command
-    from compoundai.cli.start import start_command
+
+    from dynemo.sdk.cli.serve import serve_command
+    from dynemo.sdk.cli.start import start_command
 
     server_context.service_type = "cli"
 

deploy/compoundai/sdk/src/compoundai/cli/serve.py → deploy/dynemo/sdk/src/dynemo/sdk/cli/serve.py

@@ -318,7 +318,7 @@ def serve(
         else:
             # bentoml>=1.2
             # from _bentoml_impl.server import serve_http
-            from compoundai.cli.serving import serve_http  # type: ignore
+            from dynemo.sdk.cli.serving import serve_http  # type: ignore
 
             svc.inject_config()
             serve_http(

deploy/compoundai/sdk/src/compoundai/cli/serve_nova.py → deploy/dynemo/sdk/src/dynemo/sdk/cli/serve_dynemo.py

@@ -28,8 +28,10 @@
 import click
 
 from dynemo.runtime import DistributedRuntime, dynemo_endpoint, dynemo_worker
+from dynemo.sdk import dynemo_context
 
-logger = logging.getLogger("compoundai.serve.nova")
+logger = logging.getLogger("dynemo.sdk.serve.dynemo")
+logger.setLevel(logging.INFO)
 
 
 def generate_run_id():
@@ -63,16 +65,16 @@ def main(
     worker_env: str | None,
     worker_id: int | None,
 ) -> None:
-    """Start a worker for the given service - either Nova or regular service"""
+    """Start a worker for the given service - either Dynemo or regular service"""
     from _bentoml_impl.loader import import_service
     from bentoml._internal.container import BentoMLContainer
     from bentoml._internal.context import server_context
     from bentoml._internal.log import configure_server_logging
 
     run_id = generate_run_id()
-
-    # print the contents of the environment variable BENTOML_RUNNER_MAP
-    print(f"BENTOML_RUNNER_MAP: {os.environ['BENTOML_RUNNER_MAP']}")
+    dynemo_context["service_name"] = service_name
+    dynemo_context["runner_map"] = runner_map
+    dynemo_context["worker_id"] = worker_id
 
     # Import service first to check configuration
     service = import_service(bento_identifier)
@@ -97,23 +99,24 @@ def main(
             t.cast(t.Dict[str, str], json.loads(runner_map))
         )
 
-    # Check if Nova is enabled for this service
-    if service.is_nova_component():
+    # Check if Dynemo is enabled for this service
+    if service.is_dynemo_component():
         if worker_id is not None:
             server_context.worker_index = worker_id
-        class_instance = service.inner()
 
         @dynemo_worker()
         async def worker(runtime: DistributedRuntime):
+            global dynemo_context
+            dynemo_context["runtime"] = runtime
             if service_name and service_name != service.name:
                 server_context.service_type = "service"
             else:
                 server_context.service_type = "entry_service"
 
             server_context.service_name = service.name
 
-            # Get Nova configuration and create component
-            namespace, component_name = service.nova_address()
+            # Get Dynemo configuration and create component
+            namespace, component_name = service.dynemo_address()
             logger.info(
                 f"[{run_id}] Registering component {namespace}/{component_name}"
             )
@@ -124,58 +127,60 @@ async def worker(runtime: DistributedRuntime):
                 await component.create_service()
                 logger.info(f"[{run_id}] Created {service.name} component")
 
-                # Run startup hooks before setting up endpoints
-                for name, member in vars(class_instance.__class__).items():
-                    if callable(member) and getattr(
-                        member, "__bentoml_startup_hook__", False
-                    ):
-                        logger.info(f"[{run_id}] Running startup hook: {name}")
-                        result = getattr(class_instance, name)()
-                        if inspect.isawaitable(result):
-                            await result
-                            logger.info(
-                                f"[{run_id}] Completed async startup hook: {name}"
-                            )
-                        else:
-                            logger.info(f"[{run_id}] Completed startup hook: {name}")
-
                 # Set runtime on all dependencies
                 for dep in service.dependencies.values():
                     dep.set_runtime(runtime)
                     logger.info(f"[{run_id}] Set runtime for dependency: {dep}")
 
-                # Then register all Nova endpoints
-                nova_endpoints = service.get_nova_endpoints()
-                if not nova_endpoints:
-                    error_msg = f"[{run_id}] FATAL ERROR: No Nova endpoints found in service {service.name}!"
+                # Then register all Dynemo endpoints
+                dynemo_endpoints = service.get_dynemo_endpoints()
+                if not dynemo_endpoints:
+                    error_msg = f"[{run_id}] FATAL ERROR: No Dynemo endpoints found in service {service.name}!"
                     logger.error(error_msg)
                     raise ValueError(error_msg)
 
-                print(f"[{run_id}] Nova endpoints: {nova_endpoints}")
-                for name, endpoint in nova_endpoints.items():
+                endpoints = []
+                for name, endpoint in dynemo_endpoints.items():
                     td_endpoint = component.endpoint(name)
                     logger.info(f"[{run_id}] Registering endpoint '{name}'")
+                    endpoints.append(td_endpoint)
                     # Bind an instance of inner to the endpoint
+                dynemo_context["component"] = component
+                dynemo_context["endpoints"] = endpoints
+                class_instance = service.inner()
+                twm = []
+                for name, endpoint in dynemo_endpoints.items():
                     bound_method = endpoint.func.__get__(class_instance)
                     # Only pass request type for now, use Any for response
                     # TODO: Handle a dynemo_endpoint not having types
                     # TODO: Handle multiple endpoints in a single component
                     dynemo_wrapped_method = dynemo_endpoint(endpoint.request_type, Any)(
                         bound_method
                     )
-                    result = await td_endpoint.serve_endpoint(dynemo_wrapped_method)
-                    # WARNING: unreachable code :( because serve blocks
-                    logger.info(f"[{run_id}] Result: {result}")
-                    logger.info(f"[{run_id}] Registered endpoint '{name}'")
-
-                logger.info(
-                    f"[{run_id}] Started {service.name} instance with all endpoints registered"
-                )
+                    twm.append(dynemo_wrapped_method)
+                # Run startup hooks before setting up endpoints
+                for name, member in vars(class_instance.__class__).items():
+                    if callable(member) and getattr(
+                        member, "__bentoml_startup_hook__", False
+                    ):
+                        logger.info(f"[{run_id}] Running startup hook: {name}")
+                        result = getattr(class_instance, name)()
+                        if inspect.isawaitable(result):
+                            # await on startup hook async_onstart
+                            await result
+                            logger.info(
+                                f"[{run_id}] Completed async startup hook: {name}"
+                            )
+                        else:
+                            logger.info(f"[{run_id}] Completed startup hook: {name}")
                 logger.info(
-                    f"[{run_id}] Available endpoints: {service.list_nova_endpoints()}"
+                    f"[{run_id}] Starting {service.name} instance with all registered endpoints"
                 )
+                # TODO:bis: convert to list
+                result = await endpoints[0].serve_endpoint(twm[0])
+
             except Exception as e:
-                logger.error(f"[{run_id}] Error in Nova component setup: {str(e)}")
+                logger.error(f"[{run_id}] Error in Dynemo component setup: {str(e)}")
                 raise
 
         asyncio.run(worker())