test: basic end to end

container/deps/requirements.test.txt

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+psutil>=5.0.0
 pyright
 pytest
 pytest-asyncio

pyproject.toml

@@ -157,7 +157,16 @@ markers = [
     "pre_merge: marks tests to run before merging",
     "nightly: marks tests to run nightly",
     "weekly: marks tests to run weekly",
-    "gpu: marks tests to run on GPU"
+    "gpu_1: marks tests to run on GPU",
+    "gpu_2: marks tests to run on 2GPUs",
+    "e2e: marks tests as end-to-end tests",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+    "stress: marks tests as stress tests",
+    "vllm: marks tests as requiring vllm",
+    "tensorrtllm: marks tests as requiring tensorrtllm",
+    "sglang: marks tests as requiring sglang",
+    "slow: marks tests as known to be slow"
 ]
 
 # Linting/formatting

tests/README.md

@@ -0,0 +1,98 @@
+# Dynamo Testing Framework
+
+## Overview
+
+This document outlines the testing framework for the Dynamo runtime system, including test discovery, organization, and best practices.
+
+## Directory Structure
+
+```bash
+tests/
+├── serve/              # E2E tests using dynamo serve
+│   ├── conftest.py     # test fixtures as needed for specific test area
+├── run/                # E2E tests using dynamo run
+│   ├── conftest.py     # test fixtures as needed for specific test area
+├── conftest.py         # Shared fixtures and configuration
+└── README.md           # This file
+```
+
+## Test Discovery
+
+Pytest automatically discovers tests based on their naming convention. All test files must follow this pattern:
+
+```bash
+test_<component_or_flow>.py
+```
+
+Where:
+- `component_or_flow`: The component or flow being tested (e.g., planner, kv_router)
+  - For e2e tests, this could be the API or simply "dynamo"
+
+## Running Tests
+
+To run all tests:
+```bash
+pytest
+```
+
+To run only specific tests:
+```bash
+# Run only vLLM tests
+pytest -v -m vllm
+
+# Run only e2e tests
+pytest -v -m e2e
+
+# Run tests for a specific component
+pytest -v -m planner
+
+# Run with print statements visible
+pytest -s
+```
+
+## Test Markers
+
+Markers help control which tests run under different conditions. Add these decorators to your test functions:
+
+### Frequency-based markers
+- `@pytest.mark.nightly` - Tests run nightly
+- `@pytest.mark.weekly` - Tests run weekly
+- `@pytest.mark.pre_merge` - Tests run before merging PRs
+
+### Role-based markers
+- `@pytest.mark.e2e` - End-to-end tests
+- `@pytest.mark.integration` - Integration tests
+- `@pytest.mark.unit` - Unit tests
+- `@pytest.mark.stress` - Stress/load tests
+- `@pytest.mark.benchmark` - Performance benchmark tests
+
+### Component-specific markers
+- `@pytest.mark.vllm` - Framework tests
+- `@pytest.mark.sglang` - Framework tests
+- `@pytest.mark.tensorrtllm` - Framework tests
+- `@pytest.mark.planner` - Planner component tests
+- `@pytest.mark.kv_router` - KV Router component tests
+- etc.
+
+### Execution-related markers
+- `@pytest.mark.slow` - Tests that take a long time to run
+- `@pytest.mark.skip(reason="Example: KV Manager is under development")` - Skip these tests
+- `@pytest.mark.xfail(reason="Expected to fail because...")` - Tests expected to fail
+
+## Environment Setup
+
+Tests are designed to run in the appropriate framework container built
+via ```./container/build.sh --framework X``` and run via
+```./container/run.sh --mount-workspace -it -- pytest```.
+
+
+### Environment Variables
+- `HF_TOKEN` - Your HuggingFace API token to avoid rate limits
+  - Get a token from https://huggingface.co/settings/tokens
+  - Set it before running tests: `export HF_TOKEN=your_token_here`
+
+### Model Download Cache
+
+The tests will automatically use a local cache at `~/.cache/huggingface` to avoid
+repeated downloads of model files. This cache is shared across test runs to improve performance.
+

tests/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/conftest.py

@@ -0,0 +1,76 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+import pytest
+
+from tests.utils.managed_process import ManagedProcess
+
+# Custom format inspired by your example
+LOG_FORMAT = "[TEST] %(asctime)s %(levelname)s %(name)s: %(message)s"
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format=LOG_FORMAT,
+    datefmt="%Y-%m-%dT%H:%M:%S",  # ISO 8601 UTC format with microseconds
+)
+
+
+class EtcdServer(ManagedProcess):
+    def __init__(self, request, port=2379, timeout=300):
+        port_string = str(port)
+        etcd_env = os.environ.copy()
+        etcd_env["ALLOW_NONE_AUTHENTICATION"] = "yes"
+        command = [
+            "etcd",
+            "--listen-client-urls",
+            f"http://0.0.0.0:{port_string}",
+            "--advertise-client-urls",
+            f"http://0.0.0.0:{port_string}",
+            "--data-dir",
+            "/tmp/etcd-test-data",
+        ]
+        super().__init__(
+            command=command,
+            timeout=timeout,
+            display_output=False,
+            health_check_ports=[port],
+            data_dir="/tmp/etcd-test-data",
+            log_dir=request.node.name,
+        )
+
+
+class NatsServer(ManagedProcess):
+    def __init__(self, request, port=4222, timeout=300):
+        data_dir = "/tmp/nats/jetstream"
+        command = ["nats-server", "-js", "--trace", "--store_dir", data_dir]
+        super().__init__(
+            command=command,
+            timeout=timeout,
+            display_output=False,
+            data_dir=data_dir,
+            health_check_ports=[port],
+            log_dir=request.node.name,
+        )
+
+
+@pytest.fixture()
+def runtime_services(request):
+    with NatsServer(request) as nats_process:
+        with EtcdServer(request) as etcd_process:
+            yield nats_process, etcd_process

tests/serve/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/serve/conftest.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.