Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
a37dccf
feat: updates to compoundai to get it working with kv router
biswapanda Mar 5, 2025
6058f2a
feat: updates to compoundai to get it working with kv router
biswapanda Mar 5, 2025
7b8e7dd
feat: onboarded kv rotuer and basic examples for sdk
biswapanda Mar 6, 2025
97ba44d
chore: rename to dynemo
biswapanda Mar 6, 2025
0500cd3
style: formatting
biswapanda Mar 6, 2025
84c4595
chore: add license headers
biswapanda Mar 6, 2025
f5c36b6
fix: update email and description
biswapanda Mar 6, 2025
6bc55e1
fix: rename to dynemo and doc update
biswapanda Mar 6, 2025
24b809a
fix: lint errors
biswapanda Mar 6, 2025
ae1e99f
Update container/Dockerfile.vllm_nixl
biswapanda Mar 6, 2025
f5b13bc
feat: reafactor compoundai to dynemo sdk
biswapanda Mar 6, 2025
b8f71e4
style: lint fix
biswapanda Mar 6, 2025
a26de5f
feat: rename compoundai to dyenmo.sdk
biswapanda Mar 7, 2025
4e0fd86
feat: rename compoundai to dyenmo.sdk and fix mypy issues
biswapanda Mar 7, 2025
2ad1acf
feat: update pyproject and instructions
biswapanda Mar 7, 2025
defc341
style: lint
biswapanda Mar 7, 2025
cc71998
fix skip mypy analysis on deps of bentoml
biswapanda Mar 7, 2025
a75f736
fix: install dynemo sdk in docker image as well
biswapanda Mar 7, 2025
cc5f13a
fix: container build
biswapanda Mar 7, 2025
ae0d371
Merge branch 'main' into bis/cai-examples
biswapanda Mar 7, 2025
62e755b
fix: ignore bentoml inspections
biswapanda Mar 7, 2025
7f275ae
fix: mypy
biswapanda Mar 7, 2025
d0cd734
fix: mypy
biswapanda Mar 7, 2025
a4e4a92
Merge branch 'bis/cai-examples' of https://github.com/dynemo-ai/dynem…
nnshah1 Mar 7, 2025
a2501ac
ignore pending and deprecation warning for select packages
nnshah1 Mar 7, 2025
2bf87f4
update for mypy
nnshah1 Mar 7, 2025
050ff0f
update with optional to capture none case
nnshah1 Mar 7, 2025
fce5d7b
test: add types library stub for PyYAML
biswapanda Mar 7, 2025
bca0e92
test: add types library stub for PyYAML
biswapanda Mar 7, 2025
dd704cf
fix: update downstream ci pipeline
biswapanda Mar 7, 2025
4615dcb
fix: vllm nixl image
biswapanda Mar 7, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions container/Dockerfile.vllm
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ ENV VLLM_KV_CAPI_PATH="/opt/dynemo/bindings/lib/libdynemo_llm_capi.so"
# FIXME: Copy more specific folders in for dev/debug after directory restructure
COPY . /workspace

RUN source /opt/dynemo/venv/bin/activate && \
cd /workspace/deploy/dynemo/sdk && \
uv pip install -e .

# FIXME: May want a modification with dynemo-distributed banner on entry
ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]

Expand Down
4 changes: 4 additions & 0 deletions container/Dockerfile.vllm_nixl
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,10 @@ ENV VLLM_KV_CAPI_PATH="/opt/dynemo/bindings/lib/libdynemo_llm_capi.so"

# FIXME: Copy more specific folders in for dev/debug after directory restructure
COPY . /workspace
# Install dynemo sdk for local dev and cluster deployment
RUN source /opt/dynemo/venv/bin/activate && \
cd /workspace/deploy/dynemo/sdk && \
uv pip install -e .

# FIXME: May want a modification with dynemo-distributed banner on entry
ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
Expand Down
1,160 changes: 0 additions & 1,160 deletions deploy/compoundai/sdk/uv.lock

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func TestGenerateCompoundAINIMDeployments(t *testing.T) {
wantErr: false,
},
{
name: "Test GenerateCompoundAINIMDeployments nova dependency",
name: "Test GenerateCompoundAINIMDeployments dynemo dependency",
args: args{
parentCompoundAIDeployment: &v1alpha1.CompoundAIDeployment{
ObjectMeta: metav1.ObjectMeta{
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,31 @@
# limitations under the License.

[project]
name = "compoundai"
name = "dynemo-sdk"
version = "0.1.0"
description = "Add your description here"
description = "Distributed Inference Framework - SDK"
readme = "README.md"
authors = [
{ name = "mabdulwahhab", email = "mabdulwahhab@nvidia.com" }
{ name = "NVIDIA Inc.", email = "sw-dl-dynemo@nvidia.com" },
]
requires-python = "==3.10.*"
dependencies = [
"bentoml>=1.4.1",
# TODO: currently compatible with bentoml==1.4.1,
# v1.4.2 has removed normalize_identifier which is used in compoundai
# "dynemo=={dynemo_version}",
"bentoml==1.4.1",
"types-psutil==7.0.0.20250218",
]

[project.scripts]
compoundai = "compoundai.cli.cli:cli"
dynemo-sdk = "dynemo.sdk.cli.cli:cli"

[tool.setuptools]
namespace-packages = ["dynemo"]
package-dir = {"dynemo.sdk" = "src/dynemo/sdk"}

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["src/dynemo"]
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,25 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bentoml import api
from bentoml._internal.context import server_context

from dynemo.sdk.lib.decorators import async_onstart, dynemo_api, dynemo_endpoint
from dynemo.sdk.lib.dependency import depends
from dynemo.sdk.lib.image import DYNEMO_IMAGE
from dynemo.sdk.lib.service import service

dynemo_context = {}

__all__ = [
"api",
"server_context",
"async_onstart",
"dynemo_api",
"dynemo_endpoint",
"depends",
"DYNEMO_IMAGE",
"service",
"dynemo_context",
]
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ def create_bentoml_cli() -> click.Command:
from bentoml_cli.models import model_command
from bentoml_cli.secret import secret_command
from bentoml_cli.utils import BentoMLCommandGroup, get_entry_points
from compoundai.cli.serve import serve_command
from compoundai.cli.start import start_command

from dynemo.sdk.cli.serve import serve_command
from dynemo.sdk.cli.start import start_command

server_context.service_type = "cli"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,12 @@
from typing import Any

import click
from compoundai import dynemo_context

from dynemo.runtime import DistributedRuntime, dynemo_endpoint, dynemo_worker

logger = logging.getLogger("compoundai.serve.nova")
logger = logging.getLogger("compoundai.serve.dynemo")
logger.setLevel(logging.INFO)


def generate_run_id():
Expand Down Expand Up @@ -63,16 +65,16 @@ def main(
worker_env: str | None,
worker_id: int | None,
) -> None:
"""Start a worker for the given service - either Nova or regular service"""
"""Start a worker for the given service - either Dynemo or regular service"""
from _bentoml_impl.loader import import_service
from bentoml._internal.container import BentoMLContainer
from bentoml._internal.context import server_context
from bentoml._internal.log import configure_server_logging

run_id = generate_run_id()

# print the contents of the environment variable BENTOML_RUNNER_MAP
print(f"BENTOML_RUNNER_MAP: {os.environ['BENTOML_RUNNER_MAP']}")
dynemo_context["service_name"] = service_name
dynemo_context["runner_map"] = runner_map
dynemo_context["worker_id"] = worker_id

# Import service first to check configuration
service = import_service(bento_identifier)
Expand All @@ -97,23 +99,24 @@ def main(
t.cast(t.Dict[str, str], json.loads(runner_map))
)

# Check if Nova is enabled for this service
if service.is_nova_component():
# Check if Dynemo is enabled for this service
if service.is_dynemo_component():
if worker_id is not None:
server_context.worker_index = worker_id
class_instance = service.inner()

@dynemo_worker()
async def worker(runtime: DistributedRuntime):
global dynemo_context
dynemo_context["runtime"] = runtime
if service_name and service_name != service.name:
server_context.service_type = "service"
else:
server_context.service_type = "entry_service"

server_context.service_name = service.name

# Get Nova configuration and create component
namespace, component_name = service.nova_address()
# Get Dynemo configuration and create component
namespace, component_name = service.dynemo_address()
logger.info(
f"[{run_id}] Registering component {namespace}/{component_name}"
)
Expand All @@ -124,58 +127,60 @@ async def worker(runtime: DistributedRuntime):
await component.create_service()
logger.info(f"[{run_id}] Created {service.name} component")

# Run startup hooks before setting up endpoints
for name, member in vars(class_instance.__class__).items():
if callable(member) and getattr(
member, "__bentoml_startup_hook__", False
):
logger.info(f"[{run_id}] Running startup hook: {name}")
result = getattr(class_instance, name)()
if inspect.isawaitable(result):
await result
logger.info(
f"[{run_id}] Completed async startup hook: {name}"
)
else:
logger.info(f"[{run_id}] Completed startup hook: {name}")

# Set runtime on all dependencies
for dep in service.dependencies.values():
dep.set_runtime(runtime)
logger.info(f"[{run_id}] Set runtime for dependency: {dep}")

# Then register all Nova endpoints
nova_endpoints = service.get_nova_endpoints()
if not nova_endpoints:
error_msg = f"[{run_id}] FATAL ERROR: No Nova endpoints found in service {service.name}!"
# Then register all Dynemo endpoints
dynemo_endpoints = service.get_dynemo_endpoints()
if not dynemo_endpoints:
error_msg = f"[{run_id}] FATAL ERROR: No Dynemo endpoints found in service {service.name}!"
logger.error(error_msg)
raise ValueError(error_msg)

print(f"[{run_id}] Nova endpoints: {nova_endpoints}")
for name, endpoint in nova_endpoints.items():
endpoints = []
for name, endpoint in dynemo_endpoints.items():
td_endpoint = component.endpoint(name)
logger.info(f"[{run_id}] Registering endpoint '{name}'")
endpoints.append(td_endpoint)
# Bind an instance of inner to the endpoint
dynemo_context["component"] = component
dynemo_context["endpoints"] = endpoints
class_instance = service.inner()
twm = []
for name, endpoint in dynemo_endpoints.items():
bound_method = endpoint.func.__get__(class_instance)
# Only pass request type for now, use Any for response
# TODO: Handle a dynemo_endpoint not having types
# TODO: Handle multiple endpoints in a single component
dynemo_wrapped_method = dynemo_endpoint(endpoint.request_type, Any)(
bound_method
)
result = await td_endpoint.serve_endpoint(dynemo_wrapped_method)
# WARNING: unreachable code :( because serve blocks
logger.info(f"[{run_id}] Result: {result}")
logger.info(f"[{run_id}] Registered endpoint '{name}'")

logger.info(
f"[{run_id}] Started {service.name} instance with all endpoints registered"
)
twm.append(dynemo_wrapped_method)
# Run startup hooks before setting up endpoints
for name, member in vars(class_instance.__class__).items():
if callable(member) and getattr(
member, "__bentoml_startup_hook__", False
):
logger.info(f"[{run_id}] Running startup hook: {name}")
result = getattr(class_instance, name)()
if inspect.isawaitable(result):
# await on startup hook async_onstart
await result
logger.info(
f"[{run_id}] Completed async startup hook: {name}"
)
else:
logger.info(f"[{run_id}] Completed startup hook: {name}")
logger.info(
f"[{run_id}] Available endpoints: {service.list_nova_endpoints()}"
f"[{run_id}] Starting {service.name} instance with all registered endpoints"
)
# TODO:bis: convert to list
result = await endpoints[0].serve_endpoint(twm[0])

except Exception as e:
logger.error(f"[{run_id}] Error in Nova component setup: {str(e)}")
logger.error(f"[{run_id}] Error in Dynemo component setup: {str(e)}")
raise

asyncio.run(worker())
Expand Down
Loading