Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 60 additions & 28 deletions components/src/dynamo/common/multimodal/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,40 +4,72 @@
"""Multimodal utilities for Dynamo components."""

from collections.abc import Callable
from importlib import import_module
from typing import Any

from dynamo.common.constants import EmbeddingTransferMode
from dynamo.common.multimodal.async_encoder_cache import AsyncEncoderCache
from dynamo.common.multimodal.audio_loader import AudioLoader
from dynamo.common.multimodal.embedding_transfer import (
AbstractEmbeddingReceiver,
AbstractEmbeddingSender,
LocalEmbeddingReceiver,
LocalEmbeddingSender,
NixlReadEmbeddingReceiver,
NixlReadEmbeddingSender,
NixlWriteEmbeddingReceiver,
NixlWriteEmbeddingSender,
TransferRequest,
)
from dynamo.common.multimodal.image_loader import ImageLoader
from dynamo.common.multimodal.video_loader import VideoLoader

EMBEDDING_SENDER_FACTORIES: dict[
EmbeddingTransferMode, Callable[[], AbstractEmbeddingSender]
] = {
EmbeddingTransferMode.LOCAL: LocalEmbeddingSender,
EmbeddingTransferMode.NIXL_WRITE: NixlWriteEmbeddingSender,
EmbeddingTransferMode.NIXL_READ: NixlReadEmbeddingSender,


_LAZY_EXPORTS = {
"AsyncEncoderCache": "dynamo.common.multimodal.async_encoder_cache",
"AudioLoader": "dynamo.common.multimodal.audio_loader",
"ImageLoader": "dynamo.common.multimodal.image_loader",
"VideoLoader": "dynamo.common.multimodal.video_loader",
"AbstractEmbeddingReceiver": "dynamo.common.multimodal.embedding_transfer",
"AbstractEmbeddingSender": "dynamo.common.multimodal.embedding_transfer",
"LocalEmbeddingReceiver": "dynamo.common.multimodal.embedding_transfer",
"LocalEmbeddingSender": "dynamo.common.multimodal.embedding_transfer",
"NixlReadEmbeddingReceiver": "dynamo.common.multimodal.embedding_transfer",
"NixlReadEmbeddingSender": "dynamo.common.multimodal.embedding_transfer",
"NixlWriteEmbeddingReceiver": "dynamo.common.multimodal.embedding_transfer",
"NixlWriteEmbeddingSender": "dynamo.common.multimodal.embedding_transfer",
"TransferRequest": "dynamo.common.multimodal.embedding_transfer",
}


def _load_export(name: str) -> Any:
module = import_module(_LAZY_EXPORTS[name])
value = getattr(module, name)
globals()[name] = value
return value


def __getattr__(name: str) -> Any:
if name in _LAZY_EXPORTS:
return _load_export(name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


def _lazy_factory(export_name: str, *args: Any, **kwargs: Any) -> Any:
return _load_export(export_name)(*args, **kwargs)


def _embedding_sender_factory(export_name: str) -> Callable[[], Any]:
return lambda: _lazy_factory(export_name)


def _embedding_receiver_factory(export_name: str) -> Callable[[], Any]:
return lambda: _lazy_factory(export_name)


EMBEDDING_SENDER_FACTORIES: dict[EmbeddingTransferMode, Callable[[], Any]] = {
EmbeddingTransferMode.LOCAL: _embedding_sender_factory("LocalEmbeddingSender"),
EmbeddingTransferMode.NIXL_WRITE: _embedding_sender_factory(
"NixlWriteEmbeddingSender"
),
EmbeddingTransferMode.NIXL_READ: _embedding_sender_factory("NixlReadEmbeddingSender"),
}

EMBEDDING_RECEIVER_FACTORIES: dict[
EmbeddingTransferMode, Callable[[], AbstractEmbeddingReceiver]
] = {
EmbeddingTransferMode.LOCAL: LocalEmbeddingReceiver,
EmbeddingTransferMode.NIXL_WRITE: NixlWriteEmbeddingReceiver,
EMBEDDING_RECEIVER_FACTORIES: dict[EmbeddingTransferMode, Callable[[], Any]] = {
EmbeddingTransferMode.LOCAL: _embedding_receiver_factory("LocalEmbeddingReceiver"),
EmbeddingTransferMode.NIXL_WRITE: _embedding_receiver_factory(
"NixlWriteEmbeddingReceiver"
),
# [gluo FIXME] can't use pre-registered tensor as NIXL requires descriptors
# to be at matching size, need to overwrite nixl connect library
EmbeddingTransferMode.NIXL_READ: lambda: NixlReadEmbeddingReceiver(max_items=0),
EmbeddingTransferMode.NIXL_READ: lambda: _lazy_factory(
"NixlReadEmbeddingReceiver", max_items=0
),
}

__all__ = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

import asyncio
import base64
import subprocess
import sys
import textwrap
from io import BytesIO
from unittest.mock import AsyncMock, patch

Expand All @@ -37,6 +40,33 @@
_FETCH_BYTES_PATH = "dynamo.common.multimodal.image_loader.fetch_bytes"


def test_package_image_loader_import_does_not_import_torch() -> None:
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
script = textwrap.dedent(
"""
import importlib.abc
import sys

class BlockTorchImport(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path, target=None):
if fullname == "torch" or fullname.startswith("torch."):
raise AssertionError(f"unexpected import: {fullname}")
return None

sys.meta_path.insert(0, BlockTorchImport())

from dynamo.common.http.url_validator import validate_media_url
from dynamo.common.multimodal import ImageLoader

assert ImageLoader.__name__ == "ImageLoader"
assert callable(validate_media_url)
assert "dynamo.common.multimodal.embedding_transfer" not in sys.modules
assert "dynamo.common.memory.multimodal_embedding_cache_manager" not in sys.modules
"""
)

subprocess.run([sys.executable, "-c", script], check=True)
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
Comment thread
glamr-agent marked this conversation as resolved.
Resolved


def _make_png_bytes() -> bytes:
"""Create a minimal valid PNG in memory."""
img = Image.new("RGB", (2, 2), color="red")
Expand Down
Loading