From 0b489766ff133ecae372e4e1350f318e4bf5d424 Mon Sep 17 00:00:00 2001
From: furionw <qiwa@nvidia.com>
Date: Wed, 23 Sep 2026 01:15:44 +0000
Subject: [PATCH 1/2] test(examples): cover the remote custom encoder topology
 end to end

The in-process CustomEncoder topology has a serve test; the remote one had
none, so nothing exercised the encoder handoff over a real request plane:
the existing unit tests mock both the handoff and the generator.

Add an agg_custom_remote topology that launches the example's own script and
asserts the same phrase-splice semantics as agg_custom, plus a text-only turn
so a conversation's non-image messages stay served. The wrapper script maps
the harness's dynamic system ports onto the two the example names, so parallel
topologies do not collide on its 8081/8082 defaults.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: furionw <qiwa@nvidia.com>
---
 .../launch/agg_custom_remote.sh               | 20 +++++
 tests/serve/multimodal_profiles/vllm.py       | 36 +++++++++
 tests/utils/multimodal.py                     | 79 +++++++++++++++++++
 tests/utils/test_multimodal.py                | 46 +++++++++++
 4 files changed, 181 insertions(+)
 create mode 100755 examples/custom_encoder/launch/agg_custom_remote.sh

diff --git a/examples/custom_encoder/launch/agg_custom_remote.sh b/examples/custom_encoder/launch/agg_custom_remote.sh
new file mode 100755
index 000000000000..2799a36da7a9
--- /dev/null
+++ b/examples/custom_encoder/launch/agg_custom_remote.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Serve-test entry point for the remote CustomEncoder topology.
+#
+# The serve harness launches "<directory>/launch/<script>" and hands each
+# topology dynamic ports as DYN_HTTP_PORT and DYN_SYSTEM_PORT<N>. The remote
+# example instead names its two worker ports DYN_GENERATOR_SYSTEM_PORT and
+# DYN_ORCHESTRATOR_SYSTEM_PORT, so map them here; without the mapping, parallel
+# topologies would collide on the 8081/8082 defaults.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+export DYN_GENERATOR_SYSTEM_PORT="${DYN_GENERATOR_SYSTEM_PORT:-${DYN_SYSTEM_PORT1:-8081}}"
+export DYN_ORCHESTRATOR_SYSTEM_PORT="${DYN_ORCHESTRATOR_SYSTEM_PORT:-${DYN_SYSTEM_PORT2:-8082}}"
+
+exec "$SCRIPT_DIR/../remote/launch.sh" "$@"
diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py
index 8090b845e8c7..f7781bc68203 100644
--- a/tests/serve/multimodal_profiles/vllm.py
+++ b/tests/serve/multimodal_profiles/vllm.py
@@ -24,6 +24,8 @@
     make_mixed_image_video_payload,
     make_qwen35_custom_encoder_multi_image_payload,
     make_qwen35_custom_encoder_payload,
+    make_remote_custom_encoder_payload,
+    make_remote_custom_encoder_text_only_payload,
     make_video_payload,
 )
 from tests.utils.payload_builder import (
@@ -86,6 +88,9 @@
     # not examples/backends/vllm — the TopologyConfig sets `directory` to match.
     "agg_custom": "agg_custom.sh",
     "agg_custom_qwen3_5": "agg_qwen3_5_native.sh",
+    # Remote CustomEncoder: the encoder runs in an application orchestrator
+    # worker and hands prepared embeddings to a stock aggregated generator.
+    "agg_custom_remote": "agg_custom_remote.sh",
 }
 
 VLLM_MULTIMODAL_PROFILES: list[MultimodalModelProfile] = [
@@ -754,6 +759,37 @@
                 },
                 tests=[MmCase(payload=make_custom_encoder_payload())],
             ),
+            # Same encoder and phrase as `agg_custom`, but the encoder runs in
+            # an application orchestrator worker that hands prepared embeddings
+            # to a stock aggregated generator over the request plane. Matching
+            # "42" on both topologies is what shows the handoff preserves the
+            # in-process semantics.
+            "agg_custom_remote": TopologyConfig(
+                marks=[pytest.mark.post_merge],
+                timeout_s=300,
+                directory=os.path.join(WORKSPACE_DIR, "examples/custom_encoder"),
+                env={
+                    # The harness addresses the deployment by the profile's model id,
+                    # including its text-only readiness probe, so publish under that
+                    # name instead of the example's default "remote-custom-encoder".
+                    "DYN_SERVED_MODEL_NAME": "Qwen/Qwen2.5-1.5B-Instruct",
+                    "DYN_GENERATOR_GPU": "0",
+                    "DYN_ENCODER_CLASS": (
+                        "examples.custom_encoder.hitchhikers_vision_encoder."
+                        "HitchhikersVisionEncoder"
+                    ),
+                    "PYTHONPATH": str(WORKSPACE_DIR),
+                },
+                tests=[
+                    MmCase(payload=make_remote_custom_encoder_payload()),
+                    # A turn with no media must still be served: the orchestrator
+                    # forwards it to the generator instead of requiring an image.
+                    MmCase(
+                        suffix="text_only",
+                        payload=make_remote_custom_encoder_text_only_payload(),
+                    ),
+                ],
+            ),
         },
     ),
 ]
diff --git a/tests/utils/multimodal.py b/tests/utils/multimodal.py
index 8686700fc405..1e0752239c18 100644
--- a/tests/utils/multimodal.py
+++ b/tests/utils/multimodal.py
@@ -4,6 +4,7 @@
 import base64
 import os
 import re
+from collections.abc import Mapping
 from copy import deepcopy
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -487,6 +488,84 @@ def make_custom_encoder_payload() -> ChatPayload:
     )
 
 
+class RemoteCustomEncoderChatPayload(ChatPayload):
+    """Validate generation plus the orchestrator's dummy classifier label."""
+
+    def validate(self, response: Any, content: str) -> None:
+        super().validate(response, content)
+
+        result = response.json()
+        nvext = result.get("nvext")
+        assert isinstance(nvext, Mapping), f"Expected nvext object, got {nvext!r}"
+        engine_data = nvext.get("engine_data")
+        assert isinstance(
+            engine_data, Mapping
+        ), f"Expected nvext.engine_data object, got {engine_data!r}"
+        classifier_label = engine_data.get("classifier_label")
+        assert classifier_label in {"class_a", "class_b"}, (
+            "Expected nvext.engine_data.classifier_label to contain a dummy class, "
+            f"got {classifier_label!r}"
+        )
+
+
+def make_remote_custom_encoder_payload() -> RemoteCustomEncoderChatPayload:
+    """Semantic check for the remote CustomEncoder topology.
+
+    Same phrase-splice contract as :func:`make_custom_encoder_payload`, but the
+    encoder runs in the orchestrator worker and the embeddings reach the stock
+    aggregated generator over the request plane. ``expected_log`` therefore
+    asserts the orchestrator's own encoder line rather than the decoder-side
+    ``Loaded CustomEncoder``, which only the in-process topology emits.
+    """
+    return RemoteCustomEncoderChatPayload(
+        body={
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": (
+                                "Based on The Hitchhiker's Guide to the Galaxy, "
+                                "The Answer to"
+                            ),
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": MULTIMODAL_IMG_URL},
+                        },
+                        {"type": "text", "text": " is?"},
+                    ],
+                }
+            ],
+            "max_tokens": 32,
+            "stream": False,
+            "logprobs": False,
+            "temperature": 0.0,
+            "nvext": {"extra_fields": ["engine_data"]},
+        },
+        repeat_count=1,
+        expected_response=["42"],
+        expected_log=[r"HitchhikersVisionEncoder\] ready"],
+    )
+
+
+def make_remote_custom_encoder_text_only_payload() -> ChatPayload:
+    """Text-only turn against the remote CustomEncoder topology.
+
+    The orchestrator must forward a request that carries no media straight to
+    the generator. Asserting a served answer here keeps a conversation's
+    non-image turns from regressing back into a request-level rejection.
+    """
+    return chat_payload(
+        "Based on The Hitchhiker's Guide to the Galaxy, The Answer to is?",
+        repeat_count=1,
+        expected_response=["42", "answer", "life", "Adams"],
+        max_tokens=32,
+        temperature=0.0,
+    )
+
+
 def make_qwen35_custom_encoder_payload() -> ChatPayload:
     """Single-image semantic check for the native Qwen3.5 custom encoder."""
     return Base64LazyChatPayload(
diff --git a/tests/utils/test_multimodal.py b/tests/utils/test_multimodal.py
index d09ac0e6aa96..c6a754c3caae 100644
--- a/tests/utils/test_multimodal.py
+++ b/tests/utils/test_multimodal.py
@@ -11,9 +11,11 @@
     get_multimodal_test_image_bytes,
 )
 from tests.utils.multimodal import (
+    RemoteCustomEncoderChatPayload,
     UuidPassthroughChatPayload,
     make_mixed_image_video_payload,
     make_qwen35_custom_encoder_multi_image_payload,
+    make_remote_custom_encoder_payload,
 )
 
 pytestmark = [
@@ -27,6 +29,14 @@ def _image_part(body: dict) -> dict:
     return body["messages"][0]["content"][1]
 
 
+class _Response:
+    def __init__(self, data: dict) -> None:
+        self._data = data
+
+    def json(self) -> dict:
+        return self._data
+
+
 def test_uuid_passthrough_payload_sends_fill_then_uuid_only() -> None:
     payload = UuidPassthroughChatPayload(expected_response=["green"])
 
@@ -89,6 +99,42 @@ def test_qwen35_multi_image_payload_is_order_sensitive() -> None:
     ]
 
 
+def test_remote_custom_encoder_payload_validates_classifier_label() -> None:
+    payload = make_remote_custom_encoder_payload()
+
+    assert isinstance(payload, RemoteCustomEncoderChatPayload)
+    assert payload.body["nvext"] == {"extra_fields": ["engine_data"]}
+    payload.validate(
+        _Response(
+            {
+                "nvext": {
+                    "engine_data": {
+                        "classifier_label": "class_a",
+                    }
+                }
+            }
+        ),
+        "42",
+    )
+
+
+@pytest.mark.parametrize(
+    "data",
+    [
+        {},
+        {"nvext": {}},
+        {"nvext": {"engine_data": {"classifier_label": "unexpected"}}},
+    ],
+)
+def test_remote_custom_encoder_payload_rejects_missing_or_unknown_label(
+    data: dict,
+) -> None:
+    payload = make_remote_custom_encoder_payload()
+
+    with pytest.raises(AssertionError):
+        payload.validate(_Response(data), "42")
+
+
 def test_mixed_image_video_payload_requires_video_context() -> None:
     payload = make_mixed_image_video_payload(["triangle"], frontend_decoding=True)
     content = payload.body["messages"][0]["content"]

From 8ab611d4a58912d181e9adf4422b80f5f5c6b704 Mon Sep 17 00:00:00 2001
From: furionw <qiwa@nvidia.com>
Date: Thu, 1 Oct 2026 23:08:30 -0700
Subject: [PATCH 2/2] test(examples): follow unary chat model configuration

---
 tests/serve/multimodal_profiles/vllm.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py
index f7781bc68203..7da158c152f4 100644
--- a/tests/serve/multimodal_profiles/vllm.py
+++ b/tests/serve/multimodal_profiles/vllm.py
@@ -773,21 +773,18 @@
                     # including its text-only readiness probe, so publish under that
                     # name instead of the example's default "remote-custom-encoder".
                     "DYN_SERVED_MODEL_NAME": "Qwen/Qwen2.5-1.5B-Instruct",
+                    "DYN_SERVICE_NAME": "remote-custom-encoder",
                     "DYN_GENERATOR_GPU": "0",
-                    "DYN_ENCODER_CLASS": (
-                        "examples.custom_encoder.hitchhikers_vision_encoder."
-                        "HitchhikersVisionEncoder"
-                    ),
                     "PYTHONPATH": str(WORKSPACE_DIR),
                 },
                 tests=[
-                    MmCase(payload=make_remote_custom_encoder_payload()),
                     # A turn with no media must still be served: the orchestrator
                     # forwards it to the generator instead of requiring an image.
                     MmCase(
                         suffix="text_only",
                         payload=make_remote_custom_encoder_text_only_payload(),
                     ),
+                    MmCase(payload=make_remote_custom_encoder_payload()),
                 ],
             ),
         },