Skip to content

Commit

Permalink
Fixes image type, adds document type for ChatBedrockConverse (#143)
Browse files Browse the repository at this point in the history
## Description
This PR fixes the image type supported by Bedrock Anthropic and other
models. It also adds a new type for supporting documents.

### Image sample code
```python
from langchain_core.messages import HumanMessage
from langchain_aws import ChatBedrock

llm = ChatBedrock(
    model_id="anthropic.claude-3-sonnet-20240229-v1:0",
    beta_use_converse_api=True
)

# Convert image to bytes
image_path = "images/random-image.jpg"
image_bytes = None
with open(image_path, "rb") as f:
    image_bytes = f.read()

messages = [
    HumanMessage(
        content=[
            {
                "type": "image",
                "image": {
                    "format": "jpeg",
                    "source": {
                        "bytes": image_bytes
                    }
                }
            },
            "What is this image about?"
        ]
    )
]

llm.invoke(messages)
```

### Document sample code
```python
from langchain_core.messages import HumanMessage
from langchain_aws import ChatBedrock

llm = ChatBedrock(
    model_id="anthropic.claude-3-sonnet-20240229-v1:0",
    beta_use_converse_api=True
)

# Convert document to bytes
doc_path = "documents/random-doc.pdf"
doc_bytes = None
with open(doc_path, "rb") as f:
    doc_bytes = f.read()

messages = [
    HumanMessage(
        content=[
            {
                "type": "document",
                "document": {
                    "format": "pdf",
                    "name": "random-doc",
                    "source": {
                        "bytes": doc_bytes
                    }
                }
            },
            "What is this document about?"
        ]
    )
]

llm.invoke(messages)
```



Related to #75
Fixes #132
  • Loading branch information
3coins authored Aug 20, 2024
1 parent 77ec2a2 commit 052818e
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 2 deletions.
7 changes: 5 additions & 2 deletions libs/aws/langchain_aws/chat_models/bedrock_converse.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,7 @@ def _messages_to_bedrock(
if isinstance(msg, HumanMessage):
# If there's a human, tool, human message sequence, the
# tool message will be merged with the first human message, so the second
# human message will now be preceeded by a human message and should also
# human message will now be preceded by a human message and should also
# be merged with it.
if bedrock_messages and bedrock_messages[-1]["role"] == "user":
bedrock_messages[-1]["content"].extend(content)
Expand Down Expand Up @@ -747,7 +747,7 @@ def _anthropic_to_bedrock(
elif block["type"] == "image":
# Assume block is already in bedrock format.
if "image" in block:
bedrock_content.append(block)
bedrock_content.append({"image": block["image"]})
else:
bedrock_content.append(
{
Expand All @@ -764,6 +764,9 @@ def _anthropic_to_bedrock(
bedrock_content.append(
{"image": _format_openai_image_url(block["imageUrl"]["url"])}
)
elif block["type"] == "document":
# Assume block in bedrock document format
bedrock_content.append({"document": block["document"]})
elif block["type"] == "tool_use":
bedrock_content.append(
{
Expand Down
28 changes: 28 additions & 0 deletions libs/aws/tests/unit_tests/chat_models/test_bedrock_converse.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,26 @@ def test__messages_to_bedrock() -> None:
{"type": "guard_content", "text": "hu6"},
]
),
HumanMessage(
content=[
{
"type": "document",
"document": {
"format": "pdf",
"name": "doc1",
"source": {"bytes": b"doc1_data"},
},
}
]
),
HumanMessage(
content=[
{
"type": "image",
"image": {"format": "jpeg", "source": {"bytes": b"image_data"}},
}
]
),
]
expected_messages = [
{"role": "user", "content": [{"text": "hu1"}, {"text": "hu2"}]},
Expand Down Expand Up @@ -229,6 +249,14 @@ def test__messages_to_bedrock() -> None:
},
{"guardContent": {"text": {"text": "hu5"}}},
{"guardContent": {"text": {"text": "hu6"}}},
{
"document": {
"format": "pdf",
"name": "doc1",
"source": {"bytes": b"doc1_data"},
}
},
{"image": {"format": "jpeg", "source": {"bytes": b"image_data"}}},
],
},
]
Expand Down

0 comments on commit 052818e

Please sign in to comment.