Skip to content

Community: Adding bulk_size as a setable param for OpenSearchVectorSearch #28325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 12, 2024
Merged

Community: Adding bulk_size as a setable param for OpenSearchVectorSearch #28325

Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
51843d4
let OpenSearchVectorSearch class function can accept bulk_size param …
manukychen Nov 24, 2024
9fbccec
fix classmethod self.bulk_size bug
manukychen Nov 24, 2024
11383f4
let usage of bulk size can be passed-in as an override
manukychen Dec 10, 2024
6c3bd4a
Merge branch 'master' into opensearch_bulk_size
Dec 12, 2024
0ee5af5
x
Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 10 additions & 18 deletions libs/community/langchain_community/vectorstores/opensearch_vector_search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ def __init__(
self.client = _get_opensearch_client(opensearch_url, **kwargs)
self.async_client = _get_async_opensearch_client(opensearch_url, **kwargs)
self.engine = kwargs.get("engine", "nmslib")
self.bulk_size = kwargs.get("bulk_size", 500)

@property
def embeddings(self) -> Embeddings:
Expand All @@ -413,10 +414,9 @@ def __add(
embeddings: List[List[float]],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), self.bulk_size)
index_name = kwargs.get("index_name", self.index_name)
text_field = kwargs.get("text_field", "text")
dim = len(embeddings[0])
Expand Down Expand Up @@ -454,10 +454,9 @@ async def __aadd(
embeddings: List[List[float]],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), self.bulk_size)
index_name = kwargs.get("index_name", self.index_name)
text_field = kwargs.get("text_field", "text")
dim = len(embeddings[0])
Expand Down Expand Up @@ -560,7 +559,6 @@ def add_texts(
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
"""Run more texts through the embeddings and add to the vectorstore.
Expand All @@ -587,7 +585,7 @@ def add_texts(
embeddings,
metadatas=metadatas,
ids=ids,
bulk_size=bulk_size,
bulk_size=self.bulk_size,
**kwargs,
)

Expand All @@ -596,7 +594,6 @@ async def aadd_texts(
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

break

**kwargs: Any,
) -> List[str]:
"""
Expand All @@ -609,7 +606,7 @@ async def aadd_texts(
embeddings,
metadatas=metadatas,
ids=ids,
bulk_size=bulk_size,
bulk_size=self.bulk_size,
**kwargs,
)

Expand All @@ -618,7 +615,6 @@ def add_embeddings(
text_embeddings: Iterable[Tuple[str, List[float]]],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

breaking change - can we keep this, and use the passed-in as an override? can still default to the self.bulk_size behavior if it's None

**kwargs: Any,
) -> List[str]:
"""Add the given texts and embeddings to the vectorstore.
Expand Down Expand Up @@ -646,7 +642,7 @@ def add_embeddings(
list(embeddings),
metadatas=metadatas,
ids=ids,
bulk_size=bulk_size,
bulk_size=self.bulk_size,
**kwargs,
)

Expand Down Expand Up @@ -1085,7 +1081,6 @@ def from_texts(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

break

ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1139,7 +1134,7 @@ def from_texts(
texts,
embedding,
metadatas=metadatas,
bulk_size=bulk_size,
bulk_size=cls.bulk_size,
ids=ids,
**kwargs,
)
Expand All @@ -1150,7 +1145,6 @@ async def afrom_texts(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

break

ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1204,7 +1198,7 @@ async def afrom_texts(
texts,
embedding,
metadatas=metadatas,
bulk_size=bulk_size,
bulk_size=cls.bulk_size,
ids=ids,
**kwargs,
)
Expand All @@ -1216,7 +1210,6 @@ def from_embeddings(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1285,7 +1278,7 @@ def from_embeddings(
"max_chunk_bytes",
"is_aoss",
]
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), cls.bulk_size)
dim = len(embeddings[0])
# Get the index name from either from kwargs or ENV Variable
# before falling back to random generation
Expand Down Expand Up @@ -1346,7 +1339,6 @@ async def afrom_embeddings(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1417,7 +1409,7 @@ async def afrom_embeddings(
"max_chunk_bytes",
"is_aoss",
]
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), cls.bulk_size)
dim = len(embeddings[0])
# Get the index name from either from kwargs or ENV Variable
# before falling back to random generation
Expand Down
Loading