Skip to content

Commit

Permalink
Revert "core[patch]: improve index/aindex api when batch_size<n_docs (l…
Browse files Browse the repository at this point in the history
…angchain-ai#25754)"

This reverts commit 2538963.
  • Loading branch information
KeiichiHirobe committed Dec 13, 2024
1 parent c2f1d02 commit aeec698
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 344 deletions.
28 changes: 12 additions & 16 deletions libs/core/langchain_core/indexing/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -408,18 +408,16 @@ def index(

# mypy isn't good enough to determine that source ids cannot be None
# here due to a check that's happening above, so we check again.
if any(source_id is None for source_id in source_ids):
msg = "Source ids cannot be if cleanup=='incremental'."
raise AssertionError(msg)
for source_id in source_ids:
if source_id is None:
raise AssertionError("Source ids cannot be None here.")

indexed_source_ids = cast(
Sequence[str], [source_id_assigner(doc) for doc in docs_to_index]
)
_source_ids = cast(Sequence[str], source_ids)

uids_to_delete = record_manager.list_keys(
group_ids=indexed_source_ids, before=index_start_dt
group_ids=_source_ids, before=index_start_dt
)
if indexed_source_ids and uids_to_delete:
if uids_to_delete:
# Then delete from vector store.
destination.delete(uids_to_delete)
# First delete from record store.
Expand Down Expand Up @@ -669,18 +667,16 @@ async def aindex(

# mypy isn't good enough to determine that source ids cannot be None
# here due to a check that's happening above, so we check again.
if any(source_id is None for source_id in source_ids):
msg = "Source ids cannot be if cleanup=='incremental'."
raise AssertionError(msg)
for source_id in source_ids:
if source_id is None:
raise AssertionError("Source ids cannot be None here.")

indexed_source_ids = cast(
Sequence[str], [source_id_assigner(doc) for doc in docs_to_index]
)
_source_ids = cast(Sequence[str], source_ids)

uids_to_delete = await record_manager.alist_keys(
group_ids=indexed_source_ids, before=index_start_dt
group_ids=_source_ids, before=index_start_dt
)
if indexed_source_ids and uids_to_delete:
if uids_to_delete:
# Then delete from vector store.
await destination.adelete(uids_to_delete)
# First delete from record store.
Expand Down
Loading

0 comments on commit aeec698

Please sign in to comment.