diff --git a/openrag/components/indexer/loaders/CustomDocLoader.py b/openrag/components/indexer/loaders/CustomDocLoader.py index b91206e81..3238d2c26 100644 --- a/openrag/components/indexer/loaders/CustomDocLoader.py +++ b/openrag/components/indexer/loaders/CustomDocLoader.py @@ -34,6 +34,6 @@ async def aload_document(self, file_path, metadata: dict = None): s = "" for page_num, p in enumerate(pages, start=1): - s = p.page_content.strip() + f"\n[PAGE_{page_num}]\n" + s += p.page_content.strip() + f"\n[PAGE_{page_num}]\n" return Document(page_content=s, metadata=metadata) diff --git a/openrag/components/indexer/loaders/test_customdocloader.py b/openrag/components/indexer/loaders/test_customdocloader.py new file mode 100644 index 000000000..c5c6d778e --- /dev/null +++ b/openrag/components/indexer/loaders/test_customdocloader.py @@ -0,0 +1,40 @@ +"""Regression test for CustomDocLoader page accumulation (#376). + +The previous loop body used ``s = ...`` instead of ``s += ...``, so only +the final page survived. This test confirms every page's content is now +in the returned ``Document``. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from langchain_core.documents.base import Document as LCDocument + + +@pytest.mark.asyncio +async def test_customdocloader_accumulates_all_pages(tmp_path): + from components.indexer.loaders.CustomDocLoader import CustomDocLoader + + fake_pages = [ + LCDocument(page_content="page-one"), + LCDocument(page_content="page-two"), + LCDocument(page_content="page-three"), + ] + fake_loader_instance = MagicMock() + fake_loader_instance.aload = AsyncMock(return_value=fake_pages) + fake_loader_cls = MagicMock(return_value=fake_loader_instance) + + file_path = tmp_path / "stub.docx" + file_path.write_text("ignored") + + with patch.dict(CustomDocLoader.doc_loaders, {".docx": fake_loader_cls}, clear=True): + # BaseLoader.__init__ pulls a config; we bypass it with object.__new__ + loader = object.__new__(CustomDocLoader) + result = await loader.aload_document(str(file_path), metadata={"src": "x"}) + + assert "page-one" in result.page_content + assert "page-two" in result.page_content + assert "page-three" in result.page_content + assert "[PAGE_1]" in result.page_content + assert "[PAGE_2]" in result.page_content + assert "[PAGE_3]" in result.page_content