-
Notifications
You must be signed in to change notification settings - Fork 278
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature/make parsing chunking providers (#820)
* moving towards pipeline options * move to parse / chunk providers * make run serve work. * refactored ingestion * fix import issues * cleanup
- Loading branch information
1 parent
95c0472
commit 7d9b756
Showing
27 changed files
with
1,205 additions
and
551 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,6 +7,13 @@ require_email_verification = false | |
default_admin_email = "[email protected]" | ||
default_admin_password = "change_me_immediately" | ||
|
||
[chunking] | ||
provider = "unstructured" | ||
method = "by_title" | ||
chunk_size = 512 | ||
chunk_overlap = 50 | ||
max_chunk_size = 1024 | ||
|
||
[completion] | ||
provider = "litellm" | ||
concurrent_request_limit = 16 | ||
|
@@ -37,18 +44,6 @@ concurrent_request_limit = 256 | |
[eval] | ||
provider = "None" | ||
|
||
[ingestion] | ||
excluded_parsers = [ "mp4" ] | ||
|
||
[[ingestion.override_parsers]] | ||
document_type = "pdf" | ||
parser = "PDFParser" | ||
|
||
[ingestion.text_splitter] | ||
type = "recursive_character" | ||
chunk_size = 512 | ||
chunk_overlap = 20 | ||
|
||
[kg] | ||
provider = "None" | ||
|
||
|
@@ -57,5 +52,13 @@ provider = "local" | |
log_table = "logs" | ||
log_info_table = "log_info" | ||
|
||
[parsing] | ||
provider = "r2r" | ||
excluded_parsers = ["mp4"] | ||
|
||
[[parsing.override_parsers]] | ||
document_type = "pdf" | ||
parser = "PDFParser" | ||
|
||
[prompt] | ||
provider = "r2r" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
from abc import ABC, abstractmethod | ||
from enum import Enum | ||
from typing import Any, AsyncGenerator, List, Optional | ||
|
||
from pydantic import BaseModel, Field | ||
|
||
from ..abstractions.document import Document, DocumentType | ||
from .base import Provider, ProviderConfig | ||
|
||
|
||
class Method(str, Enum): | ||
BY_TITLE = "by_title" | ||
BASIC = "basic" | ||
RECURSIVE = "recursive" | ||
|
||
|
||
class ChunkingConfig(ProviderConfig): | ||
provider: str = "r2r" | ||
method: Method = Method.RECURSIVE | ||
chunk_size: int = 512 | ||
chunk_overlap: int = 0 | ||
max_chunk_size: Optional[int] = None | ||
|
||
def validate(self) -> None: | ||
if self.provider not in self.supported_providers: | ||
raise ValueError(f"Provider {self.provider} is not supported.") | ||
|
||
@property | ||
def supported_providers(self) -> list[str]: | ||
return ["r2r", "unstructured", None] | ||
|
||
|
||
class ChunkingProvider(Provider, ABC): | ||
def __init__(self, config: ChunkingConfig): | ||
super().__init__(config) | ||
self.config = config | ||
|
||
@abstractmethod | ||
async def chunk(self, parsed_document: str) -> AsyncGenerator[str, None]: | ||
"""Chunk the parsed document using the configured chunking strategy.""" | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
from abc import ABC, abstractmethod | ||
from typing import Any, AsyncGenerator, List | ||
|
||
from pydantic import BaseModel, Field | ||
|
||
from ..abstractions.document import Document, DocumentType | ||
from .base import Provider, ProviderConfig | ||
|
||
|
||
class OverrideParser(BaseModel): | ||
document_type: DocumentType | ||
parser: str | ||
|
||
|
||
class ParsingConfig(ProviderConfig): | ||
provider: str = "r2r" | ||
excluded_parsers: List[DocumentType] = Field(default_factory=list) | ||
override_parsers: List[OverrideParser] = Field(default_factory=list) | ||
|
||
@property | ||
def supported_providers(self) -> list[str]: | ||
return ["r2r", "unstructured", None] | ||
|
||
def validate(self) -> None: | ||
if self.provider not in self.supported_providers: | ||
raise ValueError(f"Provider {self.provider} is not supported.") | ||
|
||
|
||
class ParsingProvider(Provider, ABC): | ||
def __init__(self, config: ParsingConfig): | ||
super().__init__(config) | ||
self.config = config | ||
|
||
@abstractmethod | ||
async def parse(self, document: Document) -> AsyncGenerator[Any, None]: | ||
"""Parse the document using the configured parsing strategy.""" | ||
pass | ||
|
||
@abstractmethod | ||
def get_parser_for_document_type(self, doc_type: DocumentType) -> str: | ||
"""Get the appropriate parser for a given document type.""" | ||
pass |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.