diff --git a/deepdoc/parser/pdf_parser.py b/deepdoc/parser/pdf_parser.py index 5068eed62b..4e43df7a31 100644 --- a/deepdoc/parser/pdf_parser.py +++ b/deepdoc/parser/pdf_parser.py @@ -392,7 +392,7 @@ def _naive_vertical_merge(self): b["text"].strip()[-1] in ",;:'\",、‘“;:-", len(b["text"].strip()) > 1 and b["text"].strip( )[-2] in ",;:'\",‘“、;:", - b["text"].strip()[0] in "。;?!?”)),,、:", + b_["text"].strip()[0] in "。;?!?”)),,、:", ] # features for not concating feats = [ diff --git a/rag/app/naive.py b/rag/app/naive.py index f91734b27a..f1e3e3bdb2 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -19,6 +19,8 @@ from rag.nlp import rag_tokenizer, naive_merge, tokenize_table, tokenize_chunks, find_codec from deepdoc.parser import PdfParser, ExcelParser, DocxParser from rag.settings import cron_logger +from rag.utils import num_tokens_from_string + class Docx(DocxParser): def __init__(self): @@ -149,8 +151,14 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, if not l: break txt += l - sections = txt.split("\n") - sections = [(l, "") for l in sections if l] + sections = [] + for sec in txt.split("\n"): + if num_tokens_from_string(sec) > 10 * parser_config.get("chunk_token_num", 128): + sections.append((sec[:int(len(sec)/2)], "")) + sections.append((sec[int(len(sec)/2):], "")) + else: + sections.append((sec, "")) + callback(0.8, "Finish parsing.") elif re.search(r"\.doc$", filename, re.IGNORECASE): @@ -163,7 +171,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, else: raise NotImplementedError( - "file type not supported yet(doc, docx, pdf, txt supported)") + "file type not supported yet(pdf, xlsx, doc, docx, txt supported)") st = timer() chunks = naive_merge( diff --git a/rag/nlp/rag_tokenizer.py b/rag/nlp/rag_tokenizer.py index abd9146a94..be5b724b92 100644 --- a/rag/nlp/rag_tokenizer.py +++ b/rag/nlp/rag_tokenizer.py @@ -24,7 +24,7 @@ def rkey_(self, line): def loadDict_(self, fnm): print("[HUQIE]:Build trie", fnm, file=sys.stderr) try: - of = open(fnm, "r") + of = open(fnm, "r", encoding='utf-8') while True: line = of.readline() if not line: diff --git a/requirements.txt b/requirements.txt index f242a2ce3e..c8c3cd25d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -136,3 +136,4 @@ BCEmbedding loguru==0.7.2 umap-learn fasttext==0.9.2 +volcengine diff --git a/requirements_dev.txt b/requirements_dev.txt index f29022bcac..ae0ba7a348 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -124,3 +124,4 @@ ollama==0.1.8 redis==5.0.4 fasttext==0.9.2 umap-learn +volcengine