refine text decode (#657)

### What problem does this PR solve? #651 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
infiniflow · May 7, 2024 · 7013d7f · 7013d7f
1 parent de839fc
commit 7013d7f
Show file tree

Hide file tree

Showing 7 changed files with 7 additions and 7 deletions.
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py
@@ -69,7 +69,7 @@ def row_number(fnm, binary):
 
         if fnm.split(".")[-1].lower() in ["csv", "txt"]:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
             return len(txt.split("\n"))
 
 

diff --git a/rag/app/book.py b/rag/app/book.py
@@ -91,7 +91,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         txt = ""
         if binary:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
         else:
             with open(filename, "r") as f:
                 while True:

diff --git a/rag/app/laws.py b/rag/app/laws.py
@@ -113,7 +113,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         txt = ""
         if binary:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
         else:
             with open(filename, "r") as f:
                 while True:

diff --git a/rag/app/naive.py b/rag/app/naive.py
@@ -141,7 +141,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         txt = ""
         if binary:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
         else:
             with open(filename, "r") as f:
                 while True:

diff --git a/rag/app/one.py b/rag/app/one.py
@@ -85,7 +85,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         txt = ""
         if binary:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
         else:
             with open(filename, "r") as f:
                 while True:

diff --git a/rag/app/qa.py b/rag/app/qa.py
@@ -107,7 +107,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
         txt = ""
         if binary:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
         else:
             with open(filename, "r") as f:
                 while True:

diff --git a/rag/app/table.py b/rag/app/table.py
@@ -149,7 +149,7 @@ def chunk(filename, binary=None, from_page=0, to_page=10000000000,
         txt = ""
         if binary:
             encoding = find_codec(binary)
-            txt = binary.decode(encoding)
+            txt = binary.decode(encoding, errors="ignore")
         else:
             with open(filename, "r") as f:
                 while True: