fix: check ole storage content to differentiate filetypes (#3581)

### Summary Updates the file detection logic for OLE files to check the storage content of the file to more reliable differentiate between DOC, PPT, XLS and MSG files. This corrects a bug that caused file type detection to be incorrect in cases where the `filetype` library guessed and incorrect MIME type, such as `'application/vnd.ms-excel'` for a `.msg` file. As part of this work, the `"msg"` extra was removed because the `python-oxmsg` package is now a base dependency. ### Testing Using a test `.msg` file that returns `'application/vnd.ms-excel'` from `filetype.guess_mime`. ```python from unstructured.file_utils.filetype import detect_filetype filename = "test-file.msg" detect_filetype(filename=filename) # result should be FileType.MSG ```
Unstructured-IO · Aug 30, 2024 · 6ba8135 · 6ba8135
1 parent ddb6cb6
commit 6ba8135
Show file tree

Hide file tree

Showing 53 changed files with 171 additions and 149 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -215,7 +215,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.10"]
-        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "msg", "pdf-image", "pptx", "xlsx"]
+        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "pdf-image", "pptx", "xlsx"]
     runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## 0.15.9-dev1
+## 0.15.9
 
 ### Enhancements
 
@@ -8,6 +8,7 @@
 
 ### Fixes
 
+* **Check storage contents for OLE file type detection** Updates `detect_filetype` to check the content of OLE files to more reliable differentiate DOC, PPT, XLS, and MSG files. As part of this, the `"msg"` extra was removed because the `python-oxmsg` package is now a base dependency.
 * **Fix disk space leaks and Windows errors when accessing file.name on a NamedTemporaryFile** Uses of `NamedTemporaryFile(..., delete=False)` and/or uses of `file.name` of NamedTemporaryFiles have been replaced with TemporaryFileDirectory to avoid a known issue: https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
 
 ## 0.15.8

diff --git a/Makefile b/Makefile
@@ -83,10 +83,6 @@ install-pypandoc:
 install-markdown:
 	python3 -m pip install -r requirements/extra-markdown.txt
 
-.PHONY: install-msg
-install-msg:
-	python3 -m pip install -r requirements/extra-msg.txt
-
 .PHONY: install-pdf-image
 install-pdf-image:
 	python3 -m pip install -r requirements/extra-pdf-image.txt
@@ -100,7 +96,7 @@ install-xlsx:
 	python3 -m pip install -r requirements/extra-xlsx.txt
 
 .PHONY: install-all-docs
-install-all-docs: install-base install-csv install-docx install-epub install-odt install-pypandoc install-markdown install-msg install-pdf-image install-pptx install-xlsx
+install-all-docs: install-base install-csv install-docx install-epub install-odt install-pypandoc install-markdown install-pdf-image install-pptx install-xlsx
 
 .PHONY: install-all-ingest
 install-all-ingest:
@@ -343,12 +339,6 @@ test-extra-epub:
 test-extra-markdown:
 	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_md.py
 
-.PHONY: test-extra-msg
-test-extra-msg:
-	# NOTE(scanny): exclude attachment test because partitioning attachments requires other extras
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_msg.py \
-          -k "not test_partition_msg_can_process_attachments"
-
 .PHONY: test-extra-odt
 test-extra-odt:
 	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_odt.py

diff --git a/requirements/base.in b/requirements/base.in
@@ -21,3 +21,4 @@ unstructured-client
 wrapt
 tqdm
 psutil
+python-oxmsg
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -10,7 +10,7 @@ backoff==2.2.1
     # via -r ./base.in
 beautifulsoup4==4.12.3
     # via -r ./base.in
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   httpcore
     #   httpx
@@ -23,7 +23,9 @@ charset-normalizer==3.3.2
     #   requests
     #   unstructured-client
 click==8.1.7
-    # via nltk
+    # via
+    #   nltk
+    #   python-oxmsg
 dataclasses-json==0.6.7
     # via
     #   -r ./base.in
@@ -70,6 +72,8 @@ nltk==3.9.1
     # via -r ./base.in
 numpy==1.26.4
     # via -r ./base.in
+olefile==0.47
+    # via python-oxmsg
 orderly-set==5.2.2
     # via deepdiff
 packaging==24.1
@@ -86,6 +90,8 @@ python-iso639==2024.4.27
     # via -r ./base.in
 python-magic==0.4.27
     # via -r ./base.in
+python-oxmsg==0.0.1
+    # via -r ./base.in
 rapidfuzz==3.9.6
     # via -r ./base.in
 regex==2024.7.24
@@ -120,6 +126,7 @@ typing-extensions==4.12.2
     #   anyio
     #   emoji
     #   pypdf
+    #   python-oxmsg
     #   typing-inspect
     #   unstructured-client
 typing-inspect==0.9.0
@@ -128,7 +135,7 @@ typing-inspect==0.9.0
     #   unstructured-client
 unstructured-client==0.25.5
     # via -r ./base.in
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   requests

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -34,7 +34,7 @@ bleach==6.1.0
     # via nbconvert
 build==1.2.1
     # via pip-tools
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   -c ./test.txt
@@ -130,7 +130,7 @@ jsonschema[format-nongpl]==3.2.0
     #   jupyter-events
     #   jupyterlab-server
     #   nbformat
-jupyter==1.1.0
+jupyter==1.1.1
     # via -r ./dev.in
 jupyter-client==7.4.9
     # via
@@ -370,7 +370,7 @@ typing-extensions==4.12.2
     #   -c ./test.txt
     #   anyio
     #   ipython
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt

diff --git a/requirements/extra-msg.in b/requirements/extra-msg.in
diff --git a/requirements/extra-msg.txt b/requirements/extra-msg.txt
diff --git a/requirements/extra-paddleocr.txt b/requirements/extra-paddleocr.txt
@@ -10,7 +10,7 @@ anyio==4.4.0
     #   httpx
 astor==0.8.1
     # via paddlepaddle
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   httpcore
@@ -170,7 +170,7 @@ typing-extensions==4.12.2
     #   paddlepaddle
 unstructured-paddleocr==2.8.1.0
     # via -r ./extra-paddleocr.in
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt

diff --git a/requirements/extra-pdf-image.txt b/requirements/extra-pdf-image.txt
@@ -8,7 +8,7 @@ antlr4-python3-runtime==4.9.3
     # via omegaconf
 cachetools==5.5.0
     # via google-auth
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   requests
@@ -279,7 +279,7 @@ unstructured-inference==0.7.36
     # via -r ./extra-pdf-image.in
 unstructured-pytesseract==0.3.13
     # via -r ./extra-pdf-image.in
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt

diff --git a/requirements/huggingface.txt b/requirements/huggingface.txt
@@ -4,7 +4,7 @@
 #
 #    pip-compile ./huggingface.in
 #
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   requests
@@ -103,7 +103,7 @@ typing-extensions==4.12.2
     #   -c ./base.txt
     #   huggingface-hub
     #   torch
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt

diff --git a/requirements/ingest/airtable.txt b/requirements/ingest/airtable.txt
@@ -6,7 +6,7 @@
 #
 annotated-types==0.7.0
     # via pydantic
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -36,7 +36,7 @@ typing-extensions==4.12.2
     #   pyairtable
     #   pydantic
     #   pydantic-core
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/astradb.txt b/requirements/ingest/astradb.txt
@@ -14,7 +14,7 @@ cassandra-driver==3.29.1
     # via cassio
 cassio==0.1.8
     # via astrapy
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   httpcore
@@ -91,7 +91,7 @@ typing-extensions==4.12.2
     # via
     #   -c ./ingest/../base.txt
     #   anyio
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/azure-cognitive-search.txt b/requirements/ingest/azure-cognitive-search.txt
@@ -10,7 +10,7 @@ azure-core==1.30.2
     # via azure-search-documents
 azure-search-documents==11.5.1
     # via -r ./ingest/azure-cognitive-search.in
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -38,7 +38,7 @@ typing-extensions==4.12.2
     #   -c ./ingest/../base.txt
     #   azure-core
     #   azure-search-documents
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/azure.txt b/requirements/ingest/azure.txt
@@ -27,7 +27,7 @@ azure-identity==1.17.1
     # via adlfs
 azure-storage-blob==12.22.0
     # via adlfs
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -94,7 +94,7 @@ typing-extensions==4.12.2
     #   azure-core
     #   azure-identity
     #   azure-storage-blob
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/box.txt b/requirements/ingest/box.txt
@@ -10,7 +10,7 @@ boxfs==0.3.0
     # via -r ./ingest/box.in
 boxsdk[jwt]==3.13.0
     # via boxfs
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -51,7 +51,7 @@ six==1.16.0
     # via
     #   -c ./ingest/../base.txt
     #   python-dateutil
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/chroma.txt b/requirements/ingest/chroma.txt
@@ -24,7 +24,7 @@ build==1.2.1
     # via chromadb
 cachetools==5.5.0
     # via google-auth
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   httpcore
@@ -268,7 +268,7 @@ typing-extensions==4.12.2
     #   starlette
     #   typer
     #   uvicorn
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/clarifai.txt b/requirements/ingest/clarifai.txt
@@ -4,7 +4,7 @@
 #
 #    pip-compile ./ingest/clarifai.in
 #
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -74,7 +74,7 @@ tqdm==4.66.5
     #   clarifai
 tritonclient==2.41.1
     # via clarifai
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/confluence.txt b/requirements/ingest/confluence.txt
@@ -4,13 +4,13 @@
 #
 #    pip-compile ./ingest/confluence.in
 #
-atlassian-python-api==3.41.14
+atlassian-python-api==3.41.15
     # via -r ./ingest/confluence.in
 beautifulsoup4==4.12.3
     # via
     #   -c ./ingest/../base.txt
     #   atlassian-python-api
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -45,7 +45,7 @@ soupsieve==2.6
     # via
     #   -c ./ingest/../base.txt
     #   beautifulsoup4
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt

diff --git a/requirements/ingest/databricks-volumes.txt b/requirements/ingest/databricks-volumes.txt
@@ -6,7 +6,7 @@
 #
 cachetools==5.5.0
     # via google-auth
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./ingest/../base.txt
     #   requests
@@ -34,7 +34,7 @@ requests==2.32.3
     #   databricks-sdk
 rsa==4.9
     # via google-auth
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ./ingest/../base.txt
     #   -c ./ingest/../deps/constraints.txt
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,3 +21,4 @@ unstructured-client @@
     wrapt
     tqdm
     psutil
+    python-oxmsg