Merge pull request #100 from microsoft/add_llm_tests

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
   "SpeechRecognition",
   "pathvalidate",
   "charset-normalizer",
+  "openai",
 ]
 
 [project.urls]
@@ -77,3 +78,6 @@ exclude_lines = [
   "if __name__ == .__main__.:",
   "if TYPE_CHECKING:",
 ]
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src/markitdown"]

src/markitdown/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Adam Fourney <[email protected]>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.1a1"
+__version__ = "0.0.1a2"

src/markitdown/_markitdown.py

@@ -847,8 +847,6 @@ def _get_llm_description(self, local_path, extension, client, model, prompt=None
         if prompt is None or prompt.strip() == "":
             prompt = "Write a detailed caption for this image."
 
-        sys.stderr.write(f"llm Prompt:\n{prompt}\n")
-
         data_uri = ""
         with open(local_path, "rb") as image_file:
             content_type, encoding = mimetypes.guess_type("_dummy" + extension)

tests/test_markitdown.py

@@ -11,6 +11,16 @@
 skip_remote = (
     True if os.environ.get("GITHUB_ACTIONS") else False
 )  # Don't run these tests in CI
+
+
+# Don't run the llm tests without a key and the client library
+skip_llm = False if os.environ.get("OPENAI_API_KEY") else True
+try:
+    import openai
+except ModuleNotFoundError:
+    skip_llm = True
+
+# Skip exiftool tests if not installed
 skip_exiftool = shutil.which("exiftool") is None
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")
@@ -107,6 +117,10 @@
     "髙橋淳,35,名古屋",
 ]
 
+LLM_TEST_STRINGS = [
+    "5bda1dd6",
+]
+
 
 @pytest.mark.skipif(
     skip_remote,
@@ -229,8 +243,28 @@ def test_markitdown_exiftool() -> None:
         assert target in result.text_content
 
 
+@pytest.mark.skipif(
+    skip_llm,
+    reason="do not run llm tests without a key",
+)
+def test_markitdown_llm() -> None:
+    client = openai.OpenAI()
+    markitdown = MarkItDown(llm_client=client, llm_model="gpt-4o")
+
+    result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_llm.jpg"))
+
+    for test_string in LLM_TEST_STRINGS:
+        assert test_string in result.text_content
+
+    # This is not super precise. It would also accept "red square", "blue circle",
+    # "the square is not blue", etc. But it's sufficient for this test.
+    for test_string in ["red", "circle", "blue", "square"]:
+        assert test_string in result.text_content.lower()
+
+
 if __name__ == "__main__":
     """Runs this file's tests from the command line."""
     test_markitdown_remote()
     test_markitdown_local()
     test_markitdown_exiftool()
+    test_markitdown_llm()