feat: parse to xml

.github/workflows/release_python.yml

@@ -1,6 +1,6 @@
 # This file was autogenerated by maturin v1.6.0 using:
 #    maturin generate-ci github -o ../../.github/workflows/release_pyton_pytest.yml --pytest
-# 
+#
 # Then adapted to the project
 #
 name: CI
@@ -38,7 +38,7 @@ jobs:
           python-version: '3.8'
 
       # On linux we don't use graalvm/[email protected] action to install graalvm because it will install it
-      # on the runner machine and on linux the build will happen inside a manylinux docker. 
+      # on the runner machine and on linux the build will happen inside a manylinux docker.
       # Instead, we use a script to install graalvm inside the docker container
       # the script is launched by setting the before-script-linux config option of the maturin action
       - name: Build wheels
@@ -60,7 +60,7 @@ jobs:
         with:
           name: wheels-linux-${{ matrix.platform.target }}
           path: bindings/extractous-python/dist
-      
+
       - name: pytest
         if: ${{ startsWith(matrix.platform.target, 'x86_64') }}
         shell: bash
@@ -85,7 +85,7 @@ jobs:
             apt-get update
             apt-get install tesseract-ocr tesseract-ocr-deu tesseract-ocr-ara
             apt-get install -y --no-install-recommends python3 python3-pip
-            pip3 install -U pip pytest scikit-learn
+            pip3 install -U pip pytest scikit-learn lxml
           run: |
             set -e
             pip3 install extractous --find-links bindings/extractous-python/dist --no-index --force-reinstall
@@ -206,7 +206,7 @@ jobs:
           name: wheels-sdist
           path: bindings/extractous-python/dist
 
-  # Follows the guide on https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/        
+  # Follows the guide on https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
   # We use 2 actions one to publish on PyPi on tag pushes to main brnach and the other to publish on TestPyPi on any push
   publish-to-testpypi:
     name: Publish to TestPyPI

bindings/extractous-python/src/extractor.rs

@@ -258,6 +258,55 @@ impl Extractor {
         Ok((content, py_metadata.into()))
     }
 
+    /// Extracts text from a file path. Returns a tuple with string that is of maximum length
+    /// of the extractor's `extract_string_max_length` and the metadata as dict.
+    pub fn extract_file_to_xml<'py>(
+        &self,
+        filename: &str,
+        py: Python<'py>,
+    ) -> PyResult<(String, PyObject)> {
+        let (content, metadata) = self
+            .0
+            .extract_file_to_xml(filename)
+            .map_err(|e| PyErr::new::<PyTypeError, _>(format!("{:?}", e)))?;
+
+        let py_metadata = metadata_hashmap_to_pydict(py, &metadata)?;
+        Ok((content, py_metadata.into()))
+    }
+
+    /// Extracts text from a bytearray. string that is of maximum length
+    /// of the extractor's `extract_string_max_length` and the metadata as dict.
+    pub fn extract_bytes_to_xml<'py>(
+        &self,
+        buffer: &Bound<'_, PyByteArray>,
+        py: Python<'py>,
+    ) -> PyResult<(String, PyObject)> {
+        let (content, metadata) = self
+            .0
+            .extract_bytes_to_xml(&buffer.to_vec())
+            .map_err(|e| PyErr::new::<PyTypeError, _>(format!("{:?}", e)))?;
+
+        // Create a new `StreamReader` with initial buffer capacity of ecore::DEFAULT_BUF_SIZE bytes
+        let py_metadata = metadata_hashmap_to_pydict(py, &metadata)?;
+        Ok((content, py_metadata.into()))
+    }
+
+    /// Extracts text from a URL. Returns a tuple with string that is of maximum length
+    /// of the extractor's `extract_string_max_length` and the metadata as dict.
+    pub fn extract_url_to_xml<'py>(
+        &self,
+        url: &str,
+        py: Python<'py>,
+    ) -> PyResult<(String, PyObject)> {
+        let (content, metadata) = self
+            .0
+            .extract_url_to_xml(url)
+            .map_err(|e| PyErr::new::<PyTypeError, _>(format!("{:?}", e)))?;
+
+        let py_metadata = metadata_hashmap_to_pydict(py, &metadata)?;
+        Ok((content, py_metadata.into()))
+    }
+
     fn __repr__(&self) -> String {
         format!("{:?}", self.0)
     }

bindings/extractous-python/tests/test_extract_bytes.py

@@ -3,7 +3,7 @@
 
 from extractous import Extractor
 from utils import calculate_similarity_percent, cosine_similarity, read_to_string, read_file_to_bytearray, \
-    is_expected_metadata_contained
+    is_expected_metadata_contained, extract_body_text
 
 TEST_CASES = [
     ("2022_Q3_AAPL.pdf", 0.9, 0.8),
@@ -49,6 +49,35 @@ def test_extract_bytes_to_string(file_name, target_dist, metadata_dist):
     assert percent_similarity >= metadata_dist, \
         f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"
 
+@pytest.mark.parametrize("file_name, target_dist, metadata_dist", TEST_CASES)
+def test_extract_bytes_to_xml(file_name, target_dist, metadata_dist):
+    """Test the extraction from bytes of various file types."""
+    original_filepath = f"../../test_files/documents/{file_name}"
+    expected_result_filepath = f"../../test_files/expected_result/{file_name}.txt"
+    expected_metadata_result_filepath = f"../../test_files/expected_result/{file_name}.metadata.json"
+
+    # Read expected
+    with open(expected_result_filepath, "r",  encoding="utf8") as file:
+        expected = file.read()
+    with open(expected_metadata_result_filepath, 'r', encoding="utf8") as file:
+        expected_metadata = json.load(file)
+
+    # Extract
+    file_bytes = read_file_to_bytearray(original_filepath)
+
+    extractor = Extractor()
+    result_xml, metadata = extractor.extract_bytes_to_xml(file_bytes)
+    result_text = extract_body_text(result_xml)
+
+    # Check Expected
+    assert cosine_similarity(result_text, expected) >= target_dist, \
+        f"Cosine similarity is less than {target_dist} for file: {file_name}"
+
+    # Check metadata
+    percent_similarity = calculate_similarity_percent(metadata, expected_metadata)
+    assert percent_similarity >= metadata_dist, \
+        f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"
+
 @pytest.mark.parametrize("file_name, target_dist, metadata_dist", TEST_CASES)
 def test_extract_bytes_to_stream(file_name, target_dist, metadata_dist):
     """Test the extraction from bytes of various file types."""
@@ -76,4 +105,4 @@ def test_extract_bytes_to_stream(file_name, target_dist, metadata_dist):
     # Check metadata
     percent_similarity = calculate_similarity_percent(metadata, expected_metadata)
     assert percent_similarity >= metadata_dist, \
-        f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"
+        f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"

bindings/extractous-python/tests/test_extract_file.py

@@ -2,7 +2,7 @@
 import pytest
 
 from extractous import Extractor
-from utils import calculate_similarity_percent, cosine_similarity, is_expected_metadata_contained, read_to_string
+from utils import calculate_similarity_percent, cosine_similarity, is_expected_metadata_contained, read_to_string, extract_body_text
 
 TEST_CASES = [
     ("2022_Q3_AAPL.pdf", 0.9, 0.8),
@@ -49,6 +49,34 @@ def test_extract_file_to_string(file_name, target_dist, metadata_dist):
     assert percent_similarity >= metadata_dist, \
         f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"
 
+@pytest.mark.parametrize("file_name, target_dist, metadata_dist", TEST_CASES)
+def test_extract_file_to_xml(file_name, target_dist, metadata_dist):
+    """Test the extraction and comparison of various file types."""
+    original_filepath = f"../../test_files/documents/{file_name}"
+    expected_result_filepath = f"../../test_files/expected_result/{file_name}.txt"
+    expected_metadata_result_filepath = f"../../test_files/expected_result/{file_name}.metadata.json"
+
+    # Read expected
+    with open(expected_result_filepath, "r",  encoding="utf8") as file:
+        expected = file.read()
+    with open(expected_metadata_result_filepath, 'r', encoding="utf8") as file:
+        expected_metadata = json.load(file)
+
+    # Extract
+    extractor = Extractor()
+    result_xml, metadata = extractor.extract_file_to_xml(original_filepath)
+    result_text = extract_body_text(result_xml)
+
+    # Check extracted
+    assert cosine_similarity(result_text, expected) >= target_dist, \
+        f"Cosine similarity is less than {target_dist} for file: {file_name}"
+
+    # Check metadata
+    percent_similarity = calculate_similarity_percent(metadata, expected_metadata)
+    assert percent_similarity >= metadata_dist, \
+        f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"
+
+
 
 @pytest.mark.parametrize("file_name, target_dist, metadata_dist", TEST_CASES)
 def test_extract_file_to_stream(file_name, target_dist, metadata_dist):
@@ -75,4 +103,4 @@ def test_extract_file_to_stream(file_name, target_dist, metadata_dist):
     # Check metadata
     percent_similarity = calculate_similarity_percent(metadata, expected_metadata)
     assert percent_similarity >= metadata_dist, \
-        f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"
+        f"The metadata similarity is lower than expected. Current {percent_similarity}% | filename: {file_name}"

bindings/extractous-python/tests/test_extract_url.py

@@ -16,4 +16,12 @@ def test_extract_url_to_string():
     content, metadata  = extractor.extract_url_to_string("https://www.google.com")
 
     assert "Google" in content
-    assert len(metadata.keys()) > 0
+    assert len(metadata.keys()) > 0
+
+def test_extract_url_to_xml():
+    extractor = Extractor()
+
+    content, metadata  = extractor.extract_url_to_xml("https://www.google.com")
+
+    assert "Google" in content
+    assert len(metadata.keys()) > 0

bindings/extractous-python/tests/utils.py

@@ -1,6 +1,6 @@
 from sklearn.feature_extraction.text import CountVectorizer
 from sklearn.metrics.pairwise import cosine_similarity as cosine_sim
-
+from lxml import etree
 
 def cosine_similarity(text1, text2):
     """Calculate the cosine similarity between two texts."""
@@ -78,3 +78,20 @@ def calculate_similarity_percent(expected, current):
 
     # Return the similarity percentage
     return matches / total
+
+
+def extract_body_text(xml: str) -> str:
+    """
+    Extracts and returns plain text content from the <body> section of an XML
+    string.
+    """
+    try:
+        parser = etree.XMLParser(recover=True)
+        root = etree.fromstring(xml.encode(), parser=parser)
+        ns= {"ns": "http://www.w3.org/1999/xhtml"}
+        body = root.find(".//ns:body", namespaces=ns)
+        if body is None:
+            return ""
+        return "\n".join(body.itertext()).strip()
+    except ET.ParseError as e:
+        raise ValueError(f"Invalid XML input: {e}")

extractous-core/Cargo.toml

@@ -36,6 +36,7 @@ test-case = "3.0"
 criterion = "0.5.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+quick-xml = "0.37.1"
 
 [build-dependencies]
 fs_extra = { version = "1.3.0" }

extractous-core/src/extractor.rs

@@ -162,6 +162,7 @@ impl Extractor {
             &self.pdf_config,
             &self.office_config,
             &self.ocr_config,
+            false,
         )
     }
 
@@ -174,6 +175,7 @@ impl Extractor {
             &self.pdf_config,
             &self.office_config,
             &self.ocr_config,
+            false,
         )
     }
 
@@ -186,8 +188,49 @@ impl Extractor {
             &self.pdf_config,
             &self.office_config,
             &self.ocr_config,
+            false,
         )
     }
+
+    /// Extracts text from a file path. Returns a tuple with xml string that is of maximum length
+    /// of the extractor's `extract_string_max_length` and metadata.
+    pub fn extract_file_to_xml(&self, file_path: &str) -> ExtractResult<(String, Metadata)> {
+        tika::parse_file_to_string(
+            file_path,
+            self.extract_string_max_length,
+            &self.pdf_config,
+            &self.office_config,
+            &self.ocr_config,
+            true,
+        )
+    }
+
+    /// Extracts text from a byte buffer. Returns a tuple with xml string that is of maximum length
+    /// of the extractor's `extract_string_max_length` and metadata.
+    pub fn extract_bytes_to_xml(&self, buffer: &[u8]) -> ExtractResult<(String, Metadata)> {
+        tika::parse_bytes_to_string(
+            buffer,
+            self.extract_string_max_length,
+            &self.pdf_config,
+            &self.office_config,
+            &self.ocr_config,
+            true,
+        )
+    }
+
+    /// Extracts text from a URL. Returns a tuple with xml string that is of maximum length
+    /// of the extractor's `extract_string_max_length` and metadata.
+    pub fn extract_url_to_xml(&self, url: &str) -> ExtractResult<(String, Metadata)> {
+        tika::parse_url_to_string(
+            url,
+            self.extract_string_max_length,
+            &self.pdf_config,
+            &self.office_config,
+            &self.ocr_config,
+            true,
+        )
+    }
+
 }
 
 #[cfg(test)]
@@ -197,6 +240,7 @@ mod tests {
     use std::fs::File;
     use std::io::BufReader;
     use std::io::{self, Read};
+    use std::str;
 
     const TEST_FILE: &str = "README.md";
 
@@ -292,4 +336,20 @@ mod tests {
             "Metadata should contain at least one entry"
         );
     }
+
+    #[test]
+    fn extract_file_to_xml_test() {
+        // Parse the files using extractous
+        let extractor = Extractor::new();
+        let result = extractor.extract_file_to_xml(TEST_FILE);
+        let (content, metadata) = result.unwrap();
+        assert!(
+            content.len() > 0,
+            "Metadata should contain at least one entry"
+        );
+        assert!(
+            metadata.len() > 0,
+            "Metadata should contain at least one entry"
+        );
+    }
 }