diff --git a/THIRD_PARTY_SBOM.cyclonedx.json b/THIRD_PARTY_SBOM.cyclonedx.json index b1c429b..c85f7e7 100644 --- a/THIRD_PARTY_SBOM.cyclonedx.json +++ b/THIRD_PARTY_SBOM.cyclonedx.json @@ -113,7 +113,7 @@ }, { "name": "context-engine:file-sha256", - "value": "third_party/ragflow/deepdoc/parser/docx_parser.py=e84ea01662ce60180e26dde1ca3ec36fc28a9e9d40357f0e9311ce6937a874c5" + "value": "third_party/ragflow/deepdoc/parser/docx_parser.py=b34d6b2cef003e50f23a0b2c587e451476222b44d18a0b48acc750d79abb01b0" }, { "name": "context-engine:file-sha256", diff --git a/tests/integration/test_file_import_tracer.py b/tests/integration/test_file_import_tracer.py index be473ab..6d7e15a 100644 --- a/tests/integration/test_file_import_tracer.py +++ b/tests/integration/test_file_import_tracer.py @@ -3,7 +3,6 @@ import json from collections.abc import Iterator from contextlib import contextmanager -from dataclasses import replace from datetime import UTC, datetime, timedelta from hashlib import sha256 from pathlib import Path @@ -516,42 +515,6 @@ def _job_state( migration_engine.dispose() -def _expire_redeemed_lease( - migration_configuration: DatabaseConfiguration, - scenario: _FileImportScenario, - claims: WorkerLeaseClaims, -) -> WorkerLeaseClaims: - migration_engine = create_database_engine(migration_configuration) - try: - with migration_engine.begin() as connection: - row = connection.execute( - text( - """ - UPDATE file_import_job - SET lease_issued_at = date_trunc('second', statement_timestamp()) - - interval '20 minutes', - lease_redeemed_at = date_trunc('second', statement_timestamp()) - - interval '19 minutes', - lease_expires_at = date_trunc('second', statement_timestamp()) - - interval '10 minutes' - WHERE organization_id = :org AND job_id = :job_id - RETURNING lease_issued_at, lease_expires_at - """ - ), - { - "org": scenario.organization_id, - "job_id": scenario.prepared.job_id, - }, - ).one() - finally: - migration_engine.dispose() - return replace( - claims, - issued_at=row.lease_issued_at, - expires_at=row.lease_expires_at, - ) - - def _scenario_effect_counts( migration_configuration: DatabaseConfiguration, scenario: _FileImportScenario, @@ -2200,10 +2163,16 @@ def test_expired_redeemed_lease_cannot_publish_or_record_failure( tmp_path, migration_configuration, guarded_control_engine, + lease_ttl_seconds=1, ) claims = _scenario_claims(scenario) assert _redeem_direct(guarded_worker_engine, claims) is not None - claims = _expire_redeemed_lease(migration_configuration, scenario, claims) + migration_engine = create_database_engine(migration_configuration) + try: + with migration_engine.connect() as connection: + connection.execute(text("SELECT pg_sleep(1.1)")) + finally: + migration_engine.dispose() assert ( _publish_direct( diff --git a/tests/unit/test_ragflow_document_compiler.py b/tests/unit/test_ragflow_document_compiler.py index 599e22b..0dd0cd1 100644 --- a/tests/unit/test_ragflow_document_compiler.py +++ b/tests/unit/test_ragflow_document_compiler.py @@ -6,16 +6,22 @@ import json import subprocess import sys +import warnings +import zipfile from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from types import SimpleNamespace from typing import Any, cast +from xml.sax.saxutils import escape import pytest import rfc8785 from docx import Document from docx.document import Document as DocumentType +from docx.opc.constants import CONTENT_TYPE, RELATIONSHIP_TYPE +from docx.opc.packuri import PackURI +from docx.opc.part import Part from docx.oxml import OxmlElement from pypdf import PdfWriter from pypdf.generic import Destination @@ -48,6 +54,9 @@ from third_party.ragflow.deepdoc.parser.utils import RawPdfOutline REPOSITORY_ROOT = Path(__file__).parents[2] +type _DocumentOutcome = ( + ParsedDocument[CompilationProfileRef] | DocumentCompilationFailure +) @dataclass @@ -76,17 +85,45 @@ def _docx_fixture(*, with_image: bool = False) -> bytes: run = document.add_paragraph().add_run() run._r.append(parse_xml(f"")) - output = io.BytesIO() - document.save(output) - return output.getvalue() + return _save_docx(document) def _save_docx(document: DocumentType) -> bytes: + output = io.BytesIO() + document.save(output) + canonical = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(output.getvalue())) as source_archive, + zipfile.ZipFile(canonical, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + target_archive.writestr(member, member_bytes) + return canonical.getvalue() + + +def _save_unmodified_docx(document: DocumentType) -> bytes: output = io.BytesIO() document.save(output) return output.getvalue() +def _compile_docx_at_public_seams(source: bytes) -> tuple[ + _DocumentOutcome, _DocumentOutcome +]: + return ( + compile_document_bytes( + source, + CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), + ), + compile_in_local_document_runner( + BytesArtifactSource(source), + DOCX_CONFIG_V1, + acceptance_context=acceptance_context(), + ), + ) + + def _docx_with_unsupported_body_container() -> bytes: document = Document() document.add_paragraph("Retained body text.") @@ -118,7 +155,12 @@ def _docx_with_tracked_insertion() -> bytes: return _save_docx(document) -def _docx_with_unsupported_drawing(*, in_header: bool) -> bytes: +def _docx_with_wrapped_text( + wrapper_tag: str, + hidden_text: str, + *, + in_header: bool, +) -> bytes: document = Document() document.add_paragraph("Retained body text.") paragraph = ( @@ -126,207 +168,2174 @@ def _docx_with_unsupported_drawing(*, in_header: bool) -> bytes: if in_header else document.add_paragraph() ) - paragraph.add_run()._r.append(OxmlElement("w:drawing")) + wrapper = OxmlElement(wrapper_tag) + run = OxmlElement("w:r") + text = OxmlElement("w:t") + text.text = hidden_text + run.append(text) + wrapper.append(run) + paragraph._p.append(wrapper) return _save_docx(document) -def _docx_with_nested_table() -> bytes: +def _docx_with_wrapped_table_cell_text() -> bytes: document = Document() - table = document.add_table(rows=1, cols=1) - table.cell(0, 0).text = "Outer cell" - nested = table.cell(0, 0).add_table(rows=1, cols=1) - nested.cell(0, 0).text = "Nested cell must not disappear." + document.add_paragraph("Retained body text.") + paragraph = document.add_table(rows=1, cols=1).cell(0, 0).paragraphs[0] + wrapper = OxmlElement("w:dir") + run = OxmlElement("w:r") + text = OxmlElement("w:t") + text.text = "Table-cell bidirectional text must not disappear." + run.append(text) + wrapper.append(run) + paragraph._p.append(wrapper) return _save_docx(document) -def _docx_fixture_with_blank_source_block() -> bytes: +def _docx_with_admitted_run_text() -> bytes: document = Document() - document.add_heading("Architecture", level=1) - document.add_paragraph("") - document.add_paragraph("After blank source block.") + run = document.add_paragraph().add_run("Before") + run.add_tab() + run.add_text("Middle") + run.add_break() + run._r.append(OxmlElement("w:noBreakHyphen")) + run.add_text("After") + return _save_docx(document) + + +def _docx_with_visible_token_outside_run(token_tag: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + paragraph = document.add_paragraph() + token = OxmlElement(token_tag) + if token_tag == "w:t": + token.text = "Silently omitted direct text." + paragraph._p.append(token) + return _save_docx(document) + + +def _docx_with_wrapped_footnote_text( + wrapper_tag: str, + hidden_text: str, + *, + content_type: str = CONTENT_TYPE.WML_FOOTNOTES, + with_drawing: bool = False, +) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + if wrapper_tag not in {"w:fldSimple", "w:smartTag"}: + raise ValueError("test fixture requires a supported wrapper tag") + drawing_xml = "" if with_drawing else "" + footnotes_xml = ( + '' + '' + f"<{wrapper_tag}>{escape(hidden_text)}" + f"{drawing_xml}" + "" + ).encode() + footnotes_part = Part( + PackURI("/word/footnotes.xml"), + content_type, + footnotes_xml, + document.part.package, + ) + document.part.relate_to(footnotes_part, RELATIONSHIP_TYPE.FOOTNOTES) + return _save_docx(document) + + +def _relabel_docx_part_as_binary(source: bytes, part_name: str) -> bytes: output = io.BytesIO() - document.save(output) + part_name_token = f'PartName="/{part_name}" ContentType="'.encode() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + content_type_start = member_bytes.index(part_name_token) + len( + part_name_token + ) + content_type_end = member_bytes.index(b'"', content_type_start) + member_bytes = ( + member_bytes[:content_type_start] + + b"application/octet-stream" + + member_bytes[content_type_end:] + ) + target_archive.writestr(member, member_bytes) return output.getvalue() -def _pdf_outline_fixture() -> bytes: - writer = PdfWriter() - writer.add_blank_page(width=612, height=792) - writer.add_blank_page(width=612, height=792) - root = writer.add_outline_item("Overview", 0) - writer.add_outline_item("Details", 1, parent=root) +def _docx_with_relabeled_header_xml(payload_kind: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + paragraph = document.sections[0].header.paragraphs[0] + if payload_kind in {"w:fldSimple", "w:smartTag"}: + wrapper = OxmlElement(payload_kind) + run = OxmlElement("w:r") + text = OxmlElement("w:t") + text.text = "Relabeled header text must not disappear." + run.append(text) + wrapper.append(run) + paragraph._p.append(wrapper) + elif payload_kind == "w:drawing": + paragraph.add_run()._r.append(OxmlElement("w:drawing")) + else: + raise ValueError("unknown relabeled header payload") + return _relabel_docx_part_as_binary(_save_docx(document), "word/header1.xml") + + +def _docx_with_header_disguised_as_thumbnail() -> bytes: + source = _docx_with_visible_header_text() output = io.BytesIO() - writer.write(output) - return output.getvalue() + header_relationship_type = RELATIONSHIP_TYPE.HEADER.encode() + thumbnail_relationship_type = RELATIONSHIP_TYPE.THUMBNAIL.encode() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/_rels/document.xml.rels": + assert header_relationship_type in member_bytes + member_bytes = member_bytes.replace( + header_relationship_type, + thumbnail_relationship_type, + 1, + ) + target_archive.writestr(member, member_bytes) + return _relabel_docx_part_as_binary(output.getvalue(), "word/header1.xml") + + +def _docx_with_relabeled_related_xml(part_kind: str, payload_kind: str) -> bytes: + if part_kind == "header": + return _docx_with_relabeled_header_xml(payload_kind) + if part_kind != "footnotes": + raise ValueError("unknown relabeled related part") + source = _docx_with_wrapped_footnote_text( + payload_kind if payload_kind != "w:drawing" else "w:fldSimple", + "Relabeled footnote text must not disappear.", + with_drawing=payload_kind == "w:drawing", + ) + return _relabel_docx_part_as_binary(source, "word/footnotes.xml") -def _pdf_outline_fixture_for_same_page(*, shifted: bool = False) -> bytes: - writer = PdfWriter() - page = writer.add_blank_page(width=612, height=792) - if shifted: - page.mediabox.lower_left = (-10, -10) - page.mediabox.upper_right = (602, 782) - writer.add_outline_item("First", 0) - writer.add_outline_item("Second", 0) +def _docx_with_relabeled_header_visual_and_malformed_relationships() -> bytes: + source = _docx_with_relabeled_header_xml("w:drawing") output = io.BytesIO() - writer.write(output) + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/_rels/document.xml.rels": + member_bytes = member_bytes.replace( + b" None: - artifact = _CountingArtifact(b"must not be read") +def _docx_with_visible_header_text() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + document.sections[0].header.paragraphs[0].text = ( + "Visible header text must not disappear." + ) + return _save_docx(document) - outcome = compile_in_local_document_runner( - artifact, - "pdf-layout-ocr-v1", - acceptance_context=acceptance_context(), + +def _docx_with_visible_footnote_text() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + footnotes_xml = ( + b'' + b'' + b"Visible footnote text must not disappear." + b"" + ) + footnotes_part = Part( + PackURI("/word/footnotes.xml"), + CONTENT_TYPE.WML_FOOTNOTES, + footnotes_xml, + document.part.package, ) + document.part.relate_to(footnotes_part, RELATIONSHIP_TYPE.FOOTNOTES) + return _save_docx(document) - assert type(outcome) is DocumentCompilationFailure - assert outcome.code is DocumentCompilationFailureCode.UNKNOWN_PROFILE - assert artifact.reads == 0 + +def _docx_with_empty_comments_before_visible_footnotes() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + comments_part = Part( + PackURI("/word/comments.xml"), + CONTENT_TYPE.WML_COMMENTS, + ( + b'' + ), + document.part.package, + ) + footnotes_part = Part( + PackURI("/word/footnotes.xml"), + CONTENT_TYPE.WML_FOOTNOTES, + ( + b'' + b"Later footnote text must not disappear." + b"" + ), + document.part.package, + ) + document.part.relate_to(comments_part, RELATIONSHIP_TYPE.COMMENTS) + document.part.relate_to(footnotes_part, RELATIONSHIP_TYPE.FOOTNOTES) + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + members = source_archive.infolist() + ordered_members = sorted( + members, + key=lambda member: ( + 0 + if member.filename == "word/comments.xml" + else 1 + if member.filename == "word/footnotes.xml" + else -1 + ), + ) + assert [ + member.filename + for member in ordered_members + if member.filename in {"word/comments.xml", "word/footnotes.xml"} + ] == ["word/comments.xml", "word/footnotes.xml"] + for member in ordered_members: + member_bytes = source_archive.read(member) + if member.filename == "word/_rels/document.xml.rels": + member_bytes = member_bytes.replace( + b'Target="../customXml/item1.xml"', + b'Target="/customXml/item1.xml"', + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() -def test_child_unknown_profile_returns_closed_failure() -> None: - completed = subprocess.run( - [ - sys.executable, - "-m", - "applications.document_compiler_runner", - "--profile", - "pdf-layout-ocr-v1", - ], - input=b"must not be parsed", - capture_output=True, - check=True, - timeout=30, +def _docx_with_relationship_target_escaping_package_root() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_unmodified_docx(document) + output = io.BytesIO() + replaced_target = False + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member) + if member.filename == "word/_rels/document.xml.rels": + expected = b'Target="../customXml/item1.xml"' + assert expected in member_bytes + member_bytes = member_bytes.replace( + expected, + b'Target="../../customXml/item1.xml"', + 1, + ) + replaced_target = True + target_archive.writestr(member, member_bytes) + assert replaced_target + return output.getvalue() + + +def _docx_with_malformed_footnotes_xml(*, with_header_drawing: bool) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + if with_header_drawing: + document.sections[0].header.paragraphs[0].add_run()._r.append( + OxmlElement("w:drawing") + ) + footnotes_part = Part( + PackURI("/word/footnotes.xml"), + "application/xml", + b"", + document.part.package, ) + document.part.relate_to(footnotes_part, RELATIONSHIP_TYPE.FOOTNOTES) + return _save_docx(document) - assert json.loads(completed.stdout) == { - "outcome": "failure", - "failure": {"code": "unknown_profile"}, - } +def _docx_with_malformed_part_media_type() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + generic_part = Part( + PackURI("/word/generic.xml"), + "application/xml; charset", + b"", + document.part.package, + ) + document.part.relate_to(generic_part, RELATIONSHIP_TYPE.CUSTOM_XML) + return _save_docx(document) -def test_child_enforces_artifact_bound_with_a_closed_refusal() -> None: - from applications.document_compiler_runner import MAX_DOCUMENT_ARTIFACT_BYTES - completed = subprocess.run( - [ - sys.executable, - "-m", - "applications.document_compiler_runner", - "--profile", - DOCX_CONFIG_V1, - ], - input=b"x" * (MAX_DOCUMENT_ARTIFACT_BYTES + 1), - capture_output=True, - check=True, - timeout=30, +def _docx_with_binary_ole_part(*, content_type: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + binary_part = Part( + PackURI("/word/embeddings/object1.bin"), + content_type, + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1ContextEngine OLE fixture", + document.part.package, ) + document.part.relate_to(binary_part, RELATIONSHIP_TYPE.OLE_OBJECT) + return _save_docx(document) - assert json.loads(completed.stdout) == { - "outcome": "failure", - "failure": {"code": "artifact_bound_exceeded"}, - } +def _docx_with_ole_relationship(target_kind: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + custom_xml_relationship_type = RELATIONSHIP_TYPE.CUSTOM_XML.encode() + ole_relationship_type = RELATIONSHIP_TYPE.OLE_OBJECT.encode() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/_rels/document.xml.rels": + if target_kind == "existing-xml": + assert custom_xml_relationship_type in member_bytes + member_bytes = member_bytes.replace( + custom_xml_relationship_type, + ole_relationship_type, + 1, + ) + elif target_kind == "external": + relationship = ( + b'' + ) + member_bytes = member_bytes.replace( + b"", + relationship + b"", + ) + else: + raise ValueError("unknown OLE relationship target kind") + target_archive.writestr(member, member_bytes) + return output.getvalue() -def test_owned_document_runner_has_no_network_database_or_model_imports() -> None: - paths = ( - REPOSITORY_ROOT / "adapters/parsers/ragflow_documents.py", - REPOSITORY_ROOT / "applications/document_compiler_runner.py", - REPOSITORY_ROOT / "third_party/ragflow/deepdoc/parser/docx_parser.py", - REPOSITORY_ROOT / "third_party/ragflow/deepdoc/parser/utils.py", + +def _docx_with_aliased_root_relationship_target(target: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + canonical_target = ( + "word/document.xml" + if target.startswith("word/") + else "docProps/thumbnail.jpeg" ) - forbidden = { - "common", - "huggingface_hub", - "httpx", - "os", - "psycopg", - "requests", - "socket", - "sqlalchemy", - "urllib", - } - for path in paths: - imports: set[str] = set() - tree = ast.parse(path.read_bytes(), filename=str(path)) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imports.update(alias.name.partition(".")[0] for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0: - assert node.module is not None - imports.add(node.module.partition(".")[0]) - assert imports.isdisjoint(forbidden), path + expected = f'Target="{canonical_target}"'.encode() + replacement = f'Target="{target}"'.encode() + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "_rels/.rels": + assert expected in member_bytes + member_bytes = member_bytes.replace(expected, replacement, 1) + target_archive.writestr(member, member_bytes) + return output.getvalue() -def test_docx_profile_preserves_ooxml_block_order_and_typed_locators() -> None: - source = _docx_fixture() +def _docx_with_invalid_thumbnail(thumbnail_bytes: bytes) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + replaced_thumbnail = False + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "docProps/thumbnail.jpeg": + member_bytes = thumbnail_bytes + replaced_thumbnail = True + target_archive.writestr(member, member_bytes) + assert replaced_thumbnail + return output.getvalue() - outcome = compile_document_bytes( - source, - CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), + +def _docx_with_external_thumbnail_relationship() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + if member.filename == "docProps/thumbnail.jpeg": + continue + member_bytes = source_archive.read(member.filename) + if member.filename == "_rels/.rels": + member_bytes = member_bytes.replace( + b'Target="docProps/thumbnail.jpeg"', + b'Target="https://example.invalid/thumbnail.jpeg" ' + b'TargetMode="External"', + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_external_hyperlink_relationship(target: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + relationship = ( + b'' ) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/_rels/document.xml.rels": + assert b"" in member_bytes + member_bytes = member_bytes.replace( + b"", + relationship + b"", + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() - assert type(outcome) is ParsedDocument - assert outcome.units is not None - assert [unit.kind for unit in outcome.units] == [ - DocumentStructuralKind.HEADING, - DocumentStructuralKind.PARAGRAPH, - DocumentStructuralKind.TABLE, - DocumentStructuralKind.PARAGRAPH, - ] - assert [unit.text for unit in outcome.units] == [ - "Architecture", - "First paragraph.", - "Key\tValue\nparser\tregistered", - "Last paragraph.", - ] - assert all( - type(locator) is DocxXmlLocator - for unit in outcome.units - for locator in unit.locators + +def _single_component_jpeg_thumbnail( + *, width: int, height: int, scan_component_id: int = 1 +) -> bytes: + return ( + b"\xff\xd8" + b"\xff\xc0\x00\x0b\x08" + + height.to_bytes(2, "big") + + width.to_bytes(2, "big") + + b"\x01\x01\x11\x00" + + b"\xff\xda\x00\x08\x01" + + bytes((scan_component_id,)) + + b"\x00\x00\x3f\x00" + + b"\xff\xd9" ) - docx_locators = tuple(unit.locators[0] for unit in outcome.units) - assert all(type(locator) is DocxXmlLocator for locator in docx_locators) - assert tuple( - locator.block_ordinal - for locator in docx_locators - if type(locator) is DocxXmlLocator - ) == (0, 1, 2, 3) - assert outcome.provenance.config_version == DOCX_CONFIG_V1 -def test_docx_image_is_an_honest_typed_refusal() -> None: - outcome = compile_document_bytes( - _docx_fixture(with_image=True), - CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), +def _docx_with_unsupported_drawing(*, in_header: bool) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + paragraph = ( + document.sections[0].header.paragraphs[0] + if in_header + else document.add_paragraph() ) + paragraph.add_run()._r.append(OxmlElement("w:drawing")) + return _save_docx(document) - assert type(outcome) is DocumentCompilationFailure - assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + +def _docx_with_nested_table() -> bytes: + document = Document() + table = document.add_table(rows=1, cols=1) + table.cell(0, 0).text = "Outer cell" + nested = table.cell(0, 0).add_table(rows=1, cols=1) + nested.cell(0, 0).text = "Nested cell must not disappear." + return _save_docx(document) + + +def _docx_with_boundary_whitespace() -> bytes: + document = Document() + document.add_paragraph(" leading and trailing ") + document.add_table(rows=1, cols=1).cell(0, 0).text = " cell boundary " + return _save_docx(document) + + +def _docx_with_horizontally_merged_cells() -> bytes: + document = Document() + table = document.add_table(rows=1, cols=2) + table.cell(0, 0).merge(table.cell(0, 1)).text = "Merged" + return _save_docx(document) + + +def _docx_with_office_math() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + paragraph = document.add_paragraph() + math = OxmlElement("m:oMath") + run = OxmlElement("m:r") + text = OxmlElement("m:t") + text.text = "Office Math must not disappear." + run.append(text) + math.append(run) + paragraph._p.append(math) + return _save_docx(document) + + +def _docx_with_misplaced_footnote_reference() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + document.add_paragraph()._p.append(OxmlElement("w:footnoteRef")) + return _save_docx(document) + + +def _docx_with_property_subtree_payload(payload_kind: str) -> bytes: + document = Document() + paragraph = document.add_paragraph("Retained body text.") + properties = paragraph._p.get_or_add_pPr() + if payload_kind == "footnote-reference": + properties.append(OxmlElement("w:footnoteRef")) + elif payload_kind == "simple-field": + properties.append(OxmlElement("w:fldSimple")) + elif payload_kind == "character-data": + properties.text = "Property character data must not disappear." + else: + raise ValueError("unknown property-subtree test payload") + return _save_docx(document) + + +def _docx_with_misplaced_table_structure(node_tag: str) -> bytes: + document = Document() + table = document.add_table(rows=1, cols=1) + table.cell(0, 0).text = "Represented cell" + properties = table._tbl.tblPr + if node_tag == "w:tr": + row = OxmlElement("w:tr") + properties.append(row) + parent = row + elif node_tag == "w:tc": + parent = properties + else: + raise ValueError("unknown misplaced table test node") + cell = OxmlElement("w:tc") + paragraph = OxmlElement("w:p") + run = OxmlElement("w:r") + text = OxmlElement("w:t") + text.text = "Misplaced table content must not disappear." + run.append(text) + paragraph.append(run) + cell.append(paragraph) + parent.append(cell) + return _save_docx(document) + + +def _docx_with_nonbody_office_math() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + paragraph = document.sections[0].header.paragraphs[0] + math = OxmlElement("m:oMath") + run = OxmlElement("m:r") + text = OxmlElement("m:t") + text.text = "Header Office Math must not disappear." + run.append(text) + math.append(run) + paragraph._p.append(math) + return _save_docx(document) + + +def _docx_with_unadmitted_structural_character_data() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + paragraph = document.add_paragraph()._p + paragraph.append(OxmlElement("w:pPr")) + paragraph[0].tail = "Direct paragraph data must not disappear." + return _save_docx(document) + + +def _docx_with_nested_payload_in_run_leaf() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + text = OxmlElement("w:t") + text.text = "Represented text" + text.append(OxmlElement("w:footnoteRef")) + document.add_paragraph().add_run()._r.append(text) + return _save_docx(document) + + +def _docx_with_document_sibling_office_math() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + math = OxmlElement("m:oMath") + run = OxmlElement("m:r") + text = OxmlElement("m:t") + text.text = "Document-level Office Math must not disappear." + run.append(text) + math.append(run) + document.element.insert(0, math) + return _save_docx(document) + + +def _docx_with_orphan_xml_members(*members: tuple[str, bytes]) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + target_archive.writestr(member, source_archive.read(member.filename)) + for member_name, member_bytes in members: + target_archive.writestr(member_name, member_bytes) + return output.getvalue() + + +def _docx_with_orphan_visible_text() -> bytes: + return _docx_with_orphan_xml_members( + ( + "word/orphan-visible.xml", + b'' + b"Orphan text must not disappear." + b"", + ) + ) + + +def _docx_with_orphan_drawing_and_malformed_xml() -> bytes: + return _docx_with_orphan_xml_members( + ( + "word/orphan-drawing.xml", + b'', + ), + ("word/orphan-malformed.xml", b""), + ) + + +def _docx_with_orphan_malformed_xml() -> bytes: + return _docx_with_orphan_xml_members( + ("word/orphan-malformed.xml", b"") + ) + + +def _docx_with_archive_name_collision(*, case_varied: bool) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + with zipfile.ZipFile(io.BytesIO(source)) as archive: + document_xml = archive.read("word/document.xml") + duplicate_name = "word/DOCUMENT.XML" if case_varied else "word/document.xml" + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + target_archive.writestr(member, source_archive.read(member.filename)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + target_archive.writestr(duplicate_name, document_xml) + return output.getvalue() + + +def _docx_with_manifest_key_collision(*, declaration_kind: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + with zipfile.ZipFile(io.BytesIO(source)) as archive: + manifest = archive.read("[Content_Types].xml") + if declaration_kind == "default": + declaration = b'' + elif declaration_kind == "override": + declaration = ( + b'' + ) + else: + raise ValueError("unknown manifest collision kind") + replaced_manifest = manifest.replace(b"", declaration + b"") + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + member_bytes = replaced_manifest + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_malformed_content_type_manifest( + manifest_kind: str, *, with_drawing: bool = False +) -> bytes: + document = Document() + paragraph = document.add_paragraph("Retained body text.") + if with_drawing: + paragraph.add_run()._r.append(OxmlElement("w:drawing")) + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + if manifest_kind == "root-unknown-attribute": + member_bytes = member_bytes.replace( + b"', b'">payload', 1) + elif manifest_kind == "default-unknown-attribute": + member_bytes = member_bytes.replace( + b"", + b">payload", + 1, + ) + elif manifest_kind == "nested-foreign-payload": + member_bytes = member_bytes.replace( + b"/>", + b'>payload', + 1, + ) + elif manifest_kind == "unparseable": + member_bytes = b" bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + raw_visual_xml = ( + b'' + ) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + member_bytes = b" bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + raw_visual_xml = ( + b'' + ) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/_rels/document.xml.rels": + member_bytes = member_bytes.replace( + b" bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + duplicate_name = "word/../duplicate-visual.xml" + benign_xml = ( + b'' + ) + visual_xml = ( + b'' + ) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/_rels/document.xml.rels": + member_bytes = member_bytes.replace( + b" bytes: + document = Document() + paragraph = document.add_paragraph("Retained body text.") + if with_drawing: + paragraph.add_run()._r.append(OxmlElement("w:drawing")) + if declaration_kind == "media-type": + declaration = ( + b'' + ) + elif declaration_kind == "media-type-non-ascii": + declaration = ( + ''.encode() + ) + elif declaration_kind == "media-type-emoji": + declaration = ( + ''.encode() + ) + elif declaration_kind == "media-type-wildcard": + declaration = b'' + elif declaration_kind == "media-type-slash-whitespace": + declaration = b'' + elif declaration_kind == "extension": + declaration = ( + b'' + ) + elif declaration_kind == "extension-dot": + declaration = b'' + elif declaration_kind == "extension-dot-dot": + declaration = b'' + else: + raise ValueError("unknown malformed declaration kind") + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + member_bytes = member_bytes.replace( + b"", declaration + b"" + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_raw_visual_and_malformed_related_xml() -> bytes: + document = Document() + document.add_paragraph("Retained body text.").add_run()._r.append( + OxmlElement("w:drawing") + ) + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/styles.xml": + member_bytes = b"" + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_unknown_related_xml() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + custom_part = Part( + PackURI("/word/customData.xml"), + "application/xml", + b'' + b"Unknown related XML must not disappear." + b"", + document.part.package, + ) + document.part.relate_to(custom_part, RELATIONSHIP_TYPE.CUSTOM_XML) + return _save_docx(document) + + +def _docx_with_misordered_table_structure(level: str) -> bytes: + document = Document() + table = document.add_table(rows=1, cols=1) + table.cell(0, 0).text = "Represented cell" + if level == "table": + properties = table._tbl.tblPr + table._tbl.remove(properties) + table._tbl.append(properties) + elif level == "row": + row = table.rows[0]._tr + properties = OxmlElement("w:trPr") + row.append(properties) + elif level == "cell": + cell = table.cell(0, 0)._tc + properties = cell.tcPr + cell.remove(properties) + cell.append(properties) + else: + raise ValueError("unknown table-order test level") + return _save_docx(document) + + +def _docx_with_duplicate_table_structure(structure_tag: str) -> bytes: + document = Document() + table = document.add_table(rows=1, cols=1) + table.cell(0, 0).text = "Represented cell" + if structure_tag == "w:tblGrid": + table._tbl.insert(2, OxmlElement(structure_tag)) + elif structure_tag == "w:trPr": + row = table.rows[0]._tr + row.insert(0, OxmlElement(structure_tag)) + row.insert(1, OxmlElement(structure_tag)) + elif structure_tag == "w:tcPr": + cell = table.cell(0, 0)._tc + cell.insert(1, OxmlElement(structure_tag)) + else: + raise ValueError("unknown duplicate table structure") + return _save_docx(document) + + +def _docx_with_legacy_horizontal_merge() -> bytes: + document = Document() + table = document.add_table(rows=1, cols=1) + table.cell(0, 0).text = "Merged semantics" + table.cell(0, 0)._tc.get_or_add_tcPr().append(OxmlElement("w:hMerge")) + return _save_docx(document) + + +def _docx_with_unsafe_manifest_part_name() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + declaration = ( + b'' + ) + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + member_bytes = member_bytes.replace( + b"", declaration + b"" + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_unsafe_archive_member_name() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + declaration = ( + b'' + ) + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + member_bytes = member_bytes.replace( + b"", declaration + b"" + ) + target_archive.writestr(member, member_bytes) + target_archive.writestr("word/../unrepresented.bin", b"binary") + return output.getvalue() + + +def _docx_with_orphan_binary_member() -> bytes: + return _docx_with_orphan_xml_members( + ("word/orphan.bin", b"unrepresented binary bytes") + ) + + +def _docx_without_root_document_relationship() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "_rels/.rels": + member_bytes = ( + b'' + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_hostile_root_document_relationship(relationship_kind: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + office_document_type = ( + b"http://schemas.openxmlformats.org/officeDocument/2006/" + b"relationships/officeDocument" + ) + expected_relationship = ( + b'' + ) + if relationship_kind == "wrong-type": + hostile_relationship = expected_relationship.replace( + office_document_type, + b"urn:context-engine:not-office-document", + ) + elif relationship_kind == "missing-type": + hostile_relationship = expected_relationship.replace( + b' Type="' + office_document_type + b'"', + b"", + ) + elif relationship_kind == "missing-id": + hostile_relationship = expected_relationship.replace(b' Id="rId1"', b"") + elif relationship_kind == "duplicate-id": + hostile_relationship = expected_relationship.replace(b'rId1', b'rId3') + elif relationship_kind == "invalid-target-mode": + hostile_relationship = expected_relationship.replace( + b' Target="word/document.xml"', + b' Target="word/document.xml" TargetMode="Neither"', + ) + else: + raise ValueError("unknown hostile relationship kind") + + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "_rels/.rels": + assert expected_relationship in member_bytes + member_bytes = member_bytes.replace( + expected_relationship, + hostile_relationship, + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_text_in_known_inert_member() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + payload = ( + b'' + b"Known-inert payload text must not disappear." + b"" + ) + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "word/styles.xml": + assert b"" in member_bytes + member_bytes = member_bytes.replace( + b"", + payload + b"", + ) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_aliased_manifest_part_name(alias: str) -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + source = _save_docx(document) + expected = b'PartName="/word/document.xml"' + replacement = f'PartName="{alias}"'.encode() + output = io.BytesIO() + with ( + zipfile.ZipFile(io.BytesIO(source)) as source_archive, + zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as target_archive, + ): + for member in source_archive.infolist(): + member_bytes = source_archive.read(member.filename) + if member.filename == "[Content_Types].xml": + assert expected in member_bytes + member_bytes = member_bytes.replace(expected, replacement) + target_archive.writestr(member, member_bytes) + return output.getvalue() + + +def _docx_with_unknown_main_document_sibling() -> bytes: + document = Document() + document.add_paragraph("Retained body text.") + document.element.insert(0, OxmlElement("w:unknown")) + return _save_docx(document) + + +def _docx_fixture_with_blank_source_block() -> bytes: + document = Document() + document.add_heading("Architecture", level=1) + document.add_paragraph("") + document.add_paragraph("After blank source block.") + return _save_docx(document) + + +def _pdf_outline_fixture() -> bytes: + writer = PdfWriter() + writer.add_blank_page(width=612, height=792) + writer.add_blank_page(width=612, height=792) + root = writer.add_outline_item("Overview", 0) + writer.add_outline_item("Details", 1, parent=root) + output = io.BytesIO() + writer.write(output) + return output.getvalue() + + +def _pdf_outline_fixture_for_same_page(*, shifted: bool = False) -> bytes: + writer = PdfWriter() + page = writer.add_blank_page(width=612, height=792) + if shifted: + page.mediabox.lower_left = (-10, -10) + page.mediabox.upper_right = (602, 782) + writer.add_outline_item("First", 0) + writer.add_outline_item("Second", 0) + output = io.BytesIO() + writer.write(output) + return output.getvalue() + + +def test_unknown_profile_refuses_before_artifact_bytes_are_opened() -> None: + artifact = _CountingArtifact(b"must not be read") + + outcome = compile_in_local_document_runner( + artifact, + "pdf-layout-ocr-v1", + acceptance_context=acceptance_context(), + ) + + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.UNKNOWN_PROFILE + assert artifact.reads == 0 + + +def test_child_unknown_profile_returns_closed_failure() -> None: + completed = subprocess.run( + [ + sys.executable, + "-m", + "applications.document_compiler_runner", + "--profile", + "pdf-layout-ocr-v1", + ], + input=b"must not be parsed", + capture_output=True, + check=True, + timeout=30, + ) + + assert json.loads(completed.stdout) == { + "outcome": "failure", + "failure": {"code": "unknown_profile"}, + } + + +def test_child_enforces_artifact_bound_with_a_closed_refusal() -> None: + from applications.document_compiler_runner import MAX_DOCUMENT_ARTIFACT_BYTES + + completed = subprocess.run( + [ + sys.executable, + "-m", + "applications.document_compiler_runner", + "--profile", + DOCX_CONFIG_V1, + ], + input=b"x" * (MAX_DOCUMENT_ARTIFACT_BYTES + 1), + capture_output=True, + check=True, + timeout=30, + ) + + assert json.loads(completed.stdout) == { + "outcome": "failure", + "failure": {"code": "artifact_bound_exceeded"}, + } + + +def test_owned_document_runner_has_no_network_database_or_model_imports() -> None: + paths = ( + REPOSITORY_ROOT / "adapters/parsers/ragflow_documents.py", + REPOSITORY_ROOT / "applications/document_compiler_runner.py", + REPOSITORY_ROOT / "third_party/ragflow/deepdoc/parser/docx_parser.py", + REPOSITORY_ROOT / "third_party/ragflow/deepdoc/parser/utils.py", + ) + forbidden = { + "common", + "huggingface_hub", + "httpx", + "os", + "psycopg", + "requests", + "socket", + "sqlalchemy", + "urllib", + } + for path in paths: + imports: set[str] = set() + tree = ast.parse(path.read_bytes(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.partition(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + assert node.module is not None + imports.add(node.module.partition(".")[0]) + assert imports.isdisjoint(forbidden), path + + +def test_docx_profile_preserves_ooxml_block_order_and_typed_locators() -> None: + source = _docx_fixture() + + outcome = compile_document_bytes( + source, + CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), + ) + + assert type(outcome) is ParsedDocument + assert outcome.units is not None + assert [unit.kind for unit in outcome.units] == [ + DocumentStructuralKind.HEADING, + DocumentStructuralKind.PARAGRAPH, + DocumentStructuralKind.TABLE, + DocumentStructuralKind.PARAGRAPH, + ] + assert [unit.text for unit in outcome.units] == [ + "Architecture", + "First paragraph.", + "Key\tValue\nparser\tregistered", + "Last paragraph.", + ] + assert all( + type(locator) is DocxXmlLocator + for unit in outcome.units + for locator in unit.locators + ) + docx_locators = tuple(unit.locators[0] for unit in outcome.units) + assert all(type(locator) is DocxXmlLocator for locator in docx_locators) + assert tuple( + locator.block_ordinal + for locator in docx_locators + if type(locator) is DocxXmlLocator + ) == (0, 1, 2, 3) + assert outcome.provenance.config_version == DOCX_CONFIG_V1 + + +def test_docx_image_is_an_honest_typed_refusal() -> None: + outcome = compile_document_bytes( + _docx_fixture(with_image=True), + CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), + ) + + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +@pytest.mark.parametrize( + "source_builder", + ( + _docx_with_unsupported_body_container, + _docx_with_tracked_insertion, + _docx_with_nested_table, + ), + ids=("content-control", "tracked-insertion", "nested-table"), +) +def test_docx_refuses_source_content_it_cannot_preserve( + source_builder: Callable[[], bytes], +) -> None: + outcome = compile_document_bytes( + source_builder(), + CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), + ) + + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + ("wrapper_tag", "hidden_text"), + ( + ("w:fldSimple", "Simple field text must not disappear."), + ("w:smartTag", "Smart tag text must not disappear."), + ), + ids=("simple-field", "smart-tag"), +) +@pytest.mark.parametrize("in_header", (False, True), ids=("body", "header")) +def test_docx_wrapped_text_refuses_at_parser_and_runner_seams( + wrapper_tag: str, + hidden_text: str, + in_header: bool, +) -> None: + source = _docx_with_wrapped_text( + wrapper_tag, + hidden_text, + in_header=in_header, + ) + outcomes = _compile_docx_at_public_seams(source) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + ("wrapper_tag", "hidden_text"), + ( + ("w:fldSimple", "Footnote field text must not disappear."), + ("w:smartTag", "Footnote smart tag text must not disappear."), + ), + ids=("simple-field", "smart-tag"), +) +def test_docx_wrapped_footnote_text_refuses_at_parser_and_runner_seams( + wrapper_tag: str, + hidden_text: str, +) -> None: + source = _docx_with_wrapped_footnote_text(wrapper_tag, hidden_text) + outcomes = _compile_docx_at_public_seams(source) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("part_kind", ("header", "footnotes")) +@pytest.mark.parametrize( + "payload_kind", + ("w:fldSimple", "w:smartTag", "w:drawing"), + ids=("simple-field", "smart-tag", "drawing"), +) +def test_docx_relabeled_related_xml_cannot_bypass_package_scanning( + part_kind: str, + payload_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_relabeled_related_xml(part_kind, payload_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is ( + DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + if payload_kind == "w:drawing" + else DocumentCompilationFailureCode.INVALID_ARTIFACT + ) + + +def test_docx_malformed_relationships_preserve_raw_visual_refusal_precedence( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_relabeled_header_visual_and_malformed_relationships() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +def test_docx_malformed_relationships_scan_unsafe_raw_visual_members() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_malformed_relationships_and_unsafe_raw_visual_member() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +def test_docx_malformed_relationships_scan_every_duplicate_raw_member() -> None: + direct_outcome, runner_outcome = _compile_docx_at_public_seams( + _docx_with_malformed_relationships_and_duplicate_unsafe_raw_visual_member() + ) + + assert type(direct_outcome) is DocumentCompilationFailure + assert type(runner_outcome) is DocumentCompilationFailure + assert (direct_outcome.code, runner_outcome.code) == ( + DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED, + DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED, + ) + + +def test_docx_header_cannot_hide_behind_thumbnail_relationship_at_both_seams( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_header_disguised_as_thumbnail() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + ("wrapper_tag", "hidden_text"), + ( + ("w:fldSimple", "Case-varied footnote field text must not disappear."), + ("w:smartTag", "Case-varied footnote smart tag text must not disappear."), + ), + ids=("simple-field", "smart-tag"), +) +@pytest.mark.parametrize( + "content_type", + ( + "Application/XML", + "application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+XmL", + "application/xml; charset=UTF-8", + ), + ids=("uppercase-base-xml", "mixed-case-xml-suffix", "parameterized-xml"), +) +def test_docx_case_varied_xml_media_types_still_refuse_wrapped_footnotes( + wrapper_tag: str, + hidden_text: str, + content_type: str, +) -> None: + source = _docx_with_wrapped_footnote_text( + wrapper_tag, + hidden_text, + content_type=content_type, + ) + outcomes = _compile_docx_at_public_seams(source) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "source_builder", + ( + _docx_with_visible_header_text, + _docx_with_visible_footnote_text, + lambda: _docx_with_wrapped_text( + "w:dir", + "Bidirectional text must not disappear.", + in_header=False, + ), + _docx_with_wrapped_table_cell_text, + ), + ids=("header", "footnote", "unknown-body-container", "table-cell-container"), +) +def test_docx_refuses_visible_text_it_cannot_represent_at_both_seams( + source_builder: Callable[[], bytes], +) -> None: + source = source_builder() + outcomes = _compile_docx_at_public_seams(source) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_preserves_admitted_run_text_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_admitted_run_text()) + + for outcome in outcomes: + assert type(outcome) is ParsedDocument + assert outcome.units is not None + assert [unit.text for unit in outcome.units] == ["Before\tMiddle\n-After"] + + +def test_docx_preserves_boundary_whitespace_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_boundary_whitespace()) + + for outcome in outcomes: + assert type(outcome) is ParsedDocument + assert outcome.units is not None + assert [unit.text for unit in outcome.units] == [ + " leading and trailing ", + " cell boundary ", + ] + assert outcome.units[1].table_cells == ((" cell boundary ",),) + + +def test_docx_refuses_horizontally_merged_cells_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_horizontally_merged_cells()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT @pytest.mark.parametrize( "source_builder", ( - _docx_with_unsupported_body_container, - _docx_with_tracked_insertion, - _docx_with_nested_table, + _docx_with_office_math, + _docx_with_misplaced_footnote_reference, + _docx_with_orphan_visible_text, + _docx_with_orphan_malformed_xml, ), - ids=("content-control", "tracked-insertion", "nested-table"), + ids=("office-math", "misplaced-control", "orphan-text", "orphan-malformed"), ) -def test_docx_refuses_source_content_it_cannot_preserve( +def test_docx_closed_grammar_refuses_unrepresented_xml_at_both_seams( + source_builder: Callable[[], bytes], +) -> None: + outcomes = _compile_docx_at_public_seams(source_builder()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_orphan_drawing_preserves_visual_refusal_precedence_at_both_seams( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_orphan_drawing_and_malformed_xml() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +@pytest.mark.parametrize( + "payload_kind", + ("footnote-reference", "simple-field", "character-data"), +) +def test_docx_closed_grammar_refuses_property_subtree_payloads_at_both_seams( + payload_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_property_subtree_payload(payload_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("node_tag", ("w:tr", "w:tc"), ids=("row", "cell")) +def test_docx_closed_grammar_refuses_misplaced_table_structure_at_both_seams( + node_tag: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_misplaced_table_structure(node_tag) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_refuses_nonbody_office_math_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_nonbody_office_math()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "source_builder", + ( + _docx_with_unadmitted_structural_character_data, + _docx_with_nested_payload_in_run_leaf, + _docx_with_document_sibling_office_math, + ), + ids=("structural-character-data", "nested-run-leaf", "document-sibling"), +) +def test_docx_closed_grammar_refuses_recursive_structure_bypasses_at_both_seams( + source_builder: Callable[[], bytes], +) -> None: + outcomes = _compile_docx_at_public_seams(source_builder()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("case_varied", (False, True), ids=("exact", "casefold")) +def test_docx_refuses_duplicate_archive_names_at_both_seams( + case_varied: bool, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_archive_name_collision(case_varied=case_varied) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("declaration_kind", ("default", "override")) +def test_docx_refuses_casefolded_manifest_key_collisions_at_both_seams( + declaration_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_manifest_key_collision(declaration_kind=declaration_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "manifest_kind", + ( + "root-unknown-attribute", + "root-character-data", + "default-unknown-attribute", + "override-unknown-attribute", + "declaration-character-data", + "nested-foreign-payload", + ), +) +def test_docx_refuses_malformed_content_type_manifest_at_both_seams( + manifest_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_malformed_content_type_manifest(manifest_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_manifest_failure_preserves_visual_precedence_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_malformed_content_type_manifest( + "root-unknown-attribute", + with_drawing=True, + ) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +def test_docx_unparseable_manifest_preserves_raw_visual_precedence_at_both_seams( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_malformed_content_type_manifest( + "unparseable", + with_drawing=True, + ) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +def test_docx_unparseable_manifest_scans_unsafe_raw_visual_members_at_both_seams( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_unparseable_manifest_and_unsafe_raw_visual_member() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +@pytest.mark.parametrize("declaration_kind", ("media-type", "extension")) +def test_docx_refuses_unused_malformed_content_type_declarations_at_both_seams( + declaration_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_unused_malformed_content_type_declaration(declaration_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "declaration_kind", + ( + "media-type-non-ascii", + "media-type-emoji", + "media-type-wildcard", + "media-type-slash-whitespace", + "media-type", + "extension-dot", + "extension-dot-dot", + ), + ids=( + "non-ascii-media-type", + "emoji-media-type", + "wildcard-media-type", + "slash-whitespace-media-type", + "defective-media-type", + "dot-extension", + "dot-dot-extension", + ), +) +def test_docx_refuses_manifest_tokens_outside_strict_ascii_grammar_at_both_seams( + declaration_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_unused_malformed_content_type_declaration(declaration_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("declaration_kind", ("media-type", "extension")) +def test_docx_unused_manifest_failure_preserves_visual_precedence_at_both_seams( + declaration_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_unused_malformed_content_type_declaration( + declaration_kind, + with_drawing=True, + ) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +def test_docx_raw_inventory_preserves_visual_precedence_before_document_load( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_raw_visual_and_malformed_related_xml() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +def test_docx_refuses_unknown_related_xml_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_unknown_related_xml()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("level", ("table", "row", "cell")) +def test_docx_refuses_misordered_table_structure_at_both_seams( + level: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_misordered_table_structure(level) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("structure_tag", ("w:tblGrid", "w:trPr", "w:tcPr")) +def test_docx_refuses_duplicate_table_structure_at_both_seams( + structure_tag: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_duplicate_table_structure(structure_tag) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_refuses_legacy_horizontal_merge_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_legacy_horizontal_merge()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "source_builder", + (_docx_with_unsafe_manifest_part_name, _docx_with_unsafe_archive_member_name), + ids=("manifest-part-name", "archive-member-name"), +) +def test_docx_refuses_unsafe_package_paths_at_both_seams( + source_builder: Callable[[], bytes], +) -> None: + outcomes = _compile_docx_at_public_seams(source_builder()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "source_builder", + ( + _docx_with_orphan_binary_member, + _docx_without_root_document_relationship, + _docx_with_unknown_main_document_sibling, + ), + ids=("orphan-binary", "unrelated-main", "unknown-main-sibling"), +) +def test_docx_refuses_unrepresented_package_inventory_at_both_seams( source_builder: Callable[[], bytes], ) -> None: + outcomes = _compile_docx_at_public_seams(source_builder()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "relationship_kind", + ( + "wrong-type", + "missing-type", + "missing-id", + "duplicate-id", + "invalid-target-mode", + ), +) +def test_docx_refuses_hostile_root_document_relationships_at_both_seams( + relationship_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_hostile_root_document_relationship(relationship_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "target", + ( + "word//document.xml", + "docProps//thumbnail.jpeg", + "docProps/./thumbnail.jpeg", + ), + ids=( + "document-double-slash", + "thumbnail-double-slash", + "thumbnail-dot-segment", + ), +) +def test_docx_refuses_raw_relationship_target_aliases_at_both_seams( + target: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_aliased_root_relationship_target(target) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "thumbnail_bytes", + ( + b"\xff\xd8\xff", + b'\xff\xd8\xff', + ), + ids=("truncated-marker", "drawing-after-signature"), +) +def test_docx_refuses_incomplete_jpeg_thumbnails_at_both_seams( + thumbnail_bytes: bytes, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_invalid_thumbnail(thumbnail_bytes) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_refuses_zero_dimension_jpeg_thumbnail_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_invalid_thumbnail( + _single_component_jpeg_thumbnail(width=0, height=0) + ) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_refuses_unbound_jpeg_scan_component_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_invalid_thumbnail( + _single_component_jpeg_thumbnail( + width=1, + height=1, + scan_component_id=2, + ) + ) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_refuses_external_thumbnail_relationship_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_external_thumbnail_relationship() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "target", + ( + "https://example.invalid/./resource", + "https://example.invalid/path/../resource", + ), + ids=("dot-segment", "dot-dot-segment"), +) +def test_docx_refuses_external_relationship_dot_segments_at_both_seams( + target: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_external_hyperlink_relationship(target) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_accepts_ordinary_external_relationship_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_external_hyperlink_relationship( + "https://example.invalid/path/resource?next=../resource#dot/./segment" + ) + ) + + for outcome in outcomes: + assert type(outcome) is ParsedDocument + + +def test_docx_compiles_unmodified_python_docx_bytes_at_both_seams() -> None: + document = Document() + document.add_paragraph("Unmodified python-docx package text.") + + outcomes = _compile_docx_at_public_seams(_save_unmodified_docx(document)) + + for outcome in outcomes: + assert type(outcome) is ParsedDocument + assert outcome.units is not None + assert [unit.text for unit in outcome.units] == [ + "Unmodified python-docx package text." + ] + + +def test_docx_relationship_target_cannot_escape_package_root_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_relationship_target_escaping_package_root() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_empty_unrepresented_part_does_not_hide_later_text_at_both_seams( +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_empty_comments_before_visible_footnotes() + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_refuses_text_in_known_inert_member_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_text_in_known_inert_member()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "part_name", + ( + "/word/./document.xml", + "/word//document.xml", + "/word/document.xml?alias=1", + "/word/document.xml#alias", + ), + ids=("dot-segment", "double-slash", "query", "fragment"), +) +def test_docx_refuses_aliased_manifest_part_names_at_both_seams( + part_name: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_aliased_manifest_part_name(part_name) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + "token_tag", + ("w:t", "w:tab", "w:ptab", "w:br", "w:cr", "w:noBreakHyphen"), + ids=("text", "tab", "position-tab", "break", "carriage-return", "no-break-hyphen"), +) +def test_docx_refuses_visible_tokens_outside_runs_at_both_seams( + token_tag: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_visible_token_outside_run(token_tag) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize( + ("with_header_drawing", "expected_code"), + ( + (True, DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED), + (False, DocumentCompilationFailureCode.INVALID_ARTIFACT), + ), + ids=("visual-first", "malformed-only"), +) +def test_docx_malformed_generic_xml_preserves_visual_refusal_precedence( + with_header_drawing: bool, + expected_code: DocumentCompilationFailureCode, +) -> None: + source = _docx_with_malformed_footnotes_xml( + with_header_drawing=with_header_drawing + ) + outcomes = _compile_docx_at_public_seams(source) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is expected_code + + +def test_docx_refuses_malformed_package_part_media_type_at_both_seams() -> None: + outcomes = _compile_docx_at_public_seams(_docx_with_malformed_part_media_type()) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +def test_docx_generic_xml_part_preserves_visual_refusal_precedence() -> None: outcome = compile_document_bytes( - source_builder(), + _docx_with_wrapped_footnote_text( + "w:fldSimple", + "Footnote field text must not disappear.", + with_drawing=True, + ), CompilationProfileRef("context-engine-docx-v1", DOCX_CONFIG_V1), ) assert type(outcome) is DocumentCompilationFailure - assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + assert outcome.code is DocumentCompilationFailureCode.FIGURE_NOT_SUPPORTED + + +@pytest.mark.parametrize( + "content_type", + ("application/octet-stream", 'Application/Octet-Stream; profile="xml-looking"'), + ids=("bare", "parameterized"), +) +def test_docx_refuses_unvalidated_ole_compound_file_at_both_seams( + content_type: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_binary_ole_part(content_type=content_type) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT + + +@pytest.mark.parametrize("target_kind", ("external", "existing-xml")) +def test_docx_refuses_every_ole_relationship_at_both_seams( + target_kind: str, +) -> None: + outcomes = _compile_docx_at_public_seams( + _docx_with_ole_relationship(target_kind) + ) + + for outcome in outcomes: + assert type(outcome) is DocumentCompilationFailure + assert outcome.code is DocumentCompilationFailureCode.INVALID_ARTIFACT @pytest.mark.parametrize("in_header", (False, True)) diff --git a/tests/unit/test_third_party_ragflow_registration.py b/tests/unit/test_third_party_ragflow_registration.py index 077d6a9..9ffeaf3 100644 --- a/tests/unit/test_third_party_ragflow_registration.py +++ b/tests/unit/test_third_party_ragflow_registration.py @@ -57,7 +57,9 @@ "sys", "typing", "unicodedata", + "zipfile", "docx", + "email", } diff --git a/third_party/ragflow/MODIFICATIONS.md b/third_party/ragflow/MODIFICATIONS.md index e18bc14..3d8fe55 100644 --- a/third_party/ragflow/MODIFICATIONS.md +++ b/third_party/ragflow/MODIFICATIONS.md @@ -46,10 +46,57 @@ selected helpers from the copied `MarkdownElementExtractor`; the surrounding rich Markdown compilation pipeline remains ContextEngine-owned. `docx_parser.py` is copied and patched to remove RAGFlow tokenizer, -`LazyImage`, Pandas, logging, and application constants. It now traverses -paragraph and table XML children in exact OOXML body order and returns bounded -raw blocks. ContextEngine-owned code maps those blocks into the ADR-0094 nominal -`DocxXmlLocator` family, structural units, identities, and typed refusals. +`LazyImage`, Pandas, logging, and application constants. It now inventories the +raw OPC archive and strictly parses `[Content_Types].xml` before constructing a +`python-docx` document. Every declared XML member is parsed independently and +every successfully parsed root is scanned for visuals first, so a figure +refusal takes precedence over malformed or unrepresented XML even when +`python-docx` cannot load the package. When the content-type manifest itself is +unparseable, every independently parseable raw member, including a member whose +raw ZIP name is not an admissible canonical package path, is still scanned for +visuals before the retained manifest refusal. Archive names, PartNames, media +types, case-folded declarations, and relationship reachability are validated. The +content-type manifest admits an attribute-free, text-free root containing only +exact leaf `Default` and `Override` declarations; malformed declarations remain +usable only to scan otherwise classifiable XML for visuals before the retained +manifest refusal. Every declaration's media type must use strict ASCII MIME +tokens, and its raw pre-parameter base must exactly match the parsed type and +subtype except for case; parser-normalized whitespace is never accepted. Every +extension must use the closed ASCII token grammar while excluding `.` and `..`, +even when no member uses the declaration. The +relationship grammar requires exact permitted attributes, unique non-empty +identifiers, valid target modes, internal canonical raw targets without empty +or `.` segments before resolution (canonical relative `..` segments resolve one +parent at a time and fail closed only when they escape the package root), no +`.` or `..` segment in the raw path portion of External targets, and the exact +root-to-main office-document relationship type. A manifest relabel cannot hide related XML: +non-XML-related parts outside exact admitted binary relationship classes must +parse as XML. A malformed relationship graph likewise cannot hide raw parseable +visuals: every non-manifest raw ZIP entry is scanned by entry identity before +the retained package refusal, including entries whose raw names are not +admissible canonical package paths and multiple entries that share a filename. +Thumbnail admission +requires an internal root relationship, exact `docProps/thumbnail.jpeg` target, +exact JPEG media type, nonzero dimensions, coherent frame/scan component +identifiers, and a complete bounded JPEG marker structure through +start-of-frame, start-of-scan, and end-of-image; a signature prefix alone is +never admitted. Every OLE relationship, internal or external and regardless of +target parseability, refuses because the active profile cannot represent +embedded OLE and no complete compound-file validator is registered. Orphan, +unknown, and path-ambiguous members fail closed. Admitted thumbnail bytes are +inventoried but never parsed as XML. + +The body-only profile admits one positive main-document/paragraph/run/table +grammar, including recursive property-subtree validation and exact structural +parent, order, and cardinality rules. It rejects non-body content-bearing XML, +unknown namespaces or placements, arbitrary character data in infrastructure +members, nested tables, and merged cells whose semantics cannot be represented +losslessly. Only exact core/app metadata fields admit character data. +Independently derived visible run text must equal `python-docx` paragraph text, +every visible token must be inside an admitted run, and exact paragraph and +table-cell boundary whitespace is preserved. ContextEngine-owned code maps +accepted blocks into the ADR-0094 nominal `DocxXmlLocator` family, structural +units, identities, and typed refusals. Image-bearing DOCX artifacts refuse because a bounded figure-byte policy has not been admitted; images are never silently discarded. diff --git a/third_party/ragflow/UPSTREAM.toml b/third_party/ragflow/UPSTREAM.toml index 0021f15..0ff5acc 100644 --- a/third_party/ragflow/UPSTREAM.toml +++ b/third_party/ragflow/UPSTREAM.toml @@ -36,7 +36,7 @@ sha256 = "94c8e2515d05e141fcf65e10336ceca7f9116e54b31a668637ba3f901943cb66" [[files]] upstream_path = "deepdoc/parser/docx_parser.py" vendored_path = "third_party/ragflow/deepdoc/parser/docx_parser.py" -sha256 = "e84ea01662ce60180e26dde1ca3ec36fc28a9e9d40357f0e9311ce6937a874c5" +sha256 = "b34d6b2cef003e50f23a0b2c587e451476222b44d18a0b48acc750d79abb01b0" [[files]] upstream_path = "deepdoc/parser/utils.py" diff --git a/third_party/ragflow/deepdoc/parser/docx_parser.py b/third_party/ragflow/deepdoc/parser/docx_parser.py index 445c1a6..c8ed90f 100644 --- a/third_party/ragflow/deepdoc/parser/docx_parser.py +++ b/third_party/ragflow/deepdoc/parser/docx_parser.py @@ -21,31 +21,586 @@ from __future__ import annotations from dataclasses import dataclass +from email import policy from io import BytesIO from typing import Any, Final +from zipfile import ZipFile from docx import Document from docx.document import Document as DocumentType +from docx.oxml import parse_xml from docx.oxml.ns import qn from docx.table import Table from docx.text.paragraph import Paragraph _PARAGRAPH_TAG: Final = qn("w:p") +_PARAGRAPH_PROPERTIES_TAG: Final = qn("w:pPr") _TABLE_TAG: Final = qn("w:tbl") +_TABLE_PROPERTIES_TAG: Final = qn("w:tblPr") +_TABLE_GRID_TAG: Final = qn("w:tblGrid") +_TABLE_ROW_TAG: Final = qn("w:tr") +_TABLE_ROW_PROPERTIES_TAG: Final = qn("w:trPr") +_TABLE_CELL_TAG: Final = qn("w:tc") +_TABLE_CELL_PROPERTIES_TAG: Final = qn("w:tcPr") _SECTION_PROPERTIES_TAG: Final = qn("w:sectPr") -_UNSUPPORTED_CONTENT_TAGS: Final = frozenset( +_RUN_TAG: Final = qn("w:r") +_RUN_PROPERTIES_TAG: Final = qn("w:rPr") +_TEXT_TAG: Final = qn("w:t") +_TAB_STOPS_TAG: Final = qn("w:tabs") +_TAB_TAGS: Final = frozenset({qn("w:tab"), qn("w:ptab")}) +_BREAK_TAGS: Final = frozenset({qn("w:br"), qn("w:cr")}) +_BREAK_TYPE_ATTRIBUTE: Final = qn("w:type") +_NO_BREAK_HYPHEN_TAG: Final = qn("w:noBreakHyphen") +_NON_VISIBLE_RUN_TAGS: Final = frozenset( { - qn("w:customXml"), - qn("w:del"), - qn("w:ins"), - qn("w:moveFrom"), - qn("w:moveTo"), - qn("w:sdt"), + qn("w:fldChar"), + qn("w:instrText"), + qn("w:lastRenderedPageBreak"), + _RUN_PROPERTIES_TAG, } ) +_ADMITTED_RUN_TAGS: Final = ( + frozenset({_TEXT_TAG, _NO_BREAK_HYPHEN_TAG}) + | _TAB_TAGS + | _BREAK_TAGS + | _NON_VISIBLE_RUN_TAGS +) +_VISIBLE_TOKEN_TAGS: Final = ( + frozenset({_TEXT_TAG, _NO_BREAK_HYPHEN_TAG}) | _TAB_TAGS | _BREAK_TAGS +) _UNSUPPORTED_VISUAL_TAGS: Final = frozenset( - {qn("w:drawing"), qn("w:object"), qn("w:pict")} + {qn("pic:pic"), qn("w:drawing"), qn("w:object"), qn("w:pict")} +) +_XML_CONTENT_TYPES: Final = frozenset({"application/xml", "text/xml"}) +_CONTENT_TYPES_MEMBER: Final = "[Content_Types].xml" +_CONTENT_TYPES_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/package/2006/content-types" +) +_CONTENT_TYPES_TAG: Final = f"{{{_CONTENT_TYPES_NAMESPACE}}}Types" +_CONTENT_TYPE_DEFAULT_TAG: Final = f"{{{_CONTENT_TYPES_NAMESPACE}}}Default" +_CONTENT_TYPE_OVERRIDE_TAG: Final = f"{{{_CONTENT_TYPES_NAMESPACE}}}Override" +_DOCUMENT_MEMBER: Final = "word/document.xml" +_WORDPROCESSINGML_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +) +_WORDPROCESSINGML_TAG_PREFIX: Final = f"{{{_WORDPROCESSINGML_NAMESPACE}}}" +_OFFICE_MATH_TAG_PREFIX: Final = ( + "{http://schemas.openxmlformats.org/officeDocument/2006/math}" +) +_OFFICE_MATH_CONTENT_TAGS: Final = frozenset( + { + f"{_OFFICE_MATH_TAG_PREFIX}oMath", + f"{_OFFICE_MATH_TAG_PREFIX}oMathPara", + } +) +_ADMITTED_PARAGRAPH_CHILDREN: Final = frozenset( + {_PARAGRAPH_PROPERTIES_TAG, _RUN_TAG} +) +_ADMITTED_TABLE_CHILDREN: Final = frozenset( + {_TABLE_PROPERTIES_TAG, _TABLE_GRID_TAG, _TABLE_ROW_TAG} +) +_ADMITTED_TABLE_ROW_CHILDREN: Final = frozenset( + {_TABLE_ROW_PROPERTIES_TAG, _TABLE_CELL_TAG} +) +_ADMITTED_TABLE_CELL_CHILDREN: Final = frozenset( + {_TABLE_CELL_PROPERTIES_TAG, _PARAGRAPH_TAG} +) +_MERGED_CELL_TAGS: Final = frozenset( + {qn("w:gridSpan"), qn("w:hMerge"), qn("w:vMerge")} +) +_UNREPRESENTED_PART_ROOT_TAGS: Final = frozenset( + { + qn("w:comments"), + qn("w:endnotes"), + qn("w:footnotes"), + qn("w:ftr"), + qn("w:glossaryDocument"), + qn("w:hdr"), + } +) +_RELATIONSHIPS_TAG: Final = ( + "{http://schemas.openxmlformats.org/package/2006/relationships}Relationships" +) +_RELATIONSHIP_TAG: Final = ( + "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship" +) +_OFFICE_DOCUMENT_RELATIONSHIP_TYPE: Final = ( + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/" + "officeDocument" +) +_OLE_OBJECT_RELATIONSHIP_TYPE: Final = ( + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/oleObject" +) +_THUMBNAIL_RELATIONSHIP_TYPE: Final = ( + "http://schemas.openxmlformats.org/package/2006/relationships/metadata/thumbnail" +) +_THUMBNAIL_MEMBER: Final = "docProps/thumbnail.jpeg" +_THUMBNAIL_CONTENT_TYPE: Final = "image/jpeg" +_JPEG_START_OF_IMAGE: Final = b"\xff\xd8" +_JPEG_START_OF_FRAME_MARKERS: Final = frozenset( + { + 0xC0, + 0xC1, + 0xC2, + 0xC3, + 0xC5, + 0xC6, + 0xC7, + 0xC9, + 0xCA, + 0xCB, + 0xCD, + 0xCE, + 0xCF, + } +) +_BINARY_RELATIONSHIP_TYPES: Final = frozenset( + {_OLE_OBJECT_RELATIONSHIP_TYPE, _THUMBNAIL_RELATIONSHIP_TYPE} +) +_DRAWINGML_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/drawingml/2006/main" +) +_CORE_PROPERTIES_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/package/2006/metadata/core-properties" +) +_DC_NAMESPACE: Final = "http://purl.org/dc/elements/1.1/" +_DCTERMS_NAMESPACE: Final = "http://purl.org/dc/terms/" +_EXTENDED_PROPERTIES_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties" +) +_VARIANT_TYPES_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes" +) +_BIBLIOGRAPHY_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/officeDocument/2006/bibliography" +) +_CUSTOM_XML_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/officeDocument/2006/customXml" ) +_WORD_2010_NAMESPACE: Final = ( + "http://schemas.microsoft.com/office/word/2010/wordml" +) +_OFFICE_NAMESPACE: Final = "urn:schemas-microsoft-com:office:office" +_VML_NAMESPACE: Final = "urn:schemas-microsoft-com:vml" +_MARKUP_COMPATIBILITY_NAMESPACE: Final = ( + "http://schemas.openxmlformats.org/markup-compatibility/2006" +) +_XML_SCHEMA_INSTANCE_NAMESPACE: Final = "http://www.w3.org/2001/XMLSchema-instance" +_KNOWN_INERT_XML_ROOTS: Final = { + "docProps/core.xml": frozenset( + { + "{http://schemas.openxmlformats.org/package/2006/metadata/" + "core-properties}coreProperties" + } + ), + "docProps/app.xml": frozenset( + { + "{http://schemas.openxmlformats.org/officeDocument/2006/" + "extended-properties}Properties" + } + ), + "word/styles.xml": frozenset({qn("w:styles")}), + "word/stylesWithEffects.xml": frozenset({qn("w:styles")}), + "word/settings.xml": frozenset({qn("w:settings")}), + "word/webSettings.xml": frozenset({qn("w:webSettings")}), + "word/fontTable.xml": frozenset({qn("w:fonts")}), + "word/theme/theme1.xml": frozenset( + {"{http://schemas.openxmlformats.org/drawingml/2006/main}theme"} + ), + "customXml/item1.xml": frozenset( + { + "{http://schemas.openxmlformats.org/officeDocument/2006/" + "bibliography}Sources" + } + ), + "customXml/itemProps1.xml": frozenset( + { + "{http://schemas.openxmlformats.org/officeDocument/2006/" + "customXml}datastoreItem" + } + ), + "word/numbering.xml": frozenset({qn("w:numbering")}), +} +_KNOWN_INERT_XML_ELEMENT_NAMESPACES: Final = { + "docProps/core.xml": frozenset( + {_CORE_PROPERTIES_NAMESPACE, _DC_NAMESPACE, _DCTERMS_NAMESPACE} + ), + "docProps/app.xml": frozenset( + {_EXTENDED_PROPERTIES_NAMESPACE, _VARIANT_TYPES_NAMESPACE} + ), + "word/styles.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), + "word/stylesWithEffects.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), + "word/settings.xml": frozenset( + { + _WORDPROCESSINGML_NAMESPACE, + _WORD_2010_NAMESPACE, + _OFFICE_MATH_TAG_PREFIX[1:-1], + _OFFICE_NAMESPACE, + } + ), + "word/webSettings.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), + "word/fontTable.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), + "word/theme/theme1.xml": frozenset({_DRAWINGML_NAMESPACE}), + "customXml/item1.xml": frozenset({_BIBLIOGRAPHY_NAMESPACE}), + "customXml/itemProps1.xml": frozenset({_CUSTOM_XML_NAMESPACE}), + "word/numbering.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), +} +_KNOWN_INERT_XML_TEXT_TAGS: Final = { + "docProps/core.xml": frozenset( + { + f"{{{_CORE_PROPERTIES_NAMESPACE}}}{name}" + for name in ( + "category", + "contentStatus", + "keywords", + "lastModifiedBy", + "lastPrinted", + "revision", + "version", + ) + } + | { + f"{{{_DC_NAMESPACE}}}{name}" + for name in ( + "creator", + "description", + "identifier", + "language", + "subject", + "title", + ) + } + | { + f"{{{_DCTERMS_NAMESPACE}}}{name}" for name in ("created", "modified") + } + ), + "docProps/app.xml": frozenset( + { + f"{{{_EXTENDED_PROPERTIES_NAMESPACE}}}{name}" + for name in ( + "AppVersion", + "Application", + "Characters", + "CharactersWithSpaces", + "Company", + "DocSecurity", + "HiddenSlides", + "HyperlinkBase", + "HyperlinksChanged", + "Lines", + "LinksUpToDate", + "Manager", + "MMClips", + "Notes", + "Pages", + "Paragraphs", + "PresentationFormat", + "ScaleCrop", + "SharedDoc", + "Slides", + "Template", + "TotalTime", + "Words", + ) + } + | { + f"{{{_VARIANT_TYPES_NAMESPACE}}}{name}" + for name in ( + "bool", + "bstr", + "date", + "decimal", + "filetime", + "i1", + "i2", + "i4", + "i8", + "lpstr", + "lpwstr", + "r4", + "r8", + "ui1", + "ui2", + "ui4", + "ui8", + ) + } + ), +} +_KNOWN_INERT_XML_ATTRIBUTE_NAMESPACES: Final = { + "docProps/core.xml": frozenset({"", _XML_SCHEMA_INSTANCE_NAMESPACE}), + "docProps/app.xml": frozenset({""}), + "word/styles.xml": frozenset( + {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} + ), + "word/stylesWithEffects.xml": frozenset( + {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} + ), + "word/settings.xml": frozenset( + { + "", + _WORDPROCESSINGML_NAMESPACE, + _WORD_2010_NAMESPACE, + _OFFICE_MATH_TAG_PREFIX[1:-1], + _OFFICE_NAMESPACE, + _VML_NAMESPACE, + _MARKUP_COMPATIBILITY_NAMESPACE, + } + ), + "word/webSettings.xml": frozenset({"", _MARKUP_COMPATIBILITY_NAMESPACE}), + "word/fontTable.xml": frozenset( + {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} + ), + "word/theme/theme1.xml": frozenset({""}), + "customXml/item1.xml": frozenset({""}), + "customXml/itemProps1.xml": frozenset({"", _CUSTOM_XML_NAMESPACE}), + "word/numbering.xml": frozenset( + {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} + ), +} +_PROPERTY_ROOT_PARENTS: Final = { + _PARAGRAPH_PROPERTIES_TAG: frozenset({_PARAGRAPH_TAG}), + _RUN_PROPERTIES_TAG: frozenset({_PARAGRAPH_PROPERTIES_TAG, _RUN_TAG}), + _SECTION_PROPERTIES_TAG: frozenset({qn("w:body")}), + _TABLE_PROPERTIES_TAG: frozenset({_TABLE_TAG}), + _TABLE_GRID_TAG: frozenset({_TABLE_TAG}), + _TABLE_ROW_PROPERTIES_TAG: frozenset({_TABLE_ROW_TAG}), + _TABLE_CELL_PROPERTIES_TAG: frozenset({_TABLE_CELL_TAG}), +} +_BORDER_PROPERTY_TAGS: Final = frozenset( + { + qn("w:bar"), + qn("w:between"), + qn("w:bottom"), + qn("w:end"), + qn("w:insideH"), + qn("w:insideV"), + qn("w:left"), + qn("w:right"), + qn("w:start"), + qn("w:tl2br"), + qn("w:top"), + qn("w:tr2bl"), + } +) +_MARGIN_PROPERTY_TAGS: Final = frozenset( + { + qn("w:bottom"), + qn("w:end"), + qn("w:left"), + qn("w:right"), + qn("w:start"), + qn("w:top"), + } +) +_RUN_PROPERTY_TAGS: Final = frozenset( + { + qn(f"w:{name}") + for name in ( + "b", + "bCs", + "bdr", + "caps", + "color", + "cs", + "dstrike", + "eastAsianLayout", + "effect", + "em", + "emboss", + "fitText", + "highlight", + "i", + "iCs", + "imprint", + "kern", + "lang", + "noProof", + "oMath", + "outline", + "position", + "rFonts", + "rStyle", + "rtl", + "shadow", + "shd", + "smallCaps", + "snapToGrid", + "spacing", + "specVanish", + "strike", + "sz", + "szCs", + "u", + "vanish", + "vertAlign", + "w", + "webHidden", + ) + } +) +_PROPERTY_CHILDREN: Final = { + _PARAGRAPH_PROPERTIES_TAG: frozenset( + { + qn(f"w:{name}") + for name in ( + "adjustRightInd", + "autoSpaceDE", + "autoSpaceDN", + "bidi", + "cnfStyle", + "contextualSpacing", + "divId", + "framePr", + "ind", + "jc", + "keepLines", + "keepNext", + "kinsoku", + "mirrorIndents", + "numPr", + "outlineLvl", + "overflowPunct", + "pBdr", + "pageBreakBefore", + "pStyle", + "rPr", + "shd", + "snapToGrid", + "spacing", + "suppressAutoHyphens", + "suppressLineNumbers", + "suppressOverlap", + "tabs", + "textAlignment", + "textDirection", + "textboxTightWrap", + "topLinePunct", + "widowControl", + "wordWrap", + ) + } + ), + _RUN_PROPERTIES_TAG: _RUN_PROPERTY_TAGS, + _SECTION_PROPERTIES_TAG: frozenset( + { + qn(f"w:{name}") + for name in ( + "bidi", + "cols", + "docGrid", + "endnotePr", + "footerReference", + "footnotePr", + "formProt", + "headerReference", + "lnNumType", + "noEndnote", + "paperSrc", + "pgBorders", + "pgMar", + "pgNumType", + "pgSz", + "printerSettings", + "rtlGutter", + "textDirection", + "titlePg", + "type", + "vAlign", + ) + } + ), + _TABLE_PROPERTIES_TAG: frozenset( + { + qn(f"w:{name}") + for name in ( + "bidiVisual", + "jc", + "shd", + "tblBorders", + "tblCaption", + "tblCellMar", + "tblCellSpacing", + "tblDescription", + "tblInd", + "tblLayout", + "tblLook", + "tblOverlap", + "tblStyle", + "tblStyleColBandSize", + "tblStyleRowBandSize", + "tblW", + "tblpPr", + ) + } + ), + _TABLE_GRID_TAG: frozenset({qn("w:gridCol")}), + _TABLE_ROW_PROPERTIES_TAG: frozenset( + { + qn(f"w:{name}") + for name in ( + "cantSplit", + "cnfStyle", + "divId", + "gridAfter", + "gridBefore", + "hidden", + "jc", + "tblCellSpacing", + "tblHeader", + "trHeight", + "wAfter", + "wBefore", + ) + } + ), + _TABLE_CELL_PROPERTIES_TAG: frozenset( + { + qn(f"w:{name}") + for name in ( + "cnfStyle", + "gridSpan", + "headers", + "hideMark", + "hMerge", + "noWrap", + "shd", + "tcBorders", + "tcFitText", + "tcMar", + "tcW", + "textDirection", + "vAlign", + "vMerge", + ) + } + ), + qn("w:numPr"): frozenset({qn("w:ilvl"), qn("w:numId")}), + qn("w:pBdr"): _BORDER_PROPERTY_TAGS, + qn("w:pgBorders"): _BORDER_PROPERTY_TAGS, + qn("w:tblBorders"): _BORDER_PROPERTY_TAGS, + qn("w:tcBorders"): _BORDER_PROPERTY_TAGS, + qn("w:tabs"): frozenset({qn("w:tab")}), + qn("w:tblCellMar"): _MARGIN_PROPERTY_TAGS, + qn("w:tcMar"): _MARGIN_PROPERTY_TAGS, + qn("w:cols"): frozenset({qn("w:col")}), + qn("w:footnotePr"): frozenset( + { + qn("w:numFmt"), + qn("w:numRestart"), + qn("w:numStart"), + qn("w:pos"), + } + ), + qn("w:endnotePr"): frozenset( + { + qn("w:numFmt"), + qn("w:numRestart"), + qn("w:numStart"), + qn("w:pos"), + } + ), +} class UnsupportedDocxFigureError(ValueError): @@ -56,14 +611,803 @@ def _contains_tag(element: Any, tags: frozenset[str]) -> bool: return any(node.tag in tags for node in element.iter()) -def _package_contains_visual(document: DocumentType) -> bool: - for part in document.part.package.parts: - element = getattr(part, "element", None) - if element is not None and _contains_tag(element, _UNSUPPORTED_VISUAL_TAGS): +def _is_xml_content_type(content_type: object) -> bool: + if type(content_type) is not str or not content_type.isascii(): + raise ValueError("DOCX package part has a malformed media type") + parsed = policy.default.header_factory("Content-Type", content_type) + raw_media_type = content_type.partition(";")[0] + if ( + parsed.defects + or not _is_mime_token(parsed.maintype) + or not _is_mime_token(parsed.subtype) + or raw_media_type.casefold() != f"{parsed.maintype}/{parsed.subtype}" + ): + raise ValueError("DOCX package part has a malformed media type") + media_type = f"{parsed.maintype}/{parsed.subtype}" + return media_type in _XML_CONTENT_TYPES or parsed.subtype.endswith("+xml") + + +def _is_mime_token(value: object) -> bool: + return ( + type(value) is str + and bool(value) + and value != "*" + and value.isascii() + and all( + character.isalnum() or character in "!#$%&'*+-.^_`|~" + for character in value + ) + ) + + +def _is_package_extension(value: object) -> bool: + return ( + type(value) is str + and bool(value) + and value not in {".", ".."} + and value.isascii() + and all( + character.isalnum() or character in "!#$&'*+-.^_`|~" + for character in value + ) + ) + + +def _normalized_package_path(value: object, *, leading_slash: bool) -> str: + if type(value) is not str or not value: + raise ValueError("DOCX package path is malformed") + if leading_slash: + if not value.startswith("/") or value.startswith("//"): + raise ValueError("DOCX package PartName is not absolute") + value = value[1:] + elif value.startswith("/"): + raise ValueError("DOCX archive member name is absolute") + if "\\" in value or "?" in value or "#" in value or value.endswith("/"): + raise ValueError("DOCX package path is not canonical") + segments = value.split("/") + if any( + not segment + or segment in {".", ".."} + or "%" in segment + for segment in segments + ): + raise ValueError("DOCX package path contains an unsafe segment") + return "/".join(segments) + + +def _content_type_declarations( + element: Any, +) -> tuple[dict[str, str], dict[str, str], bool]: + has_malformed_declaration = ( + element.tag != _CONTENT_TYPES_TAG + or element.attrib + or _has_direct_character_data(element) + ) + defaults: dict[str, str] = {} + overrides: dict[str, str] = {} + for declaration in element.iterchildren(): + if declaration.tag == _CONTENT_TYPE_DEFAULT_TAG: + permitted_attributes = {"Extension", "ContentType"} + key = declaration.get("Extension") + target = defaults + elif declaration.tag == _CONTENT_TYPE_OVERRIDE_TAG: + permitted_attributes = {"PartName", "ContentType"} + part_name = declaration.get("PartName") + try: + key = _normalized_package_path(part_name, leading_slash=True) + except ValueError: + has_malformed_declaration = True + continue + target = overrides + else: + has_malformed_declaration = True + continue + content_type = declaration.get("ContentType") + if ( + set(declaration.attrib) != permitted_attributes + or len(declaration) + or _has_direct_character_data(declaration) + or type(key) is not str + or not key + or (target is defaults and not _is_package_extension(key)) + or type(content_type) is not str + or not content_type + ): + has_malformed_declaration = True + if type(key) is not str or not key or type(content_type) is not str: + continue + try: + _is_xml_content_type(content_type) + except ValueError: + has_malformed_declaration = True + continue + normalized_key = key.casefold() + if normalized_key in target: + has_malformed_declaration = True + continue + target[normalized_key] = content_type + return defaults, overrides, has_malformed_declaration + + +def _relationship_part_source(member_name: str) -> str | None: + if member_name == "_rels/.rels": + return "" + parent, separator, filename = member_name.rpartition("/_rels/") + if separator and filename.endswith(".rels"): + return f"{parent}/{filename.removesuffix('.rels')}" + return None + + +def _resolved_relationship_target(source_part: str, target: str) -> str: + if target.startswith("/"): + return _normalized_package_path(target, leading_slash=True) + parent = source_part.rpartition("/")[0] + segments = [segment for segment in parent.split("/") if segment] + target_segments = target.split("/") + if any(not segment or segment == "." for segment in target_segments): + raise ValueError("DOCX relationship target is not canonical") + for segment in target_segments: + if "\\" in segment or "%" in segment: + raise ValueError("DOCX relationship target is not canonical") + if segment == "..": + if not segments: + raise ValueError("DOCX relationship target escapes the package root") + segments.pop() + continue + segments.append(segment) + return _normalized_package_path("/".join(segments), leading_slash=False) + + +def _related_member_names( + elements: tuple[tuple[str, Any], ...], +) -> tuple[frozenset[str], dict[str, frozenset[str]]]: + relationship_targets: dict[str, tuple[tuple[str, str], ...]] = {} + root_main_relationships = 0 + for member_name, element in elements: + source_part = _relationship_part_source(member_name) + if source_part is None: + continue + if source_part in relationship_targets: + raise ValueError("DOCX package has duplicate relationship parts") + if element.tag != _RELATIONSHIPS_TAG: + raise ValueError("DOCX relationship part has an invalid root") + if element.attrib or _has_direct_character_data(element): + raise ValueError("DOCX relationship root is outside the grammar") + relationship_ids: set[str] = set() + internal_targets: list[tuple[str, str]] = [] + for relationship in element.iterchildren(): + if relationship.tag != _RELATIONSHIP_TAG: + raise ValueError("DOCX relationship part has an invalid child") + if len(relationship) or _has_direct_character_data(relationship): + raise ValueError("DOCX relationship is outside the grammar") + attribute_names = frozenset(relationship.attrib) + if not attribute_names.issubset({"Id", "Type", "Target", "TargetMode"}): + raise ValueError("DOCX relationship has an unknown attribute") + relationship_id = relationship.get("Id") + relationship_type = relationship.get("Type") + target = relationship.get("Target") + target_mode = relationship.get("TargetMode", "Internal") + if ( + type(relationship_id) is not str + or not relationship_id + or relationship_id in relationship_ids + or type(relationship_type) is not str + or not relationship_type + or type(target) is not str + or not target + or target_mode not in {"Internal", "External"} + ): + raise ValueError("DOCX relationship is malformed") + relationship_ids.add(relationship_id) + target_path = target.split("#", 1)[0].split("?", 1)[0] + if any(segment == "." for segment in target_path.split("/")) or ( + target_mode == "External" + and any(segment == ".." for segment in target_path.split("/")) + ): + raise ValueError("DOCX relationship target is not canonical") + if relationship_type == _OLE_OBJECT_RELATIONSHIP_TYPE: + raise ValueError("DOCX OLE relationships are outside the grammar") + if ( + relationship_type == _THUMBNAIL_RELATIONSHIP_TYPE + and target_mode == "External" + ): + raise ValueError("DOCX thumbnail relationship must be internal") + if target_mode == "External": + continue + resolved = _resolved_relationship_target(source_part, target) + if relationship_type == _THUMBNAIL_RELATIONSHIP_TYPE and ( + source_part != "" or resolved != _THUMBNAIL_MEMBER + ): + raise ValueError("DOCX thumbnail relationship is outside the grammar") + if source_part == "" and resolved == _DOCUMENT_MEMBER: + if relationship_type != _OFFICE_DOCUMENT_RELATIONSHIP_TYPE: + raise ValueError( + "DOCX main document has the wrong relationship type" + ) + root_main_relationships += 1 + internal_targets.append((resolved, relationship_type)) + relationship_targets[source_part] = tuple(internal_targets) + if root_main_relationships != 1: + raise ValueError("DOCX package must relate exactly one main document") + + related: set[str] = set() + relationship_types: dict[str, set[str]] = {} + reachable_sources = {""} + while reachable_sources: + source_part = reachable_sources.pop() + for target, relationship_type in relationship_targets.get(source_part, ()): + relationship_types.setdefault(target, set()).add(relationship_type) + if target not in related: + related.add(target) + reachable_sources.add(target) + return frozenset(related), { + target: frozenset(types) for target, types in relationship_types.items() + } + + +def _is_complete_jpeg(value: bytes) -> bool: + if not value.startswith(_JPEG_START_OF_IMAGE): + return False + offset = len(_JPEG_START_OF_IMAGE) + in_entropy_data = False + frame_component_ids: frozenset[int] | None = None + scanned_component_ids: set[int] = set() + has_start_of_scan = False + while offset < len(value): + if in_entropy_data: + marker_prefix = value.find(b"\xff", offset) + if marker_prefix < 0: + return False + offset = marker_prefix + elif value[offset] != 0xFF: + return False + while offset < len(value) and value[offset] == 0xFF: + offset += 1 + if offset >= len(value): + return False + marker = value[offset] + offset += 1 + if in_entropy_data and (marker == 0x00 or 0xD0 <= marker <= 0xD7): + continue + in_entropy_data = False + if marker == 0xD9: + return ( + frame_component_ids is not None + and has_start_of_scan + and scanned_component_ids == frame_component_ids + and not any(value[offset:]) + ) + if marker in {0x00, 0x01, 0xD8} or 0xD0 <= marker <= 0xD7: + return False + if offset + 2 > len(value): + return False + segment_length = int.from_bytes(value[offset : offset + 2], "big") + segment_end = offset + segment_length + if segment_length < 2 or segment_end > len(value): + return False + if marker in _JPEG_START_OF_FRAME_MARKERS: + if segment_length < 11 or frame_component_ids is not None: + return False + component_count = value[offset + 7] + if component_count == 0 or segment_length != 8 + 3 * component_count: + return False + height = int.from_bytes(value[offset + 3 : offset + 5], "big") + width = int.from_bytes(value[offset + 5 : offset + 7], "big") + component_ids = frozenset( + value[offset + 8 + component_offset * 3] + for component_offset in range(component_count) + ) + if width == 0 or height == 0 or len(component_ids) != component_count: + return False + frame_component_ids = component_ids + elif marker == 0xDA: + if frame_component_ids is None or segment_length < 8: + return False + component_count = value[offset + 2] + if component_count == 0 or segment_length != 6 + 2 * component_count: + return False + scan_component_ids = frozenset( + value[offset + 3 + component_offset * 2] + for component_offset in range(component_count) + ) + if ( + len(scan_component_ids) != component_count + or not scan_component_ids.issubset(frame_component_ids) + ): + return False + scanned_component_ids.update(scan_component_ids) + has_start_of_scan = True + in_entropy_data = True + offset = segment_end + return False + + +def _package_xml_elements( + source: bytes, +) -> tuple[tuple[tuple[str, Any], ...], frozenset[str], bool]: + elements: list[tuple[str, Any]] = [] + parsed_member_names: set[str] = set() + package_members: list[tuple[str, bytes, str | None]] = [] + has_malformed_part = False + with ZipFile(BytesIO(source)) as archive: + members = archive.infolist() + member_bytes_by_identity = { + id(member): archive.read(member) + for member in members + if not member.is_dir() and member.filename != _CONTENT_TYPES_MEMBER + } + normalized_members: list[tuple[Any, str]] = [] + for member in members: + try: + normalized_members.append( + ( + member, + _normalized_package_path( + member.filename, leading_slash=False + ), + ) + ) + except ValueError: + has_malformed_part = True + member_names = tuple(name for _, name in normalized_members) + if len(member_names) != len({name.casefold() for name in member_names}): + has_malformed_part = True + try: + content_types = parse_xml(archive.read(_CONTENT_TYPES_MEMBER)) + elements.append((_CONTENT_TYPES_MEMBER, content_types)) + defaults, overrides, malformed_manifest = _content_type_declarations( + content_types + ) + has_malformed_part = has_malformed_part or malformed_manifest + except Exception: + for member in members: + member_bytes = member_bytes_by_identity.get(id(member)) + if member_bytes is None: + continue + try: + elements.append((member.filename, parse_xml(member_bytes))) + except Exception: + continue + return tuple(elements), frozenset(), True + declared_override_names = set(overrides) + if not declared_override_names.issubset( + {name.casefold() for name in member_names} + ): + has_malformed_part = True + for member, member_name in normalized_members: + if member.is_dir() or member_name == _CONTENT_TYPES_MEMBER: + continue + member_bytes = member_bytes_by_identity[id(member)] + content_type = overrides.get(member_name.casefold()) + if content_type is None and "." in member_name.rsplit("/", 1)[-1]: + extension = member_name.rsplit(".", 1)[-1].casefold() + content_type = defaults.get(extension) + if content_type is None: + package_members.append((member_name, member_bytes, None)) + has_malformed_part = True + continue + is_xml = _is_xml_content_type(content_type) + package_members.append((member_name, member_bytes, content_type)) + if not is_xml and _relationship_part_source(member_name) is None: + continue + try: + elements.append((member_name, parse_xml(member_bytes))) + parsed_member_names.add(member_name) + except Exception: + has_malformed_part = True + continue + try: + related_members, relationship_types = _related_member_names(tuple(elements)) + except ValueError: + related_members = frozenset() + relationship_types = {} + has_malformed_part = True + for member in members: + member_bytes = member_bytes_by_identity.get(id(member)) + if member_bytes is None: + continue + try: + elements.append((member.filename, parse_xml(member_bytes))) + except Exception: + continue + for member_name, member_bytes, content_type in package_members: + if member_name in parsed_member_names or member_name not in related_members: + continue + member_relationship_types = relationship_types.get(member_name, frozenset()) + if member_relationship_types and member_relationship_types.issubset( + _BINARY_RELATIONSHIP_TYPES + ): + if _OLE_OBJECT_RELATIONSHIP_TYPE in member_relationship_types: + has_malformed_part = True + if _THUMBNAIL_RELATIONSHIP_TYPE in member_relationship_types and ( + member_name != _THUMBNAIL_MEMBER + or type(content_type) is not str + or content_type.casefold() != _THUMBNAIL_CONTENT_TYPE + or not _is_complete_jpeg(member_bytes) + ): + has_malformed_part = True + continue + try: + elements.append((member_name, parse_xml(member_bytes))) + parsed_member_names.add(member_name) + except Exception: + has_malformed_part = True + archive_members = frozenset(member_names) + if not related_members.issubset(archive_members): + has_malformed_part = True + for member_name in archive_members: + if member_name == _CONTENT_TYPES_MEMBER: + continue + relationship_source = _relationship_part_source(member_name) + if relationship_source is not None: + if relationship_source and relationship_source not in related_members: + has_malformed_part = True + elif member_name not in related_members: + has_malformed_part = True + return tuple(elements), related_members, has_malformed_part + + +def _elements_contain_tag( + elements: tuple[tuple[str, Any], ...], tags: frozenset[str] +) -> bool: + return any(_contains_tag(element, tags) for _, element in elements) + + +def _ooxml_visible_text(element: Any) -> str: + text: list[str] = [] + for run in element.iter(_RUN_TAG): + for node in run.iterdescendants(): + if node.tag == _TEXT_TAG: + text.append(node.text or "") + elif node.tag in _TAB_TAGS: + text.append("\t") + elif node.tag == _NO_BREAK_HYPHEN_TAG: + text.append("-") + elif node.tag in _BREAK_TAGS and ( + node.tag == qn("w:cr") + or node.get(_BREAK_TYPE_ATTRIBUTE) in (None, "textWrapping") + ): + text.append("\n") + return "".join(text) + + +def _contains_unadmitted_run_content(element: Any) -> bool: + return any( + child.tag not in _ADMITTED_RUN_TAGS + for run in element.iter(_RUN_TAG) + for child in run.iterchildren() + ) + + +def _contains_visible_token_outside_run(element: Any) -> bool: + return any( + not any(ancestor.tag == _RUN_TAG for ancestor in token.iterancestors()) + for token in element.iter() + if token.tag in _VISIBLE_TOKEN_TAGS + and not ( + token.tag == qn("w:tab") + and any( + ancestor.tag == _TAB_STOPS_TAG for ancestor in token.iterancestors() + ) + ) + ) + + +def _contains_unrepresented_package_text( + elements: tuple[tuple[str, Any], ...] +) -> bool: + for member_name, element in elements: + if member_name == _DOCUMENT_MEMBER: + body = element.find(qn("w:body")) + if any( + child is not body + and ( + _ooxml_visible_text(child) + or ( + type(child.tag) is str + and child.tag.startswith(_OFFICE_MATH_TAG_PREFIX) + ) + ) + for child in element.iterchildren() + ): + return True + elif _ooxml_visible_text(element): return True return False +def _has_direct_character_data(node: Any) -> bool: + return any(text.strip() for text in node.xpath("text()")) + + +def _xml_namespace(name: object) -> str | None: + if type(name) is not str: + return None + if not name.startswith("{"): + return "" + namespace, separator, _ = name[1:].partition("}") + return namespace if separator and namespace else None + + +def _known_inert_member_uses_admitted_grammar( + member_name: str, element: Any +) -> bool: + expected_roots = _KNOWN_INERT_XML_ROOTS.get(member_name) + element_namespaces = _KNOWN_INERT_XML_ELEMENT_NAMESPACES.get(member_name) + attribute_namespaces = _KNOWN_INERT_XML_ATTRIBUTE_NAMESPACES.get(member_name) + if ( + expected_roots is None + or element_namespaces is None + or attribute_namespaces is None + or element.tag not in expected_roots + ): + return False + admitted_text_tags = _KNOWN_INERT_XML_TEXT_TAGS.get(member_name, frozenset()) + for node in element.iter(): + if _xml_namespace(node.tag) not in element_namespaces: + return False + if any( + _xml_namespace(attribute_name) not in attribute_namespaces + for attribute_name in node.attrib + ): + return False + if _has_direct_character_data(node) and node.tag not in admitted_text_tags: + return False + return True + + +def _main_document_uses_admitted_root_grammar( + elements: tuple[tuple[str, Any], ...] +) -> bool: + roots = tuple( + element for member_name, element in elements if member_name == _DOCUMENT_MEMBER + ) + if len(roots) != 1: + return False + root = roots[0] + children = tuple(root.iterchildren()) + return ( + root.tag == qn("w:document") + and not _has_direct_character_data(root) + and len(children) == 1 + and children[0].tag == qn("w:body") + ) + + +def _property_subtree_uses_admitted_grammar(node: Any) -> bool: + if _has_direct_character_data(node): + return False + admitted_children = _PROPERTY_CHILDREN.get(node.tag, frozenset()) + children = tuple(node.iterchildren()) + return all( + child.tag in admitted_children + and _property_subtree_uses_admitted_grammar(child) + for child in children + ) + + +def _property_subtrees_use_admitted_grammar(body: Any) -> bool: + for root_tag, admitted_parents in _PROPERTY_ROOT_PARENTS.items(): + for root in body.iter(root_tag): + parent = root.getparent() + if ( + parent is None + or parent.tag not in admitted_parents + or not _property_subtree_uses_admitted_grammar(root) + ): + return False + return True + + +def _contains_unrepresented_package_structure( + elements: tuple[tuple[str, Any], ...], related_members: frozenset[str] +) -> bool: + for member_name, element in elements: + if member_name == _DOCUMENT_MEMBER: + continue + if member_name == _CONTENT_TYPES_MEMBER: + continue + relationship_source = _relationship_part_source(member_name) + if relationship_source is not None: + if element.tag != _RELATIONSHIPS_TAG: + return True + continue + if member_name not in related_members: + return True + expected_inert_roots = _KNOWN_INERT_XML_ROOTS.get(member_name) + if expected_inert_roots is not None: + if not _known_inert_member_uses_admitted_grammar(member_name, element): + return True + continue + if element.tag in _UNREPRESENTED_PART_ROOT_TAGS: + if any( + type(node.tag) is str + and node.tag.startswith(_OFFICE_MATH_TAG_PREFIX) + for node in element.iter() + ): + return True + if element.tag not in {qn("w:hdr"), qn("w:ftr")}: + if len(element) > 0 or bool(element.text and element.text.strip()): + return True + continue + for paragraph in element.iterchildren(): + if paragraph.tag != _PARAGRAPH_TAG: + return True + for child in paragraph.iterchildren(): + if child.tag == _PARAGRAPH_PROPERTIES_TAG: + if not _property_subtree_uses_admitted_grammar(child): + return True + elif child.tag == _RUN_TAG: + run_children = tuple(child.iterchildren()) + if any(run_child.tag != _RUN_PROPERTIES_TAG for run_child in run_children): + return True + if any( + not _property_subtree_uses_admitted_grammar(run_child) + for run_child in run_children + ): + return True + else: + return True + continue + return True + return False + + +def _body_uses_closed_admitted_grammar(document: DocumentType) -> bool: + body = document.element.body + if any( + type(node.tag) is not str + or not node.tag.startswith(_WORDPROCESSINGML_TAG_PREFIX) + for node in body.iter() + ): + return False + body_children = tuple(body.iterchildren()) + section_indexes = tuple( + index + for index, child in enumerate(body_children) + if child.tag == _SECTION_PROPERTIES_TAG + ) + if len(section_indexes) > 1 or ( + section_indexes and section_indexes[0] != len(body_children) - 1 + ): + return False + if any( + child.tag not in {_PARAGRAPH_TAG, _TABLE_TAG, _SECTION_PROPERTIES_TAG} + for child in body_children + ): + return False + if not _property_subtrees_use_admitted_grammar(body): + return False + for paragraph in body.iter(_PARAGRAPH_TAG): + parent = paragraph.getparent() + if parent is None or parent.tag not in {qn("w:body"), _TABLE_CELL_TAG}: + return False + children = tuple(paragraph.iterchildren()) + if _has_direct_character_data(paragraph): + return False + if any(child.tag not in _ADMITTED_PARAGRAPH_CHILDREN for child in children): + return False + property_indexes = tuple( + index + for index, child in enumerate(children) + if child.tag == _PARAGRAPH_PROPERTIES_TAG + ) + if len(property_indexes) > 1 or ( + property_indexes and property_indexes[0] != 0 + ): + return False + for run in body.iter(_RUN_TAG): + parent = run.getparent() + if parent is None or parent.tag != _PARAGRAPH_TAG: + return False + children = tuple(run.iterchildren()) + if _has_direct_character_data(run): + return False + if any(child.tag not in _ADMITTED_RUN_TAGS for child in children): + return False + if any( + child.tag != _RUN_PROPERTIES_TAG + and ( + len(child) > 0 + or ( + child.tag not in {_TEXT_TAG, qn("w:instrText")} + and _has_direct_character_data(child) + ) + ) + for child in children + ): + return False + for table in body.iter(_TABLE_TAG): + table_children = tuple(table.iterchildren()) + if ( + table.getparent() is not body + or _has_direct_character_data(table) + or any(child.tag not in _ADMITTED_TABLE_CHILDREN for child in table_children) + ): + return False + table_property_indexes = tuple( + index + for index, child in enumerate(table_children) + if child.tag == _TABLE_PROPERTIES_TAG + ) + table_grid_indexes = tuple( + index + for index, child in enumerate(table_children) + if child.tag == _TABLE_GRID_TAG + ) + if ( + len(table_property_indexes) != 1 + or table_property_indexes[0] != 0 + or len(table_grid_indexes) != 1 + or table_grid_indexes[0] != 1 + or not any(child.tag == _TABLE_ROW_TAG for child in table_children) + ): + return False + for row in table.iterchildren(_TABLE_ROW_TAG): + row_children = tuple(row.iterchildren()) + if _has_direct_character_data(row) or any( + child.tag not in _ADMITTED_TABLE_ROW_CHILDREN for child in row_children + ): + return False + row_property_indexes = tuple( + index + for index, child in enumerate(row_children) + if child.tag == _TABLE_ROW_PROPERTIES_TAG + ) + if ( + len(row_property_indexes) > 1 + or (row_property_indexes and row_property_indexes[0] != 0) + or not any(child.tag == _TABLE_CELL_TAG for child in row_children) + ): + return False + for cell in table.iter(_TABLE_CELL_TAG): + cell_children = tuple(cell.iterchildren()) + if _has_direct_character_data(cell) or any( + child.tag not in _ADMITTED_TABLE_CELL_CHILDREN + for child in cell_children + ): + return False + property_indexes = tuple( + index + for index, child in enumerate(cell_children) + if child.tag == _TABLE_CELL_PROPERTIES_TAG + ) + if ( + len(property_indexes) != 1 + or property_indexes[0] != 0 + or not any(child.tag == _PARAGRAPH_TAG for child in cell_children) + ): + return False + if any( + row.getparent() is None + or row.getparent().tag != _TABLE_TAG + or row.getparent().getparent() is not body + for row in body.iter(_TABLE_ROW_TAG) + ): + return False + if any( + cell.getparent() is None + or cell.getparent().tag != _TABLE_ROW_TAG + or cell.getparent().getparent() is None + or cell.getparent().getparent().tag != _TABLE_TAG + or cell.getparent().getparent().getparent() is not body + for cell in body.iter(_TABLE_CELL_TAG) + ): + return False + return True + + +def _body_paragraph_text_is_lossless(document: DocumentType) -> bool: + body = document.element.body + return all( + any(ancestor.tag == _PARAGRAPH_TAG for ancestor in run.iterancestors()) + for run in body.iter(_RUN_TAG) + ) and all( + Paragraph(paragraph, document).text == _ooxml_visible_text(paragraph) + for paragraph in body.iter(_PARAGRAPH_TAG) + ) + + @dataclass(frozen=True, slots=True) class RawDocxBlock: """One bounded block in OOXML body order.""" @@ -83,20 +1427,47 @@ class RAGFlowDocxParser: def __call__(self, source: bytes) -> tuple[RawDocxBlock, ...]: if type(source) is not bytes: raise TypeError("DOCX parser source must be exact bytes") - document = Document(BytesIO(source)) - if not isinstance(document, DocumentType): - raise ValueError("DOCX parser did not construct an exact document") - if _package_contains_visual(document): + package_elements, related_members, has_malformed_part = ( + _package_xml_elements(source) + ) + if _elements_contain_tag(package_elements, _UNSUPPORTED_VISUAL_TAGS): raise UnsupportedDocxFigureError( "DOCX profile does not admit visual objects" ) + if has_malformed_part: + raise ValueError("DOCX contains a malformed XML package part") + if _DOCUMENT_MEMBER not in related_members: + raise ValueError("DOCX main document is not related from the package root") + if not _main_document_uses_admitted_root_grammar(package_elements): + raise ValueError("DOCX main document root is outside the admitted grammar") + if _contains_unrepresented_package_structure( + package_elements, related_members + ): + raise ValueError("DOCX contains unrepresented package structure") + document = Document(BytesIO(source)) + if not isinstance(document, DocumentType): + raise ValueError("DOCX parser did not construct an exact document") + if not _body_uses_closed_admitted_grammar(document): + raise ValueError("DOCX body is outside the closed admitted grammar") + if any( + _contains_unadmitted_run_content(element) + for _, element in package_elements + ): + raise ValueError("DOCX contains unsupported run content") + if any( + _contains_visible_token_outside_run(element) + for _, element in package_elements + ): + raise ValueError("DOCX contains visible content outside a run") + if _contains_unrepresented_package_text(package_elements): + raise ValueError("DOCX contains text outside the represented body") + if not _body_paragraph_text_is_lossless(document): + raise ValueError("DOCX paragraph text cannot be represented losslessly") blocks: list[RawDocxBlock] = [] for block_ordinal, child in enumerate(document.element.body.iterchildren()): - if _contains_tag(child, _UNSUPPORTED_CONTENT_TAGS): - raise ValueError("DOCX contains an unsupported content container") if child.tag == _PARAGRAPH_TAG: paragraph = Paragraph(child, document) - text = paragraph.text.strip() + text = paragraph.text has_figure = bool(child.xpath(".//pic:pic")) if text or has_figure: style_name = ( @@ -117,9 +1488,11 @@ def __call__(self, source: bytes) -> tuple[RawDocxBlock, ...]: elif child.tag == _TABLE_TAG: if any(node.tag == _TABLE_TAG for node in child.iterdescendants()): raise ValueError("DOCX profile does not admit nested tables") + if _contains_tag(child, _MERGED_CELL_TAGS): + raise ValueError("DOCX profile does not admit merged table cells") table = Table(child, document) rows = tuple( - tuple(cell.text.strip() for cell in row.cells) + tuple(cell.text for cell in row.cells) for row in table.rows ) if rows: diff --git a/third_party/ragflow/patches/issue-204-docx-parser.patch b/third_party/ragflow/patches/issue-204-docx-parser.patch index fb568b0..85fe056 100644 --- a/third_party/ragflow/patches/issue-204-docx-parser.patch +++ b/third_party/ragflow/patches/issue-204-docx-parser.patch @@ -1,6 +1,8 @@ +diff --git a/deepdoc/parser/docx_parser.py b/deepdoc/parser/docx_parser.py +index 48f8db8..c8ed90f 100644 --- a/deepdoc/parser/docx_parser.py +++ b/deepdoc/parser/docx_parser.py -@@ -13,170 +13,130 @@ +@@ -13,170 +13,1503 @@ # See the License for the specific language governing permissions and # limitations under the License. # @@ -12,18 +14,599 @@ -import pandas as pd -from collections import Counter -from rag.nlp import rag_tokenizer --from io import BytesIO ++"""Patched RAGFlow DOCX block extraction with no application dependencies.""" ++ ++from __future__ import annotations ++ ++from dataclasses import dataclass ++from email import policy + from io import BytesIO -import logging -from common.constants import MAXIMUM_PAGE_NUMBER -from docx.image.exceptions import ( - InvalidImageStreamError, - UnexpectedEndOfFileError, - UnrecognizedImageError, --) ++from typing import Any, Final ++from zipfile import ZipFile ++ ++from docx import Document ++from docx.document import Document as DocumentType ++from docx.oxml import parse_xml ++from docx.oxml.ns import qn ++from docx.table import Table ++from docx.text.paragraph import Paragraph ++ ++_PARAGRAPH_TAG: Final = qn("w:p") ++_PARAGRAPH_PROPERTIES_TAG: Final = qn("w:pPr") ++_TABLE_TAG: Final = qn("w:tbl") ++_TABLE_PROPERTIES_TAG: Final = qn("w:tblPr") ++_TABLE_GRID_TAG: Final = qn("w:tblGrid") ++_TABLE_ROW_TAG: Final = qn("w:tr") ++_TABLE_ROW_PROPERTIES_TAG: Final = qn("w:trPr") ++_TABLE_CELL_TAG: Final = qn("w:tc") ++_TABLE_CELL_PROPERTIES_TAG: Final = qn("w:tcPr") ++_SECTION_PROPERTIES_TAG: Final = qn("w:sectPr") ++_RUN_TAG: Final = qn("w:r") ++_RUN_PROPERTIES_TAG: Final = qn("w:rPr") ++_TEXT_TAG: Final = qn("w:t") ++_TAB_STOPS_TAG: Final = qn("w:tabs") ++_TAB_TAGS: Final = frozenset({qn("w:tab"), qn("w:ptab")}) ++_BREAK_TAGS: Final = frozenset({qn("w:br"), qn("w:cr")}) ++_BREAK_TYPE_ATTRIBUTE: Final = qn("w:type") ++_NO_BREAK_HYPHEN_TAG: Final = qn("w:noBreakHyphen") ++_NON_VISIBLE_RUN_TAGS: Final = frozenset( ++ { ++ qn("w:fldChar"), ++ qn("w:instrText"), ++ qn("w:lastRenderedPageBreak"), ++ _RUN_PROPERTIES_TAG, ++ } ++) ++_ADMITTED_RUN_TAGS: Final = ( ++ frozenset({_TEXT_TAG, _NO_BREAK_HYPHEN_TAG}) ++ | _TAB_TAGS ++ | _BREAK_TAGS ++ | _NON_VISIBLE_RUN_TAGS ++) ++_VISIBLE_TOKEN_TAGS: Final = ( ++ frozenset({_TEXT_TAG, _NO_BREAK_HYPHEN_TAG}) | _TAB_TAGS | _BREAK_TAGS ++) ++_UNSUPPORTED_VISUAL_TAGS: Final = frozenset( ++ {qn("pic:pic"), qn("w:drawing"), qn("w:object"), qn("w:pict")} ++) ++_XML_CONTENT_TYPES: Final = frozenset({"application/xml", "text/xml"}) ++_CONTENT_TYPES_MEMBER: Final = "[Content_Types].xml" ++_CONTENT_TYPES_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/package/2006/content-types" ++) ++_CONTENT_TYPES_TAG: Final = f"{{{_CONTENT_TYPES_NAMESPACE}}}Types" ++_CONTENT_TYPE_DEFAULT_TAG: Final = f"{{{_CONTENT_TYPES_NAMESPACE}}}Default" ++_CONTENT_TYPE_OVERRIDE_TAG: Final = f"{{{_CONTENT_TYPES_NAMESPACE}}}Override" ++_DOCUMENT_MEMBER: Final = "word/document.xml" ++_WORDPROCESSINGML_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/wordprocessingml/2006/main" ++) ++_WORDPROCESSINGML_TAG_PREFIX: Final = f"{{{_WORDPROCESSINGML_NAMESPACE}}}" ++_OFFICE_MATH_TAG_PREFIX: Final = ( ++ "{http://schemas.openxmlformats.org/officeDocument/2006/math}" ++) ++_OFFICE_MATH_CONTENT_TAGS: Final = frozenset( ++ { ++ f"{_OFFICE_MATH_TAG_PREFIX}oMath", ++ f"{_OFFICE_MATH_TAG_PREFIX}oMathPara", ++ } ++) ++_ADMITTED_PARAGRAPH_CHILDREN: Final = frozenset( ++ {_PARAGRAPH_PROPERTIES_TAG, _RUN_TAG} ++) ++_ADMITTED_TABLE_CHILDREN: Final = frozenset( ++ {_TABLE_PROPERTIES_TAG, _TABLE_GRID_TAG, _TABLE_ROW_TAG} ++) ++_ADMITTED_TABLE_ROW_CHILDREN: Final = frozenset( ++ {_TABLE_ROW_PROPERTIES_TAG, _TABLE_CELL_TAG} ++) ++_ADMITTED_TABLE_CELL_CHILDREN: Final = frozenset( ++ {_TABLE_CELL_PROPERTIES_TAG, _PARAGRAPH_TAG} ++) ++_MERGED_CELL_TAGS: Final = frozenset( ++ {qn("w:gridSpan"), qn("w:hMerge"), qn("w:vMerge")} ++) ++_UNREPRESENTED_PART_ROOT_TAGS: Final = frozenset( ++ { ++ qn("w:comments"), ++ qn("w:endnotes"), ++ qn("w:footnotes"), ++ qn("w:ftr"), ++ qn("w:glossaryDocument"), ++ qn("w:hdr"), ++ } ++) ++_RELATIONSHIPS_TAG: Final = ( ++ "{http://schemas.openxmlformats.org/package/2006/relationships}Relationships" ++) ++_RELATIONSHIP_TAG: Final = ( ++ "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship" ++) ++_OFFICE_DOCUMENT_RELATIONSHIP_TYPE: Final = ( ++ "http://schemas.openxmlformats.org/officeDocument/2006/relationships/" ++ "officeDocument" ++) ++_OLE_OBJECT_RELATIONSHIP_TYPE: Final = ( ++ "http://schemas.openxmlformats.org/officeDocument/2006/relationships/oleObject" ++) ++_THUMBNAIL_RELATIONSHIP_TYPE: Final = ( ++ "http://schemas.openxmlformats.org/package/2006/relationships/metadata/thumbnail" ++) ++_THUMBNAIL_MEMBER: Final = "docProps/thumbnail.jpeg" ++_THUMBNAIL_CONTENT_TYPE: Final = "image/jpeg" ++_JPEG_START_OF_IMAGE: Final = b"\xff\xd8" ++_JPEG_START_OF_FRAME_MARKERS: Final = frozenset( ++ { ++ 0xC0, ++ 0xC1, ++ 0xC2, ++ 0xC3, ++ 0xC5, ++ 0xC6, ++ 0xC7, ++ 0xC9, ++ 0xCA, ++ 0xCB, ++ 0xCD, ++ 0xCE, ++ 0xCF, ++ } ++) ++_BINARY_RELATIONSHIP_TYPES: Final = frozenset( ++ {_OLE_OBJECT_RELATIONSHIP_TYPE, _THUMBNAIL_RELATIONSHIP_TYPE} ++) ++_DRAWINGML_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/drawingml/2006/main" ++) ++_CORE_PROPERTIES_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/package/2006/metadata/core-properties" + ) -from rag.utils.lazy_image import LazyImage -+"""Patched RAGFlow DOCX block extraction with no application dependencies.""" ++_DC_NAMESPACE: Final = "http://purl.org/dc/elements/1.1/" ++_DCTERMS_NAMESPACE: Final = "http://purl.org/dc/terms/" ++_EXTENDED_PROPERTIES_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties" ++) ++_VARIANT_TYPES_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes" ++) ++_BIBLIOGRAPHY_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/officeDocument/2006/bibliography" ++) ++_CUSTOM_XML_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/officeDocument/2006/customXml" ++) ++_WORD_2010_NAMESPACE: Final = ( ++ "http://schemas.microsoft.com/office/word/2010/wordml" ++) ++_OFFICE_NAMESPACE: Final = "urn:schemas-microsoft-com:office:office" ++_VML_NAMESPACE: Final = "urn:schemas-microsoft-com:vml" ++_MARKUP_COMPATIBILITY_NAMESPACE: Final = ( ++ "http://schemas.openxmlformats.org/markup-compatibility/2006" ++) ++_XML_SCHEMA_INSTANCE_NAMESPACE: Final = "http://www.w3.org/2001/XMLSchema-instance" ++_KNOWN_INERT_XML_ROOTS: Final = { ++ "docProps/core.xml": frozenset( ++ { ++ "{http://schemas.openxmlformats.org/package/2006/metadata/" ++ "core-properties}coreProperties" ++ } ++ ), ++ "docProps/app.xml": frozenset( ++ { ++ "{http://schemas.openxmlformats.org/officeDocument/2006/" ++ "extended-properties}Properties" ++ } ++ ), ++ "word/styles.xml": frozenset({qn("w:styles")}), ++ "word/stylesWithEffects.xml": frozenset({qn("w:styles")}), ++ "word/settings.xml": frozenset({qn("w:settings")}), ++ "word/webSettings.xml": frozenset({qn("w:webSettings")}), ++ "word/fontTable.xml": frozenset({qn("w:fonts")}), ++ "word/theme/theme1.xml": frozenset( ++ {"{http://schemas.openxmlformats.org/drawingml/2006/main}theme"} ++ ), ++ "customXml/item1.xml": frozenset( ++ { ++ "{http://schemas.openxmlformats.org/officeDocument/2006/" ++ "bibliography}Sources" ++ } ++ ), ++ "customXml/itemProps1.xml": frozenset( ++ { ++ "{http://schemas.openxmlformats.org/officeDocument/2006/" ++ "customXml}datastoreItem" ++ } ++ ), ++ "word/numbering.xml": frozenset({qn("w:numbering")}), ++} ++_KNOWN_INERT_XML_ELEMENT_NAMESPACES: Final = { ++ "docProps/core.xml": frozenset( ++ {_CORE_PROPERTIES_NAMESPACE, _DC_NAMESPACE, _DCTERMS_NAMESPACE} ++ ), ++ "docProps/app.xml": frozenset( ++ {_EXTENDED_PROPERTIES_NAMESPACE, _VARIANT_TYPES_NAMESPACE} ++ ), ++ "word/styles.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), ++ "word/stylesWithEffects.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), ++ "word/settings.xml": frozenset( ++ { ++ _WORDPROCESSINGML_NAMESPACE, ++ _WORD_2010_NAMESPACE, ++ _OFFICE_MATH_TAG_PREFIX[1:-1], ++ _OFFICE_NAMESPACE, ++ } ++ ), ++ "word/webSettings.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), ++ "word/fontTable.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), ++ "word/theme/theme1.xml": frozenset({_DRAWINGML_NAMESPACE}), ++ "customXml/item1.xml": frozenset({_BIBLIOGRAPHY_NAMESPACE}), ++ "customXml/itemProps1.xml": frozenset({_CUSTOM_XML_NAMESPACE}), ++ "word/numbering.xml": frozenset({_WORDPROCESSINGML_NAMESPACE}), ++} ++_KNOWN_INERT_XML_TEXT_TAGS: Final = { ++ "docProps/core.xml": frozenset( ++ { ++ f"{{{_CORE_PROPERTIES_NAMESPACE}}}{name}" ++ for name in ( ++ "category", ++ "contentStatus", ++ "keywords", ++ "lastModifiedBy", ++ "lastPrinted", ++ "revision", ++ "version", ++ ) ++ } ++ | { ++ f"{{{_DC_NAMESPACE}}}{name}" ++ for name in ( ++ "creator", ++ "description", ++ "identifier", ++ "language", ++ "subject", ++ "title", ++ ) ++ } ++ | { ++ f"{{{_DCTERMS_NAMESPACE}}}{name}" for name in ("created", "modified") ++ } ++ ), ++ "docProps/app.xml": frozenset( ++ { ++ f"{{{_EXTENDED_PROPERTIES_NAMESPACE}}}{name}" ++ for name in ( ++ "AppVersion", ++ "Application", ++ "Characters", ++ "CharactersWithSpaces", ++ "Company", ++ "DocSecurity", ++ "HiddenSlides", ++ "HyperlinkBase", ++ "HyperlinksChanged", ++ "Lines", ++ "LinksUpToDate", ++ "Manager", ++ "MMClips", ++ "Notes", ++ "Pages", ++ "Paragraphs", ++ "PresentationFormat", ++ "ScaleCrop", ++ "SharedDoc", ++ "Slides", ++ "Template", ++ "TotalTime", ++ "Words", ++ ) ++ } ++ | { ++ f"{{{_VARIANT_TYPES_NAMESPACE}}}{name}" ++ for name in ( ++ "bool", ++ "bstr", ++ "date", ++ "decimal", ++ "filetime", ++ "i1", ++ "i2", ++ "i4", ++ "i8", ++ "lpstr", ++ "lpwstr", ++ "r4", ++ "r8", ++ "ui1", ++ "ui2", ++ "ui4", ++ "ui8", ++ ) ++ } ++ ), ++} ++_KNOWN_INERT_XML_ATTRIBUTE_NAMESPACES: Final = { ++ "docProps/core.xml": frozenset({"", _XML_SCHEMA_INSTANCE_NAMESPACE}), ++ "docProps/app.xml": frozenset({""}), ++ "word/styles.xml": frozenset( ++ {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} ++ ), ++ "word/stylesWithEffects.xml": frozenset( ++ {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} ++ ), ++ "word/settings.xml": frozenset( ++ { ++ "", ++ _WORDPROCESSINGML_NAMESPACE, ++ _WORD_2010_NAMESPACE, ++ _OFFICE_MATH_TAG_PREFIX[1:-1], ++ _OFFICE_NAMESPACE, ++ _VML_NAMESPACE, ++ _MARKUP_COMPATIBILITY_NAMESPACE, ++ } ++ ), ++ "word/webSettings.xml": frozenset({"", _MARKUP_COMPATIBILITY_NAMESPACE}), ++ "word/fontTable.xml": frozenset( ++ {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} ++ ), ++ "word/theme/theme1.xml": frozenset({""}), ++ "customXml/item1.xml": frozenset({""}), ++ "customXml/itemProps1.xml": frozenset({"", _CUSTOM_XML_NAMESPACE}), ++ "word/numbering.xml": frozenset( ++ {_WORDPROCESSINGML_NAMESPACE, _MARKUP_COMPATIBILITY_NAMESPACE} ++ ), ++} ++_PROPERTY_ROOT_PARENTS: Final = { ++ _PARAGRAPH_PROPERTIES_TAG: frozenset({_PARAGRAPH_TAG}), ++ _RUN_PROPERTIES_TAG: frozenset({_PARAGRAPH_PROPERTIES_TAG, _RUN_TAG}), ++ _SECTION_PROPERTIES_TAG: frozenset({qn("w:body")}), ++ _TABLE_PROPERTIES_TAG: frozenset({_TABLE_TAG}), ++ _TABLE_GRID_TAG: frozenset({_TABLE_TAG}), ++ _TABLE_ROW_PROPERTIES_TAG: frozenset({_TABLE_ROW_TAG}), ++ _TABLE_CELL_PROPERTIES_TAG: frozenset({_TABLE_CELL_TAG}), ++} ++_BORDER_PROPERTY_TAGS: Final = frozenset( ++ { ++ qn("w:bar"), ++ qn("w:between"), ++ qn("w:bottom"), ++ qn("w:end"), ++ qn("w:insideH"), ++ qn("w:insideV"), ++ qn("w:left"), ++ qn("w:right"), ++ qn("w:start"), ++ qn("w:tl2br"), ++ qn("w:top"), ++ qn("w:tr2bl"), ++ } ++) ++_MARGIN_PROPERTY_TAGS: Final = frozenset( ++ { ++ qn("w:bottom"), ++ qn("w:end"), ++ qn("w:left"), ++ qn("w:right"), ++ qn("w:start"), ++ qn("w:top"), ++ } ++) ++_RUN_PROPERTY_TAGS: Final = frozenset( ++ { ++ qn(f"w:{name}") ++ for name in ( ++ "b", ++ "bCs", ++ "bdr", ++ "caps", ++ "color", ++ "cs", ++ "dstrike", ++ "eastAsianLayout", ++ "effect", ++ "em", ++ "emboss", ++ "fitText", ++ "highlight", ++ "i", ++ "iCs", ++ "imprint", ++ "kern", ++ "lang", ++ "noProof", ++ "oMath", ++ "outline", ++ "position", ++ "rFonts", ++ "rStyle", ++ "rtl", ++ "shadow", ++ "shd", ++ "smallCaps", ++ "snapToGrid", ++ "spacing", ++ "specVanish", ++ "strike", ++ "sz", ++ "szCs", ++ "u", ++ "vanish", ++ "vertAlign", ++ "w", ++ "webHidden", ++ ) ++ } ++) ++_PROPERTY_CHILDREN: Final = { ++ _PARAGRAPH_PROPERTIES_TAG: frozenset( ++ { ++ qn(f"w:{name}") ++ for name in ( ++ "adjustRightInd", ++ "autoSpaceDE", ++ "autoSpaceDN", ++ "bidi", ++ "cnfStyle", ++ "contextualSpacing", ++ "divId", ++ "framePr", ++ "ind", ++ "jc", ++ "keepLines", ++ "keepNext", ++ "kinsoku", ++ "mirrorIndents", ++ "numPr", ++ "outlineLvl", ++ "overflowPunct", ++ "pBdr", ++ "pageBreakBefore", ++ "pStyle", ++ "rPr", ++ "shd", ++ "snapToGrid", ++ "spacing", ++ "suppressAutoHyphens", ++ "suppressLineNumbers", ++ "suppressOverlap", ++ "tabs", ++ "textAlignment", ++ "textDirection", ++ "textboxTightWrap", ++ "topLinePunct", ++ "widowControl", ++ "wordWrap", ++ ) ++ } ++ ), ++ _RUN_PROPERTIES_TAG: _RUN_PROPERTY_TAGS, ++ _SECTION_PROPERTIES_TAG: frozenset( ++ { ++ qn(f"w:{name}") ++ for name in ( ++ "bidi", ++ "cols", ++ "docGrid", ++ "endnotePr", ++ "footerReference", ++ "footnotePr", ++ "formProt", ++ "headerReference", ++ "lnNumType", ++ "noEndnote", ++ "paperSrc", ++ "pgBorders", ++ "pgMar", ++ "pgNumType", ++ "pgSz", ++ "printerSettings", ++ "rtlGutter", ++ "textDirection", ++ "titlePg", ++ "type", ++ "vAlign", ++ ) ++ } ++ ), ++ _TABLE_PROPERTIES_TAG: frozenset( ++ { ++ qn(f"w:{name}") ++ for name in ( ++ "bidiVisual", ++ "jc", ++ "shd", ++ "tblBorders", ++ "tblCaption", ++ "tblCellMar", ++ "tblCellSpacing", ++ "tblDescription", ++ "tblInd", ++ "tblLayout", ++ "tblLook", ++ "tblOverlap", ++ "tblStyle", ++ "tblStyleColBandSize", ++ "tblStyleRowBandSize", ++ "tblW", ++ "tblpPr", ++ ) ++ } ++ ), ++ _TABLE_GRID_TAG: frozenset({qn("w:gridCol")}), ++ _TABLE_ROW_PROPERTIES_TAG: frozenset( ++ { ++ qn(f"w:{name}") ++ for name in ( ++ "cantSplit", ++ "cnfStyle", ++ "divId", ++ "gridAfter", ++ "gridBefore", ++ "hidden", ++ "jc", ++ "tblCellSpacing", ++ "tblHeader", ++ "trHeight", ++ "wAfter", ++ "wBefore", ++ ) ++ } ++ ), ++ _TABLE_CELL_PROPERTIES_TAG: frozenset( ++ { ++ qn(f"w:{name}") ++ for name in ( ++ "cnfStyle", ++ "gridSpan", ++ "headers", ++ "hideMark", ++ "hMerge", ++ "noWrap", ++ "shd", ++ "tcBorders", ++ "tcFitText", ++ "tcMar", ++ "tcW", ++ "textDirection", ++ "vAlign", ++ "vMerge", ++ ) ++ } ++ ), ++ qn("w:numPr"): frozenset({qn("w:ilvl"), qn("w:numId")}), ++ qn("w:pBdr"): _BORDER_PROPERTY_TAGS, ++ qn("w:pgBorders"): _BORDER_PROPERTY_TAGS, ++ qn("w:tblBorders"): _BORDER_PROPERTY_TAGS, ++ qn("w:tcBorders"): _BORDER_PROPERTY_TAGS, ++ qn("w:tabs"): frozenset({qn("w:tab")}), ++ qn("w:tblCellMar"): _MARGIN_PROPERTY_TAGS, ++ qn("w:tcMar"): _MARGIN_PROPERTY_TAGS, ++ qn("w:cols"): frozenset({qn("w:col")}), ++ qn("w:footnotePr"): frozenset( ++ { ++ qn("w:numFmt"), ++ qn("w:numRestart"), ++ qn("w:numStart"), ++ qn("w:pos"), ++ } ++ ), ++ qn("w:endnotePr"): frozenset( ++ { ++ qn("w:numFmt"), ++ qn("w:numRestart"), ++ qn("w:numStart"), ++ qn("w:pos"), ++ } ++ ), ++} -+from __future__ import annotations -class RAGFlowDocxParser: - def get_picture(self, document, paragraph): @@ -37,14 +620,133 @@ - continue - embed = embed[0] - image_blob = None -- try: ++class UnsupportedDocxFigureError(ValueError): ++ """The closed DOCX profile encountered an unsupported visual object.""" ++ ++ ++def _contains_tag(element: Any, tags: frozenset[str]) -> bool: ++ return any(node.tag in tags for node in element.iter()) ++ ++ ++def _is_xml_content_type(content_type: object) -> bool: ++ if type(content_type) is not str or not content_type.isascii(): ++ raise ValueError("DOCX package part has a malformed media type") ++ parsed = policy.default.header_factory("Content-Type", content_type) ++ raw_media_type = content_type.partition(";")[0] ++ if ( ++ parsed.defects ++ or not _is_mime_token(parsed.maintype) ++ or not _is_mime_token(parsed.subtype) ++ or raw_media_type.casefold() != f"{parsed.maintype}/{parsed.subtype}" ++ ): ++ raise ValueError("DOCX package part has a malformed media type") ++ media_type = f"{parsed.maintype}/{parsed.subtype}" ++ return media_type in _XML_CONTENT_TYPES or parsed.subtype.endswith("+xml") ++ ++ ++def _is_mime_token(value: object) -> bool: ++ return ( ++ type(value) is str ++ and bool(value) ++ and value != "*" ++ and value.isascii() ++ and all( ++ character.isalnum() or character in "!#$%&'*+-.^_`|~" ++ for character in value ++ ) ++ ) ++ ++ ++def _is_package_extension(value: object) -> bool: ++ return ( ++ type(value) is str ++ and bool(value) ++ and value not in {".", ".."} ++ and value.isascii() ++ and all( ++ character.isalnum() or character in "!#$&'*+-.^_`|~" ++ for character in value ++ ) ++ ) ++ ++ ++def _normalized_package_path(value: object, *, leading_slash: bool) -> str: ++ if type(value) is not str or not value: ++ raise ValueError("DOCX package path is malformed") ++ if leading_slash: ++ if not value.startswith("/") or value.startswith("//"): ++ raise ValueError("DOCX package PartName is not absolute") ++ value = value[1:] ++ elif value.startswith("/"): ++ raise ValueError("DOCX archive member name is absolute") ++ if "\\" in value or "?" in value or "#" in value or value.endswith("/"): ++ raise ValueError("DOCX package path is not canonical") ++ segments = value.split("/") ++ if any( ++ not segment ++ or segment in {".", ".."} ++ or "%" in segment ++ for segment in segments ++ ): ++ raise ValueError("DOCX package path contains an unsafe segment") ++ return "/".join(segments) ++ ++ ++def _content_type_declarations( ++ element: Any, ++) -> tuple[dict[str, str], dict[str, str], bool]: ++ has_malformed_declaration = ( ++ element.tag != _CONTENT_TYPES_TAG ++ or element.attrib ++ or _has_direct_character_data(element) ++ ) ++ defaults: dict[str, str] = {} ++ overrides: dict[str, str] = {} ++ for declaration in element.iterchildren(): ++ if declaration.tag == _CONTENT_TYPE_DEFAULT_TAG: ++ permitted_attributes = {"Extension", "ContentType"} ++ key = declaration.get("Extension") ++ target = defaults ++ elif declaration.tag == _CONTENT_TYPE_OVERRIDE_TAG: ++ permitted_attributes = {"PartName", "ContentType"} ++ part_name = declaration.get("PartName") + try: - related_part = document.part.related_parts[embed] - except Exception as e: - logging.warning(f"Skipping image due to unexpected error getting related_part: {e}") -- continue -+from dataclasses import dataclass -+from io import BytesIO -+from typing import Any, Final ++ key = _normalized_package_path(part_name, leading_slash=True) ++ except ValueError: ++ has_malformed_declaration = True + continue ++ target = overrides ++ else: ++ has_malformed_declaration = True ++ continue ++ content_type = declaration.get("ContentType") ++ if ( ++ set(declaration.attrib) != permitted_attributes ++ or len(declaration) ++ or _has_direct_character_data(declaration) ++ or type(key) is not str ++ or not key ++ or (target is defaults and not _is_package_extension(key)) ++ or type(content_type) is not str ++ or not content_type ++ ): ++ has_malformed_declaration = True ++ if type(key) is not str or not key or type(content_type) is not str: ++ continue ++ try: ++ _is_xml_content_type(content_type) ++ except ValueError: ++ has_malformed_declaration = True ++ continue ++ normalized_key = key.casefold() ++ if normalized_key in target: ++ has_malformed_declaration = True ++ continue ++ target[normalized_key] = content_type ++ return defaults, overrides, has_malformed_declaration - try: - image = related_part.image @@ -59,12 +761,7 @@ - logging.info(f"Damaged image encountered, attempting blob fallback: {e}") - except Exception as e: - logging.warning(f"Unexpected error getting image, attempting blob fallback: {e}") -+from docx import Document -+from docx.document import Document as DocumentType -+from docx.oxml.ns import qn -+from docx.table import Table -+from docx.text.paragraph import Paragraph - +- - if image_blob is None: - image_blob = getattr(related_part, "blob", None) - if image_blob: @@ -72,33 +769,15 @@ - if not image_blobs: - return None - return LazyImage(image_blobs) -+_PARAGRAPH_TAG: Final = qn("w:p") -+_TABLE_TAG: Final = qn("w:tbl") -+_SECTION_PROPERTIES_TAG: Final = qn("w:sectPr") -+_UNSUPPORTED_CONTENT_TAGS: Final = frozenset( -+ { -+ qn("w:customXml"), -+ qn("w:del"), -+ qn("w:ins"), -+ qn("w:moveFrom"), -+ qn("w:moveTo"), -+ qn("w:sdt"), -+ } -+) -+_UNSUPPORTED_VISUAL_TAGS: Final = frozenset( -+ {qn("w:drawing"), qn("w:object"), qn("w:pict")} -+) - +- - def __extract_table_content(self, tb): - df = [] - for row in tb.rows: - df.append([c.text for c in row.cells]) - return self.__compose_table_content(pd.DataFrame(df)) - +- - def __compose_table_content(self, df): -+class UnsupportedDocxFigureError(ValueError): -+ """The closed DOCX profile encountered an unsupported visual object.""" - +- - def blockType(b): - pattern = [ - ("^(20|19)[0-9]{2}[年/-][0-9]{1,2}[月/-][0-9]{1,2}日*$", "Dt"), @@ -123,25 +802,17 @@ - return "Tx" - else: - return "Lx" - +- - if len(tks) == 1 and rag_tokenizer.tag(tks[0]) == "nr": - return "Nr" -+def _contains_tag(element: Any, tags: frozenset[str]) -> bool: -+ return any(node.tag in tags for node in element.iter()) - +- - return "Ot" - +- - if len(df) < 2: - return [] - max_type = Counter([blockType(str(df.iloc[i, j])) for i in range(1, len(df)) for j in range(len(df.iloc[i, :]))]) - max_type = max(max_type.items(), key=lambda x: x[1])[0] -+def _package_contains_visual(document: DocumentType) -> bool: -+ for part in document.part.package.parts: -+ element = getattr(part, "element", None) -+ if element is not None and _contains_tag(element, _UNSUPPORTED_VISUAL_TAGS): -+ return True -+ return False - +- - colnm = len(df.iloc[0, :]) - hdrows = [0] # header is not necessarily appear in the first line - if max_type == "Nu": @@ -150,11 +821,96 @@ - tys = max(tys.items(), key=lambda x: x[1])[0] - if tys != max_type: - hdrows.append(r) - +- - lines = [] - for i in range(1, len(df)): - if i in hdrows: -- continue ++ ++def _relationship_part_source(member_name: str) -> str | None: ++ if member_name == "_rels/.rels": ++ return "" ++ parent, separator, filename = member_name.rpartition("/_rels/") ++ if separator and filename.endswith(".rels"): ++ return f"{parent}/{filename.removesuffix('.rels')}" ++ return None ++ ++ ++def _resolved_relationship_target(source_part: str, target: str) -> str: ++ if target.startswith("/"): ++ return _normalized_package_path(target, leading_slash=True) ++ parent = source_part.rpartition("/")[0] ++ segments = [segment for segment in parent.split("/") if segment] ++ target_segments = target.split("/") ++ if any(not segment or segment == "." for segment in target_segments): ++ raise ValueError("DOCX relationship target is not canonical") ++ for segment in target_segments: ++ if "\\" in segment or "%" in segment: ++ raise ValueError("DOCX relationship target is not canonical") ++ if segment == "..": ++ if not segments: ++ raise ValueError("DOCX relationship target escapes the package root") ++ segments.pop() ++ continue ++ segments.append(segment) ++ return _normalized_package_path("/".join(segments), leading_slash=False) ++ ++ ++def _related_member_names( ++ elements: tuple[tuple[str, Any], ...], ++) -> tuple[frozenset[str], dict[str, frozenset[str]]]: ++ relationship_targets: dict[str, tuple[tuple[str, str], ...]] = {} ++ root_main_relationships = 0 ++ for member_name, element in elements: ++ source_part = _relationship_part_source(member_name) ++ if source_part is None: ++ continue ++ if source_part in relationship_targets: ++ raise ValueError("DOCX package has duplicate relationship parts") ++ if element.tag != _RELATIONSHIPS_TAG: ++ raise ValueError("DOCX relationship part has an invalid root") ++ if element.attrib or _has_direct_character_data(element): ++ raise ValueError("DOCX relationship root is outside the grammar") ++ relationship_ids: set[str] = set() ++ internal_targets: list[tuple[str, str]] = [] ++ for relationship in element.iterchildren(): ++ if relationship.tag != _RELATIONSHIP_TAG: ++ raise ValueError("DOCX relationship part has an invalid child") ++ if len(relationship) or _has_direct_character_data(relationship): ++ raise ValueError("DOCX relationship is outside the grammar") ++ attribute_names = frozenset(relationship.attrib) ++ if not attribute_names.issubset({"Id", "Type", "Target", "TargetMode"}): ++ raise ValueError("DOCX relationship has an unknown attribute") ++ relationship_id = relationship.get("Id") ++ relationship_type = relationship.get("Type") ++ target = relationship.get("Target") ++ target_mode = relationship.get("TargetMode", "Internal") ++ if ( ++ type(relationship_id) is not str ++ or not relationship_id ++ or relationship_id in relationship_ids ++ or type(relationship_type) is not str ++ or not relationship_type ++ or type(target) is not str ++ or not target ++ or target_mode not in {"Internal", "External"} ++ ): ++ raise ValueError("DOCX relationship is malformed") ++ relationship_ids.add(relationship_id) ++ target_path = target.split("#", 1)[0].split("?", 1)[0] ++ if any(segment == "." for segment in target_path.split("/")) or ( ++ target_mode == "External" ++ and any(segment == ".." for segment in target_path.split("/")) ++ ): ++ raise ValueError("DOCX relationship target is not canonical") ++ if relationship_type == _OLE_OBJECT_RELATIONSHIP_TYPE: ++ raise ValueError("DOCX OLE relationships are outside the grammar") ++ if ( ++ relationship_type == _THUMBNAIL_RELATIONSHIP_TYPE ++ and target_mode == "External" ++ ): ++ raise ValueError("DOCX thumbnail relationship must be internal") ++ if target_mode == "External": + continue - hr = [r - i for r in hdrows] - hr = [r for r in hr if r < 0] - t = len(hr) - 1 @@ -178,24 +934,167 @@ - cells = [] - for j in range(len(df.iloc[i, :])): - if not str(df.iloc[i, j]): -- continue ++ resolved = _resolved_relationship_target(source_part, target) ++ if relationship_type == _THUMBNAIL_RELATIONSHIP_TYPE and ( ++ source_part != "" or resolved != _THUMBNAIL_MEMBER ++ ): ++ raise ValueError("DOCX thumbnail relationship is outside the grammar") ++ if source_part == "" and resolved == _DOCUMENT_MEMBER: ++ if relationship_type != _OFFICE_DOCUMENT_RELATIONSHIP_TYPE: ++ raise ValueError( ++ "DOCX main document has the wrong relationship type" ++ ) ++ root_main_relationships += 1 ++ internal_targets.append((resolved, relationship_type)) ++ relationship_targets[source_part] = tuple(internal_targets) ++ if root_main_relationships != 1: ++ raise ValueError("DOCX package must relate exactly one main document") ++ ++ related: set[str] = set() ++ relationship_types: dict[str, set[str]] = {} ++ reachable_sources = {""} ++ while reachable_sources: ++ source_part = reachable_sources.pop() ++ for target, relationship_type in relationship_targets.get(source_part, ()): ++ relationship_types.setdefault(target, set()).add(relationship_type) ++ if target not in related: ++ related.add(target) ++ reachable_sources.add(target) ++ return frozenset(related), { ++ target: frozenset(types) for target, types in relationship_types.items() ++ } ++ ++ ++def _is_complete_jpeg(value: bytes) -> bool: ++ if not value.startswith(_JPEG_START_OF_IMAGE): ++ return False ++ offset = len(_JPEG_START_OF_IMAGE) ++ in_entropy_data = False ++ frame_component_ids: frozenset[int] | None = None ++ scanned_component_ids: set[int] = set() ++ has_start_of_scan = False ++ while offset < len(value): ++ if in_entropy_data: ++ marker_prefix = value.find(b"\xff", offset) ++ if marker_prefix < 0: ++ return False ++ offset = marker_prefix ++ elif value[offset] != 0xFF: ++ return False ++ while offset < len(value) and value[offset] == 0xFF: ++ offset += 1 ++ if offset >= len(value): ++ return False ++ marker = value[offset] ++ offset += 1 ++ if in_entropy_data and (marker == 0x00 or 0xD0 <= marker <= 0xD7): ++ continue ++ in_entropy_data = False ++ if marker == 0xD9: ++ return ( ++ frame_component_ids is not None ++ and has_start_of_scan ++ and scanned_component_ids == frame_component_ids ++ and not any(value[offset:]) ++ ) ++ if marker in {0x00, 0x01, 0xD8} or 0xD0 <= marker <= 0xD7: ++ return False ++ if offset + 2 > len(value): ++ return False ++ segment_length = int.from_bytes(value[offset : offset + 2], "big") ++ segment_end = offset + segment_length ++ if segment_length < 2 or segment_end > len(value): ++ return False ++ if marker in _JPEG_START_OF_FRAME_MARKERS: ++ if segment_length < 11 or frame_component_ids is not None: ++ return False ++ component_count = value[offset + 7] ++ if component_count == 0 or segment_length != 8 + 3 * component_count: ++ return False ++ height = int.from_bytes(value[offset + 3 : offset + 5], "big") ++ width = int.from_bytes(value[offset + 5 : offset + 7], "big") ++ component_ids = frozenset( ++ value[offset + 8 + component_offset * 3] ++ for component_offset in range(component_count) ++ ) ++ if width == 0 or height == 0 or len(component_ids) != component_count: ++ return False ++ frame_component_ids = component_ids ++ elif marker == 0xDA: ++ if frame_component_ids is None or segment_length < 8: ++ return False ++ component_count = value[offset + 2] ++ if component_count == 0 or segment_length != 6 + 2 * component_count: ++ return False ++ scan_component_ids = frozenset( ++ value[offset + 3 + component_offset * 2] ++ for component_offset in range(component_count) ++ ) ++ if ( ++ len(scan_component_ids) != component_count ++ or not scan_component_ids.issubset(frame_component_ids) ++ ): ++ return False ++ scanned_component_ids.update(scan_component_ids) ++ has_start_of_scan = True ++ in_entropy_data = True ++ offset = segment_end ++ return False ++ ++ ++def _package_xml_elements( ++ source: bytes, ++) -> tuple[tuple[tuple[str, Any], ...], frozenset[str], bool]: ++ elements: list[tuple[str, Any]] = [] ++ parsed_member_names: set[str] = set() ++ package_members: list[tuple[str, bytes, str | None]] = [] ++ has_malformed_part = False ++ with ZipFile(BytesIO(source)) as archive: ++ members = archive.infolist() ++ member_bytes_by_identity = { ++ id(member): archive.read(member) ++ for member in members ++ if not member.is_dir() and member.filename != _CONTENT_TYPES_MEMBER ++ } ++ normalized_members: list[tuple[Any, str]] = [] ++ for member in members: ++ try: ++ normalized_members.append( ++ ( ++ member, ++ _normalized_package_path( ++ member.filename, leading_slash=False ++ ), ++ ) ++ ) ++ except ValueError: ++ has_malformed_part = True ++ member_names = tuple(name for _, name in normalized_members) ++ if len(member_names) != len({name.casefold() for name in member_names}): ++ has_malformed_part = True ++ try: ++ content_types = parse_xml(archive.read(_CONTENT_TYPES_MEMBER)) ++ elements.append((_CONTENT_TYPES_MEMBER, content_types)) ++ defaults, overrides, malformed_manifest = _content_type_declarations( ++ content_types ++ ) ++ has_malformed_part = has_malformed_part or malformed_manifest ++ except Exception: ++ for member in members: ++ member_bytes = member_bytes_by_identity.get(id(member)) ++ if member_bytes is None: ++ continue ++ try: ++ elements.append((member.filename, parse_xml(member_bytes))) ++ except Exception: + continue - cells.append(headers[j] + str(df.iloc[i, j])) - lines.append(";".join(cells)) -+@dataclass(frozen=True, slots=True) -+class RawDocxBlock: -+ """One bounded block in OOXML body order.""" - +- - if colnm > 3: - return lines - return ["\n".join(lines)] -+ kind: str -+ block_ordinal: int -+ text: str -+ style_name: str | None -+ xml: bytes -+ table_cells: tuple[tuple[str, ...], ...] = () -+ has_figure: bool = False - +- - def __call__(self, fnm, from_page=0, to_page=MAXIMUM_PAGE_NUMBER): - self.doc = Document(fnm) if isinstance(fnm, str) else Document(BytesIO(fnm)) - pn = 0 # parsed page @@ -203,36 +1102,522 @@ - for p in self.doc.paragraphs: - if pn > to_page: - break - +- - runs_within_single_paragraph = [] # save runs within the range of pages - for run in p.runs: - if pn > to_page: - break - if from_page <= pn < to_page and p.text.strip(): - runs_within_single_paragraph.append(run.text) # append run.text first -+class RAGFlowDocxParser: -+ """Narrow patched parser retained from the approved RAGFlow source region.""" - +- - # wrap page break checker into a static method - if "lastRenderedPageBreak" in run._element.xml: - pn += 1 +- +- secs.append(("".join(runs_within_single_paragraph), p.style.name if hasattr(p.style, "name") else "")) # then concat run.text as part of the paragraph +- +- tbls = [self.__extract_table_content(tb) for tb in self.doc.tables] +- return secs, tbls ++ return tuple(elements), frozenset(), True ++ declared_override_names = set(overrides) ++ if not declared_override_names.issubset( ++ {name.casefold() for name in member_names} ++ ): ++ has_malformed_part = True ++ for member, member_name in normalized_members: ++ if member.is_dir() or member_name == _CONTENT_TYPES_MEMBER: ++ continue ++ member_bytes = member_bytes_by_identity[id(member)] ++ content_type = overrides.get(member_name.casefold()) ++ if content_type is None and "." in member_name.rsplit("/", 1)[-1]: ++ extension = member_name.rsplit(".", 1)[-1].casefold() ++ content_type = defaults.get(extension) ++ if content_type is None: ++ package_members.append((member_name, member_bytes, None)) ++ has_malformed_part = True ++ continue ++ is_xml = _is_xml_content_type(content_type) ++ package_members.append((member_name, member_bytes, content_type)) ++ if not is_xml and _relationship_part_source(member_name) is None: ++ continue ++ try: ++ elements.append((member_name, parse_xml(member_bytes))) ++ parsed_member_names.add(member_name) ++ except Exception: ++ has_malformed_part = True ++ continue ++ try: ++ related_members, relationship_types = _related_member_names(tuple(elements)) ++ except ValueError: ++ related_members = frozenset() ++ relationship_types = {} ++ has_malformed_part = True ++ for member in members: ++ member_bytes = member_bytes_by_identity.get(id(member)) ++ if member_bytes is None: ++ continue ++ try: ++ elements.append((member.filename, parse_xml(member_bytes))) ++ except Exception: ++ continue ++ for member_name, member_bytes, content_type in package_members: ++ if member_name in parsed_member_names or member_name not in related_members: ++ continue ++ member_relationship_types = relationship_types.get(member_name, frozenset()) ++ if member_relationship_types and member_relationship_types.issubset( ++ _BINARY_RELATIONSHIP_TYPES ++ ): ++ if _OLE_OBJECT_RELATIONSHIP_TYPE in member_relationship_types: ++ has_malformed_part = True ++ if _THUMBNAIL_RELATIONSHIP_TYPE in member_relationship_types and ( ++ member_name != _THUMBNAIL_MEMBER ++ or type(content_type) is not str ++ or content_type.casefold() != _THUMBNAIL_CONTENT_TYPE ++ or not _is_complete_jpeg(member_bytes) ++ ): ++ has_malformed_part = True ++ continue ++ try: ++ elements.append((member_name, parse_xml(member_bytes))) ++ parsed_member_names.add(member_name) ++ except Exception: ++ has_malformed_part = True ++ archive_members = frozenset(member_names) ++ if not related_members.issubset(archive_members): ++ has_malformed_part = True ++ for member_name in archive_members: ++ if member_name == _CONTENT_TYPES_MEMBER: ++ continue ++ relationship_source = _relationship_part_source(member_name) ++ if relationship_source is not None: ++ if relationship_source and relationship_source not in related_members: ++ has_malformed_part = True ++ elif member_name not in related_members: ++ has_malformed_part = True ++ return tuple(elements), related_members, has_malformed_part ++ ++ ++def _elements_contain_tag( ++ elements: tuple[tuple[str, Any], ...], tags: frozenset[str] ++) -> bool: ++ return any(_contains_tag(element, tags) for _, element in elements) ++ ++ ++def _ooxml_visible_text(element: Any) -> str: ++ text: list[str] = [] ++ for run in element.iter(_RUN_TAG): ++ for node in run.iterdescendants(): ++ if node.tag == _TEXT_TAG: ++ text.append(node.text or "") ++ elif node.tag in _TAB_TAGS: ++ text.append("\t") ++ elif node.tag == _NO_BREAK_HYPHEN_TAG: ++ text.append("-") ++ elif node.tag in _BREAK_TAGS and ( ++ node.tag == qn("w:cr") ++ or node.get(_BREAK_TYPE_ATTRIBUTE) in (None, "textWrapping") ++ ): ++ text.append("\n") ++ return "".join(text) ++ ++ ++def _contains_unadmitted_run_content(element: Any) -> bool: ++ return any( ++ child.tag not in _ADMITTED_RUN_TAGS ++ for run in element.iter(_RUN_TAG) ++ for child in run.iterchildren() ++ ) ++ ++ ++def _contains_visible_token_outside_run(element: Any) -> bool: ++ return any( ++ not any(ancestor.tag == _RUN_TAG for ancestor in token.iterancestors()) ++ for token in element.iter() ++ if token.tag in _VISIBLE_TOKEN_TAGS ++ and not ( ++ token.tag == qn("w:tab") ++ and any( ++ ancestor.tag == _TAB_STOPS_TAG for ancestor in token.iterancestors() ++ ) ++ ) ++ ) ++ ++ ++def _contains_unrepresented_package_text( ++ elements: tuple[tuple[str, Any], ...] ++) -> bool: ++ for member_name, element in elements: ++ if member_name == _DOCUMENT_MEMBER: ++ body = element.find(qn("w:body")) ++ if any( ++ child is not body ++ and ( ++ _ooxml_visible_text(child) ++ or ( ++ type(child.tag) is str ++ and child.tag.startswith(_OFFICE_MATH_TAG_PREFIX) ++ ) ++ ) ++ for child in element.iterchildren() ++ ): ++ return True ++ elif _ooxml_visible_text(element): ++ return True ++ return False ++ ++ ++def _has_direct_character_data(node: Any) -> bool: ++ return any(text.strip() for text in node.xpath("text()")) ++ ++ ++def _xml_namespace(name: object) -> str | None: ++ if type(name) is not str: ++ return None ++ if not name.startswith("{"): ++ return "" ++ namespace, separator, _ = name[1:].partition("}") ++ return namespace if separator and namespace else None ++ ++ ++def _known_inert_member_uses_admitted_grammar( ++ member_name: str, element: Any ++) -> bool: ++ expected_roots = _KNOWN_INERT_XML_ROOTS.get(member_name) ++ element_namespaces = _KNOWN_INERT_XML_ELEMENT_NAMESPACES.get(member_name) ++ attribute_namespaces = _KNOWN_INERT_XML_ATTRIBUTE_NAMESPACES.get(member_name) ++ if ( ++ expected_roots is None ++ or element_namespaces is None ++ or attribute_namespaces is None ++ or element.tag not in expected_roots ++ ): ++ return False ++ admitted_text_tags = _KNOWN_INERT_XML_TEXT_TAGS.get(member_name, frozenset()) ++ for node in element.iter(): ++ if _xml_namespace(node.tag) not in element_namespaces: ++ return False ++ if any( ++ _xml_namespace(attribute_name) not in attribute_namespaces ++ for attribute_name in node.attrib ++ ): ++ return False ++ if _has_direct_character_data(node) and node.tag not in admitted_text_tags: ++ return False ++ return True ++ ++ ++def _main_document_uses_admitted_root_grammar( ++ elements: tuple[tuple[str, Any], ...] ++) -> bool: ++ roots = tuple( ++ element for member_name, element in elements if member_name == _DOCUMENT_MEMBER ++ ) ++ if len(roots) != 1: ++ return False ++ root = roots[0] ++ children = tuple(root.iterchildren()) ++ return ( ++ root.tag == qn("w:document") ++ and not _has_direct_character_data(root) ++ and len(children) == 1 ++ and children[0].tag == qn("w:body") ++ ) ++ ++ ++def _property_subtree_uses_admitted_grammar(node: Any) -> bool: ++ if _has_direct_character_data(node): ++ return False ++ admitted_children = _PROPERTY_CHILDREN.get(node.tag, frozenset()) ++ children = tuple(node.iterchildren()) ++ return all( ++ child.tag in admitted_children ++ and _property_subtree_uses_admitted_grammar(child) ++ for child in children ++ ) ++ ++ ++def _property_subtrees_use_admitted_grammar(body: Any) -> bool: ++ for root_tag, admitted_parents in _PROPERTY_ROOT_PARENTS.items(): ++ for root in body.iter(root_tag): ++ parent = root.getparent() ++ if ( ++ parent is None ++ or parent.tag not in admitted_parents ++ or not _property_subtree_uses_admitted_grammar(root) ++ ): ++ return False ++ return True ++ ++ ++def _contains_unrepresented_package_structure( ++ elements: tuple[tuple[str, Any], ...], related_members: frozenset[str] ++) -> bool: ++ for member_name, element in elements: ++ if member_name == _DOCUMENT_MEMBER: ++ continue ++ if member_name == _CONTENT_TYPES_MEMBER: ++ continue ++ relationship_source = _relationship_part_source(member_name) ++ if relationship_source is not None: ++ if element.tag != _RELATIONSHIPS_TAG: ++ return True ++ continue ++ if member_name not in related_members: ++ return True ++ expected_inert_roots = _KNOWN_INERT_XML_ROOTS.get(member_name) ++ if expected_inert_roots is not None: ++ if not _known_inert_member_uses_admitted_grammar(member_name, element): ++ return True ++ continue ++ if element.tag in _UNREPRESENTED_PART_ROOT_TAGS: ++ if any( ++ type(node.tag) is str ++ and node.tag.startswith(_OFFICE_MATH_TAG_PREFIX) ++ for node in element.iter() ++ ): ++ return True ++ if element.tag not in {qn("w:hdr"), qn("w:ftr")}: ++ if len(element) > 0 or bool(element.text and element.text.strip()): ++ return True ++ continue ++ for paragraph in element.iterchildren(): ++ if paragraph.tag != _PARAGRAPH_TAG: ++ return True ++ for child in paragraph.iterchildren(): ++ if child.tag == _PARAGRAPH_PROPERTIES_TAG: ++ if not _property_subtree_uses_admitted_grammar(child): ++ return True ++ elif child.tag == _RUN_TAG: ++ run_children = tuple(child.iterchildren()) ++ if any(run_child.tag != _RUN_PROPERTIES_TAG for run_child in run_children): ++ return True ++ if any( ++ not _property_subtree_uses_admitted_grammar(run_child) ++ for run_child in run_children ++ ): ++ return True ++ else: ++ return True ++ continue ++ return True ++ return False ++ ++ ++def _body_uses_closed_admitted_grammar(document: DocumentType) -> bool: ++ body = document.element.body ++ if any( ++ type(node.tag) is not str ++ or not node.tag.startswith(_WORDPROCESSINGML_TAG_PREFIX) ++ for node in body.iter() ++ ): ++ return False ++ body_children = tuple(body.iterchildren()) ++ section_indexes = tuple( ++ index ++ for index, child in enumerate(body_children) ++ if child.tag == _SECTION_PROPERTIES_TAG ++ ) ++ if len(section_indexes) > 1 or ( ++ section_indexes and section_indexes[0] != len(body_children) - 1 ++ ): ++ return False ++ if any( ++ child.tag not in {_PARAGRAPH_TAG, _TABLE_TAG, _SECTION_PROPERTIES_TAG} ++ for child in body_children ++ ): ++ return False ++ if not _property_subtrees_use_admitted_grammar(body): ++ return False ++ for paragraph in body.iter(_PARAGRAPH_TAG): ++ parent = paragraph.getparent() ++ if parent is None or parent.tag not in {qn("w:body"), _TABLE_CELL_TAG}: ++ return False ++ children = tuple(paragraph.iterchildren()) ++ if _has_direct_character_data(paragraph): ++ return False ++ if any(child.tag not in _ADMITTED_PARAGRAPH_CHILDREN for child in children): ++ return False ++ property_indexes = tuple( ++ index ++ for index, child in enumerate(children) ++ if child.tag == _PARAGRAPH_PROPERTIES_TAG ++ ) ++ if len(property_indexes) > 1 or ( ++ property_indexes and property_indexes[0] != 0 ++ ): ++ return False ++ for run in body.iter(_RUN_TAG): ++ parent = run.getparent() ++ if parent is None or parent.tag != _PARAGRAPH_TAG: ++ return False ++ children = tuple(run.iterchildren()) ++ if _has_direct_character_data(run): ++ return False ++ if any(child.tag not in _ADMITTED_RUN_TAGS for child in children): ++ return False ++ if any( ++ child.tag != _RUN_PROPERTIES_TAG ++ and ( ++ len(child) > 0 ++ or ( ++ child.tag not in {_TEXT_TAG, qn("w:instrText")} ++ and _has_direct_character_data(child) ++ ) ++ ) ++ for child in children ++ ): ++ return False ++ for table in body.iter(_TABLE_TAG): ++ table_children = tuple(table.iterchildren()) ++ if ( ++ table.getparent() is not body ++ or _has_direct_character_data(table) ++ or any(child.tag not in _ADMITTED_TABLE_CHILDREN for child in table_children) ++ ): ++ return False ++ table_property_indexes = tuple( ++ index ++ for index, child in enumerate(table_children) ++ if child.tag == _TABLE_PROPERTIES_TAG ++ ) ++ table_grid_indexes = tuple( ++ index ++ for index, child in enumerate(table_children) ++ if child.tag == _TABLE_GRID_TAG ++ ) ++ if ( ++ len(table_property_indexes) != 1 ++ or table_property_indexes[0] != 0 ++ or len(table_grid_indexes) != 1 ++ or table_grid_indexes[0] != 1 ++ or not any(child.tag == _TABLE_ROW_TAG for child in table_children) ++ ): ++ return False ++ for row in table.iterchildren(_TABLE_ROW_TAG): ++ row_children = tuple(row.iterchildren()) ++ if _has_direct_character_data(row) or any( ++ child.tag not in _ADMITTED_TABLE_ROW_CHILDREN for child in row_children ++ ): ++ return False ++ row_property_indexes = tuple( ++ index ++ for index, child in enumerate(row_children) ++ if child.tag == _TABLE_ROW_PROPERTIES_TAG ++ ) ++ if ( ++ len(row_property_indexes) > 1 ++ or (row_property_indexes and row_property_indexes[0] != 0) ++ or not any(child.tag == _TABLE_CELL_TAG for child in row_children) ++ ): ++ return False ++ for cell in table.iter(_TABLE_CELL_TAG): ++ cell_children = tuple(cell.iterchildren()) ++ if _has_direct_character_data(cell) or any( ++ child.tag not in _ADMITTED_TABLE_CELL_CHILDREN ++ for child in cell_children ++ ): ++ return False ++ property_indexes = tuple( ++ index ++ for index, child in enumerate(cell_children) ++ if child.tag == _TABLE_CELL_PROPERTIES_TAG ++ ) ++ if ( ++ len(property_indexes) != 1 ++ or property_indexes[0] != 0 ++ or not any(child.tag == _PARAGRAPH_TAG for child in cell_children) ++ ): ++ return False ++ if any( ++ row.getparent() is None ++ or row.getparent().tag != _TABLE_TAG ++ or row.getparent().getparent() is not body ++ for row in body.iter(_TABLE_ROW_TAG) ++ ): ++ return False ++ if any( ++ cell.getparent() is None ++ or cell.getparent().tag != _TABLE_ROW_TAG ++ or cell.getparent().getparent() is None ++ or cell.getparent().getparent().tag != _TABLE_TAG ++ or cell.getparent().getparent().getparent() is not body ++ for cell in body.iter(_TABLE_CELL_TAG) ++ ): ++ return False ++ return True ++ ++ ++def _body_paragraph_text_is_lossless(document: DocumentType) -> bool: ++ body = document.element.body ++ return all( ++ any(ancestor.tag == _PARAGRAPH_TAG for ancestor in run.iterancestors()) ++ for run in body.iter(_RUN_TAG) ++ ) and all( ++ Paragraph(paragraph, document).text == _ooxml_visible_text(paragraph) ++ for paragraph in body.iter(_PARAGRAPH_TAG) ++ ) ++ ++ ++@dataclass(frozen=True, slots=True) ++class RawDocxBlock: ++ """One bounded block in OOXML body order.""" ++ ++ kind: str ++ block_ordinal: int ++ text: str ++ style_name: str | None ++ xml: bytes ++ table_cells: tuple[tuple[str, ...], ...] = () ++ has_figure: bool = False ++ ++ ++class RAGFlowDocxParser: ++ """Narrow patched parser retained from the approved RAGFlow source region.""" ++ + def __call__(self, source: bytes) -> tuple[RawDocxBlock, ...]: + if type(source) is not bytes: + raise TypeError("DOCX parser source must be exact bytes") -+ document = Document(BytesIO(source)) -+ if not isinstance(document, DocumentType): -+ raise ValueError("DOCX parser did not construct an exact document") -+ if _package_contains_visual(document): ++ package_elements, related_members, has_malformed_part = ( ++ _package_xml_elements(source) ++ ) ++ if _elements_contain_tag(package_elements, _UNSUPPORTED_VISUAL_TAGS): + raise UnsupportedDocxFigureError( + "DOCX profile does not admit visual objects" + ) ++ if has_malformed_part: ++ raise ValueError("DOCX contains a malformed XML package part") ++ if _DOCUMENT_MEMBER not in related_members: ++ raise ValueError("DOCX main document is not related from the package root") ++ if not _main_document_uses_admitted_root_grammar(package_elements): ++ raise ValueError("DOCX main document root is outside the admitted grammar") ++ if _contains_unrepresented_package_structure( ++ package_elements, related_members ++ ): ++ raise ValueError("DOCX contains unrepresented package structure") ++ document = Document(BytesIO(source)) ++ if not isinstance(document, DocumentType): ++ raise ValueError("DOCX parser did not construct an exact document") ++ if not _body_uses_closed_admitted_grammar(document): ++ raise ValueError("DOCX body is outside the closed admitted grammar") ++ if any( ++ _contains_unadmitted_run_content(element) ++ for _, element in package_elements ++ ): ++ raise ValueError("DOCX contains unsupported run content") ++ if any( ++ _contains_visible_token_outside_run(element) ++ for _, element in package_elements ++ ): ++ raise ValueError("DOCX contains visible content outside a run") ++ if _contains_unrepresented_package_text(package_elements): ++ raise ValueError("DOCX contains text outside the represented body") ++ if not _body_paragraph_text_is_lossless(document): ++ raise ValueError("DOCX paragraph text cannot be represented losslessly") + blocks: list[RawDocxBlock] = [] + for block_ordinal, child in enumerate(document.element.body.iterchildren()): -+ if _contains_tag(child, _UNSUPPORTED_CONTENT_TAGS): -+ raise ValueError("DOCX contains an unsupported content container") + if child.tag == _PARAGRAPH_TAG: + paragraph = Paragraph(child, document) -+ text = paragraph.text.strip() ++ text = paragraph.text + has_figure = bool(child.xpath(".//pic:pic")) + if text or has_figure: + style_name = ( @@ -253,9 +1638,11 @@ + elif child.tag == _TABLE_TAG: + if any(node.tag == _TABLE_TAG for node in child.iterdescendants()): + raise ValueError("DOCX profile does not admit nested tables") ++ if _contains_tag(child, _MERGED_CELL_TAGS): ++ raise ValueError("DOCX profile does not admit merged table cells") + table = Table(child, document) + rows = tuple( -+ tuple(cell.text.strip() for cell in row.cells) ++ tuple(cell.text for cell in row.cells) + for row in table.rows + ) + if rows: @@ -273,9 +1660,6 @@ + elif child.tag != _SECTION_PROPERTIES_TAG: + raise ValueError("DOCX contains an unsupported body element") + return tuple(blocks) - -- secs.append(("".join(runs_within_single_paragraph), p.style.name if hasattr(p.style, "name") else "")) # then concat run.text as part of the paragraph - -- tbls = [self.__extract_table_content(tb) for tb in self.doc.tables] -- return secs, tbls ++ ++ +__all__ = ["RAGFlowDocxParser", "RawDocxBlock", "UnsupportedDocxFigureError"]