Please confirm.
diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 4eab1ff65..27e47c238 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -107,6 +107,18 @@ def test_chunk_by_dom_skips_malformed_image_data() -> None: assert [c.unit_type for c in chunks] == ["dom"] +def test_chunk_by_dom_skips_script_and_style_images() -> None: + tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" + html = ( + f'' + f'' + "Visible.
" + ) + chunks = chunk_by_dom(html) + assert [c.unit_type for c in chunks] == ["dom"] + assert chunks[0].text == "Visible." + + def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: turns = [ ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"), diff --git a/tests/test_embedded_image_payload.py b/tests/test_embedded_image_payload.py new file mode 100644 index 000000000..f57566137 --- /dev/null +++ b/tests/test_embedded_image_payload.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import base64 + +from lineageweave.embedded_image_payload import ( + decode_data_uri_image, + looks_like_raster_image, + source_offset, +) + +_TINY_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" +) +_TINY_PNG = base64.b64decode(_TINY_PNG_B64) +_JPEG_BYTES = b"\xff\xd8\xff\x00" +_GIF87_BYTES = b"GIF87a" + b"\x00" * 2 +_GIF89_BYTES = b"GIF89a" + b"\x00" * 2 +_WEBP_BYTES = b"RIFF\x00\x00\x00\x00WEBP" +_AVIF_BYTES = b"\x00\x00\x00\x00ftypavif\x00\x00\x00\x00" +_AVIS_BYTES = b"\x00\x00\x00\x00ftypavis\x00\x00\x00\x00" +_MIF1_BYTES = b"\x00\x00\x00\x00ftypmif1\x00\x00\x00\x00" + + +def test_looks_like_raster_image_accepts_png_signature() -> None: + assert looks_like_raster_image("image/png", _TINY_PNG) is True + + +def test_looks_like_raster_image_rejects_ascii_labeled_as_png() -> None: + assert looks_like_raster_image("image/png", b"Hello") is False + + +def test_looks_like_raster_image_rejects_empty_payload() -> None: + assert looks_like_raster_image("image/png", b"") is False + + +def test_looks_like_raster_image_accepts_jpeg_gif_webp_avif_signatures() -> None: + assert looks_like_raster_image("image/jpeg", _JPEG_BYTES) is True + assert looks_like_raster_image("image/jpg", _JPEG_BYTES) is True + assert looks_like_raster_image("image/gif", _GIF87_BYTES) is True + assert looks_like_raster_image("image/gif", _GIF89_BYTES) is True + assert looks_like_raster_image("image/webp", _WEBP_BYTES) is True + assert looks_like_raster_image("image/avif", _AVIF_BYTES) is True + assert looks_like_raster_image("image/avif", _AVIS_BYTES) is True + assert looks_like_raster_image("image/avif", _MIF1_BYTES) is True + + +def test_looks_like_raster_image_rejects_wrong_magic_and_unknown_type() -> None: + assert looks_like_raster_image("image/jpeg", b"not-a-jpeg") is False + assert looks_like_raster_image("image/gif", b"GIF8xa") is False + assert looks_like_raster_image("image/webp", b"RIFF....NOTW") is False + assert looks_like_raster_image("image/webp", b"RIFF") is False + assert looks_like_raster_image("image/avif", b"xxxxftypxxxx") is False + assert looks_like_raster_image("image/avif", b"short") is False + assert looks_like_raster_image("image/svg+xml", _TINY_PNG) is False + + +def test_decode_data_uri_image_rejects_svg_and_remote_src() -> None: + assert decode_data_uri_image("https://example.test/invoice.png") is None + assert ( + decode_data_uri_image( + "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPg==" + ) + is None + ) + + +def test_decode_data_uri_image_rejects_missing_comma_or_base64_marker() -> None: + assert decode_data_uri_image("data:image/png;base64") is None + assert decode_data_uri_image(f"data:image/png,{_TINY_PNG_B64}") is None + + +def test_decode_data_uri_image_rejects_unpadded_and_wrong_magic() -> None: + assert decode_data_uri_image("data:image/png;base64,YQ") is None + assert decode_data_uri_image("data:image/png;base64,AAAA") is None + + +def test_decode_data_uri_image_accepts_newlines_inside_png_payload() -> None: + wrapped = f"data:image/png;base64,{_TINY_PNG_B64[:24]}\n{_TINY_PNG_B64[24:]}" + decoded = decode_data_uri_image(wrapped) + assert decoded == ("image/png", _TINY_PNG) + + +def test_decode_data_uri_image_accepts_jpeg_alias() -> None: + encoded = base64.b64encode(_JPEG_BYTES).decode("ascii") + assert decode_data_uri_image(f"data:image/jpg;base64,{encoded}") == ( + "image/jpg", + _JPEG_BYTES, + ) + + +def test_decode_data_uri_image_accepts_gif_webp_and_avif() -> None: + for mime_type, payload in ( + ("image/gif", _GIF89_BYTES), + ("image/webp", _WEBP_BYTES), + ("image/avif", _AVIF_BYTES), + ): + encoded = base64.b64encode(payload).decode("ascii") + assert decode_data_uri_image(f"data:{mime_type};base64,{encoded}") == ( + mime_type, + payload, + ) + + +def test_source_offset_maps_htmlparser_getpos() -> None: + source = "ab\ncd" + assert source_offset(source, 1, 0) == 0 + assert source_offset(source, 2, 1) == 4 + assert source_offset(source, 0, 0) == 0 + assert source_offset(source, 9, 0) == len(source) diff --git a/tests/test_image_content.py b/tests/test_image_content.py index 033202be6..8015c1b6d 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -1,9 +1,12 @@ from __future__ import annotations import base64 +from pathlib import Path import pytest +from lineageweave.chunking import chunk_by_dom +from lineageweave.embedded_image_payload import decode_data_uri_image from lineageweave.image_content import ( ImageContentClient, ImageDescriptionParseError, @@ -55,6 +58,57 @@ def test_extract_base64_images_empty_document_yields_no_images() -> None: assert extract_base64_images("No images here.
") == [] +def test_extract_base64_images_skips_svg_and_unpadded_payloads() -> None: + svg = ( + '
'
+ )
+ assert extract_base64_images(svg) == []
+ assert extract_base64_images('
') == []
+ assert decode_data_uri_image(f"data:image/png;base64,{_TINY_PNG_B64}") is not None
+
+
+def test_extract_base64_images_skips_style_script_and_src_less_tags() -> None:
+ html = (
+ f''
+ f''
+ "
'
+ )
+ images = extract_base64_images(html)
+ assert len(images) == 1
+ assert images[0].data == base64.b64decode(_TINY_PNG_B64)
+
+
+def test_invoice_fixture_is_one_visible_png_for_every_extractor() -> None:
+ """Outlook-style invoice HTML must not resurrect the base64 wall.
+
+ The same file is read by the TypeScript popup splitter. All three
+ extractors must see one raster PNG, ignore the commented copy, the
+ remote URL, the SVG, and the CSS background, and keep surrounding
+ sentences readable.
+ """
+ html = (Path(__file__).parent / "fixtures" / "synthetic_invoice_embedded_image.html").read_text(
+ encoding="utf-8"
+ )
+ images = extract_base64_images(html)
+ chunks = chunk_by_dom(html)
+ image_chunks = [chunk for chunk in chunks if chunk.unit_type == "image"]
+ text = " ".join(chunk.text for chunk in chunks if chunk.unit_type == "dom")
+
+ assert len(images) == 1
+ assert len(image_chunks) == 1
+ assert images[0].mime_type == "image/png"
+ assert images[0].data == base64.b64decode(_TINY_PNG_B64)
+ assert images[0].position == html.find("