diff --git a/backend/services/email_parser.py b/backend/services/email_parser.py index 2d32454ad..be8bee1c4 100644 --- a/backend/services/email_parser.py +++ b/backend/services/email_parser.py @@ -2,7 +2,8 @@ from email.message import Message from pathlib import Path import datetime -from email.utils import formataddr, getaddresses +import re +from email.utils import getaddresses from email.utils import parsedate_to_datetime from typing import NotRequired, TypedDict from .attachment_parser import parse_email_attachment @@ -39,13 +40,39 @@ def _sanitize_display_text(text: str) -> str: return strip_html_markup(_sanitize_nul(text)) +# Mirror email.utils.formataddr's RFC 5322 display-name quoting: the specials +# that force a quoted-string, and the characters escaped inside one. +_ADDRESS_SPECIALS_RE = re.compile(r'[()<>@,;:\\".\[\]]') +_ADDRESS_QUOTED_ESCAPE_RE = re.compile(r'["\\]') + + +def _format_display_address(display_name: str, address: str) -> str: + """Formats an already-decoded display name and address for storage. + + Mirrors ``email.utils.formataddr`` quoting for RFC 5322 special characters + but keeps ``display_name`` literal instead of re-encoding a non-ASCII name + as an RFC 2047 encoded-word. The ``From``/``To``/``Reply-To`` headers arrive + already header-decoded (``policy.default``), and these values are stored for + human display, not re-emitted as message headers, so ``formataddr`` would + corrupt a decoded name (e.g. Korean) back into ``=?utf-8?b?...?=``. + """ + if not display_name: + return address + if _ADDRESS_SPECIALS_RE.search(display_name): + escaped_name = _ADDRESS_QUOTED_ESCAPE_RE.sub(r"\\\g<0>", display_name) + return f'"{escaped_name}" <{address}>' + return f"{display_name} <{address}>" + + def _sanitize_address_display_text(text: str) -> str: sanitized_parts: list[str] = [] for display_name, address in getaddresses([text]): safe_display_name = _sanitize_display_text(display_name).strip() safe_address = _sanitize_nul(address).strip() if safe_address: - sanitized_parts.append(formataddr((safe_display_name, safe_address))) + sanitized_parts.append( + _format_display_address(safe_display_name, safe_address) + ) elif safe_display_name: sanitized_parts.append(safe_display_name) if sanitized_parts: @@ -136,6 +163,14 @@ def _extract_date(msg: Message) -> datetime.datetime: if not parsed_date: parsed_date = datetime.datetime.now(datetime.timezone.utc) + elif parsed_date.tzinfo is None: + # RFC 5322 section 3.3: a "-0000" zone means the time zone is unknown, + # for which parsedate_to_datetime returns a naive datetime. Every other + # branch here yields a timezone-aware datetime, and mixing naive with + # aware datetimes raises TypeError on comparison/sorting and misbinds the + # instant when stored in a timestamptz column. Treat the unknown zone as + # UTC so the returned value is always timezone-aware. + parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc) return parsed_date diff --git a/backend/services/threading_service.py b/backend/services/threading_service.py index 01c738432..c500e4d35 100644 --- a/backend/services/threading_service.py +++ b/backend/services/threading_service.py @@ -31,16 +31,32 @@ def generate_email_fingerprint( def normalize_message_id(value: str | None) -> str | None: - """Return the canonical persisted form for a Message-ID-like header.""" + """Return the canonical persisted form for a Message-ID-like header. + + A Message-ID (RFC 5322 section 3.6.4) carries no interior whitespace, but + header unfolding (RFC 5322 section 2.2.3) can leave interior spaces or tabs + when a folded header is rejoined -- e.g. ```` unfolds + to ````. Collapsing all interior whitespace keeps the + folded and unfolded forms of the same Message-ID equal, so de-duplication + and threading never split one message into two over a fold boundary. + """ if value is None: return None - normalized = str(value).strip().strip("<>").strip() + stripped = str(value).strip().strip("<>") + normalized = "".join(stripped.split()) return normalized or None def extract_reference_ids(value: str | None) -> list[str]: - """Extract canonical message IDs from a References header in header order.""" + """Extract canonical message IDs from a ``1*msg-id`` header in header order. + + RFC 5322 defines both References (section 3.6.4) and In-Reply-To + (section 3.6.4) as ``1*msg-id`` -- one or more angle-bracketed Message-IDs, + each optionally surrounded by CFWS -- so this extractor applies to either + header. Ids are canonicalized with :func:`normalize_message_id` and + de-duplicated while preserving header order. + """ if not value: return [] @@ -107,19 +123,21 @@ async def assign_thread_id( Determine the thread_id for a new email based on in_reply_to and references. If no existing match is found, generate a new thread_id. """ - in_reply_to = normalize_message_id(email_data.get("in_reply_to")) + # In-Reply-To (RFC 5322 section 3.6.4) is 1*msg-id, exactly like References, + # and each id may be wrapped in CFWS. Parse it with the same multi-id + # extractor rather than treating the whole header as one opaque Message-ID, + # so a reply that names several parents -- or a single id trailed by a + # comment -- still threads onto an existing ancestor instead of splitting off. + in_reply_to_ids = extract_reference_ids(email_data.get("in_reply_to")) references = extract_reference_ids(email_data.get("references")) existing_candidates = [] # Optimization: Use a set for O(1) membership checks to prevent O(n^2) deduplication of candidates seen = set() - if in_reply_to: - existing_candidates.append(in_reply_to) - seen.add(in_reply_to) - for ref in references: - if ref not in seen: - seen.add(ref) - existing_candidates.append(ref) + for candidate in (*in_reply_to_ids, *references): + if candidate not in seen: + seen.add(candidate) + existing_candidates.append(candidate) if existing_candidates: thread_ids_by_message_id = await _find_existing_thread_ids( @@ -138,8 +156,8 @@ async def assign_thread_id( if references: return references[0] - if in_reply_to: - return in_reply_to + if in_reply_to_ids: + return in_reply_to_ids[0] msg_id = normalize_message_id(email_data.get("message_id")) if msg_id: diff --git a/backend/tests/test_email_parser.py b/backend/tests/test_email_parser.py index e44d03127..da098bc34 100644 --- a/backend/tests/test_email_parser.py +++ b/backend/tests/test_email_parser.py @@ -1,12 +1,23 @@ +import base64 import datetime import os import tempfile from email.message import Message -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest -from services.email_parser import _extract_thread_id, _sanitize_nul, parse_eml -from services.exceptions import EmailParseError +from services.email_parser import ( + EmailParseError, + _attachment_part_content, + _extract_thread_id, + _format_display_address, + _process_multipart_body, + _process_singlepart_body, + _sanitize_address_display_text, + _sanitize_nul, + parse_eml, + parse_eml_bytes, +) def test_parse_eml_basic(): @@ -109,6 +120,107 @@ def test_parse_eml_strips_active_html_from_address_display_fields(): os.unlink(temp_path) +def test_parse_eml_stores_non_ascii_display_names_decoded(): + # RFC 2047: non-ASCII From/To/Reply-To display names arrive as encoded-words + # (e.g. =?UTF-8?B?...?=). policy.default header-decodes them; the stored + # display fields must keep the decoded text rather than re-encoding it back + # into an encoded-word (formataddr's behavior), which would render every + # non-ASCII sender/recipient as garbled =?utf-8?...?= bytes in the UI. + from_name = "박성호" + to_name = "김천" + reply_name = "응답" + subject_text = "회 테스트" + + def encoded_word(text: str) -> bytes: + token = base64.b64encode(text.encode("utf-8")).decode("ascii") + return f"=?UTF-8?B?{token}?=".encode("ascii") + + eml_content = ( + b"Message-ID: \r\n" + b"From: " + encoded_word(from_name) + b" \r\n" + b"To: " + encoded_word(to_name) + b" \r\n" + b"Reply-To: " + encoded_word(reply_name) + b" \r\n" + b"Subject: " + encoded_word(subject_text) + b"\r\n" + b"Date: Mon, 27 Apr 2026 10:00:00 +0000\r\n" + b"\r\n" + b"Plain body" + ) + + with tempfile.NamedTemporaryFile(delete=False, suffix=".eml") as f: + f.write(eml_content) + temp_path = f.name + + try: + parsed = parse_eml(temp_path) + assert parsed["sender"] == f"{from_name} " + assert parsed["recipients"] == f"{to_name} " + assert parsed["reply_to"] == f"{reply_name} " + assert parsed["subject"] == subject_text + assert "=?" not in parsed["sender"] + assert "=?" not in parsed["recipients"] + finally: + os.unlink(temp_path) + + +def test_sanitize_address_display_text_keeps_decoded_unicode_and_quotes_specials(): + # A decoded non-ASCII name stays literal (formataddr would re-encode it). + assert ( + _sanitize_address_display_text("박성호 ") + == "박성호 " + ) + # A display name containing an RFC 5322 special is quoted so a ", "-joined + # multi-address value stays unambiguous. + assert ( + _sanitize_address_display_text('"Doe, John" ') + == '"Doe, John" ' + ) + # Multiple addresses with mixed scripts are each formatted and comma-joined. + assert ( + _sanitize_address_display_text("박성호 , Bob ") + == "박성호 , Bob " + ) + + +def test_format_display_address_escapes_quotes_and_handles_empty_name(): + # No display name -> bare address. + assert _format_display_address("", "a@x.com") == "a@x.com" + # Non-ASCII name kept literal. + assert _format_display_address("박성호", "s@x.com") == "박성호 " + # Embedded quotes/backslashes are escaped inside the quoted-string, matching + # email.utils.formataddr's escaping. + assert ( + _format_display_address('Fancy "Q"', "q@x.com") == '"Fancy \\"Q\\"" ' + ) + + +def test_process_multipart_body_ignores_non_string_part_content(): + # get_content() can return a non-str (e.g. undecodable bytes) even for a + # text/* part; the isinstance guard must drop it rather than concatenate + # bytes into the plain/html body. + plain_part = MagicMock() + plain_part.get_content_type.return_value = "text/plain" + plain_part.get_filename.return_value = None + plain_part.get_content.return_value = b"not-a-str" + html_part = MagicMock() + html_part.get_content_type.return_value = "text/html" + html_part.get_filename.return_value = None + html_part.get_content.return_value = b"not-a-str" + msg = MagicMock() + msg.walk.return_value = [plain_part, html_part] + + assert _process_multipart_body(msg) == ("", "", []) + + +def test_process_singlepart_body_ignores_non_string_content(): + # A single-part message whose get_content() returns a non-str yields an + # empty body rather than a stringified bytes value. + msg = MagicMock() + msg.get_content_type.return_value = "text/plain" + msg.get_content.return_value = b"not-a-str" + + assert _process_singlepart_body(msg) == ("", "", []) + + def test_parse_eml_strips_active_html_from_attachment_display_fields(): eml_content = b"""Message-ID: From: sender@test.com @@ -325,6 +437,33 @@ def test_parse_eml_missing_and_malformed_date(): os.unlink(temp_path2) +def test_parse_eml_unknown_timezone_date_is_timezone_aware(): + # RFC 5322 section 3.3: a "-0000" zone means the time zone is unknown, for + # which parsedate_to_datetime returns a *naive* datetime. Every other parse + # path yields an aware datetime, so the parser must normalize this to aware + # too -- otherwise sorting/comparing it against another message's date raises + # "can't compare offset-naive and offset-aware datetimes" and it misbinds the + # instant in a timestamptz column. + eml_content = b"""Message-ID: +From: test@test.com +To: recipient@test.com +Subject: Unknown zone +Date: Mon, 27 Apr 2026 10:00:00 -0000 + +Test.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".eml") as f: + f.write(eml_content) + temp_path = f.name + + try: + parsed = parse_eml(temp_path) + assert parsed["date"].tzinfo is not None + # must not raise offset-naive/aware TypeError + assert parsed["date"] <= datetime.datetime.now(datetime.timezone.utc) + finally: + os.unlink(temp_path) + + def test_parse_eml_io_error(): with pytest.raises(EmailParseError): parse_eml("/path/to/nonexistent/file.eml") @@ -381,6 +520,70 @@ def test_extract_thread_id_uses_first_reference_from_long_header(): assert _extract_thread_id(msg, "") == "" +def test_sanitize_address_display_text_keeps_name_only_and_falls_back_to_text(): + # A token with a display name but an empty address part keeps the name + # (rather than dropping it), and a header that yields no address at all + # falls back to the sanitized raw text. + assert _sanitize_address_display_text("Display Name <>") == "Display Name" + assert _sanitize_address_display_text("") == "" + + +def test_attachment_part_content_falls_back_to_raw_payload_on_decode_error(): + # A part whose get_content() cannot decode (unknown charset / malformed + # transfer-encoding) falls back to the raw decoded payload, and to "" when + # the payload is absent, instead of propagating the decode error. + raw_part = MagicMock() + raw_part.get_content.side_effect = LookupError("unknown charset") + raw_part.get_payload.return_value = b"raw-bytes" + assert _attachment_part_content(raw_part) == b"raw-bytes" + + empty_part = MagicMock() + empty_part.get_content.side_effect = ValueError("bad encoding") + empty_part.get_payload.return_value = None + assert _attachment_part_content(empty_part) == "" + + +def test_parse_eml_bytes_parses_provider_bytes_and_wraps_parse_errors(): + parsed = parse_eml_bytes( + b"Message-ID: \r\n" + b"From: sender@test.com\r\n" + b"To: user@test.com\r\n" + b"Subject: Bytes\r\n\r\n" + b"Body" + ) + assert parsed["message_id"] == "" + assert parsed["subject"] == "Bytes" + + # A parser failure is wrapped as the sanitized public EmailParseError rather + # than leaking the internal exception chain at the ingest boundary. + with patch( + "services.email_parser.message_from_bytes", side_effect=ValueError("boom") + ): + with pytest.raises(EmailParseError): + parse_eml_bytes(b"anything") + + +def test_extract_thread_id_falls_through_whitespace_only_headers(): + # A References/In-Reply-To header that unfolds to only whitespace is present + # but yields no token when split; _extract_thread_id must fall through to the + # next source rather than return a blank thread id. + fell_to_in_reply_to = Message() + fell_to_in_reply_to["References"] = " " + fell_to_in_reply_to["In-Reply-To"] = "" + assert ( + _extract_thread_id(fell_to_in_reply_to, "") + == "" + ) + + fell_to_message_id = Message() + fell_to_message_id["References"] = " " + fell_to_message_id["In-Reply-To"] = " \t " + assert ( + _extract_thread_id(fell_to_message_id, "") + == "" + ) + + def test_parse_eml_extracts_reply_to_header(): eml_content = b"""Message-ID: From: Sender Name diff --git a/backend/tests/test_threading_service.py b/backend/tests/test_threading_service.py index 9c0d03450..2372ceaf1 100644 --- a/backend/tests/test_threading_service.py +++ b/backend/tests/test_threading_service.py @@ -1,6 +1,12 @@ import pytest -from services.threading_service import assign_thread_id +from services.threading_service import ( + _find_existing_thread_ids, + assign_thread_id, + extract_reference_ids, + generate_email_fingerprint, + normalize_message_id, +) class _Result: @@ -143,3 +149,226 @@ async def test_existing_thread_lookup_is_scoped_to_owner_and_organization(): query_text = str(session.queries[-1]).lower() assert "email_records.user_id" in query_text assert "email_records.organization_id" in query_text + + +def test_normalize_message_id_strips_brackets_and_outer_whitespace(): + assert normalize_message_id("") == "abc@example.com" + assert normalize_message_id(" ") == "abc@example.com" + assert normalize_message_id("< abc@example.com >") == "abc@example.com" + assert normalize_message_id("<>") == "abc@example.com" + assert normalize_message_id("abc@example.com") == "abc@example.com" + + +def test_normalize_message_id_handles_empty_and_none(): + assert normalize_message_id(None) is None + assert normalize_message_id("") is None + assert normalize_message_id(" ") is None + assert normalize_message_id("<>") is None + + +def test_normalize_message_id_collapses_interior_unfolding_whitespace(): + # RFC 5322 section 2.2.3 header unfolding can leave interior whitespace when + # a folded Message-ID is rejoined; RFC 5322 section 3.6.4 msg-id carries + # none, so the folded and unfolded forms must normalize to the same value or + # dedup/threading would treat one message as two. + canonical = normalize_message_id("") + assert normalize_message_id("") == canonical + assert normalize_message_id("") == canonical + assert normalize_message_id("") == canonical + assert normalize_message_id("") == canonical + + +def test_extract_reference_ids_normalizes_folded_whitespace_and_dedupes(): + header = " \r\n " + # The first two are the same id split over a fold boundary, so only two + # distinct references remain, in header order. + assert extract_reference_ids(header) == ["a@x.com", "b@x.com"] + + +def test_extract_reference_ids_drops_bracketed_whitespace_only_ids(): + # "< >" / "<\t>" are bracketed but whitespace-only: they normalize to nothing + # and must be dropped, not carried as empty thread candidates. + assert extract_reference_ids("< > <\t>") == ["a@x.com"] + + +def test_extract_reference_ids_falls_back_to_whitespace_split_without_brackets(): + # A References value with no angle brackets (some non-conforming clients) + # falls back to a whitespace split rather than yielding nothing. + assert extract_reference_ids("a@x.com b@x.com") == ["a@x.com", "b@x.com"] + + +@pytest.mark.asyncio +async def test_find_existing_thread_ids_returns_empty_without_candidates(): + session = _SequentialSession([]) + result = await _find_existing_thread_ids( + session, [], user_id="testuser", organization_id="org-acme" + ) + assert result == {} + assert session.execute_count == 0 + + +@pytest.mark.asyncio +async def test_find_existing_thread_ids_dedupes_overlapping_bracket_targets(): + # A bare id and its already-bracketed form collapse to one target set, so the + # shared "" lookup key is not enqueued twice. + session = _QueryCapturingSession([[("", "thread-a")]]) + result = await _find_existing_thread_ids( + session, + ["", "a@x.com"], + user_id="testuser", + organization_id="org-acme", + ) + assert result == {"a@x.com": "thread-a"} + + +@pytest.mark.asyncio +async def test_find_existing_thread_ids_skips_rows_with_blank_thread_or_message_id(): + # A stored row with no thread_id is skipped, and a row whose message_id + # normalizes to nothing is skipped; neither pollutes the returned map. + session = _SequentialSession( + [ + [ + ("", None), + ("", "thread-b"), + ("", "thread-c"), + ] + ] + ) + result = await _find_existing_thread_ids( + session, + ["a@x.com", "c@x.com"], + user_id="testuser", + organization_id="org-acme", + ) + assert result == {"c@x.com": "thread-c"} + + +@pytest.mark.asyncio +async def test_assign_thread_id_uses_a_later_candidate_when_the_first_has_no_thread(): + # The immediate parent (in_reply_to) is not yet imported, but an older + # reference is: the lookup loop must skip the unmatched first candidate and + # return the matched later one, not fall through to the deterministic root. + session = _SequentialSession([[("", "thread-older")]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": "", + "references": "", + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "thread-older" + + +def test_generate_email_fingerprint_is_deterministic_case_insensitive_and_field_sensitive(): + baseline = generate_email_fingerprint( + "Quarterly plan", "Mon, 01 Jun 2026 09:00:00 +0000", "a@x.com", "b@y.com" + ) + # 1. deterministic + SHA-256 hex digest + assert baseline == generate_email_fingerprint( + "Quarterly plan", "Mon, 01 Jun 2026 09:00:00 +0000", "a@x.com", "b@y.com" + ) + assert len(baseline) == 64 + assert all(character in "0123456789abcdef" for character in baseline) + # 2. lower-cased + outer-whitespace-stripped components collapse to one key + assert ( + generate_email_fingerprint( + " QUARTERLY PLAN ", "Mon, 01 Jun 2026 09:00:00 +0000", "A@X.com", " b@Y.com " + ) + == baseline + ) + # 3. None components are treated as empty (no crash) and stay distinct + all_empty = generate_email_fingerprint(None, None, None, None) + assert len(all_empty) == 64 + assert all_empty != baseline + # 4. any changed component changes the fingerprint (no field is dropped) + assert ( + generate_email_fingerprint( + "Quarterly plan", "Mon, 01 Jun 2026 09:00:00 +0000", "a@x.com", "c@z.com" + ) + != baseline + ) + + +@pytest.mark.asyncio +async def test_assign_thread_id_generates_fresh_uuid_when_no_identifiers_present(): + # An email with no in_reply_to, no references, and no message_id has nothing + # to thread on, so a fresh uuid4 root is minted and no lookup is issued. + session = _SequentialSession([]) + + thread_id = await assign_thread_id( + session, + {"message_id": None, "in_reply_to": None, "references": None}, + user_id="testuser", + organization_id="org-acme", + ) + + assert len(thread_id) == 32 + assert all(character in "0123456789abcdef" for character in thread_id) + assert session.execute_count == 0 + + +@pytest.mark.asyncio +async def test_multi_id_in_reply_to_threads_on_any_existing_parent(): + # RFC 5322 section 3.6.4: In-Reply-To is 1*msg-id, so it may carry more than + # one parent id (a reply that joins two messages). Threading must consider + # every parent, not treat the whole header as one opaque id -- otherwise a + # multi-id In-Reply-To never matches an existing thread and the reply splits + # off on its own. + session = _SequentialSession([[("", "thread-xyz")]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": " ", + "references": None, + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "thread-xyz" + + +@pytest.mark.asyncio +async def test_multi_id_in_reply_to_fallback_uses_first_parent_as_root(): + session = _SequentialSession([[]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": " ", + "references": None, + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "first@example.com" + + +@pytest.mark.asyncio +async def test_in_reply_to_with_cfws_comment_extracts_bare_msg_id(): + # RFC 5322 sections 3.6.4 / 3.2.2 permit CFWS (e.g. a trailing comment) around + # a msg-id. The comment text must not leak into the id, or the reply is + # threaded/deduped against a garbage id and splits from its parent thread. + session = _SequentialSession([[("", "thread-123")]]) + + thread_id = await assign_thread_id( + session, + { + "message_id": "", + "in_reply_to": " (sent from my phone)", + "references": None, + }, + user_id="testuser", + organization_id="org-acme", + ) + + assert thread_id == "thread-123" diff --git a/docs/research/email-ingest-threading/README.md b/docs/research/email-ingest-threading/README.md new file mode 100644 index 000000000..96bf77fbd --- /dev/null +++ b/docs/research/email-ingest-threading/README.md @@ -0,0 +1,106 @@ +# Email Ingest & Threading — Standards Basis and Design Rationale + +This pack grounds the email-ingest correctness work on +`ContextualWisdomLab/naruon#1192` +(`backend/services/threading_service.py`, `backend/services/email_parser.py`): +Message-ID interior-whitespace normalization, unknown-zone (`-0000`) date +normalization, In-Reply-To `1*msg-id` (multi-parent + CFWS) parsing, and +RFC 2047 encoded-word decoding of non-ASCII display names. + +## Standards basis (RFC 5322 message format + RFC 2047 header encoding) + +Each fix is anchored to a specific clause of the relevant standard: + +- **Header unfolding** — RFC 5322 §2.2.3. When a folded header is rejoined, + interior whitespace/tabs can survive. `normalize_message_id` collapses that + interior whitespace so the folded and unfolded forms of one Message-ID map to + a single de-dup/threading key. +- **Message-ID** — RFC 5322 §3.6.4 (`msg-id = "<" id-left "@" id-right ">"`). + A well-formed Message-ID carries no interior whitespace, so collapsing it is a + no-op for conforming input and only repairs unfolded input. +- **In-Reply-To / References** — RFC 5322 §3.6.4 defines both as `1*msg-id` + (one or more angle-bracketed ids, each optionally wrapped in CFWS, §3.2.2). + `assign_thread_id` therefore parses In-Reply-To with the same multi-id + extractor used for References, so a reply naming several parents — or a single + id trailed by a comment — threads onto an existing ancestor instead of + splitting off on a garbage key. +- **Date / unknown zone** — RFC 5322 §3.3 defines a `-0000` zone as "time zone + unknown". `email.utils.parsedate_to_datetime` returns a naive datetime for + that case; `_extract_date` treats the unknown zone as UTC so every ingested + date is timezone-aware and safe to sort and to store in a `timestamptz` + column. +- **Non-ASCII display names** — RFC 2047 (MIME Part Three) defines the + `=?charset?enc?text?=` encoded-word so non-ASCII text can appear in structured + headers. `From` / `To` / `Reply-To` display names arrive header-decoded under + `email.policy.default`, so `_sanitize_address_display_text` must store the + decoded name and must **not** re-encode it. `email.utils.formataddr` re-encodes + any non-ASCII display name back into an encoded-word, which stored a garbled + `=?utf-8?...?=` value for every non-ASCII (e.g. Korean) sender/recipient; + `_format_display_address` keeps the decoded name literal while preserving + formataddr's RFC 5322 quoting/escaping for display-name specials. + +## Design rationale — header-based, precision-first threading + +Naruon reconstructs conversations from the RFC 5322 reference graph +(References / In-Reply-To), deterministically, and deliberately does **not** +fall back to subject- or content-based grouping. This is a precision/recall +trade-off, not an oversight, and the rejected alternative is grounded in the +conversation-threading literature: + +- Content- and coherence-model approaches to thread reconstruction are + probabilistic and improve *recall* on broken reference chains, but carry an + inherent false-merge (precision) cost: Mohiuddin, Joty, and Nguyen (2018) + reconstruct thread trees by scoring candidate structures with a neural + coherence model, and even their best model reaches only ~30% thread-level + reconstruction accuracy — useful for recall on missing links, but far from the + certainty a mailbox view requires. +- Email threads are also large and topically heterogeneous in practice (Kooti, + Aiello, Grbovic, Lerman, & Mantrach, 2015, characterize replying behavior + over 16 billion messages; Zhang, Celikyilmaz, Gao, & Bansal, 2021, curate + 2,549 real email threads for EmailSum), so a wrong content-based merge is both + likely and costly at scale. +- Naruon therefore optimizes for *precision* — never merging unrelated messages — + because a wrong merge silently corrupts a user's mailbox view. The invariant is + pinned by `test_forwarded_subject_alone_does_not_merge_unrelated_thread`. +- The #1192 In-Reply-To fix improves *recall on the header-complete path* (it + no longer drops multi-parent / CFWS replies) with **zero** precision cost, + because it stays entirely within the deterministic header graph. + +The cited papers are bookmarked in the shared alphaXiv library folder +"CWL · Naruon email/threading standards grounding". + +## References (APA 7) + +- Resnick, P. (Ed.). (2008). *Internet message format* (RFC 5322). RFC Editor. + https://www.rfc-editor.org/rfc/rfc5322.txt +- Moore, K. (1996). *MIME (Multipurpose Internet Mail Extensions) part three: + Message header extensions for non-ASCII text* (RFC 2047). RFC Editor. + https://www.rfc-editor.org/rfc/rfc2047.txt +- Kooti, F., Aiello, L. M., Grbovic, M., Lerman, K., & Mantrach, A. (2015). + *Evolution of conversations in the age of email overload* [Preprint]. arXiv. + https://arxiv.org/abs/1504.00704 +- Mohiuddin, T., Joty, S., & Nguyen, D. T. (2018). *Coherence modeling of + asynchronous conversations: A neural entity grid approach* [Preprint]. arXiv. + https://arxiv.org/abs/1805.02275 +- Zhang, S., Celikyilmaz, A., Gao, J., & Bansal, M. (2021). *EmailSum: + Abstractive email thread summarization* [Preprint]. arXiv. + https://arxiv.org/abs/2107.14691 + +## Preservation notes + +- Original standard text and paper PDFs are referenced by their canonical + RFC Editor / arXiv URLs above and are bookmarked in the alphaXiv library + folder named in the design-rationale section. They are not committed here + because this sandbox's outbound proxy blocks `rfc-editor.org` and `arxiv.org`; + when a network-enabled run can fetch them, drop the RFC text into + `standards/` and the PDFs into `pdfs/` following the sibling packs' layout. +- Git LFS is intentionally not used, consistent with the other + `docs/research/*` packs. + +## Governance notes + +- Work item: `ContextualWisdomLab/naruon#1192` (RFC 5322 / RFC 2047 email-ingest + correctness + coverage). +- Verification: threading/parser suites pass with `--noconftest`; + `threading_service.py` at 100% and `email_parser.py` at 98% branch coverage + (the RFC 2047 `_format_display_address` helper is fully covered).