diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec84c36f..169b80083 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ ## [Unreleased] +- 예상되는 NewsDOM payload-size 거부는 `warning`이 아닌 `info` 운영 이벤트로 기록하고, + 영속 `provider_payload_size_exceeded` 상태를 고객 안내의 기준으로 유지합니다. ADR-0005 + PDF DOM 업로드 계약도 ADR 색인에 복원했으며, 현재 직접 업로드 20MiB 경계와 향후 + 64MiB 정렬 제안을 분리해 문서 추적성을 유지합니다. +- 첨부파일 파싱 소스 바이트 한도를 이메일 import 전송 계약과 같은 64MiB로 정렬했습니다. 20MiB 초과 64MiB 이하 첨부는 숨은 parser 제한으로 늦게 거부되지 않으며, 64MiB 초과 첨부는 기존처럼 `parse_size_limit_exceeded`로 fail-closed 합니다. 미지원 바이너리는 계약이 정의한 `unsupported_content_type` 또는 `unsupported_binary` 메타데이터 상태를 유지합니다. 결정과 고객 다음 행동은 [ADR-0006](docs/adr/0006-bounded-attachment-parse-source-contract.md) 및 [doctoring 문서](docs/doctoring/bounded-attachment-parse-source-contract.md)에 기록했습니다. - 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. diff --git a/backend/api/data.py b/backend/api/data.py index dccd85890..1d205a7f5 100644 --- a/backend/api/data.py +++ b/backend/api/data.py @@ -34,6 +34,7 @@ from services.newsdom_pdf_recognition import ( PDF_DOM_RECOGNITION_PENDING_STATUS, ) +from services.newsdom_client import NEWSDOM_MAX_PARSE_UPLOAD_BYTES from services.ontology_service import ontology_service from services.webdav_service import webdav_service @@ -44,7 +45,7 @@ # with the NewsDOM sidecar's own MAX_PARSE_UPLOAD_BYTES (20 MiB): accepting more # would let a caller stash a pending document the configured sidecar will always # reject while the base64 copy inflates the database. -_MAX_PDF_DOM_UPLOAD_BYTES = 20 * 1024 * 1024 +_MAX_PDF_DOM_UPLOAD_BYTES = NEWSDOM_MAX_PARSE_UPLOAD_BYTES ATTACHMENT_PARSE_BREAKDOWN_EVIDENCE_SOURCE = ( "email_attachments.content_type, " "email_attachments.parse_content_type, " diff --git a/backend/services/attachment_parser.py b/backend/services/attachment_parser.py index 868b9b183..cf0a274c4 100644 --- a/backend/services/attachment_parser.py +++ b/backend/services/attachment_parser.py @@ -15,7 +15,10 @@ "application/x-binary", } MAX_ATTACHMENT_PARSE_SOURCE_CHARS = 1_000_000 -MAX_ATTACHMENT_PARSE_SOURCE_BYTES = 20 * 1024 * 1024 +# Keep deferred attachment recognition aligned with the authenticated upload +# transport. Unsupported binaries remain metadata-only; recognized/deferred +# formats may retain at most this bounded source payload for a worker. +MAX_ATTACHMENT_PARSE_SOURCE_BYTES = 64 * 1024 * 1024 @dataclass(frozen=True) diff --git a/backend/services/newsdom_client.py b/backend/services/newsdom_client.py index 09b94fc8a..4a9738a37 100644 --- a/backend/services/newsdom_client.py +++ b/backend/services/newsdom_client.py @@ -34,6 +34,9 @@ _LOCAL_DEV_HOSTNAMES = {"localhost", "localhost.localdomain"} _LOCAL_DEV_IP_LITERALS = {"127.0.0.1", "::1"} _DEFAULT_PARSE_TIMEOUT_SECONDS = 300.0 +# The deployed NewsDOM ``/parse`` contract accepts at most 20 MiB. Keep this +# boundary explicit so deferred 64 MiB retention cannot remain pending forever. +NEWSDOM_MAX_PARSE_UPLOAD_BYTES = 20 * 1024 * 1024 class NewsdomConfigurationError(RuntimeError): @@ -44,6 +47,10 @@ class NewsdomRequestError(RuntimeError): """Raised when the NewsDOM sidecar cannot fulfil a parse request.""" +class NewsdomPayloadTooLargeError(NewsdomRequestError): + """Raised before network I/O when a PDF exceeds the sidecar contract.""" + + class NewsdomEmptyRecognitionError(NewsdomRequestError): """Raised when a 200 sidecar response carries no usable recognized text. @@ -393,6 +400,10 @@ async def request_pdf_dom( """ if not pdf_bytes: raise NewsdomRequestError("Cannot recognize an empty PDF payload") + if len(pdf_bytes) > NEWSDOM_MAX_PARSE_UPLOAD_BYTES: + raise NewsdomPayloadTooLargeError( + "NewsDOM PDF payload exceeds the 20 MiB parse upload contract" + ) validated = await validate_newsdom_base_url_details_async(base_url) if validated is None: diff --git a/backend/services/newsdom_worker.py b/backend/services/newsdom_worker.py index 26e4cffbb..094ade6dd 100644 --- a/backend/services/newsdom_worker.py +++ b/backend/services/newsdom_worker.py @@ -33,6 +33,7 @@ from services.content_graph import ParseResult from services.newsdom_client import ( NewsdomConfigurationError, + NewsdomPayloadTooLargeError, NewsdomRequestError, request_pdf_dom, ) @@ -257,6 +258,18 @@ async def process_pending_attachment( exc, ) return RESULT_PENDING + except NewsdomPayloadTooLargeError as exc: + attachment.parse_status = PDF_DOM_RECOGNITION_FAILED_STATUS + attachment.parse_error_code = "provider_payload_size_exceeded" + # A bounded, expected admission rejection is operational information, + # not an infrastructure warning; the persisted error code remains the + # customer-visible source of truth. + logger.info( + "NewsDOM attachment %s exceeds the provider payload contract: %s", + getattr(attachment, "id", "?"), + exc, + ) + return RESULT_FAILED except (NewsdomRequestError, ValueError) as exc: attachment.parse_status = PDF_DOM_RECOGNITION_FAILED_STATUS attachment.parse_error_code = "recognition_failed" diff --git a/backend/tests/test_attachment_parser.py b/backend/tests/test_attachment_parser.py index ad2dd892d..fcd492665 100644 --- a/backend/tests/test_attachment_parser.py +++ b/backend/tests/test_attachment_parser.py @@ -160,6 +160,11 @@ def test_unsupported_binary_attachment_is_visible_without_raw_bytes(): assert result.parse_error_code == "unsupported_content_type" +def test_attachment_source_budget_accepts_payloads_above_twenty_mib(): + assert MAX_ATTACHMENT_PARSE_SOURCE_BYTES == 64 * 1024 * 1024 + assert MAX_ATTACHMENT_PARSE_SOURCE_BYTES > 20 * 1024 * 1024 + + def test_pdf_attachment_is_deferred_pending_newsdom_recognition(): raw = b"%PDF-1.7 raw bytes" result = parse_email_attachment( diff --git a/backend/tests/test_newsdom_client.py b/backend/tests/test_newsdom_client.py index cda13ec35..6fe7d7885 100644 --- a/backend/tests/test_newsdom_client.py +++ b/backend/tests/test_newsdom_client.py @@ -9,7 +9,9 @@ from core.config import settings from services.newsdom_client import ( NEWSDOM_BASE_URL_NOT_ALLOWED, + NEWSDOM_MAX_PARSE_UPLOAD_BYTES, NewsdomConfigurationError, + NewsdomPayloadTooLargeError, NewsdomRequestError, _normalize_newsdom_base_url, request_pdf_dom, @@ -86,6 +88,18 @@ async def test_request_pdf_dom_rejects_empty_payload(newsdom_allowlist): ) +@pytest.mark.asyncio +async def test_request_pdf_dom_rejects_payload_above_sidecar_contract_before_network( + newsdom_allowlist, +): + with pytest.raises(NewsdomPayloadTooLargeError): + await request_pdf_dom( + base_url="https://newsdom.example.com", + api_token=None, + pdf_bytes=b"%PDF-" + b"A" * NEWSDOM_MAX_PARSE_UPLOAD_BYTES, + ) + + @pytest.mark.asyncio async def test_request_pdf_dom_raises_config_error_without_base_url(newsdom_allowlist): with pytest.raises(NewsdomConfigurationError): diff --git a/backend/tests/test_newsdom_worker.py b/backend/tests/test_newsdom_worker.py index 0693d395c..6d3fb3d96 100644 --- a/backend/tests/test_newsdom_worker.py +++ b/backend/tests/test_newsdom_worker.py @@ -8,13 +8,14 @@ import asyncio import base64 +import logging from types import SimpleNamespace import pytest from db.models import Attachment, Document, Email from services.content_graph import ContentSegment, ParseResult -from services.newsdom_client import NewsdomConfigurationError +from services.newsdom_client import NewsdomConfigurationError, NewsdomPayloadTooLargeError from services.newsdom_pdf_recognition import ( PDF_DOM_RECOGNITION_FAILED_STATUS, PDF_DOM_RECOGNITION_PENDING_STATUS, @@ -220,6 +221,28 @@ async def request_fn(**_kwargs): assert attachment.parse_status != "parsed" +@pytest.mark.asyncio +async def test_attachment_above_provider_limit_fails_instead_of_remaining_pending(caplog): + attachment = _pending_attachment() + + async def oversized_request(**_kwargs): + raise NewsdomPayloadTooLargeError("provider limit") + + with caplog.at_level(logging.INFO, logger="services.newsdom_worker"): + result = await process_pending_attachment( + session=object(), + attachment=attachment, + config_resolver=await _resolver_with(_config()), + request_fn=oversized_request, + ) + + assert result == RESULT_FAILED + assert attachment.parse_status == PDF_DOM_RECOGNITION_FAILED_STATUS + assert attachment.parse_error_code == "provider_payload_size_exceeded" + records = [record for record in caplog.records if "exceeds the provider payload contract" in record.message] + assert records and all(record.levelno == logging.INFO for record in records) + + @pytest.mark.asyncio async def test_attachment_orphan_and_rejected_configuration_stay_visible(): orphan = Attachment( diff --git a/docs/adr/0005-bounded-pdf-dom-upload-contract.md b/docs/adr/0005-bounded-pdf-dom-upload-contract.md new file mode 100644 index 000000000..3fc6f48e4 --- /dev/null +++ b/docs/adr/0005-bounded-pdf-dom-upload-contract.md @@ -0,0 +1,55 @@ +# ADR-0005: Bounded PDF DOM upload contract + +**Status:** Proposed +**Date:** 2026-08-20 +**Decision owner:** Naruon maintainers +**Scope:** Signed `POST /api/data/documents/pdf-dom-recognition` uploads +**Figma File ID:** N/A — backend upload contract; no visual surface. + +## Context + +Naruon email imports and deferred attachment admission use a 64 MiB bounded +transport budget. The direct Data workspace PDF-DOM endpoint and its NewsDOM +sidecar contract remain independently bounded at 20 MiB on the current +protected branch; this record preserves the proposed alignment without +claiming that the separate transport change has shipped. + +## Decision + +Retain the current 20 MiB direct PDF-DOM upload and decoder boundary until the +separate transport change is reviewed and integrated. Keep the signed-session +boundary, PDF signature validation, one-byte-over-limit read, base64 +persistence contract, and `413` response unchanged. A future alignment to the +64 MiB import budget requires sidecar confirmation, capacity evidence, and a +new current-head review; this ADR does not authorize that change. + +## Consequences + +- Email and manual PDF ingestion currently have explicit, separately governed + bounded contracts (64 MiB import/deferred admission; 20 MiB direct DOM). +- Workspace quotas, background-worker limits, and database-capacity monitoring + remain required because temporary content can be larger. +- No unbounded upload is introduced; malformed or non-PDF payloads continue to + fail closed before recognition. + +## Alternatives rejected + +### Align the manual endpoint immediately + +Deferred until the sidecar and storage capacity contract are independently +verified; changing only the Naruon endpoint would create a customer-visible +failure later in recognition. + +### Remove the upload limit + +Rejected because request and database resource use must remain bounded at the +authenticated trust boundary. + +## References (APA 7th) + +Internet Engineering Task Force. (2022). *HTTP semantics (RFC 9110).* RFC +Editor. https://www.rfc-editor.org/rfc/rfc9110 + +National Institute of Standards and Technology. (2025). *Secure software +development framework (SSDF) version 1.2* (NIST Special Publication 800-218 +Rev. 1, Initial Public Draft). https://doi.org/10.6028/NIST.SP.800-218r1.ipd diff --git a/docs/adr/0006-bounded-attachment-parse-source-contract.md b/docs/adr/0006-bounded-attachment-parse-source-contract.md new file mode 100644 index 000000000..a34bca1b0 --- /dev/null +++ b/docs/adr/0006-bounded-attachment-parse-source-contract.md @@ -0,0 +1,65 @@ +# ADR-0006: Bounded attachment parse-source contract + +- **Status:** Accepted for Naruon attachment ingestion +- **Date:** 2026-08-25 +- **Figma file ID:** N/A — backend ingestion and evidence contract; no UX surface +- **Owners:** Naruon ingestion and data-quality maintainers + +## Context + +Naruon accepts authenticated email imports up to 64 MiB, while the deferred +attachment parser previously rejected source payloads above 20 MiB. That split +made a valid upload fail later in parsing and prevented a customer from knowing +whether a file was rejected by transport, parser admission, or an unsupported +format. Unsupported binaries are intentionally not parsed inline and must remain +metadata-only until a separately reviewed parser is available. + +## Decision + +Use one 64 MiB upper bound for attachment source bytes retained for a deferred +recognition worker. The parser continues to: + +1. accept only the existing authenticated import transport; +2. retain validated PDF bytes only for the deferred NewsDOM path; +3. return `parse_size_limit_exceeded` without raw content above 64 MiB; +4. return `unsupported_content_type` with `unsupported_binary` and no raw bytes + for an unparseable content type; and +5. preserve the parser key, parse status, and error code for the Data quality + evidence surface. + +This is a bounded admission contract, not a promise that every binary format +is parseable. Adding a new parser requires its own dependency, sandbox, +provenance, and regression review. + +## Consequences + +- Attachments larger than 20 MiB and no larger than 64 MiB can reach deferred + recognition consistently with the import transport. +- A 64 MiB raw source can expand when base64-encoded in the existing deferred + content column; the database/object-lifecycle work must move this payload to + object storage before materially increasing the bound again. +- Unsupported binaries remain visible in scoped quality counts without exposing + their bytes, identifiers, or provider content. +- The contract is independent of any Figma design and has no Storybook scene. + +## Verification + +- `backend/tests/test_attachment_parser.py` asserts the 64 MiB boundary is + above the former 20 MiB parser limit and preserves unsupported-binary + metadata-only behavior. +- The import transport remains covered by + `backend/tests/test_email_import_service.py`. +- The PDF DOM upload contract is being integrated separately by stacked PR + #1427; this ADR governs the attachment-parser source budget and remains + valid if that transport change is merged independently. + +## References (APA 7th) + +Fielding, R. T., Nottingham, M., & Reschke, J. (Eds.). (2022). *HTTP +semantics* (RFC 9110). Internet Engineering Task Force. +https://www.rfc-editor.org/rfc/rfc9110.html + +Souppaya, M., Scarfone, K., & Dodson, D. (2022). *Secure software development +framework (SSDF) version 1.1: Recommendations for mitigating the risk of +software vulnerabilities* (NIST Special Publication 800-218). National Institute +of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218 diff --git a/docs/adr/README.md b/docs/adr/README.md index 4d461fff6..4757a8267 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -13,6 +13,8 @@ govern implementation. | [ADR-0002](0002-fitted-topic-artifact-consumption.md) | Conditionally consume only a versioned fitted topic artifact through a fail-closed adapter | Proposed | Target `PLANNED`; runtime `BLOCKED-UPSTREAM` | | [ADR-0003](0003-separate-topic-measurement-from-agenda-generation.md) | Keep statistical measurement separate from agenda generation | Proposed | Target and future capability `PLANNED`; no implementation authorization | | [ADR-0004](0004-status-weighted-calendar-conflicts.md) | Evaluate CalDAV VEVENT overlaps by occupying status; cancelled does not occupy | Accepted | `ACCEPTED-NARUON-POLICY`; advisory evaluate API only | +| [ADR-0005](0005-bounded-pdf-dom-upload-contract.md) | Keep the current 20MiB direct PDF DOM contract until separately reviewed 64MiB alignment | Proposed | `PLANNED`; no transport expansion is authorized | +| [ADR-0006](0006-bounded-attachment-parse-source-contract.md) | Align deferred attachment source admission with the authenticated 64 MiB import contract while keeping unsupported binaries metadata-only | Accepted | `ACCEPTED-NARUON-INGESTION`; bounded deferred recognition | The complete topic-intelligence requirements, architecture, contract, UML, conceptual ERD, security, test, and operability graph is indexed at diff --git a/docs/doctoring/bounded-attachment-parse-source-contract.md b/docs/doctoring/bounded-attachment-parse-source-contract.md new file mode 100644 index 000000000..3b8959b77 --- /dev/null +++ b/docs/doctoring/bounded-attachment-parse-source-contract.md @@ -0,0 +1,39 @@ +# Bounded attachment parse-source contract + +## Customer outcome + +An email attachment between 20 MiB and 64 MiB is not rejected by a hidden +parser-only limit after transport admission. The Data workspace still reports +unsupported formats explicitly, so the next action is to add a reviewed parser +or use the original provider file rather than treating metadata as extracted +content. + +## Contract + +`MAX_ATTACHMENT_PARSE_SOURCE_BYTES` is 64 MiB, matching the authenticated email +import budget. The parser is fail-closed: + +- supported text formats are parsed inline within the existing character bound; +- PDF bytes are retained only for bounded deferred NewsDOM recognition; +- unsupported binary formats return `unsupported_content_type`, + `unsupported_binary`, and empty content; +- oversized source bytes return `parse_size_limit_exceeded` and empty content. + +This preserves provenance without claiming that an unsupported file was parsed. +The quality surface exposes the parser key and status, not raw attachment bytes, +message IDs, attachment IDs, credentials, or customer payloads. + +## Evidence and next action + +The parser boundary is tested in +`backend/tests/test_attachment_parser.py`. The import transport is tested in +`backend/tests/test_email_import_service.py`. If a customer needs a currently +unsupported format, add a dedicated parser proposal with sandbox, dependency, +provenance, and exact-head regression evidence before changing the registry. + +## Research traceability + +The bounded transport and fail-closed error contract are aligned with HTTP +representation semantics (Fielding et al., 2022) and secure development +verification practices (Souppaya et al., 2022). See +[`ADR-0006`](../adr/0006-bounded-attachment-parse-source-contract.md).