From 71671c605c4d0f065897eb86c7f85654048c4332 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 23:12:16 +0900 Subject: [PATCH 1/7] fix(api): align PDF upload transport budget --- CHANGELOG.md | 4 ++ docs/adr/0003-bounded-pdf-upload-transport.md | 44 +++++++++++++++++++ docs/adr/README.md | 1 + .../doctoring/bounded-pdf-upload-transport.md | 21 +++++++++ src/newsdom_api/main.py | 4 +- tests/test_parse_endpoint.py | 5 +++ 6 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 docs/adr/0003-bounded-pdf-upload-transport.md create mode 100644 docs/doctoring/bounded-pdf-upload-transport.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2398ea5c..71a07ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > acceptance are aligned for an actual 0.3.0 publication. ### Changed +- Align `/parse` with Naruon's signed PDF DOM transport at a bounded 64 MiB. + The endpoint still validates authentication before multipart parsing and + rejects the first byte above the ceiling with `413`. + - `/parse`를 언어 선택형 파서로 일반화: MinerU `-l japan`/`-m ocr` 하드코딩을 제거하고 optional form 필드 `language`(MinerU 3.4.4 공식 기본 `ch`, 공개 언어군/alias 검증)와 `mode`(`auto`/`ocr`/`txt`, 기본 `auto`)로 파라미터화. `mode=auto`는 born-digital PDF가 강제 OCR을 건너뛰도록 함. 기존 입력 `language=japan&mode=ocr`는 공식 규약대로 `ch`/`ocr`로 정규화됨. - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음. diff --git a/docs/adr/0003-bounded-pdf-upload-transport.md b/docs/adr/0003-bounded-pdf-upload-transport.md new file mode 100644 index 00000000..3fc0c65a --- /dev/null +++ b/docs/adr/0003-bounded-pdf-upload-transport.md @@ -0,0 +1,44 @@ +# ADR-0003: Bounded PDF upload transport + +**Status:** Accepted +**Date:** 2026-08-21 +**Decision owner:** NewsDOM maintainers +**Scope:** Authenticated `POST /parse` upload boundary +**Figma File ID:** N/A — sidecar API contract; no visual surface. + +## Context + +Naruon's direct PDF DOM upload contract is bounded at 64 MiB, but the owning +NewsDOM sidecar still rejected the same customer PDF above 20 MiB. That +cross-service mismatch made the equivalent email and manual workflows behave +differently and caused a customer-visible failure after the request crossed a +service boundary. + +## Decision + +Set `MAX_PARSE_UPLOAD_BYTES` to 64 MiB. Keep bearer authentication before +multipart body parsing, the streaming first-byte-over-limit check, PDF +signature validation, temporary-file cleanup, and the `413 Payload Too Large` +response unchanged. + +Naruon remains the consumer-side owner of its signed persistence boundary. This +ADR only changes the sidecar's transport ceiling; parser runtime, concurrency, +storage quotas, and deployment capacity remain separate controls. + +## Consequences + +- Customers can use the same bounded 64 MiB expectation for direct and sidecar + PDF DOM ingestion. +- A larger valid upload can reach the parser, so deployment capacity and parser + timeout controls remain mandatory. +- No unbounded body read is introduced; the endpoint continues to stop on the + first byte above the limit. + +## References (APA 7th) + +Internet Engineering Task Force. (2022). *HTTP semantics (RFC 9110).* RFC + Editor. https://www.rfc-editor.org/rfc/rfc9110 + +National Institute of Standards and Technology. (2025). *Secure software + development framework (SSDF) version 1.2* (NIST Special Publication 800-218 + Rev. 1, Initial Public Draft). https://doi.org/10.6028/NIST.SP.800-218r1.ipd diff --git a/docs/adr/README.md b/docs/adr/README.md index 6efe928f..08de7c94 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -7,6 +7,7 @@ This directory contains Architecture Decision Records (ADRs) for the `newsdom-ap | ADR | Title | Status | Date | | --------------------------------------------------------- | --------------------------------------------- | -------- | ---------- | | [0001](0001-defer-openssf-best-practices-enrollment.md) | Defer OpenSSF Best Practices Enrollment | Accepted | 2026-04-24 | +| [0003](0003-bounded-pdf-upload-transport.md) | Align the bounded PDF transport with Naruon | Accepted | 2026-08-21 | ## ADR Status diff --git a/docs/doctoring/bounded-pdf-upload-transport.md b/docs/doctoring/bounded-pdf-upload-transport.md new file mode 100644 index 00000000..72b53818 --- /dev/null +++ b/docs/doctoring/bounded-pdf-upload-transport.md @@ -0,0 +1,21 @@ +# Doctoring record: bounded PDF upload transport + +**Observed gap:** The live NewsDOM sidecar accepted only 20 MiB while the +Naruon direct PDF DOM contract allowed 64 MiB, so equivalent customer workflows +were inconsistent. + +**Correction:** `MAX_PARSE_UPLOAD_BYTES` and its boundary tests now use 64 MiB. +Authentication remains checked before multipart parsing, and streaming input +still fails closed at the first byte over the bound. + +**Evidence:** `tests/test_parse_endpoint.py` covers the 64 MiB contract and the +unknown-size streaming over-limit path. Full coverage and exact-head hosted +checks remain required before merge. + +**References (APA 7th):** + +- Internet Engineering Task Force. (2022). *HTTP semantics (RFC 9110).* RFC + Editor. https://www.rfc-editor.org/rfc/rfc9110 +- National Institute of Standards and Technology. (2025). *Secure software + development framework (SSDF) version 1.2* (NIST Special Publication 800-218 + Rev. 1, Initial Public Draft). https://doi.org/10.6028/NIST.SP.800-218r1.ipd diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index f61aafc2..960a8637 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -41,7 +41,9 @@ from .schemas import HealthResponse, ParseResponse, ReadinessResponse from .service import parse_pdf -MAX_PARSE_UPLOAD_BYTES = 20 * 1024 * 1024 +# Keep the sidecar transport ceiling aligned with Naruon's direct PDF DOM +# upload. The streaming read still rejects the first byte above this bound. +MAX_PARSE_UPLOAD_BYTES = 64 * 1024 * 1024 MAX_AUTHORIZATION_HEADER_BYTES = MAX_BEARER_HEADER_BYTES UNSUPPORTED_MEDIA_DETAIL = "Unsupported Media Type" PAYLOAD_TOO_LARGE_DETAIL = "Payload Too Large" diff --git a/tests/test_parse_endpoint.py b/tests/test_parse_endpoint.py index 1491ada0..a4290770 100644 --- a/tests/test_parse_endpoint.py +++ b/tests/test_parse_endpoint.py @@ -358,6 +358,11 @@ def fake_parse_pdf_bytes(file_path, filename, **kwargs): assert response.json()["detail"] == "Payload Too Large" +def test_parse_endpoint_budget_matches_naruon_transport_contract(): + """Keep the sidecar upload ceiling aligned with Naruon's PDF transport.""" + assert MAX_PARSE_UPLOAD_BYTES == 64 * 1024 * 1024 + + @pytest.mark.asyncio async def test_parse_endpoint_rejects_large_file_without_size_metadata(): upload = _ReadTrackingUpload(b"%PDF-" + (b"x" * MAX_PARSE_UPLOAD_BYTES)) From 7147df8748241e47a7eb6d6f0f18f5b20c9c7e54 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:55:55 +0900 Subject: [PATCH 2/7] docs(adr): normalize upload decision formatting --- docs/adr/0003-bounded-pdf-upload-transport.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/adr/0003-bounded-pdf-upload-transport.md b/docs/adr/0003-bounded-pdf-upload-transport.md index 3fc0c65a..0a426bff 100644 --- a/docs/adr/0003-bounded-pdf-upload-transport.md +++ b/docs/adr/0003-bounded-pdf-upload-transport.md @@ -1,9 +1,9 @@ # ADR-0003: Bounded PDF upload transport -**Status:** Accepted -**Date:** 2026-08-21 -**Decision owner:** NewsDOM maintainers -**Scope:** Authenticated `POST /parse` upload boundary +**Status:** Accepted +**Date:** 2026-08-21 +**Decision owner:** NewsDOM maintainers +**Scope:** Authenticated `POST /parse` upload boundary **Figma File ID:** N/A — sidecar API contract; no visual surface. ## Context From 547998a2a4332bd6fb111b9987d12ecf1ab4a27f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 01:30:17 +0900 Subject: [PATCH 3/7] test(parse): accept exact upload boundary --- tests/test_parse_endpoint.py | 44 ++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/test_parse_endpoint.py b/tests/test_parse_endpoint.py index a4290770..89fce227 100644 --- a/tests/test_parse_endpoint.py +++ b/tests/test_parse_endpoint.py @@ -32,17 +32,35 @@ class _ReadTrackingUpload: filename = "fixture.pdf" size = 10 * 1024 * 1024 - def __init__(self, payload: bytes): + def __init__(self, payload: bytes | int): self._payload = payload self._offset = 0 self.read_sizes: list[int] = [] + self.bytes_returned = 0 async def read(self, size: int = -1) -> bytes: self.read_sizes.append(size) - if size < 0: - size = len(self._payload) - self._offset - chunk = self._payload[self._offset : self._offset + size] + if isinstance(self._payload, int): + remaining = self._payload - self._offset + if remaining <= 0: + return b"" + if size < 0: + size = remaining + count = min(size, remaining) + prefix = b"%PDF-" + chunk = b"" + if self._offset < len(prefix): + prefix_count = min(count, len(prefix) - self._offset) + chunk = prefix[self._offset : self._offset + prefix_count] + count -= prefix_count + if count: + chunk += b"x" * count + else: + if size < 0: + size = len(self._payload) - self._offset + chunk = self._payload[self._offset : self._offset + size] self._offset += len(chunk) + self.bytes_returned += len(chunk) return chunk @@ -363,6 +381,24 @@ def test_parse_endpoint_budget_matches_naruon_transport_contract(): assert MAX_PARSE_UPLOAD_BYTES == 64 * 1024 * 1024 +@pytest.mark.asyncio +async def test_parse_endpoint_accepts_exact_upload_budget(monkeypatch): + """Accept a valid streamed PDF whose final byte is exactly at the limit.""" + monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None) + monkeypatch.setattr( + "newsdom_api.main.parse_pdf", + lambda file_path, filename, **kwargs: {"document_id": "fixture", "pages": []}, + ) + + upload = _ReadTrackingUpload(MAX_PARSE_UPLOAD_BYTES) + upload.size = MAX_PARSE_UPLOAD_BYTES + + result = await parse(upload) + + assert result == {"document_id": "fixture", "pages": []} + assert upload.bytes_returned == MAX_PARSE_UPLOAD_BYTES + + @pytest.mark.asyncio async def test_parse_endpoint_rejects_large_file_without_size_metadata(): upload = _ReadTrackingUpload(b"%PDF-" + (b"x" * MAX_PARSE_UPLOAD_BYTES)) From 03ec64c2cb31a5f567c89422983b6bd020b0a8b3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 01:47:29 +0900 Subject: [PATCH 4/7] docs(adr): complete record index --- docs/adr/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/adr/README.md b/docs/adr/README.md index 08de7c94..32eee56c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -7,7 +7,8 @@ This directory contains Architecture Decision Records (ADRs) for the `newsdom-ap | ADR | Title | Status | Date | | --------------------------------------------------------- | --------------------------------------------- | -------- | ---------- | | [0001](0001-defer-openssf-best-practices-enrollment.md) | Defer OpenSSF Best Practices Enrollment | Accepted | 2026-04-24 | -| [0003](0003-bounded-pdf-upload-transport.md) | Align the bounded PDF transport with Naruon | Accepted | 2026-08-21 | +| [0002](0002-single-maintainer-review-exception.md) | Single-maintainer protected-branch review exception | Accepted | 2026-04-24 | +| [0003](0003-bounded-pdf-upload-transport.md) | Bounded PDF upload transport | Accepted | 2026-08-21 | ## ADR Status From b06b844853fb9d631f16d6b09020df4bc4160aaf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 20:59:40 -0700 Subject: [PATCH 5/7] docs(api): document 64 MiB parse upload limit --- manual/api-reference.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manual/api-reference.md b/manual/api-reference.md index 7ceec967..89160c8b 100644 --- a/manual/api-reference.md +++ b/manual/api-reference.md @@ -32,6 +32,10 @@ FastAPI는 OpenAPI 기반의 대화형 API 문서를 자동으로 생성합니 #### 요청 매개변수 (Request Body) - **`file`** (`UploadFile`, 필수): 변환할 PDF 바이너리 파일 데이터 (`multipart/form-data`) +업로드 본문은 최대 64 MiB까지 허용됩니다. 한도를 초과하면 서버가 임시 파일을 +MinerU에 전달하지 않고 `413 Payload Too Large`를 반환하므로, 고객은 PDF를 +분할한 뒤 각 파일을 다시 업로드해야 합니다. + #### cURL 테스트 예제 ```bash From 93383e9538aa713dbb559910c4b297136594a2eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 21:00:54 -0700 Subject: [PATCH 6/7] test(parse): prove first-byte-over-limit rejection --- tests/test_parse_upload_budget_contract.py | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/test_parse_upload_budget_contract.py diff --git a/tests/test_parse_upload_budget_contract.py b/tests/test_parse_upload_budget_contract.py new file mode 100644 index 00000000..3665eaeb --- /dev/null +++ b/tests/test_parse_upload_budget_contract.py @@ -0,0 +1,49 @@ +"""Exact streaming-boundary regressions for the public PDF upload budget.""" + +import pytest +from fastapi import HTTPException + +from newsdom_api.main import MAX_PARSE_UPLOAD_BYTES, parse + + +class _VirtualPdfUpload: + """Stream a bounded synthetic PDF without allocating the full payload.""" + + content_type = "application/pdf" + filename = "fixture.pdf" + size = None + + def __init__(self, total_bytes: int) -> None: + self.total_bytes = total_bytes + self.bytes_returned = 0 + + async def read(self, size: int = -1) -> bytes: + """Return at most ``size`` bytes while preserving a valid PDF prefix.""" + remaining = self.total_bytes - self.bytes_returned + if remaining <= 0: + return b"" + count = remaining if size < 0 else min(size, remaining) + prefix = b"%PDF-" + start = self.bytes_returned + chunk = b"" + if start < len(prefix): + prefix_count = min(count, len(prefix) - start) + chunk = prefix[start : start + prefix_count] + count -= prefix_count + if count: + chunk += b"x" * count + self.bytes_returned += len(chunk) + return chunk + + +@pytest.mark.asyncio +async def test_streaming_upload_rejects_exact_first_byte_over_budget() -> None: + """Reject after consuming exactly the first byte beyond the 64 MiB ceiling.""" + upload = _VirtualPdfUpload(MAX_PARSE_UPLOAD_BYTES + 1) + + with pytest.raises(HTTPException) as exc_info: + await parse(upload) + + assert exc_info.value.status_code == 413 + assert exc_info.value.detail == "Payload Too Large" + assert upload.bytes_returned == MAX_PARSE_UPLOAD_BYTES + 1 From 585bb4e0fb719ab6a576cf46d1ef12b77872557b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 04:58:53 -0700 Subject: [PATCH 7/7] docs(api): clarify PDF file upload ceiling --- manual/api-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/api-reference.md b/manual/api-reference.md index 89160c8b..f8bcf5a4 100644 --- a/manual/api-reference.md +++ b/manual/api-reference.md @@ -32,7 +32,7 @@ FastAPI는 OpenAPI 기반의 대화형 API 문서를 자동으로 생성합니 #### 요청 매개변수 (Request Body) - **`file`** (`UploadFile`, 필수): 변환할 PDF 바이너리 파일 데이터 (`multipart/form-data`) -업로드 본문은 최대 64 MiB까지 허용됩니다. 한도를 초과하면 서버가 임시 파일을 +PDF 파일은 최대 64 MiB까지 허용됩니다. 한도를 초과하면 서버가 임시 파일을 MinerU에 전달하지 않고 `413 Payload Too Large`를 반환하므로, 고객은 PDF를 분할한 뒤 각 파일을 다시 업로드해야 합니다.