From 51bbcecc996ad27803aa295c9da296658b91358e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 10 Apr 2026 04:03:37 +0900 Subject: [PATCH 1/3] ci: add clusterfuzzlite smoke integration for dom normalization --- .clusterfuzzlite/Dockerfile | 4 ++ .clusterfuzzlite/build.sh | 20 ++++++ .clusterfuzzlite/project.yaml | 1 + .github/workflows/clusterfuzzlite.yml | 32 ++++++++++ README.md | 6 ++ .../dom_builder_fuzzer/mineru_sample.json | 6 ++ fuzzers/dom_builder_fuzzer.py | 62 +++++++++++++++++++ pyproject.toml | 2 +- tests/test_fuzzing_integration.py | 43 +++++++++++++ tests/test_project_metadata.py | 6 ++ 10 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 .clusterfuzzlite/Dockerfile create mode 100644 .clusterfuzzlite/build.sh create mode 100644 .clusterfuzzlite/project.yaml create mode 100644 .github/workflows/clusterfuzzlite.yml create mode 100644 fuzzers/corpus/dom_builder_fuzzer/mineru_sample.json create mode 100644 fuzzers/dom_builder_fuzzer.py create mode 100644 tests/test_fuzzing_integration.py diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000..dcad3676 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,4 @@ +FROM gcr.io/oss-fuzz-base/base-builder-python + +WORKDIR /src/newsdom-api +COPY . . diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100644 index 00000000..159f5af0 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +cd "$SRC/newsdom-api" + +pip3 install . pyinstaller atheris + +for fuzzer in $(find fuzzers -name '*_fuzzer.py'); do + fuzzer_basename=$(basename -s .py "$fuzzer") + fuzzer_package="${fuzzer_basename}.pkg" + + pyinstaller --distpath "$OUT" --onefile --name "$fuzzer_package" "$fuzzer" + + cat >"$OUT/$fuzzer_basename" < list[dict[str, Any]]: + """Return a MinerU-like content list or an empty list.""" + + if not isinstance(candidate, list): + return [] + return [item for item in candidate if isinstance(item, dict)] + + +def exercise_dom_builder(raw_bytes: bytes) -> None: + """Exercise build_dom with bytes that may or may not decode into JSON blocks.""" + + try: + decoded = raw_bytes.decode("utf-8", errors="ignore") + candidate = json.loads(decoded) + except Exception: + return + build_dom(_coerce_content_list(candidate), document_id="fuzz") + + +def _run_smoke(seed_path: Path) -> None: + """Run one deterministic normalization pass from a known corpus seed.""" + + sample = json.loads(seed_path.read_text(encoding="utf-8")) + build_dom(_coerce_content_list(sample), document_id="smoke") + + +def main(argv: list[str] | None = None) -> int: + """Run either deterministic smoke mode or Atheris fuzz mode.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--smoke", type=Path) + args = parser.parse_args(argv) + + if args.smoke is not None: + _run_smoke(args.smoke) + return 0 + + import atheris + + def test_one_input(data: bytes) -> None: + exercise_dom_builder(data) + + atheris.Setup(sys.argv, test_one_input) + atheris.Fuzz() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index a4676094..931fccbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "DOM-style parser API for scanned Japanese newspaper PDFs" readme = "README.md" requires-python = ">=3.10,<3.14" -license = {text = "MIT"} +license = "MIT" authors = [{name = "Seongho Bae"}] dependencies = [ "fastapi>=0.115,<1.0", diff --git a/tests/test_fuzzing_integration.py b/tests/test_fuzzing_integration.py new file mode 100644 index 00000000..5d0cf7b6 --- /dev/null +++ b/tests/test_fuzzing_integration.py @@ -0,0 +1,43 @@ +import subprocess +import sys +from pathlib import Path + + +def test_clusterfuzzlite_integration_files_exist(): + assert Path(".clusterfuzzlite/project.yaml").exists() + assert Path(".clusterfuzzlite/Dockerfile").exists() + assert Path(".clusterfuzzlite/build.sh").exists() + assert Path(".github/workflows/clusterfuzzlite.yml").exists() + assert Path("fuzzers/dom_builder_fuzzer.py").exists() + assert Path("fuzzers/corpus/dom_builder_fuzzer/mineru_sample.json").exists() + + +def test_clusterfuzzlite_workflow_runs_pinned_python_code_change_fuzzing(): + text = Path(".github/workflows/clusterfuzzlite.yml").read_text(encoding="utf-8") + assert ( + "google/clusterfuzzlite/actions/build_fuzzers@52ecc61cb587ee99c26825a112a21abf19c7448c" + in text + ) + assert ( + "google/clusterfuzzlite/actions/run_fuzzers@52ecc61cb587ee99c26825a112a21abf19c7448c" + in text + ) + assert "language: python" in text + assert "mode: code-change" in text + assert "fuzz-seconds: 300" in text + + +def test_dom_builder_fuzzer_smoke_mode_runs_without_cluster(): + completed = subprocess.run( + [ + sys.executable, + "fuzzers/dom_builder_fuzzer.py", + "--smoke", + "tests/fixtures/mineru_sample.json", + ], + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + assert "Traceback" not in completed.stderr diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index eb6bb158..deab61b9 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -9,3 +9,9 @@ def test_project_metadata_does_not_bundle_mineru_extra(): def test_docs_theme_range_stays_below_warning_release(): text = Path("pyproject.toml").read_text(encoding="utf-8") assert '"mkdocs-material>=9.6,<9.7"' in text + + +def test_project_uses_spdx_license_string_not_deprecated_table(): + text = Path("pyproject.toml").read_text(encoding="utf-8") + assert 'license = "MIT"' in text + assert 'license = {text = "MIT"}' not in text From 782d3ca74a88db48980c025530f57155624bc4a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 10 Apr 2026 04:10:32 +0900 Subject: [PATCH 2/3] fix: place clusterfuzzlite build script where compile expects it --- .clusterfuzzlite/Dockerfile | 1 + tests/test_fuzzing_integration.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile index dcad3676..32f28c01 100644 --- a/.clusterfuzzlite/Dockerfile +++ b/.clusterfuzzlite/Dockerfile @@ -2,3 +2,4 @@ FROM gcr.io/oss-fuzz-base/base-builder-python WORKDIR /src/newsdom-api COPY . . +COPY .clusterfuzzlite/build.sh /src/build.sh diff --git a/tests/test_fuzzing_integration.py b/tests/test_fuzzing_integration.py index 5d0cf7b6..9f7f59f6 100644 --- a/tests/test_fuzzing_integration.py +++ b/tests/test_fuzzing_integration.py @@ -27,6 +27,11 @@ def test_clusterfuzzlite_workflow_runs_pinned_python_code_change_fuzzing(): assert "fuzz-seconds: 300" in text +def test_clusterfuzzlite_dockerfile_places_build_script_at_src_root(): + text = Path(".clusterfuzzlite/Dockerfile").read_text(encoding="utf-8") + assert "COPY .clusterfuzzlite/build.sh /src/build.sh" in text + + def test_dom_builder_fuzzer_smoke_mode_runs_without_cluster(): completed = subprocess.run( [ From 39b2662ad28749db4cdccf9726f020c16313c7cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 10 Apr 2026 04:35:28 +0900 Subject: [PATCH 3/3] chore: pin new Docker and fuzz dependencies by digest --- .clusterfuzzlite/Dockerfile | 7 ++++++- .clusterfuzzlite/build.sh | 3 ++- Dockerfile | 11 ++++++++--- Dockerfile.nvidia | 14 +++++++++----- pyproject.toml | 7 +++++++ tests/test_docker_delivery.py | 6 +++++- tests/test_fuzzing_integration.py | 8 ++++++++ tests/test_project_metadata.py | 12 +++++++++++- 8 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile index 32f28c01..b07dfc02 100644 --- a/.clusterfuzzlite/Dockerfile +++ b/.clusterfuzzlite/Dockerfile @@ -1,5 +1,10 @@ -FROM gcr.io/oss-fuzz-base/base-builder-python +ARG UV_IMAGE=ghcr.io/astral-sh/uv@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 + +FROM ${UV_IMAGE} AS uv-bin + +FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:60e8ef87f2c0367254ff979a4dea61dad2684b001e3e666e2cc6fe992064dbbf WORKDIR /src/newsdom-api +COPY --from=uv-bin /uv /uvx /usr/local/bin/ COPY . . COPY .clusterfuzzlite/build.sh /src/build.sh diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh index 159f5af0..27a583a6 100644 --- a/.clusterfuzzlite/build.sh +++ b/.clusterfuzzlite/build.sh @@ -3,7 +3,8 @@ set -euo pipefail cd "$SRC/newsdom-api" -pip3 install . pyinstaller atheris +uv sync --frozen --extra fuzz +export PATH="$SRC/newsdom-api/.venv/bin:$PATH" for fuzzer in $(find fuzzers -name '*_fuzzer.py'); do fuzzer_basename=$(basename -s .py "$fuzzer") diff --git a/Dockerfile b/Dockerfile index 0098a97a..93779d42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,9 @@ -FROM python:3.12-slim AS builder +ARG PYTHON_BASE=python:3.12-slim@sha256:5072b08ad74609c5329ab4085a96dfa873de565fb4751a4cfcd7dcc427661df0 +ARG UV_IMAGE=ghcr.io/astral-sh/uv@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 + +FROM ${UV_IMAGE} AS uv-bin + +FROM ${PYTHON_BASE} AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -6,14 +11,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app -RUN python -m pip install --no-cache-dir "uv==0.11.3" +COPY --from=uv-bin /uv /uvx /bin/ COPY pyproject.toml uv.lock README.md ./ COPY src/ src/ RUN uv sync --frozen --no-dev -FROM python:3.12-slim AS runtime +FROM ${PYTHON_BASE} AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia index e65f7f2b..e3585734 100644 --- a/Dockerfile.nvidia +++ b/Dockerfile.nvidia @@ -1,4 +1,9 @@ -FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04 AS builder +ARG NVIDIA_BASE=nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04@sha256:46cb48a4abfbc40c836fe57bc05a07101b6458fffc63bbdfd6a50db98c9358bd +ARG UV_IMAGE=ghcr.io/astral-sh/uv@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823 + +FROM ${UV_IMAGE} AS uv-bin + +FROM --platform=linux/amd64 ${NVIDIA_BASE} AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -10,15 +15,14 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends python3 python3-pip python3-venv ca-certificates && \ rm -rf /var/lib/apt/lists/* -RUN python3 -m pip install --no-cache-dir "uv==0.11.3" +COPY --from=uv-bin /uv /uvx /usr/local/bin/ COPY pyproject.toml uv.lock README.md ./ COPY src/ src/ -RUN uv sync --python python3 --frozen --no-dev && \ - uv pip install --python .venv/bin/python "mineru[pipeline]==3.0.9" +RUN uv sync --python python3 --frozen --no-dev --extra nvidia -FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04 AS runtime +FROM --platform=linux/amd64 ${NVIDIA_BASE} AS runtime ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ diff --git a/pyproject.toml b/pyproject.toml index 931fccbd..2b0c1cd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,13 @@ docs = [ "mkdocs>=1.6,<2.0", "mkdocs-material>=9.6,<9.7", ] +nvidia = [ + "mineru[pipeline]==3.0.9 ; platform_system == 'Linux' and platform_machine == 'x86_64'", +] +fuzz = [ + "atheris==3.0.0 ; platform_system == 'Linux' and python_version >= '3.11'", + "pyinstaller==6.16.0", +] [tool.setuptools] package-dir = {"" = "src"} diff --git a/tests/test_docker_delivery.py b/tests/test_docker_delivery.py index 1413572a..9aaa6923 100644 --- a/tests/test_docker_delivery.py +++ b/tests/test_docker_delivery.py @@ -20,6 +20,8 @@ def test_dockerfile_uses_project_metadata_and_src_layout(): assert "pyproject.toml" in text assert "uv.lock" in text assert "src/" in text + assert "python:3.12-slim@sha256:" in text + assert "ghcr.io/astral-sh/uv@sha256:" in text def test_dockerfile_runs_uvicorn_with_healthcheck_and_external_mineru_path(): @@ -36,7 +38,9 @@ def test_dockerfile_runs_uvicorn_with_healthcheck_and_external_mineru_path(): def test_nvidia_dockerfile_installs_mineru_pipeline_stack(): text = Path("Dockerfile.nvidia").read_text(encoding="utf-8") - assert "mineru[pipeline]==3.0.9" in text + assert "nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04@sha256:" in text + assert "ghcr.io/astral-sh/uv@sha256:" in text + assert "--extra nvidia" in text assert "NEWSDOM_MINERU_BIN" in text diff --git a/tests/test_fuzzing_integration.py b/tests/test_fuzzing_integration.py index 9f7f59f6..6803cad3 100644 --- a/tests/test_fuzzing_integration.py +++ b/tests/test_fuzzing_integration.py @@ -29,9 +29,17 @@ def test_clusterfuzzlite_workflow_runs_pinned_python_code_change_fuzzing(): def test_clusterfuzzlite_dockerfile_places_build_script_at_src_root(): text = Path(".clusterfuzzlite/Dockerfile").read_text(encoding="utf-8") + assert "gcr.io/oss-fuzz-base/base-builder-python@sha256:" in text + assert "ghcr.io/astral-sh/uv@sha256:" in text assert "COPY .clusterfuzzlite/build.sh /src/build.sh" in text +def test_clusterfuzzlite_build_script_uses_locked_uv_fuzz_extra(): + text = Path(".clusterfuzzlite/build.sh").read_text(encoding="utf-8") + assert "uv sync --frozen --extra fuzz" in text + assert "pip3 install . pyinstaller atheris" not in text + + def test_dom_builder_fuzzer_smoke_mode_runs_without_cluster(): completed = subprocess.run( [ diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index deab61b9..bd990252 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -3,7 +3,8 @@ def test_project_metadata_does_not_bundle_mineru_extra(): text = Path("pyproject.toml").read_text(encoding="utf-8") - assert "mineru[pipeline]" not in text + dependencies_section = text.split("dependencies = [", 1)[1].split("]", 1)[0] + assert "mineru[pipeline]" not in dependencies_section def test_docs_theme_range_stays_below_warning_release(): @@ -15,3 +16,12 @@ def test_project_uses_spdx_license_string_not_deprecated_table(): text = Path("pyproject.toml").read_text(encoding="utf-8") assert 'license = "MIT"' in text assert 'license = {text = "MIT"}' not in text + + +def test_project_declares_locked_nvidia_and_fuzz_extras(): + text = Path("pyproject.toml").read_text(encoding="utf-8") + assert "nvidia = [" in text + assert '"mineru[pipeline]==3.0.9 ;' in text + assert "fuzz = [" in text + assert '"atheris==3.0.0 ;' in text + assert '"pyinstaller==6.16.0"' in text