From f0f943fb5f670a63a8c00c4db5c1dac64a36c403 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 24 Apr 2026 09:11:03 +0900 Subject: [PATCH] feat(ci): Implement Docker-based CI and OCR benchmark harness - Implements the OCR benchmark harness and its unit test. - Introduces a Dockerfile to create a stable test environment. - Overhauls the CI workflow to build and use this Docker image for tests, resolving persistent dependency and environment issues. This includes fixing the image tag to be lowercase. --- .github/workflows/tests.yml | 53 +++++++++---- Dockerfile.test | 34 ++++++++ tests/test_benchmark_ocr.py | 77 ++++++++++++++++++ tools/benchmark_ocr.py | 154 ++++++++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+), 15 deletions(-) create mode 100644 Dockerfile.test create mode 100644 tests/test_benchmark_ocr.py create mode 100644 tools/benchmark_ocr.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 00d1aeb7..146b3f50 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,30 +7,53 @@ on: permissions: contents: read + packages: write jobs: - pytest: + build-test-image: + name: Build Test Image runs-on: ubuntu-latest - env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + outputs: + image_name: ${{ steps.image_name.outputs.name }} steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Setup Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 with: - python-version: '3.10' + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Setup uv - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 + - name: Define image name and tag + id: image_name + run: echo "name=ghcr.io/${{ github.repository_owner,, }}/newsdom-api/test:${{ github.sha }}" | tee -a $GITHUB_OUTPUT + + - name: Build and push test image + uses: docker/build-push-action@v5 with: - version: '0.11.3' + context: . + file: ./Dockerfile.test + push: true + tags: ${{ steps.image_name.outputs.name }} + cache-from: type=gha + cache-to: type=gha,mode=max - - name: Install dependencies - run: uv sync --frozen --all-extras + pytest: + name: Run Pytest in Container + runs-on: ubuntu-latest + needs: build-test-image + container: + image: ${{ needs.build-test-image.outputs.image_name }} + env: + PYTHONWARNINGS: error + steps: + - name: Checkout + uses: actions/checkout@v4 - - name: Run tests with warnings as errors - env: - PYTHONWARNINGS: error + - name: Run tests run: uv run pytest diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..0b1ae896 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,34 @@ +# Docker image for running the test suite in a controlled and reliable environment +FROM python:3.10-slim + +# Set timezone to avoid interactive prompts during package installations +ENV TZ=Asia/Tokyo +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +WORKDIR /app + +# Install system dependencies, including Git, Tesseract for future use, and Japanese fonts +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + tesseract-ocr \ + tesseract-ocr-jpn \ + fonts-ipafont-gothic \ + && rm -rf /var/lib/apt/lists/* + +# Copy only necessary files for dependency installation first to leverage caching +COPY pyproject.toml uv.lock ./ + +# Install uv via pip +RUN pip install uv==0.1.41 + +# Install all project dependencies into a virtual environment +RUN uv sync --frozen --all-extras + +# Copy the rest of the application source code +COPY . . + +# Add the virtual environment's bin to the PATH to make scripts like 'mineru' available +ENV PATH="/app/.venv/bin:$PATH" + +# Verify that the 'mineru' executable is now on the PATH +RUN which mineru diff --git a/tests/test_benchmark_ocr.py b/tests/test_benchmark_ocr.py new file mode 100644 index 00000000..c0d0b4d0 --- /dev/null +++ b/tests/test_benchmark_ocr.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from tools import benchmark_ocr + + +@pytest.fixture +def mock_pdf_dir(tmp_path: Path) -> Path: + pdf_dir = tmp_path / "pdfs" + pdf_dir.mkdir() + (pdf_dir / "doc1.pdf").write_text("dummy pdf 1") + (pdf_dir / "doc2.pdf").write_text("dummy pdf 2") + (pdf_dir / "doc3.pdf").write_text("dummy pdf 3") + return pdf_dir + + +def test_benchmark_ocr_harness(mock_pdf_dir: Path, tmp_path: Path): + """ + The benchmark harness should run a set of engines against a corpus of PDFs + and correctly aggregate success, failure, and timeout results. + """ + output_path = tmp_path / "results.json" + + # Mock engine runners + mock_engine1 = MagicMock(return_value={"status": "success", "page_count": 2}) + mock_engine2 = MagicMock() + mock_engine2.side_effect = [ + subprocess.TimeoutExpired(cmd="engine2", timeout=10), + RuntimeError("OCR process failed"), + {"status": "success", "page_count": 1}, + ] + + engines = { + "engine1": mock_engine1, + "engine2": mock_engine2, + } + + with patch.dict(benchmark_ocr.OCR_ENGINES, engines): + benchmark_ocr.main( + [ + "--fixtures-dir", + str(mock_pdf_dir), + "--output", + str(output_path), + "--engines", + "engine1", + "engine2", + ] + ) + + assert output_path.exists() + results = json.loads(output_path.read_text()) + + # Assert engine1 (always succeeds) + assert results["engine1"]["success"] == 3 + assert results["engine1"]["failed"] == 0 + assert results["engine1"]["timed_out"] == 0 + assert "doc1.pdf" in results["engine1"]["results"] + assert results["engine1"]["results"]["doc1.pdf"]["status"] == "success" + + # Assert engine2 (fails, times out, then succeeds) + assert results["engine2"]["success"] == 1 + assert results["engine2"]["failed"] == 1 + assert results["engine2"]["timed_out"] == 1 + assert results["engine2"]["results"]["doc1.pdf"]["status"] == "timed_out" + assert results["engine2"]["results"]["doc2.pdf"]["status"] == "failed" + assert "OCR process failed" in results["engine2"]["results"]["doc2.pdf"]["error"] + assert results["engine2"]["results"]["doc3.pdf"]["status"] == "success" + + # Check summary + assert results["summary"]["total_files"] == 3 + assert results["summary"]["total_runs"] == 6 diff --git a/tools/benchmark_ocr.py b/tools/benchmark_ocr.py new file mode 100644 index 00000000..44fc5f0b --- /dev/null +++ b/tools/benchmark_ocr.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import argparse +import json +import subprocess +import time +from pathlib import Path +from typing import Any, Callable + +from newsdom_api.service import parse_pdf_bytes + + +def run_mineru_engine(pdf_path: Path) -> dict[str, Any]: + """ + Run the 'mineru' OCR engine on a single PDF file. + + This function acts as a wrapper around the existing 'parse_pdf_bytes' service, + adapting its output for the benchmark harness. + + Args: + pdf_path: The path to the PDF file to process. + + Returns: + A dictionary containing the processing status and key metrics. + """ + response = parse_pdf_bytes(pdf_path.read_bytes(), filename=pdf_path.name) + return { + "status": "success", + "page_count": len(response.pages), + "article_count": sum(len(p.articles) for p in response.pages), + } + + +OCR_ENGINES: dict[str, Callable[[Path], dict[str, Any]]] = { + "mineru": run_mineru_engine, +} + + +def run_benchmark( + fixtures_dir: Path, output_path: Path, engines: list[str] +) -> None: + """ + Run the OCR benchmark against a directory of PDFs for a list of engines. + + It iterates through each PDF and runs each specified engine, capturing + success, failure, or timeout, along with performance metrics. The + aggregated results are written to a JSON file. + + Args: + fixtures_dir: The directory containing the PDF files to benchmark. + output_path: The path to write the final JSON results file. + engines: A list of engine names to run. + """ + pdf_paths = sorted(list(fixtures_dir.glob("*.pdf"))) + if not pdf_paths: + raise FileNotFoundError(f"No PDF files found in {fixtures_dir}") + + all_results: dict[str, Any] = { + "summary": { + "total_files": len(pdf_paths), + "total_runs": len(pdf_paths) * len(engines), + } + } + + for engine_name in engines: + if engine_name not in OCR_ENGINES: + raise ValueError(f"Unknown engine: {engine_name}") + + print(f"--- Running benchmark for engine: {engine_name} ---") + engine_runner = OCR_ENGINES[engine_name] + engine_results: dict[str, Any] = { + "success": 0, + "failed": 0, + "timed_out": 0, + "results": {}, + } + + for pdf_path in pdf_paths: + start_time = time.monotonic() + try: + result = engine_runner(pdf_path) + duration = time.monotonic() - start_time + engine_results["success"] += 1 + engine_results["results"][pdf_path.name] = { + "status": "success", + "duration": round(duration, 2), + **result, + } + print(f" [SUCCESS] {pdf_path.name} in {duration:.2f}s") + + except (RuntimeError, subprocess.CalledProcessError) as e: + duration = time.monotonic() - start_time + engine_results["failed"] += 1 + engine_results["results"][pdf_path.name] = { + "status": "failed", + "duration": round(duration, 2), + "error": str(e), + } + print(f" [FAILED] {pdf_path.name} in {duration:.2f}s") + + except subprocess.TimeoutExpired: + duration = time.monotonic() - start_time + engine_results["timed_out"] += 1 + engine_results["results"][pdf_path.name] = { + "status": "timed_out", + "duration": round(duration, 2), + } + print(f" [TIMED OUT] {pdf_path.name} after {duration:.2f}s") + + all_results[engine_name] = engine_results + + output_path.write_text( + json.dumps(all_results, ensure_ascii=False, indent=2), encoding="utf-8" + ) + print(f"\nBenchmark results written to {output_path}") + + +def main(argv: list[str] | None = None) -> None: + """ + Main entry point for the OCR benchmark script. + + Parses command-line arguments and initiates the benchmark run. + + Args: + argv: Optional list of command-line arguments for testing. + """ + parser = argparse.ArgumentParser( + description="Run OCR benchmark against a directory of PDFs." + ) + parser.add_argument( + "--fixtures-dir", + type=Path, + required=True, + help="Directory containing PDF files.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Path to write the JSON results file.", + ) + parser.add_argument( + "--engines", + nargs="+", + default=["mineru"], + help=f"Space-separated list of engines to benchmark. Available: {', '.join(OCR_ENGINES.keys())}", + ) + args = parser.parse_args(argv) + + run_benchmark(args.fixtures_dir, args.output, args.engines) + + +if __name__ == "__main__": + main()