Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .clusterfuzzlite/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
FROM gcr.io/oss-fuzz-base/base-builder-python
ARG UV_IMAGE=ghcr.io/astral-sh/uv@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823

FROM ${UV_IMAGE} AS uv-bin

FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:60e8ef87f2c0367254ff979a4dea61dad2684b001e3e666e2cc6fe992064dbbf

WORKDIR /src/newsdom-api
COPY --from=uv-bin /uv /uvx /usr/local/bin/
COPY . .
COPY .clusterfuzzlite/build.sh /src/build.sh
3 changes: 2 additions & 1 deletion .clusterfuzzlite/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ set -euo pipefail

cd "$SRC/newsdom-api"

pip3 install . pyinstaller atheris
uv sync --frozen --extra fuzz
export PATH="$SRC/newsdom-api/.venv/bin:$PATH"

for fuzzer in $(find fuzzers -name '*_fuzzer.py'); do
fuzzer_basename=$(basename -s .py "$fuzzer")
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/clusterfuzzlite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ jobs:
with:
language: python
sanitizer: address
github-token: ${{ github.token }}

- name: Run fuzzers
uses: google/clusterfuzzlite/actions/run_fuzzers@52ecc61cb587ee99c26825a112a21abf19c7448c
Expand Down
11 changes: 8 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
FROM python:3.12-slim AS builder
ARG PYTHON_BASE=python:3.12-slim@sha256:5072b08ad74609c5329ab4085a96dfa873de565fb4751a4cfcd7dcc427661df0
ARG UV_IMAGE=ghcr.io/astral-sh/uv@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823

FROM ${UV_IMAGE} AS uv-bin

FROM ${PYTHON_BASE} AS builder

ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1

WORKDIR /app

RUN python -m pip install --no-cache-dir "uv==0.11.3"
COPY --from=uv-bin /uv /uvx /bin/

COPY pyproject.toml uv.lock README.md ./
COPY src/ src/

RUN uv sync --frozen --no-dev

FROM python:3.12-slim AS runtime
FROM ${PYTHON_BASE} AS runtime

ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
Expand Down
11 changes: 8 additions & 3 deletions Dockerfile.nvidia
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04 AS builder
ARG NVIDIA_BASE=nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04@sha256:46cb48a4abfbc40c836fe57bc05a07101b6458fffc63bbdfd6a50db98c9358bd
ARG UV_IMAGE=ghcr.io/astral-sh/uv@sha256:90bbb3c16635e9627f49eec6539f956d70746c409209041800a0280b93152823

FROM ${UV_IMAGE} AS uv-bin

FROM --platform=linux/amd64 ${NVIDIA_BASE} AS builder

ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
Expand All @@ -10,15 +15,15 @@ RUN apt-get update && \
apt-get install -y --no-install-recommends python3 python3-pip python3-venv ca-certificates && \
rm -rf /var/lib/apt/lists/*

RUN python3 -m pip install --no-cache-dir "uv==0.11.3"
COPY --from=uv-bin /uv /uvx /usr/local/bin/

COPY pyproject.toml uv.lock README.md ./
COPY src/ src/

RUN uv sync --python python3 --frozen --no-dev && \
uv pip install --python .venv/bin/python "mineru[pipeline]==3.0.9"

FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04 AS runtime
FROM --platform=linux/amd64 ${NVIDIA_BASE} AS runtime

ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
Expand Down
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ docs = [
"mkdocs>=1.6,<2.0",
"mkdocs-material>=9.6,<9.7",
]
fuzz = [
"atheris==3.0.0 ; platform_system == 'Linux' and python_version >= '3.11'",
"pyinstaller==6.16.0",
]

[tool.setuptools]
package-dir = {"" = "src"}
Expand Down
6 changes: 5 additions & 1 deletion tests/test_docker_delivery.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ def test_dockerfile_uses_project_metadata_and_src_layout():
assert "pyproject.toml" in text
assert "uv.lock" in text
assert "src/" in text
assert "python:3.12-slim@sha256:" in text
assert "ghcr.io/astral-sh/uv@sha256:" in text


def test_dockerfile_runs_uvicorn_with_healthcheck_and_external_mineru_path():
Expand All @@ -36,7 +38,9 @@ def test_dockerfile_runs_uvicorn_with_healthcheck_and_external_mineru_path():

def test_nvidia_dockerfile_installs_mineru_pipeline_stack():
text = Path("Dockerfile.nvidia").read_text(encoding="utf-8")
assert "mineru[pipeline]==3.0.9" in text
assert "nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04@sha256:" in text
assert "ghcr.io/astral-sh/uv@sha256:" in text
assert 'uv pip install --python .venv/bin/python "mineru[pipeline]==3.0.9"' in text
assert "NEWSDOM_MINERU_BIN" in text


Expand Down
14 changes: 14 additions & 0 deletions tests/test_fuzzing_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,20 @@ def test_clusterfuzzlite_workflow_runs_pinned_python_code_change_fuzzing():
assert "language: python" in text
assert "mode: code-change" in text
assert "fuzz-seconds: 300" in text
assert "github-token: ${{ github.token }}" in text


def test_clusterfuzzlite_dockerfile_places_build_script_at_src_root():
text = Path(".clusterfuzzlite/Dockerfile").read_text(encoding="utf-8")
assert "gcr.io/oss-fuzz-base/base-builder-python@sha256:" in text
assert "ghcr.io/astral-sh/uv@sha256:" in text
assert "COPY .clusterfuzzlite/build.sh /src/build.sh" in text


def test_clusterfuzzlite_build_script_uses_locked_uv_fuzz_extra():
text = Path(".clusterfuzzlite/build.sh").read_text(encoding="utf-8")
assert "uv sync --frozen --extra fuzz" in text
assert "pip3 install . pyinstaller atheris" not in text


def test_dom_builder_fuzzer_smoke_mode_runs_without_cluster():
Expand Down
11 changes: 10 additions & 1 deletion tests/test_project_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

def test_project_metadata_does_not_bundle_mineru_extra():
text = Path("pyproject.toml").read_text(encoding="utf-8")
assert "mineru[pipeline]" not in text
dependencies_section = text.split("dependencies = [", 1)[1].split("]", 1)[0]
assert "mineru[pipeline]" not in dependencies_section


def test_docs_theme_range_stays_below_warning_release():
Expand All @@ -15,3 +16,11 @@ def test_project_uses_spdx_license_string_not_deprecated_table():
text = Path("pyproject.toml").read_text(encoding="utf-8")
assert 'license = "MIT"' in text
assert 'license = {text = "MIT"}' not in text


def test_project_declares_locked_fuzz_extra_without_bundling_nvidia_stack():
text = Path("pyproject.toml").read_text(encoding="utf-8")
assert "fuzz = [" in text
assert '"atheris==3.0.0 ;' in text
assert '"pyinstaller==6.16.0"' in text
assert "nvidia = [" not in text
108 changes: 107 additions & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading