Skip to content

Commit

Permalink
[python] Switch to mesonpy build tool
Browse files Browse the repository at this point in the history 
We need to switch build tools for the Cython extension in preparation
for our migration to C++. This compilcates the build process a bit
becasue we now need to use different tools to build source
distributions (tar.gz) and binary distributions (wheels / .whl).
  • Loading branch information
@olafmersmann
olafmersmann committed Dec 6, 2024
1 parent bd14da3 commit 5a5d938
Show file tree
Hide file tree
Showing 12 changed files with 383 additions and 251 deletions.
113 changes: 57 additions & 56 deletions .github/workflows/source_build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,59 +6,60 @@ jobs:
source-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
- name: Setup Python
run: python -m pip install --upgrade -r scripts/requirements.txt
- name: Fabricate COCO
run: python scripts/fabricate
- name: Run core unit tests
working-directory: ${{github.workspace}}/test/unit-test/
run: |
cmake -B build .
cmake --build build
ctest --test-dir build
- name: Run core integration tests
working-directory: ${{github.workspace}}/test/integration-test/
run: |
cmake -B build .
cmake --build build
ctest --test-dir build
- name: Build coco-experiment Python package
run: python -m build -s -o dist build/python
- name: Archive C source package
uses: actions/upload-artifact@v4
with:
name: dist-c
path: dist/cocoex-c-*.zip
- name: Archive Java source package
uses: actions/upload-artifact@v4
with:
name: dist-java
path: dist/cocoex-java-*.zip
- name: Archive Matlab source package
uses: actions/upload-artifact@v4
with:
name: dist-matlab
path: dist/cocoex-matlab-*.zip
- name: Archive Octave source package
uses: actions/upload-artifact@v4
with:
name: dist-octave
path: dist/cocoex-octave-*.zip
- name: Archive Rust source package
uses: actions/upload-artifact@v4
with:
name: dist-rust
path: dist/cocoex-rust-*.zip
- name: Archive cocoex Python source package
uses: actions/upload-artifact@v4
with:
name: dist-python-sdist
path: |
dist/coco_experiment*.tar.gz
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
- name: Setup Python
run: python -m pip install --upgrade -r scripts/requirements.txt
- name: Fabricate COCO
run: python scripts/fabricate
- name: Run core unit tests
working-directory: ${{github.workspace}}/test/unit-test/
run: |
cmake -B build .
cmake --build build
ctest --test-dir build
- name: Run core integration tests
working-directory: ${{github.workspace}}/test/integration-test/
run: |
cmake -B build .
cmake --build build
ctest --test-dir build
- name: Build coco-experiment Python package
working-directory: ${{github.workspace}}/build/python
run: python -m hatchling build -t sdist -d ../../dist/
- name: Archive C source package
uses: actions/upload-artifact@v4
with:
name: dist-c
path: dist/cocoex-c-*.zip
- name: Archive Java source package
uses: actions/upload-artifact@v4
with:
name: dist-java
path: dist/cocoex-java-*.zip
- name: Archive Matlab source package
uses: actions/upload-artifact@v4
with:
name: dist-matlab
path: dist/cocoex-matlab-*.zip
- name: Archive Octave source package
uses: actions/upload-artifact@v4
with:
name: dist-octave
path: dist/cocoex-octave-*.zip
- name: Archive Rust source package
uses: actions/upload-artifact@v4
with:
name: dist-rust
path: dist/cocoex-rust-*.zip
- name: Archive cocoex Python source package
uses: actions/upload-artifact@v4
with:
name: dist-python-sdist
path: |
dist/coco_experiment*.tar.gz
187 changes: 94 additions & 93 deletions .github/workflows/test_python.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ on:
build_only:
description: "only run wheel builds job"
required: false
default: 'false'
default: "false"
type: string

jobs:
python-wheels-build:
# This strategy is taken straight from the numpy wheels.yaml CI job
#
#
# Since we depend on numpy, there's no reason to try and build a binary cocoex wheel
# on platform that don't have a compiled numpy available.
strategy:
Expand All @@ -23,52 +23,55 @@ jobs:
- [ubuntu-20.04, manylinux_x86_64]
- [ubuntu-20.04, musllinux_x86_64]
- [macos-14, macosx_arm64]
- [windows-2019, win_amd64]
- [windows-2019, win_amd64]
python:
- ["cp39", "3.9"]
- ["cp310", "3.10"]
- ["cp311", "3.11"]
- ["cp312", "3.12"]
#- ["cp313", "3.13"]
- ["pp39", "pp"]
exclude:
- buildplat: [ ubuntu-20.04, musllinux_x86_64 ]
- buildplat: [ubuntu-20.04, musllinux_x86_64]
python: ["pp39", "pp"]
runs-on: ${{ matrix.buildplat[0] }}
defaults:
run:
working-directory: ${{github.workspace}}
steps:
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Download Python source package
uses: actions/download-artifact@v4
with:
name: dist-python-sdist
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.16.2
- name: Build cocoex wheel
shell: bash
run: python -m cibuildwheel --output-dir dist/ coco_experiment-*.tar.gz
env:
CIBW_PRERELEASE_PYTHONS: True
CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
CIBW_BUILD_VERBOSITY: 2
- name: Archive Python wheels
uses: actions/upload-artifact@v4
with:
name: dist-python-wheels-${{ matrix.python[1] }}-${{ matrix.buildplat[1] }}
path: dist/coco_experiment*.whl

steps:
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Download Python source package
uses: actions/download-artifact@v4
with:
name: dist-python-sdist
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.16.2
- name: Build cocoex wheel
shell: bash
run: python -m cibuildwheel --output-dir dist/ coco_experiment-*.tar.gz
env:
CIBW_PRERELEASE_PYTHONS: True
# Required for MSVC builds on Windows, otherwise meson picks up mingw64.
CIBW_CONFIG_SETTINGS_WINDOWS: "setup-args=--vsenv"
CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
CIBW_BUILD_VERBOSITY: 1
- name: Archive Python wheels
uses: actions/upload-artifact@v4
with:
name: dist-python-wheels-${{ matrix.python[1] }}-${{ matrix.buildplat[1] }}
path: dist/coco_experiment*.whl

python-wheels-test:
needs: python-wheels-build
if: ${{ inputs.build_only == 'false' }}
strategy:
fail-fast: false
matrix:
buildplat:
- [ubuntu-20.04, manylinux_x86_64]
- [ubuntu-20.04, manylinux_x86_64]
- [macos-14, macosx_arm64]
- [windows-2019, win_amd64]
python: ["3.9", "3.10", "3.11", "3.12"]
Expand All @@ -77,68 +80,66 @@ jobs:
run:
working-directory: ${{github.workspace}}
steps:
- uses: actions/download-artifact@v4
with:
name: dist-python-sdist
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-python-wheels-${{ matrix.python }}-${{ matrix.buildplat[1] }}
path: wheels
- name: List available wheels
run: ls -l wheels/
- uses: actions/setup-python@v5
with:
python-version: ${{matrix.python}}
- name: Setup Python
run: python -m pip install --upgrade pip wheel
- name: Install cocoex
shell: bash
run: python -m pip install wheels/coco_experiment*.whl
- name: Run cocoex test
shell: bash
run: |
tar xf dist/coco_experiment-*.tar.gz --strip-components=1
python -m pip install pytest pytest-cov
pytest --cov=cocoex test/
python -m pip install doctest
python -c "import doctest, cocoex; doctest.testmod(cocoex)"
- uses: actions/download-artifact@v4
with:
name: dist-python-sdist
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-python-wheels-${{ matrix.python }}-${{ matrix.buildplat[1] }}
path: wheels
- name: List available wheels
run: ls -l wheels/
- uses: actions/setup-python@v5
with:
python-version: ${{matrix.python}}
- name: Setup Python
run: python -m pip install --upgrade pip wheel
- name: Install cocoex
shell: bash
run: python -m pip install wheels/coco_experiment*.whl
- name: Run cocoex test
shell: bash
run: |
tar xf dist/coco_experiment-*.tar.gz --strip-components=1
python -m pip install pytest pytest-cov
pytest --cov=cocoex test/
python scripts/run-doctest.py
python-wheels-example:
needs: python-wheels-build
if: ${{ inputs.build_only == 'false' }}
strategy:
fail-fast: false
matrix:
buildplat:
- [ubuntu-20.04, manylinux_x86_64]
- [ubuntu-20.04, manylinux_x86_64]
- [macos-14, macosx_arm64]
- [windows-2019, win_amd64]
- [windows-2019, win_amd64]
python: ["3.9", "3.10", "3.11", "3.12"]
runs-on: ${{ matrix.buildplat[0] }}
defaults:
run:
working-directory: ${{github.workspace}}
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: dist-python-wheels-${{ matrix.python }}-${{ matrix.buildplat[1] }}
path: wheels
- uses: actions/setup-python@v5
with:
python-version: ${{matrix.python}}
- name: Setup Python
run: python -m pip install --upgrade pip wheel
- name: Install cocoex and cocopp
shell: bash
run: |
python -m pip install wheels/coco_experiment*.whl
python -m pip install cocopp
- name: Install scipy for example experiment
run: python -m pip install scipy
- name: Run example experiment
run: python build/python/example/example_experiment2.py
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: dist-python-wheels-${{ matrix.python }}-${{ matrix.buildplat[1] }}
path: wheels
- uses: actions/setup-python@v5
with:
python-version: ${{matrix.python}}
- name: Setup Python
run: python -m pip install --upgrade pip wheel
- name: Install cocoex and cocopp
shell: bash
run: |
python -m pip install wheels/coco_experiment*.whl
python -m pip install cocopp
- name: Install scipy for example experiment
run: python -m pip install scipy
- name: Run example experiment
run: python build/python/example/example_experiment2.py

merge-python-wheels:
runs-on: ubuntu-latest
Expand All @@ -156,19 +157,19 @@ jobs:
runs-on: "ubuntu-latest"
if: ${{ inputs.build_only == 'false' }}
steps:
- uses: actions/download-artifact@v4
with:
name: dist-python-sdist
path: dist
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Setup Python
run: python -m pip install --upgrade pip wheel
- name: Lint with Ruff
shell: bash
continue-on-error: true
run: |
pip install ruff
tar xf dist/coco_experiment*.tar.gz --strip-components=1
ruff check --output-format=github .
- uses: actions/download-artifact@v4
with:
name: dist-python-sdist
path: dist
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Setup Python
run: python -m pip install --upgrade pip wheel
- name: Lint with Ruff
shell: bash
continue-on-error: true
run: |
pip install ruff
tar xf dist/coco_experiment*.tar.gz --strip-components=1
ruff check --output-format=github .
Loading

0 comments on commit 5a5d938

Please sign in to comment.