General tidy and restrict cffi < 1.17 due to static condensation demo issue (#3343)

jhale · web-flow · commit 06e90b8506d9 · 2024-08-09T14:37:08.000Z
* CFFI 1.17 seems to have issues with numba.

* Clean up RedHat build.

* Can people stop adding optional dependencies in CI runs?

* Tidy.

* Tidy.

* General tidying.

* Fix.

* Remove logguru exception

* Tidy up Dockerfile.

* Fix.

* Install cython for petsc4py build

* Setuptools

* Tweaks.

* Fix.

* Always build docs once and only push if on DOLFINx with main or v* tag.

* Add back documentation artifacts.

* Fix.

* Fix.

* Fix.

* Need these, not installed (!).

* petsc4py needs numpy.

* Split apart numba/cffi and petsc/cffi definitions

* Missing cffi!

* Revert.

* Try unpinning CFFI.

* Does seem to be necessary.

* Improve docstring.
diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml
@@ -33,6 +33,7 @@ jobs:
     name: Lint
     steps:
       - uses: actions/checkout@v4
+      
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -60,7 +61,7 @@ jobs:
         run: |
           cd cpp
           clang-format --version
-          find . -type f \( -name "*.cpp" -o -name "*.h" \) ! -name "loguru.cpp" | xargs clang-format --dry-run --Werror
+          find . -type f \( -name "*.cpp" -o -name "*.h" \) | xargs clang-format --dry-run --Werror
       - name: clang-format Python binding checks (non-blocking)
         continue-on-error: true
         run: |
@@ -83,6 +84,8 @@ jobs:
 
     name: Build and test
     steps:
+      - uses: actions/checkout@v4
+      
       - name: Install dependencies
         run: |
           sudo apt-get update
@@ -93,7 +96,8 @@ jobs:
           python-version: "3.12"
 
       - name: Install Python build dependencies
-        run: pip install mpi4py nanobind scikit-build-core[pyproject] --upgrade
+        run: |
+          pip install --upgrade -r python/build-requirements.txt
 
       - name: Install FEniCS Python components (default branches/tags)
         if: github.event_name != 'workflow_dispatch'
@@ -108,8 +112,6 @@ jobs:
           pip install git+https://github.com/FEniCS/basix.git@${{ github.event.inputs.basix_ref }}
           pip install git+https://github.com/FEniCS/ffcx.git@${{ github.event.inputs.ffcx_ref }}
 
-      - uses: actions/checkout@v4
-
       - name: Configure and install C++
         run: |
           cmake -G Ninja -DDOLFINX_ENABLE_PETSC=false -DCMAKE_BUILD_TYPE=Developer -B build -S cpp/
@@ -129,12 +131,12 @@ jobs:
           mpirun -n 3 ctest -V --output-on-failure -R unittests
 
       - name: Build Python interface
-        run: pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type="Debug" python/
+        run: pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type="Debug" 'python/[test]'
 
-      - name: Install Python demo/test dependencies
-        run: pip install matplotlib pyamg pytest pytest-xdist scipy
       - name: Run demos (Python, serial)
-        run: python -m pytest -n auto -m serial --durations=10 python/demo/test.py
+        run: |
+          pip install pytest-xdist
+          python -m pytest -n auto -m serial --durations=10 python/demo/test.py
       - name: Run demos (Python, MPI (np=3))
         run: python -m pytest -m mpi --num-proc=3 python/demo/test.py
       - name: Run unit tests
@@ -165,9 +167,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install required Python packages (docs)
-        run: pip install breathe jupytext matplotlib myst_parser sphinx sphinx_rtd_theme
-
       - name: Install FEniCS Python components (default branches/tags)
         if: github.event_name != 'workflow_dispatch'
         run: |
@@ -188,14 +187,6 @@ jobs:
           cmake --build build
           cmake --install build
 
-      - name: Build C++ interface documentation
-        run: |
-          export DOLFINX_VERSION=`cmake -L build | grep DOXYGEN_DOLFINX_VERSION | cut -f2 -d "="`
-          echo $DOLFINX_VERSION
-          cd cpp/doc
-          doxygen Doxyfile
-          make html
-
       - name: Build C++ unit tests
         run: |
           cmake -G Ninja -DCMAKE_BUILD_TYPE=Developer -B build/test/ -S cpp/test/
@@ -216,26 +207,22 @@ jobs:
           ctest -V -R demo -R serial
           ctest -V -R demo -R mpi_2
 
-      - name: Install required Python (build Python interface)
-        run: pip install nanobind scikit-build-core[pyproject]
+      - name: Install Python build dependencies 
+        run: pip install -r python/build-requirements.txt 
 
       - name: Build Python interface
         run: |
-          pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type="Debug" python/
-      - name: Build Python interface documentation
-        run: |
-          cd python/doc
-          python -m sphinx -W -b html source/ build/html/
+          pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type="Debug" 'python/[test]'
 
       - name: Set default DOLFINx JIT options
         run: |
           mkdir -p ~/.config/dolfinx
           echo '{ "cffi_extra_compile_args": ["-g0", "-O0" ] }' > ~/.config/dolfinx/dolfinx_jit_options.json
 
-      - name: Install Python demo/test dependencies
-        run: pip install numba pyamg pytest pytest-xdist scipy
       - name: Run demos (Python, serial)
-        run: python -m pytest -n auto -m serial --durations=10 python/demo/test.py
+        run: |
+          pip install pytest-xdist
+          python -m pytest -n auto -m serial --durations=10 python/demo/test.py
       - name: Run demos (Python, MPI (np=3))
         run: python -m pytest -m mpi --num-proc=3 python/demo/test.py
 
@@ -244,9 +231,7 @@ jobs:
       - name: Run Python unit tests (MPI, np=3)
         run: mpirun -np 3 python -m pytest -m "petsc4py or adios2" python/test/unit/
 
-  publish-docs:
-    if: ${{ github.repository == 'FEniCS/dolfinx' && ( github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }}
-    needs: build-with-petsc
+  build-and-publish-docs:
     runs-on: ubuntu-latest
     container: "ghcr.io/fenics/test-env:current-openmpi"
     env:
@@ -255,13 +240,10 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
       PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
 
-    name: Publish docs
+    name: Build and publish docs
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install required Python packages (docs)
-        run: pip install breathe jupytext matplotlib myst_parser sphinx sphinx_rtd_theme
-
       - name: Install FEniCS Python components (default branches/tags)
         if: github.event_name != 'workflow_dispatch'
         run: |
@@ -281,34 +263,55 @@ jobs:
           cmake --build build
           cmake --install build
 
+      - name: Build Python interface
+        run: |
+          pip install -r python/build-requirements.txt
+          pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type="Debug" 'python/[docs]'
+
       - name: Build C++ interface documentation
         run: |
           export DOLFINX_VERSION=`cmake -L build | grep DOXYGEN_DOLFINX_VERSION | cut -f2 -d "="`
           echo $DOLFINX_VERSION
           cd cpp/doc
           doxygen Doxyfile
           make html
-
-      - name: Install required Python (build Python interface)
-        run: pip install nanobind scikit-build-core[pyproject]
-
-      - name: Build Python interface
-        run: pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type="Debug" python/
+      - name: Upload C++ Doxygen documentation artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-cpp-doxygen
+          path: cpp/doc/html 
+          retention-days: 2 
+      - name: Upload C++ Sphinx documentation artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-cpp-sphinx
+          path: cpp/doc/build/html 
+          retention-days: 2 
+      
       - name: Build Python interface documentation
         run: |
           cd python/doc
           python -m sphinx -W -b html source/ build/html/
+      - name: Upload Python documentation artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-python
+          path: python/doc/build/html 
+          retention-days: 2 
 
       - name: Checkout FEniCS/docs
+        if: ${{ github.repository == 'FEniCS/dolfinx' && ( github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }}
         uses: actions/checkout@v4
         with:
           repository: "FEniCS/docs"
           path: "docs"
           ssh-key: "${{ secrets.SSH_GITHUB_DOCS_PRIVATE_KEY }}"
       - name: Set version name
+        if: ${{ github.repository == 'FEniCS/dolfinx' && ( github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }}
         run: |
           echo "VERSION_NAME=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Copy documentation into repository
+        if: ${{ github.repository == 'FEniCS/dolfinx' && ( github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }}
         run: |
           cd docs
           git rm -r --ignore-unmatch dolfinx/${{ env.VERSION_NAME }}/cpp
@@ -320,6 +323,7 @@ jobs:
           cp -r ../cpp/doc/html/* dolfinx/${{ env.VERSION_NAME }}/cpp/doxygen
           cp -r ../python/doc/build/html/* dolfinx/${{ env.VERSION_NAME }}/python
       - name: Commit and push documentation to FEniCS/docs
+        if: ${{ github.repository == 'FEniCS/dolfinx' && ( github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }}
         run: |
           cd docs
           git config --global user.email "fenics@github.com"
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -25,11 +25,10 @@ jobs:
           brew install adios2 boost cmake hdf5-mpi make ninja open-mpi pkg-config pugixml spdlog # FEniCS
           brew install bison flex gfortran # PETSc
 
-      - name: Install Python dependencies
+      - name: Install Python dependencies (petsc4py)
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install cffi cppimport cython numba numpy pytest pytest-xdist scipy
-          python -m pip install --no-build-isolation mpi4py
+          python -m pip install mpi4py numpy
+          python -m pip install cython setuptools wheel
 
       - name: Install minimal PETSc and petsc4py
         run: |
@@ -91,7 +90,7 @@ jobs:
         working-directory: dolfinx
         run: |
           python -m pip install -r python/build-requirements.txt
-          python -m pip install --check-build-dependencies --no-build-isolation python/
+          python -m pip install --check-build-dependencies --no-build-isolation 'python/[test]'
 
       - name: Basic test
         run: |
@@ -100,7 +99,10 @@ jobs:
 
       - name: Run Python unit tests (serial)
         working-directory: dolfinx
-        run: python3 -m pytest -n=auto --durations=50 python/test/unit/
+        run: |
+          python -m pip install pytest-xdist
+          python3 -m pytest -n=auto --durations=50 python/test/unit/
       - name: Run Python unit tests (MPI, np=3)
         working-directory: dolfinx
-        run: mpirun -np 3 python3 -m pytest python/test/unit/
+        run: |
+          mpirun -np 3 python3 -m pytest python/test/unit/
diff --git a/.github/workflows/pyvista.yml b/.github/workflows/pyvista.yml
@@ -38,15 +38,15 @@ jobs:
 
       - name: Install FEniCS Python components
         run: |
-          python3 -m pip install git+https://github.com/FEniCS/ufl.git
-          python3 -m pip install git+https://github.com/FEniCS/basix.git
-          python3 -m pip install git+https://github.com/FEniCS/ffcx.git
+          pip install git+https://github.com/FEniCS/ufl.git
+          pip install git+https://github.com/FEniCS/basix.git
+          pip install git+https://github.com/FEniCS/ffcx.git
           apt-get update
           apt-get install -y --no-install-recommends libgl1-mesa-dev xvfb  # pyvista
           apt-get install -y --no-install-recommends libqt5gui5t64 libgl1 # pyvistaqt
-          python3 -m pip install pyvista==${PYVISTA_VERSION}
-          python3 -m pip install pyqt5 pyvistaqt==${PYVISTA_QT_VERSION}
-          python3 -m pip install matplotlib ipython nanobind scikit-build-core[pyproject] pytest pytest-xdist scipy numba
+          pip install pyvista==${PYVISTA_VERSION}
+          pip install pyqt5 pyvistaqt==${PYVISTA_QT_VERSION}
+          pip install --no-build-isolation -r python/build-requirements.txt
 
       - name: Configure C++
         run: cmake -G Ninja -DCMAKE_BUILD_TYPE=Developer -B build -S cpp/
@@ -56,10 +56,12 @@ jobs:
           cmake --install build
 
       - name: Build Python interface
-        run: pip -v install --check-build-dependencies --config-settings=build-dir="build" --config-settings=cmake.build-type="Debug" --no-build-isolation python/
+        run: pip -v install --check-build-dependencies --config-settings=build-dir="build" --config-settings=cmake.build-type="Debug" --no-build-isolation 'python/[test]'
 
       - name: Run pyvista demos (Python, serial)
-        run: python3 -m pytest -v -n 2 -m serial --durations=10 python/demo/test.py
+        run: |
+          pip install pytest-xdist
+          python3 -m pytest -v -n 2 -m serial --durations=10 python/demo/test.py
 
       - name: Run pyvista demos (Python, MPI (np=2))
         run: python3 -m pytest -v -m mpi --num-proc=2 python/demo/test.py
diff --git a/.github/workflows/redhat.yml b/.github/workflows/redhat.yml
@@ -24,13 +24,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install Python dependencies
+      - name: Install Python build dependencies
         run: |
-          python3 -m pip install mpi4py nanobind scikit-build-core[pyproject] --upgrade
+          python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel
 
       - name: Install FEniCS Python components
         run: |
-          python3 -m pip install --no-cache-dir --upgrade pip setuptools
           python3 -m pip install git+https://github.com/FEniCS/ufl.git
           python3 -m pip install git+https://github.com/FEniCS/basix.git
           python3 -m pip install git+https://github.com/FEniCS/ffcx.git
@@ -64,21 +63,20 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install cffi==1.16.0 numba scikit-build-core[pyproject] pytest pytest-xdist scipy matplotlib
       - name: Build Python interface (editable install)
         run: |
-          python3 -m pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type=Debug --config-settings=build-dir="build" -e python/
+          python3 -m pip install --upgrade -r python/build-requirements.txt
+          python3 -m pip install --check-build-dependencies --no-build-isolation --config-settings=cmake.build-type=Debug --config-settings=build-dir="build" -e 'python/[test]'
 
       - name: Set default DOLFINx JIT options
         run: |
           mkdir -p ~/.config/dolfinx
           echo '{ "cffi_extra_compile_args": ["-g0", "-O0" ] }' > ~/.config/dolfinx/dolfinx_jit_options.json
 
-      - name: Install test Python dependencies
-        run: python3 -m pip install matplotlib numba pytest pytest-xdist scipy
-
       - name: Run demos (Python, serial)
-        run: python3 -m pytest -n auto -m serial --durations=10 python/demo/test.py
+        run: |
+          python3 -m pip install pytest-xdist
+          python3 -m pytest -n auto -m serial --durations=10 python/demo/test.py
       - name: Run demos (Python, MPI (np=2))
         run: python3 -m pytest -m mpi --num-proc=2 python/demo/test.py
 
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -58,9 +58,9 @@ jobs:
           echo "$HOME/.sonar/build-wrapper-linux-x86" >> $GITHUB_PATH
       - name: Install FEniCS Python components
         run: |
-          python3 -m pip install --break-system-packages git+https://github.com/FEniCS/ufl.git
-          python3 -m pip install --break-system-packages git+https://github.com/FEniCS/basix.git
-          python3 -m pip install --break-system-packages git+https://github.com/FEniCS/ffcx.git
+          python3 -m pip install git+https://github.com/FEniCS/ufl.git
+          python3 -m pip install git+https://github.com/FEniCS/basix.git
+          python3 -m pip install git+https://github.com/FEniCS/ffcx.git
       - name: Run build-wrapper
         run: |
           mkdir build
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -60,12 +60,13 @@ jobs:
 
       - name: Install build dependencies (workaround)
         run: |
-          python -m pip install scikit-build-core[pyproject] git+https://github.com/jhale/nanobind.git@jhale/msvc2022-workaround setuptools wheel
+          python -m pip install git+https://github.com/jhale/nanobind.git@jhale/msvc2022-workaround
+          python -m pip install scikit-build-core[pyproject] setuptools wheel
       - name: Install Basix (Python)
         working-directory: basix
         run: |
           cd python
-          python -m pip -v install --no-build-isolation --no-cache-dir .[ci] --config-settings=cmake.args=-DBasix_DIR=D:/a/basix/install/lib/cmake/basix
+          python -m pip -v install --check-build-dependencies --no-build-isolation --no-cache-dir .[ci] --config-settings=cmake.args=-DBasix_DIR=D:/a/basix/install/lib/cmake/basix
           cd ../
 
       - name: Checkout FFCx
@@ -140,6 +141,7 @@ jobs:
       - name: Run units tests (Python, serial)
         working-directory: dolfinx
         run: |
+          pip install pytest-xdist
           python -m pytest -n auto -m "not petsc4py and not adios2" python/test/unit
       - name: Run units tests (Python, MPI, np=3)
         working-directory: dolfinx
@@ -149,7 +151,6 @@ jobs:
       - name: Run Python demos (serial)
         working-directory: dolfinx
         run: |
-          pip install pyamg matplotlib numba scipy # TO REMOVE
           cd python/demo
           python3 -m pytest -n auto -m serial --durations=10 test.py
       - name: Run Python demos (MPI, np=3)
diff --git a/docker/Dockerfile.test-env b/docker/Dockerfile.test-env
diff --git a/python/dolfinx/utils.py b/python/dolfinx/utils.py
diff --git a/python/pyproject.toml b/python/pyproject.toml
