From cc997fa8fd5f8290e77217ba80c4798206384b4e Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 1 Jul 2022 23:40:46 +0100 Subject: [PATCH 01/12] automate sdist and wheel build and publish --- .github/workflows/ci-build.yml | 91 ++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/ci-build.yml diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 0000000000..9932a62d43 --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,91 @@ +# Reference: +# - https://github.com/actions/checkout +# - https://github.com/actions/download-artifact +# - https://github.com/actions/upload-artifact +# - https://github.com/pypa/build +# - https://github.com/pypa/gh-action-pypi-publish +# - https://test.pypi.org/help/#apitoken + +name: ci-build + +on: + pull_request: + + push: + tags: + - "v*" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: "Build sdist and wheel" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: "Building" + shell: bash + run: | + pipx run build --sdist --wheel + + - uses: actions/upload-artifact@v3 + with: + name: pypi-artifacts + path: ${{ github.workspace }}/dist/* + + show-artifacts: + needs: [build] + name: "Show artifacts" + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v3 + with: + name: pypi-artifacts + path: ${{ github.workspace }}/dist + + - shell: bash + run: | + ls -l ${{ github.workspace }}/dist + + publish-artifacts-test-pypi: + needs: [build] + name: "Publish to Test PyPI" + runs-on: ubuntu-latest + # upload to Test PyPI for every commit on main branch + if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' + steps: + - uses: actions/download-artifact@v3 + with: + name: pypi-artifacts + path: ${{ github.workspace }}/dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + skip_existing: true + print_hash: true + + publish-artifacts-pypi: + needs: [build] + name: "Publish to PyPI" + runs-on: ubuntu-latest + # upload to PyPI for every tag starting with 'v' + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v3 + with: + name: pypi-artifacts + path: ${{ github.workspace }}/dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + print_hash: true \ No newline at end of file From 3f28aa35db96c01a9e94d337c4a50d398110ac2e Mon Sep 17 00:00:00 2001 From: Bill Little Date: Sat, 2 Jul 2022 00:14:34 +0100 Subject: [PATCH 02/12] update names --- .github/workflows/ci-tests.yml | 2 +- .github/workflows/{ci-build.yml => ci-wheels.yml} | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) rename .github/workflows/{ci-build.yml => ci-wheels.yml} (93%) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4cdc3f5e94..2657e1e42a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -3,7 +3,7 @@ # - https://github.com/actions/checkout # - https://github.com/marketplace/actions/setup-miniconda -name: ci-tests +name: ci-tests on: push: diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-wheels.yml similarity index 93% rename from .github/workflows/ci-build.yml rename to .github/workflows/ci-wheels.yml index 9932a62d43..7feb80211c 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-wheels.yml @@ -6,7 +6,7 @@ # - https://github.com/pypa/gh-action-pypi-publish # - https://test.pypi.org/help/#apitoken -name: ci-build +name: ci-wheels on: pull_request: @@ -21,14 +21,14 @@ concurrency: jobs: build: - name: "Build sdist and wheel" + name: "build sdist & wheel" runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: "Building" + - name: "building" shell: bash run: | pipx run build --sdist --wheel @@ -40,7 +40,7 @@ jobs: show-artifacts: needs: [build] - name: "Show artifacts" + name: "show artifacts" runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 @@ -54,7 +54,7 @@ jobs: publish-artifacts-test-pypi: needs: [build] - name: "Publish to Test PyPI" + name: "publish to test.pypi" runs-on: ubuntu-latest # upload to Test PyPI for every commit on main branch if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' @@ -74,7 +74,7 @@ jobs: publish-artifacts-pypi: needs: [build] - name: "Publish to PyPI" + name: "publish to pypi" runs-on: ubuntu-latest # upload to PyPI for every tag starting with 'v' if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') From 13824d8b33d48098590ab25aa532475a60b8efd4 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 5 Jul 2022 15:34:17 +0100 Subject: [PATCH 03/12] tidy package manifest and discovery --- MANIFEST.in | 19 ++++++++----------- setup.cfg | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 81d7165199..9c7f8b0ba7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,15 +1,13 @@ # Top-level files include CHANGES COPYING COPYING.LESSER +prune .github +exclude .gitignore -# Files from setup.py package_data that are not automatically added to source distributions -recursive-include lib/iris/tests/results *.cml *.cdl *.txt *.xml *.json -recursive-include lib/iris/etc * -include lib/iris/tests/stock/file_headers/* +# Files required for conda package management +recursive-include requirements *.yml *.lock -recursive-include requirements * - -# File required to build docs -recursive-include docs Makefile *.js *.png *.py *.rst +# Files required to build docs +recursive-include docs * prune docs/src/_build prune docs/src/generated prune docs/gallery_tests @@ -18,7 +16,6 @@ prune docs/gallery_tests include tools/generate_std_names.py include etc/cf-standard-name-table.xml -global-exclude *.pyc +global-exclude *.py[cod] global-exclude __pycache__ -global-exclude iris_image_test_output -exclude lib/iris/_version.py \ No newline at end of file +exclude lib/iris/_version.py diff --git a/setup.cfg b/setup.cfg index 66e865572e..a60d107835 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,7 +54,7 @@ install_requires = numpy>=1.19 scipy xxhash -packages = find: +packages = find_namespace: package_dir = =lib python_requires = From 0adba3a0294c131c5884fe8e8d3b6962528afaac Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 5 Jul 2022 16:12:41 +0100 Subject: [PATCH 04/12] add note to dev docs --- docs/src/developers_guide/release.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/src/developers_guide/release.rst b/docs/src/developers_guide/release.rst index 182ab482b3..d9c10826b8 100644 --- a/docs/src/developers_guide/release.rst +++ b/docs/src/developers_guide/release.rst @@ -123,6 +123,14 @@ conda package on the `conda-forge Anaconda channel`_. Update PyPI ----------- +.. note:: + + As part of our Continuous-Integration (CI), the building and publishing of + PyPI artifacts is now automated by a dedicated GitHub Action. + + The following instructions **no longer** require to be performed manually, + but remain part of the documentation for reference purposed only. + Update the `scitools-iris`_ project on PyPI with the latest Iris release. To do this perform the following steps. From 42a7a594b36193e62027443b371651aa216aa9e2 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 12 Jul 2022 21:27:28 +0100 Subject: [PATCH 05/12] update MANIFEST.in --- MANIFEST.in | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9c7f8b0ba7..ad28df9c7c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ prune .github exclude .gitignore # Files required for conda package management -recursive-include requirements *.yml *.lock +recursive-include requirements * # Files required to build docs recursive-include docs * @@ -17,5 +17,4 @@ include tools/generate_std_names.py include etc/cf-standard-name-table.xml global-exclude *.py[cod] -global-exclude __pycache__ -exclude lib/iris/_version.py +global-exclude __pycache__ \ No newline at end of file From d7c61d323cc156eeda4b620b893e7ff72c70f87c Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 12 Jul 2022 21:25:14 +0100 Subject: [PATCH 06/12] Update .github/workflows/ci-wheels.yml Co-authored-by: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> --- .github/workflows/ci-wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 7feb80211c..45efd9fa36 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -14,6 +14,10 @@ on: push: tags: - "v*" + branches-ignore: + - "auto-update-lockfiles" + - "pre-commit-ci-update-config" + - "dependabot/*" concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 234174659493392472eadc8b941c61fae25550f2 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 12 Jul 2022 21:31:32 +0100 Subject: [PATCH 07/12] add unshallow comment to rtd --- .readthedocs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 8ec8ab145c..58d5b26769 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,6 +6,9 @@ build: python: mambaforge-4.10 jobs: post_checkout: + # The SciTools/iris repository is shallow i.e., has a .git/shallow, + # therefore complete the repository with a full history in order + # to allow setuptools-scm to correctly auto-discover the version. - git fetch --unshallow - git fetch --all From 0565db35014956d4f6a2c276dfe33c04811d3d54 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 12 Jul 2022 21:32:54 +0100 Subject: [PATCH 08/12] reinstate docs release --- docs/src/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/conf.py b/docs/src/conf.py index 4f1eae7403..6b11e83a8c 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -89,7 +89,9 @@ def autolog(message): version = get_version("scitools-iris") if version.endswith("+dirty"): version = version[: -len("+dirty")] +release = version autolog(f"Iris Version = {version}") +autolog(f"Iris Release = {release}") # -- General configuration --------------------------------------------------- From 41b2e8de875c276e706fad9136771f9279b3dbdb Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 12 Jul 2022 21:36:22 +0100 Subject: [PATCH 09/12] add pypa build comment --- .github/workflows/ci-wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 45efd9fa36..6341a53ed1 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -35,6 +35,8 @@ jobs: - name: "building" shell: bash run: | + # require build with explicit --sdist and --wheel in order to + # get correct version associated with sdist and bdist artifacts pipx run build --sdist --wheel - uses: actions/upload-artifact@v3 From 373afbbc614526fb9b9b918a7a923f42fdd56ab8 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 12 Jul 2022 22:29:37 +0100 Subject: [PATCH 10/12] test wheel --- .github/workflows/ci-wheels.yml | 77 +++++++++++++++++++++++++++++++-- noxfile.py | 28 ++++++++++++ 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 6341a53ed1..265489883f 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -14,7 +14,7 @@ on: push: tags: - "v*" - branches-ignore: + branches-ignore: - "auto-update-lockfiles" - "pre-commit-ci-update-config" - "dependabot/*" @@ -44,8 +44,77 @@ jobs: name: pypi-artifacts path: ${{ github.workspace }}/dist/* + test-wheel: + needs: build + name: "test wheel (py${{ matrix.python-version }})" + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] + session: ["wheel"] + env: + ENV_NAME: "ci-wheels" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v3 + with: + name: pypi-artifacts + path: ${{ github.workspace }}/dist + + - name: "environment configure" + env: + # Maximum cache period (in weeks) before forcing a cache refresh. + CACHE_WEEKS: 2 + run: | + echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} + echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + + - name: "conda package cache" + uses: ./.github/workflows/composite/conda-pkg-cache + with: + cache_build: 0 + cache_period: ${{ env.CACHE_PERIOD }} + env_name: ${{ env.ENV_NAME }} + + - name: "conda install" + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-version: latest + channels: conda-forge,defaults + activate-environment: ${{ env.ENV_NAME }} + auto-update-conda: false + use-only-tar-bz2: true + + - name: "conda environment cache" + uses: ./.github/workflows/composite/conda-env-cache + with: + cache_build: 0 + cache_period: ${{ env.CACHE_PERIOD }} + env_name: ${{ env.ENV_NAME }} + install_packages: "nox pip" + + - name: "nox cache" + uses: ./.github/workflows/composite/nox-cache + with: + cache_build: 0 + env_name: ${{ env.ENV_NAME }} + lock_file: ${{ env.LOCK_FILE }} + + - name: "nox install and test wheel" + env: + PY_VER: ${{ matrix.python-version }} + run: | + nox --session ${{ matrix.session }} -- --verbose + show-artifacts: - needs: [build] + needs: build name: "show artifacts" runs-on: ubuntu-latest steps: @@ -59,7 +128,7 @@ jobs: ls -l ${{ github.workspace }}/dist publish-artifacts-test-pypi: - needs: [build] + needs: test-wheel name: "publish to test.pypi" runs-on: ubuntu-latest # upload to Test PyPI for every commit on main branch @@ -79,7 +148,7 @@ jobs: print_hash: true publish-artifacts-pypi: - needs: [build] + needs: test-wheel name: "publish to pypi" runs-on: ubuntu-latest # upload to PyPI for every tag starting with 'v' diff --git a/noxfile.py b/noxfile.py index 2b1df8fb00..7527f78ac5 100755 --- a/noxfile.py +++ b/noxfile.py @@ -271,6 +271,34 @@ def linkcheck(session: nox.sessions.Session): ) +@nox.session(python=PY_VER, venv_backend="conda") +def wheel(session: nox.sessions.Session): + """ + Perform iris local wheel install and import test. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + """ + prepare_venv(session) + session.cd("dist") + fname = list(Path(".").glob("scitools_iris-*.whl")) + if len(fname) == 0: + raise ValueError("Cannot find wheel.") + if len(fname) > 1: + emsg = f"Expected to find 1 wheel, found {len(fname)} instead." + raise ValueError(emsg) + session.install(fname[0].name) + session.run( + "python", + "-c", + "import iris; print(f'{iris.__version__=}')", + external=True, + ) + + @nox.session @nox.parametrize( "run_type", From 893e16654601a1c5736fc893e5d0ac1949596b63 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Wed, 13 Jul 2022 08:25:20 +0100 Subject: [PATCH 11/12] added whatnew + tweak nox wheel emsgs --- docs/src/whatsnew/latest.rst | 3 +++ noxfile.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 761833ba15..7ff731d4cb 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -254,6 +254,9 @@ This document explains the changes made to Iris for this release #. `@bjlittle`_ and `@trexfeathers`_ (reviewer) adopted `setuptools-scm`_ for automated ``iris`` package versioning. (:pull:`4841`) +#. `@bjlittle`_ and `@trexfeathers`_ (reviewer) added building, testing and + publishing of ``iris`` PyPI ``sdist`` and binary ``wheels`` as part of + our GitHub Continuous-Integration. (:pull`4849`) .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/noxfile.py b/noxfile.py index 7527f78ac5..8aabf862fb 100755 --- a/noxfile.py +++ b/noxfile.py @@ -286,9 +286,11 @@ def wheel(session: nox.sessions.Session): session.cd("dist") fname = list(Path(".").glob("scitools_iris-*.whl")) if len(fname) == 0: - raise ValueError("Cannot find wheel.") + raise ValueError("Cannot find wheel to install.") if len(fname) > 1: - emsg = f"Expected to find 1 wheel, found {len(fname)} instead." + emsg = ( + f"Expected to find 1 wheel to install, found {len(fname)} instead." + ) raise ValueError(emsg) session.install(fname[0].name) session.run( From cd88820c4fbf1ae6f2d25f9ed05ab0dd985064fb Mon Sep 17 00:00:00 2001 From: Bill Little Date: Wed, 13 Jul 2022 12:10:19 +0100 Subject: [PATCH 12/12] Update docs/src/developers_guide/release.rst --- docs/src/developers_guide/release.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/developers_guide/release.rst b/docs/src/developers_guide/release.rst index d9c10826b8..25a426e20b 100644 --- a/docs/src/developers_guide/release.rst +++ b/docs/src/developers_guide/release.rst @@ -129,7 +129,7 @@ Update PyPI PyPI artifacts is now automated by a dedicated GitHub Action. The following instructions **no longer** require to be performed manually, - but remain part of the documentation for reference purposed only. + but remain part of the documentation for reference purposes only. Update the `scitools-iris`_ project on PyPI with the latest Iris release.