diff --git a/.github/workflows/benchmarks_run.yml b/.github/workflows/benchmarks_run.yml index c91e7841a1..36e7c5e0fb 100644 --- a/.github/workflows/benchmarks_run.yml +++ b/.github/workflows/benchmarks_run.yml @@ -66,7 +66,6 @@ jobs: IRIS_TEST_DATA_PATH: benchmarks/iris-test-data IRIS_TEST_DATA_VERSION: "2.28" # Lets us manually bump the cache to rebuild - ENV_CACHE_BUILD: "0" TEST_DATA_CACHE_BUILD: "2" steps: @@ -79,15 +78,10 @@ jobs: - name: Install run dependencies run: pip install asv nox!=2025.05.01 - - name: Cache environment directories - id: cache-env-dir - uses: actions/cache@v4 - with: - path: | - .nox - benchmarks/.asv/env - $CONDA/pkgs - key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. + - name: Cache Conda package directory + uses: ./.github/workflows/composite/conda-pkg-cache - name: Cache test data directory id: cache-test-data diff --git a/.github/workflows/benchmarks_validate.yml b/.github/workflows/benchmarks_validate.yml index e3f090b32c..26930b501d 100644 --- a/.github/workflows/benchmarks_validate.yml +++ b/.github/workflows/benchmarks_validate.yml @@ -21,10 +21,6 @@ jobs: validate: runs-on: ubuntu-latest - env: - # Lets us manually bump the cache to rebuild - ENV_CACHE_BUILD: "0" - steps: - name: Checkout repo uses: actions/checkout@v5 @@ -34,15 +30,10 @@ jobs: - name: Install run dependencies run: pip install asv nox!=2025.05.01 - - name: Cache environment directories - id: cache-env-dir - uses: actions/cache@v4 - with: - path: | - .nox - benchmarks/.asv/env - $CONDA/pkgs - key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. + - name: Cache Conda package directory + uses: ./.github/workflows/composite/conda-pkg-cache - name: Validate setup run: nox -s benchmarks -- validate diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5df55143bd..9cb036031d 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -73,12 +73,12 @@ jobs: env_name: ${{ env.ENV_NAME }} version: ${{ env.IRIS_TEST_DATA_VERSION }} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache with: - cache_build: 6 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} + python_version: ${{ matrix.python-version }} - name: "conda install" uses: conda-incubator/setup-miniconda@v3 @@ -87,14 +87,12 @@ jobs: channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false + # This aligns the top-level env with the env that Nox will create + # below - meaning optimal use of package caching. + python-version: ${{ matrix.python-version }} - - name: "conda environment cache" - uses: ./.github/workflows/composite/conda-env-cache - with: - cache_build: 6 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} - install_packages: "cartopy nox pip" + - name: "install cartopy + nox + pip" + run: conda install --quiet --name ${{ env.ENV_NAME }} cartopy nox pip - name: "conda info" run: | @@ -108,13 +106,6 @@ jobs: cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache - with: - cache_build: 6 - env_name: ${{ env.ENV_NAME }} - lock_file: ${{ env.LOCK_FILE }} - # TODO: drop use of site.cfg and explicit use of mplrc - name: "iris configure" env: diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 7d7eea830b..24342c3db4 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -74,36 +74,26 @@ jobs: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} echo "LOCK_FILE=requirements/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache with: - cache_build: 0 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} + python_version: ${{ matrix.python-version }} - name: "conda install" uses: conda-incubator/setup-miniconda@v3 with: miniforge-version: latest - channels: conda-forge,defaults + channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false - use-only-tar-bz2: true + # This aligns the top-level env with the env that Nox will create + # below - meaning optimal use of package caching. + python-version: ${{ matrix.python-version }} - - name: "conda environment cache" - uses: ./.github/workflows/composite/conda-env-cache - with: - cache_build: 0 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} - install_packages: "nox pip" - - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache - with: - cache_build: 1 - env_name: ${{ env.ENV_NAME }} - lock_file: ${{ env.LOCK_FILE }} + - name: "install nox + pip" + run: conda install --quiet --name ${{ env.ENV_NAME }} nox pip - name: "nox install and test wheel" env: diff --git a/.github/workflows/composite/conda-env-cache/action.yml b/.github/workflows/composite/conda-env-cache/action.yml deleted file mode 100644 index 15eaaec63c..0000000000 --- a/.github/workflows/composite/conda-env-cache/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "conda-env-cache" -description: "create and cache the conda environment" - -# -# Assumes the environment contains the following variables: -# - CONDA -# -inputs: - cache_build: - description: "conda environment cache build number" - required: false - default: "0" - cache_period: - description: "conda environment cache timestamp" - required: true - env_name: - description: "environment name" - required: true - install_packages: - description: "conda packages to install into environment" - required: true - -runs: - using: "composite" - steps: - - uses: actions/cache@v4 - id: conda-env-cache - with: - path: ${{ env.CONDA }}/envs/${{ inputs.env_name }} - key: ${{ runner.os }}-conda-env-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }} - - - if: steps.conda-env-cache.outputs.cache-hit != 'true' - shell: bash - run: | - conda install --quiet --name ${{ inputs.env_name }} ${{ inputs.install_packages }} diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 48c4470e44..616c76df96 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -1,22 +1,42 @@ name: "conda-pkg-cache" -description: "cache the conda environment packages" +description: "standardise conda package cache to ~/conda_pkgs_dir and cache across workflow runs" inputs: - cache_build: - description: "conda environment cache build number" + python_version: + description: "conda package cache python version" required: false - default: "0" - cache_period: - description: "conda environment cache timestamp" - required: true - env_name: - description: "environment name" - required: true + default: "3.13" runs: using: "composite" steps: - - uses: actions/cache@v4 + - id: set-cache-period + shell: bash + env: + # Maximum cache period (in weeks) before forcing a cache refresh.. + CACHE_WEEKS: 2 + run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> $GITHUB_OUTPUT + + - id: define-cache-key + shell: bash + env: + # Lets us manually bump the cache to rebuild. + CACHE_BUILD: "0" + run: | + OS="${{ runner.os }}" + PERIOD="${{ steps.set-cache-period.outputs.CACHE_PERIOD }}" + BUILD="${{ env.CACHE_BUILD }}" + PY="${{ inputs.python_version }}" + REQ="${{ hashFiles('requirements/locks/**') }}" + CACHE_KEY="conda_pkgs_dir-${OS}-p${PERIOD}-b${BUILD}-py${PY}-${REQ}" + echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV + + - id: set-pkg-dir + shell: bash + run: conda config --add pkgs_dirs ~/conda_pkgs_dir + + - id: conda-pkg-cache + uses: actions/cache@v4 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-pkgs-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }} + key: ${{ env.CACHE_KEY }} diff --git a/.github/workflows/composite/nox-cache/action.yml b/.github/workflows/composite/nox-cache/action.yml deleted file mode 100644 index 00387331e7..0000000000 --- a/.github/workflows/composite/nox-cache/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "nox cache" -description: "cache the nox test environments" - -inputs: - cache_build: - description: "nox cache build number" - required: false - default: "0" - env_name: - description: "environment name" - required: true - lock_file: - description: "conda-lock environment requirements filename" - required: true - -runs: - using: "composite" - steps: - - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/.nox - key: ${{ runner.os }}-nox-${{ inputs.env_name }}-s${{ matrix.session }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }} diff --git a/lib/iris/tests/test_coding_standards.py b/lib/iris/tests/test_coding_standards.py index f51a531721..13abceb858 100644 --- a/lib/iris/tests/test_coding_standards.py +++ b/lib/iris/tests/test_coding_standards.py @@ -83,6 +83,7 @@ def test_python_versions(): nox_file = root_dir / "noxfile.py" ci_wheels_file = workflows_dir / "ci-wheels.yml" ci_tests_file = workflows_dir / "ci-tests.yml" + ci_pkg_cache_file = workflows_dir / "composite" / "conda-pkg-cache" / "action.yml" benchmark_runner_file = benchmarks_dir / "bm_runner.py" text_searches: List[Tuple[Path, str]] = [ @@ -107,6 +108,7 @@ def test_python_versions(): f'{" " * 8}session: ["doctest", "gallery"]' ), ), + (ci_pkg_cache_file, f'default: "{latest_supported}"'), (benchmark_runner_file, f'python_version = "{latest_supported}"'), ]