Merge-forward to FEATURE_split_attrs

.github/workflows/benchmark.yml

@@ -6,6 +6,12 @@ on:
   schedule:
     # Runs every day at 23:00.
     - cron: "0 23 * * *"
+  workflow_dispatch:
+    inputs:
+      first_commit:
+        description: "Argument to be passed to the overnight benchmark script."
+        required: false
+        type: string
 
 jobs:
   benchmark:
@@ -15,7 +21,7 @@ jobs:
     env:
       IRIS_TEST_DATA_LOC_PATH: benchmarks
       IRIS_TEST_DATA_PATH: benchmarks/iris-test-data
-      IRIS_TEST_DATA_VERSION: "2.15"
+      IRIS_TEST_DATA_VERSION: "2.18"
       # Lets us manually bump the cache to rebuild
       ENV_CACHE_BUILD: "0"
       TEST_DATA_CACHE_BUILD: "2"
@@ -64,7 +70,12 @@ jobs:
 
       - name: Run overnight benchmarks
         run: |
-          first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1)
+          first_commit=${{ inputs.first_commit }}
+          if [ "$first_commit" == "" ]
+          then
+            first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1)
+          fi
+
           if [ "$first_commit" != "" ]
           then
             nox --session="benchmarks(overnight)" -- $first_commit

.github/workflows/ci-tests.yml

@@ -46,7 +46,7 @@ jobs:
             session: "tests"
 
     env:
-      IRIS_TEST_DATA_VERSION: "2.16"
+      IRIS_TEST_DATA_VERSION: "2.18"
       ENV_NAME: "ci-tests"
 
     steps:

.github/workflows/refresh-lockfiles.yml

@@ -1,13 +1,5 @@
-# This workflow periodically creates new environment lock files based on the newest
-# available packages and dependencies.
-#
-# Environment specifications are given as conda environment.yml files found in
-# `requirements/ci/py**.yml`.  These state the packages required, the conda channels
-# that the packages will be pulled from, and any versions of packages that need to be
-# pinned at specific versions.
-#
-# For environments that have changed, a pull request will be made and submitted
-# to the main branch
+# Updates the environment lock files. See the called workflow in the
+# scitools/workflows repo for more details.
 
 name: Refresh Lockfiles
 
@@ -20,98 +12,7 @@ on:
     # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule
     - cron: "1 0 * * 6"
 
-
 jobs:
-
-  get_python_matrix:
-    # Determines which Python versions should be included in the matrix used in
-    # the gen_lockfiles job.
-    if: "github.repository == 'SciTools/iris'"
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.get_py.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@v3
-      - id: get_py
-        run: echo "::set-output name=matrix::$(ls -1 requirements/ci/py*.yml | xargs -n1 basename | sed 's/....$//' | jq -cnR '[inputs]')"
-
-  gen_lockfiles:
-    # this is a matrix job: it splits to create new lockfiles for each
-    # of the CI test python versions.
-    if: "github.repository == 'SciTools/iris'"
-    runs-on: ubuntu-latest
-    needs: get_python_matrix
-
-    strategy:
-      matrix:
-        python: ${{ fromJSON(needs.get_python_matrix.outputs.matrix) }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: install requirements
-        run: |
-          source $CONDA/bin/activate base
-          conda install -y -c conda-forge conda-libmamba-solver conda-lock
-      - name: generate lockfile
-        env:
-          CONDA_EXPERIMENTAL_SOLVER: libmamba
-        run: |
-          $CONDA/bin/conda-lock lock -k explicit -p linux-64 -f requirements/ci/${{matrix.python}}.yml
-          mv conda-linux-64.lock ${{matrix.python}}-linux-64.lock
-      - name: output lockfile
-        uses: actions/upload-artifact@v3
-        with:
-          path: ${{matrix.python}}-linux-64.lock
-
-  create_pr:
-    # once the matrix job has completed all the lock files will have been uploaded as artifacts.
-    # Download the artifacts, add them to the repo, and create a PR.
-    if: "github.repository == 'SciTools/iris'"
-    runs-on: ubuntu-latest
-    needs: gen_lockfiles
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: get artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: artifacts
-
-      - name: Update lock files in repo
-        run: |
-          cp artifacts/artifact/*.lock requirements/ci/nox.lock
-          rm -r artifacts
-
-      - name: "Generate token"
-        uses: tibdex/github-app-token@v1
-        id: generate-token
-        with:
-          app_id: ${{ secrets.AUTH_APP_ID }}
-          private_key: ${{ secrets.AUTH_APP_PRIVATE_KEY }}
-
-      - name: Create Pull Request
-        id: cpr
-        uses: peter-evans/create-pull-request@671dc9c9e0c2d73f07fa45a3eb0220e1622f0c5f
-        with:
-          token: ${{ steps.generate-token.outputs.token }}
-          commit-message: Updated environment lockfiles
-          committer: "Lockfile bot <[email protected]>"
-          author: "Lockfile bot <[email protected]>"
-          delete-branch: true
-          branch: auto-update-lockfiles
-          title: "[iris.ci] environment lockfiles auto-update"
-          body: |
-            Lockfiles updated to the latest resolvable environment.
-
-            ### If the CI tasks fail, create a new branch based on this PR and add the required fixes to that branch.
-          labels: |
-            New: Pull Request
-            Bot
-
-      - name: Check Pull Request
-        if: steps.cpr.outputs.pull-request-number != ''
-        run: |
-          echo "pull-request #${{ steps.cpr.outputs.pull-request-number }}"
-          echo "pull-request URL ${{ steps.cpr.outputs.pull-request-url }}"
-          echo "pull-request operation [${{ steps.cpr.outputs.pull-request-operation }}]"
-          echo "pull-request head SHA ${{ steps.cpr.outputs.pull-request-head-sha }}"
+  refresh_lockfiles:
+    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@main
+    secrets: inherit

.github/workflows/stale.yml

@@ -14,7 +14,7 @@ jobs:
     if: "github.repository == 'SciTools/iris'"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v6
+      - uses: actions/stale@v7
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ minimum_pre_commit_version: 1.21.0
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
         # Prevent giant files from being committed.
     -   id: check-added-large-files
@@ -29,32 +29,31 @@ repos:
     -   id: no-commit-to-branch
 
 -   repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 23.1.0
     hooks:
     -   id: black
         pass_filenames: false
         args: [--config=./pyproject.toml, .]
 
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
     -   id: flake8
         types: [file, python]
         args: [--config=./setup.cfg]
 
 -   repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
     -   id: isort
         types: [file, python]
         args: [--filter-files]
 
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.12.1
+    rev: 1.13.0
     hooks:
     -   id: blacken-docs
         types: [file, rst]
-        additional_dependencies: [black==21.6b0]
 
 -   repo: https://github.com/aio-libs/sort-all
     rev: v1.2.0

README.md

@@ -54,3 +54,24 @@ For documentation see the
 developer version or the most recent released
 <a href="https://scitools-iris.readthedocs.io/en/stable/">stable</a> version.
 </p>
+
+## [#ShowYourStripes](https://showyourstripes.info/s/globe)
+
+<h4 align="center">
+  <a href="https://showyourstripes.info/s/globe">
+    <img src="https://raw.githubusercontent.com/ed-hawkins/show-your-stripes/master/2021/GLOBE---1850-2021-MO.png"
+         height="50" width="800"
+         alt="#showyourstripes Global 1850-2021"></a>
+</h4>
+
+**Graphics and Lead Scientist**: [Ed Hawkins](http://www.met.reading.ac.uk/~ed/home/index.php), National Centre for Atmospheric Science, University of Reading.
+
+**Data**: Berkeley Earth, NOAA, UK Met Office, MeteoSwiss, DWD, SMHI, UoR, Meteo France & ZAMG.
+
+<p>
+<a href="https://showyourstripes.info/s/globe">#ShowYourStripes</a> is distributed under a
+<a href="https://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>
+<a href="https://creativecommons.org/licenses/by/4.0/">
+  <img src="https://i.creativecommons.org/l/by/4.0/80x15.png" alt="creative-commons-by" style="border-width:0"></a>
+</p>
+

benchmarks/benchmarks/experimental/ugrid/__init__.py

@@ -50,7 +50,7 @@ def time_create(self, *params):
 
 class Connectivity(UGridCommon):
     def setup(self, n_faces):
-        self.array = np.zeros([n_faces, 3], dtype=np.int)
+        self.array = np.zeros([n_faces, 3], dtype=int)
         super().setup(n_faces)
 
     def create(self):

benchmarks/benchmarks/generate_data/__init__.py

@@ -113,7 +113,7 @@ def load_realised():
     file loading, but some benchmarks are only meaningful if starting with real
     arrays.
     """
-    from iris.fileformats.netcdf import _get_cf_var_data as pre_patched
+    from iris.fileformats.netcdf.loader import _get_cf_var_data as pre_patched
 
     def patched(cf_var, filename):
         return as_concrete_data(pre_patched(cf_var, filename))

benchmarks/benchmarks/import_iris.py

@@ -5,10 +5,30 @@
 # licensing details.
 from importlib import import_module, reload
 
+################
+# Prepare info for reset_colormaps:
+
+# Import and capture colormaps.
+from matplotlib import colormaps  # isort:skip
+
+_COLORMAPS_ORIG = set(colormaps)
+
+# Import iris.palette, which modifies colormaps.
+import iris.palette
+
+# Derive which colormaps have been added by iris.palette.
+_COLORMAPS_MOD = set(colormaps)
+COLORMAPS_EXTRA = _COLORMAPS_MOD - _COLORMAPS_ORIG
+
+# Touch iris.palette to prevent linters complaining.
+_ = iris.palette
+
+################
+
 
 class Iris:
     @staticmethod
-    def _import(module_name):
+    def _import(module_name, reset_colormaps=False):
         """
         Have experimented with adding sleep() commands into the imported
         modules. The results reveal:
@@ -25,6 +45,13 @@ def _import(module_name):
         and the repetitions are therefore no faster than the first run.
         """
         mod = import_module(module_name)
+
+        if reset_colormaps:
+            # Needed because reload() will attempt to register new colormaps a
+            #  second time, which errors by default.
+            for cm_name in COLORMAPS_EXTRA:
+                colormaps.unregister(cm_name)
+
         reload(mod)
 
     def time_iris(self):
@@ -205,7 +232,7 @@ def time_iterate(self):
         self._import("iris.iterate")
 
     def time_palette(self):
-        self._import("iris.palette")
+        self._import("iris.palette", reset_colormaps=True)
 
     def time_plot(self):
         self._import("iris.plot")

benchmarks/benchmarks/regridding.py

@@ -12,8 +12,11 @@
 # importing anything else
 from iris import tests  # isort:skip
 
+import numpy as np
+
 import iris
-from iris.analysis import AreaWeighted
+from iris.analysis import AreaWeighted, PointInCell
+from iris.coords import AuxCoord
 
 
 class HorizontalChunkedRegridding:
@@ -53,3 +56,48 @@ def time_regrid_area_w_new_grid(self) -> None:
         out = self.chunked_cube.regrid(self.template_cube, self.scheme_area_w)
         # Realise data
         out.data
+
+
+class CurvilinearRegridding:
+    def setup(self) -> None:
+        # Prepare a cube and a template
+
+        cube_file_path = tests.get_data_path(
+            ["NetCDF", "regrid", "regrid_xyt.nc"]
+        )
+        self.cube = iris.load_cube(cube_file_path)
+
+        # Make the source cube curvilinear
+        x_coord = self.cube.coord("longitude")
+        y_coord = self.cube.coord("latitude")
+        xx, yy = np.meshgrid(x_coord.points, y_coord.points)
+        self.cube.remove_coord(x_coord)
+        self.cube.remove_coord(y_coord)
+        x_coord_2d = AuxCoord(
+            xx,
+            standard_name=x_coord.standard_name,
+            units=x_coord.units,
+            coord_system=x_coord.coord_system,
+        )
+        y_coord_2d = AuxCoord(
+            yy,
+            standard_name=y_coord.standard_name,
+            units=y_coord.units,
+            coord_system=y_coord.coord_system,
+        )
+        self.cube.add_aux_coord(x_coord_2d, (1, 2))
+        self.cube.add_aux_coord(y_coord_2d, (1, 2))
+
+        template_file_path = tests.get_data_path(
+            ["NetCDF", "regrid", "regrid_template_global_latlon.nc"]
+        )
+        self.template_cube = iris.load_cube(template_file_path)
+
+        # Prepare a regridding scheme
+        self.scheme_pic = PointInCell()
+
+    def time_regrid_pic(self) -> None:
+        # Regrid the cube onto the template.
+        out = self.cube.regrid(self.template_cube, self.scheme_pic)
+        # Realise the data
+        out.data

docs/Makefile

@@ -20,11 +20,6 @@ html-quick:
         echo "make html-quick in $$i..."; \
         (cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) html-quick); done
 
-spelling:
-	@for i in $(SUBDIRS); do \
-        echo "make spelling in $$i..."; \
-        (cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) spelling); done
-
 all:
 	@for i in $(SUBDIRS); do \
 		echo "make all in $$i..."; \
@@ -55,3 +50,8 @@ linkcheck:
 		echo "Running linkcheck in $$i..."; \
 		(cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) linkcheck); done
 
+show:
+	@for i in $(SUBDIRS); do \
+		echo "Running show in $$i..."; \
+		(cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) show); done
+