Update pandas_ndim branch from main

.github/ISSUE_TEMPLATE/config.yml

@@ -4,3 +4,6 @@ contact_links:
   -  name: 💬 Iris GitHub Discussions
      url: https://github.com/SciTools/iris/discussions
      about: Engage with the Iris community to discuss your issue
+  -  name: ❓ Usage Question
+     url: https://github.com/SciTools/iris/discussions/categories/q-a
+     about: Raise a question about how to use Iris in the Q&A section of Discussions

.github/workflows/benchmark.yml

@@ -6,6 +6,12 @@ on:
   schedule:
     # Runs every day at 23:00.
     - cron: "0 23 * * *"
+  workflow_dispatch:
+    inputs:
+      first_commit:
+        description: "Argument to be passed to the overnight benchmark script."
+        required: false
+        type: string
 
 jobs:
   benchmark:
@@ -15,7 +21,7 @@ jobs:
     env:
       IRIS_TEST_DATA_LOC_PATH: benchmarks
       IRIS_TEST_DATA_PATH: benchmarks/iris-test-data
-      IRIS_TEST_DATA_VERSION: "2.15"
+      IRIS_TEST_DATA_VERSION: "2.18"
       # Lets us manually bump the cache to rebuild
       ENV_CACHE_BUILD: "0"
       TEST_DATA_CACHE_BUILD: "2"
@@ -64,7 +70,12 @@ jobs:
 
       - name: Run overnight benchmarks
         run: |
-          first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1)
+          first_commit=${{ inputs.first_commit }}
+          if [ "$first_commit" == "" ]
+          then
+            first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1)
+          fi
+
           if [ "$first_commit" != "" ]
           then
             nox --session="benchmarks(overnight)" -- $first_commit

.github/workflows/ci-tests.yml

@@ -46,7 +46,7 @@ jobs:
             session: "tests"
 
     env:
-      IRIS_TEST_DATA_VERSION: "2.15"
+      IRIS_TEST_DATA_VERSION: "2.18"
       ENV_NAME: "ci-tests"
 
     steps:

.github/workflows/ci-wheels.yml

@@ -163,4 +163,4 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
-        print_hash: true
+        print_hash: true

.github/workflows/refresh-lockfiles.yml

@@ -1,13 +1,5 @@
-# This workflow periodically creates new environment lock files based on the newest
-# available packages and dependencies.
-#
-# Environment specifications are given as conda environment.yml files found in
-# `requirements/ci/py**.yml`.  These state the packages required, the conda channels
-# that the packages will be pulled from, and any versions of packages that need to be
-# pinned at specific versions.
-#
-# For environments that have changed, a pull request will be made and submitted
-# to the main branch
+# Updates the environment lock files. See the called workflow in the
+# scitools/workflows repo for more details.
 
 name: Refresh Lockfiles
 
@@ -20,96 +12,7 @@ on:
     # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule
     - cron: "1 0 * * 6"
 
-
 jobs:
-
-  get_python_matrix:
-    # Determines which Python versions should be included in the matrix used in
-    # the gen_lockfiles job.
-    if: "github.repository == 'SciTools/iris'"
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.get_py.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@v3
-      - id: get_py
-        run: echo "::set-output name=matrix::$(ls -1 requirements/ci/py*.yml | xargs -n1 basename | sed 's/....$//' | jq -cnR '[inputs]')"
-
-  gen_lockfiles:
-    # this is a matrix job: it splits to create new lockfiles for each
-    # of the CI test python versions.
-    if: "github.repository == 'SciTools/iris'"
-    runs-on: ubuntu-latest
-    needs: get_python_matrix
-
-    strategy:
-      matrix:
-        python: ${{ fromJSON(needs.get_python_matrix.outputs.matrix) }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: install conda-lock
-        run: |
-          source $CONDA/bin/activate base
-          conda install -y -c conda-forge conda-lock
-      - name: generate lockfile
-        run: |
-          $CONDA/bin/conda-lock lock -k explicit -p linux-64 -f requirements/ci/${{matrix.python}}.yml
-          mv conda-linux-64.lock ${{matrix.python}}-linux-64.lock
-      - name: output lockfile
-        uses: actions/upload-artifact@v3
-        with:
-          path: ${{matrix.python}}-linux-64.lock
-
-  create_pr:
-    # once the matrix job has completed all the lock files will have been uploaded as artifacts.
-    # Download the artifacts, add them to the repo, and create a PR.
-    if: "github.repository == 'SciTools/iris'"
-    runs-on: ubuntu-latest
-    needs: gen_lockfiles
-
-    steps:
-      - uses: actions/checkout@v3
-      - name: get artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: artifacts
-
-      - name: Update lock files in repo
-        run: |
-          cp artifacts/artifact/*.lock requirements/ci/nox.lock
-          rm -r artifacts
-
-      - name: "Generate token"
-        uses: tibdex/github-app-token@v1
-        id: generate-token
-        with:
-          app_id: ${{ secrets.AUTH_APP_ID }}
-          private_key: ${{ secrets.AUTH_APP_PRIVATE_KEY }}
-
-      - name: Create Pull Request
-        id: cpr
-        uses: peter-evans/create-pull-request@18f90432bedd2afd6a825469ffd38aa24712a91d
-        with:
-          token: ${{ steps.generate-token.outputs.token }}
-          commit-message: Updated environment lockfiles
-          committer: "Lockfile bot <[email protected]>"
-          author: "Lockfile bot <[email protected]>"
-          delete-branch: true
-          branch: auto-update-lockfiles
-          title: "[iris.ci] environment lockfiles auto-update"
-          body: |
-            Lockfiles updated to the latest resolvable environment.
-
-            ### If the CI tasks fail, create a new branch based on this PR and add the required fixes to that branch.
-          labels: |
-            New: Pull Request
-            Bot
-
-      - name: Check Pull Request
-        if: steps.cpr.outputs.pull-request-number != ''
-        run: |
-          echo "pull-request #${{ steps.cpr.outputs.pull-request-number }}"
-          echo "pull-request URL ${{ steps.cpr.outputs.pull-request-url }}"
-          echo "pull-request operation [${{ steps.cpr.outputs.pull-request-operation }}]"
-          echo "pull-request head SHA ${{ steps.cpr.outputs.pull-request-head-sha }}"
+  refresh_lockfiles:
+    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@main
+    secrets: inherit

.github/workflows/stale.yml

@@ -14,7 +14,7 @@ jobs:
     if: "github.repository == 'SciTools/iris'"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v5
+      - uses: actions/stale@v6
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     -   id: no-commit-to-branch
 
 -   repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 22.10.0
     hooks:
     -   id: black
         pass_filenames: false

benchmarks/benchmarks/generate_data/__init__.py

@@ -113,7 +113,7 @@ def load_realised():
     file loading, but some benchmarks are only meaningful if starting with real
     arrays.
     """
-    from iris.fileformats.netcdf import _get_cf_var_data as pre_patched
+    from iris.fileformats.netcdf.loader import _get_cf_var_data as pre_patched
 
     def patched(cf_var, filename):
         return as_concrete_data(pre_patched(cf_var, filename))

benchmarks/benchmarks/import_iris.py

@@ -5,10 +5,30 @@
 # licensing details.
 from importlib import import_module, reload
 
+################
+# Prepare info for reset_colormaps:
+
+# Import and capture colormaps.
+from matplotlib import colormaps  # isort:skip
+
+_COLORMAPS_ORIG = set(colormaps)
+
+# Import iris.palette, which modifies colormaps.
+import iris.palette
+
+# Derive which colormaps have been added by iris.palette.
+_COLORMAPS_MOD = set(colormaps)
+COLORMAPS_EXTRA = _COLORMAPS_MOD - _COLORMAPS_ORIG
+
+# Touch iris.palette to prevent linters complaining.
+_ = iris.palette
+
+################
+
 
 class Iris:
     @staticmethod
-    def _import(module_name):
+    def _import(module_name, reset_colormaps=False):
         """
         Have experimented with adding sleep() commands into the imported
         modules. The results reveal:
@@ -25,6 +45,13 @@ def _import(module_name):
         and the repetitions are therefore no faster than the first run.
         """
         mod = import_module(module_name)
+
+        if reset_colormaps:
+            # Needed because reload() will attempt to register new colormaps a
+            #  second time, which errors by default.
+            for cm_name in COLORMAPS_EXTRA:
+                colormaps.unregister(cm_name)
+
         reload(mod)
 
     def time_iris(self):
@@ -205,7 +232,7 @@ def time_iterate(self):
         self._import("iris.iterate")
 
     def time_palette(self):
-        self._import("iris.palette")
+        self._import("iris.palette", reset_colormaps=True)
 
     def time_plot(self):
         self._import("iris.plot")

benchmarks/benchmarks/regridding.py

@@ -12,8 +12,11 @@
 # importing anything else
 from iris import tests  # isort:skip
 
+import numpy as np
+
 import iris
-from iris.analysis import AreaWeighted
+from iris.analysis import AreaWeighted, PointInCell
+from iris.coords import AuxCoord
 
 
 class HorizontalChunkedRegridding:
@@ -53,3 +56,48 @@ def time_regrid_area_w_new_grid(self) -> None:
         out = self.chunked_cube.regrid(self.template_cube, self.scheme_area_w)
         # Realise data
         out.data
+
+
+class CurvilinearRegridding:
+    def setup(self) -> None:
+        # Prepare a cube and a template
+
+        cube_file_path = tests.get_data_path(
+            ["NetCDF", "regrid", "regrid_xyt.nc"]
+        )
+        self.cube = iris.load_cube(cube_file_path)
+
+        # Make the source cube curvilinear
+        x_coord = self.cube.coord("longitude")
+        y_coord = self.cube.coord("latitude")
+        xx, yy = np.meshgrid(x_coord.points, y_coord.points)
+        self.cube.remove_coord(x_coord)
+        self.cube.remove_coord(y_coord)
+        x_coord_2d = AuxCoord(
+            xx,
+            standard_name=x_coord.standard_name,
+            units=x_coord.units,
+            coord_system=x_coord.coord_system,
+        )
+        y_coord_2d = AuxCoord(
+            yy,
+            standard_name=y_coord.standard_name,
+            units=y_coord.units,
+            coord_system=y_coord.coord_system,
+        )
+        self.cube.add_aux_coord(x_coord_2d, (1, 2))
+        self.cube.add_aux_coord(y_coord_2d, (1, 2))
+
+        template_file_path = tests.get_data_path(
+            ["NetCDF", "regrid", "regrid_template_global_latlon.nc"]
+        )
+        self.template_cube = iris.load_cube(template_file_path)
+
+        # Prepare a regridding scheme
+        self.scheme_pic = PointInCell()
+
+    def time_regrid_pic(self) -> None:
+        # Regrid the cube onto the template.
+        out = self.cube.regrid(self.template_cube, self.scheme_pic)
+        # Realise the data
+        out.data

docs/Makefile

@@ -55,3 +55,8 @@ linkcheck:
 		echo "Running linkcheck in $$i..."; \
 		(cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) linkcheck); done
 
+show:
+	@for i in $(SUBDIRS); do \
+		echo "Running show in $$i..."; \
+		(cd $$i; $(MAKE) $(MFLAGS) $(MYMAKEFLAGS) show); done
+