diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 35dd3ee282..b1539860ed 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -15,7 +15,7 @@ jobs: env: IRIS_TEST_DATA_LOC_PATH: benchmarks IRIS_TEST_DATA_PATH: benchmarks/iris-test-data - IRIS_TEST_DATA_VERSION: "2.5" + IRIS_TEST_DATA_VERSION: "2.12" # Lets us manually bump the cache to rebuild ENV_CACHE_BUILD: "0" TEST_DATA_CACHE_BUILD: "2" diff --git a/.github/workflows/ci-docs-tests.yml b/.github/workflows/ci-docs-tests.yml index 9eb849f287..95ea891733 100644 --- a/.github/workflows/ci-docs-tests.yml +++ b/.github/workflows/ci-docs-tests.yml @@ -37,7 +37,7 @@ jobs: python-version: ["3.8"] env: - IRIS_TEST_DATA_VERSION: "2.10" + IRIS_TEST_DATA_VERSION: "2.12" ENV_NAME: "ci-docs-tests" steps: diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8af50f188a..7f3ce7ae38 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -38,7 +38,7 @@ jobs: python-version: ["3.8"] env: - IRIS_TEST_DATA_VERSION: "2.10" + IRIS_TEST_DATA_VERSION: "2.12" ENV_NAME: "ci-tests" steps: diff --git a/.github/workflows/refresh-lockfiles.yml b/.github/workflows/refresh-lockfiles.yml index f93858e03d..e817131c5e 100644 --- a/.github/workflows/refresh-lockfiles.yml +++ b/.github/workflows/refresh-lockfiles.yml @@ -41,7 +41,7 @@ jobs: if: "github.repository == 'SciTools/iris'" runs-on: ubuntu-latest needs: get_python_matrix - + strategy: matrix: python: ${{ fromJSON(needs.get_python_matrix.outputs.matrix) }} @@ -60,30 +60,38 @@ jobs: uses: actions/upload-artifact@v3 with: path: ${{matrix.python}}-linux-64.lock - + create_pr: # once the matrix job has completed all the lock files will have been uploaded as artifacts. # Download the artifacts, add them to the repo, and create a PR. if: "github.repository == 'SciTools/iris'" runs-on: ubuntu-latest needs: gen_lockfiles - + steps: - uses: actions/checkout@v3 - name: get artifacts uses: actions/download-artifact@v3 with: path: artifacts - + - name: Update lock files in repo run: | cp artifacts/artifact/*.lock requirements/ci/nox.lock rm -r artifacts - + + - name: "Generate token" + uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.AUTH_APP_ID }} + private_key: ${{ secrets.AUTH_APP_PRIVATE_KEY }} + - name: Create Pull Request id: cpr uses: peter-evans/create-pull-request@923ad837f191474af6b1721408744feb989a4c27 with: + token: ${{ steps.generate-token.outputs.token }} commit-message: Updated environment lockfiles committer: "Lockfile bot " author: "Lockfile bot " @@ -94,10 +102,6 @@ jobs: Lockfiles updated to the latest resolvable environment. ### If the CI tasks fail, create a new branch based on this PR and add the required fixes to that branch. - - ### If the PR CI tasks have not run, please close and re-open this PR to trigger them. - - Reference: create-pull-request GHA [triggering further workflow runs](https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs) labels: | New: Pull Request Bot diff --git a/docs/src/developers_guide/assets/developer-settings-github-apps.png b/docs/src/developers_guide/assets/developer-settings-github-apps.png new file mode 100644 index 0000000000..a63994d087 Binary files /dev/null and b/docs/src/developers_guide/assets/developer-settings-github-apps.png differ diff --git a/docs/src/developers_guide/assets/download-pem.png b/docs/src/developers_guide/assets/download-pem.png new file mode 100644 index 0000000000..cbceb1304d Binary files /dev/null and b/docs/src/developers_guide/assets/download-pem.png differ diff --git a/docs/src/developers_guide/assets/generate-key.png b/docs/src/developers_guide/assets/generate-key.png new file mode 100644 index 0000000000..ac894dc71b Binary files /dev/null and b/docs/src/developers_guide/assets/generate-key.png differ diff --git a/docs/src/developers_guide/assets/gha-token-example.png b/docs/src/developers_guide/assets/gha-token-example.png new file mode 100644 index 0000000000..cba1cf6935 Binary files /dev/null and b/docs/src/developers_guide/assets/gha-token-example.png differ diff --git a/docs/src/developers_guide/assets/install-app.png b/docs/src/developers_guide/assets/install-app.png new file mode 100644 index 0000000000..31259de588 Binary files /dev/null and b/docs/src/developers_guide/assets/install-app.png differ diff --git a/docs/src/developers_guide/assets/install-iris-actions.png b/docs/src/developers_guide/assets/install-iris-actions.png new file mode 100644 index 0000000000..db16dee55b Binary files /dev/null and b/docs/src/developers_guide/assets/install-iris-actions.png differ diff --git a/docs/src/developers_guide/assets/installed-app.png b/docs/src/developers_guide/assets/installed-app.png new file mode 100644 index 0000000000..ab87032393 Binary files /dev/null and b/docs/src/developers_guide/assets/installed-app.png differ diff --git a/docs/src/developers_guide/assets/iris-actions-secret.png b/docs/src/developers_guide/assets/iris-actions-secret.png new file mode 100644 index 0000000000..f32456d0f2 Binary files /dev/null and b/docs/src/developers_guide/assets/iris-actions-secret.png differ diff --git a/docs/src/developers_guide/assets/iris-github-apps.png b/docs/src/developers_guide/assets/iris-github-apps.png new file mode 100644 index 0000000000..50753532b7 Binary files /dev/null and b/docs/src/developers_guide/assets/iris-github-apps.png differ diff --git a/docs/src/developers_guide/assets/iris-secrets-created.png b/docs/src/developers_guide/assets/iris-secrets-created.png new file mode 100644 index 0000000000..19b0ba11dc Binary files /dev/null and b/docs/src/developers_guide/assets/iris-secrets-created.png differ diff --git a/docs/src/developers_guide/assets/iris-security-actions.png b/docs/src/developers_guide/assets/iris-security-actions.png new file mode 100644 index 0000000000..7cbe3a7dc2 Binary files /dev/null and b/docs/src/developers_guide/assets/iris-security-actions.png differ diff --git a/docs/src/developers_guide/assets/iris-settings.png b/docs/src/developers_guide/assets/iris-settings.png new file mode 100644 index 0000000000..70714235c2 Binary files /dev/null and b/docs/src/developers_guide/assets/iris-settings.png differ diff --git a/docs/src/developers_guide/assets/org-perms-members.png b/docs/src/developers_guide/assets/org-perms-members.png new file mode 100644 index 0000000000..99fd8985e2 Binary files /dev/null and b/docs/src/developers_guide/assets/org-perms-members.png differ diff --git a/docs/src/developers_guide/assets/repo-perms-contents.png b/docs/src/developers_guide/assets/repo-perms-contents.png new file mode 100644 index 0000000000..4c325c334d Binary files /dev/null and b/docs/src/developers_guide/assets/repo-perms-contents.png differ diff --git a/docs/src/developers_guide/assets/repo-perms-pull-requests.png b/docs/src/developers_guide/assets/repo-perms-pull-requests.png new file mode 100644 index 0000000000..812f5ef951 Binary files /dev/null and b/docs/src/developers_guide/assets/repo-perms-pull-requests.png differ diff --git a/docs/src/developers_guide/assets/scitools-settings.png b/docs/src/developers_guide/assets/scitools-settings.png new file mode 100644 index 0000000000..8d7e728ab5 Binary files /dev/null and b/docs/src/developers_guide/assets/scitools-settings.png differ diff --git a/docs/src/developers_guide/assets/user-perms.png b/docs/src/developers_guide/assets/user-perms.png new file mode 100644 index 0000000000..607c7dcdb6 Binary files /dev/null and b/docs/src/developers_guide/assets/user-perms.png differ diff --git a/docs/src/developers_guide/assets/webhook-active.png b/docs/src/developers_guide/assets/webhook-active.png new file mode 100644 index 0000000000..538362f335 Binary files /dev/null and b/docs/src/developers_guide/assets/webhook-active.png differ diff --git a/docs/src/developers_guide/contributing_getting_involved.rst b/docs/src/developers_guide/contributing_getting_involved.rst index f4e677cea2..9ec6559114 100644 --- a/docs/src/developers_guide/contributing_getting_involved.rst +++ b/docs/src/developers_guide/contributing_getting_involved.rst @@ -50,6 +50,7 @@ If you are new to using GitHub we recommend reading the contributing_documentation contributing_codebase_index contributing_changes + github_app release diff --git a/docs/src/developers_guide/github_app.rst b/docs/src/developers_guide/github_app.rst new file mode 100644 index 0000000000..338166fd76 --- /dev/null +++ b/docs/src/developers_guide/github_app.rst @@ -0,0 +1,272 @@ +.. include:: ../common_links.inc + +Token GitHub App +---------------- + +.. note:: + + This section of the documentation is applicable only to GitHub `SciTools`_ + Organisation **owners** and **administrators**. + +This section describes how to create, configure, install and use our `SciTools`_ +GitHub App for generating tokens for use with *GitHub Actions* (GHA). + + +Background +^^^^^^^^^^ + +Our GitHub *Continuous Integration* (CI) workflows require fully reproducible +`conda`_ environments to test ``iris`` and build our documentation. + +The ``iris`` `refresh-lockfiles`_ GHA workflow uses the `conda-lock`_ package to routinely +generate a platform specific ``lockfile`` containing all the package dependencies +required by ``iris`` for a specific version of ``python``. + +The environment lockfiles created by the `refresh-lockfiles`_ GHA are contributed +back to ``iris`` though a pull-request that is automatically generated using the +third-party `create-pull-request`_ GHA. By default, pull-requests created by such an +action using the standard ``GITHUB_TOKEN`` **cannot** trigger other workflows, such +as our CI. + +As a result, we use a dedicated authentication **GitHub App** to securely generate tokens +for the `create-pull-request`_ GHA, which then permits our full suite of CI testing workflows +to be triggered against the lockfiles pull-request. Ensuring that the CI is triggered gives us +confidence that the proposed new lockfiles have not introduced a package level incompatibility +or issue within ``iris``. See :ref:`use gha`. + + +Create GitHub App +^^^^^^^^^^^^^^^^^ + +The **GitHub App** is created for the sole purpose of generating tokens for use with actions, +and **must** be owned by the `SciTools`_ organisation. + +To create a minimal `GitHub App`_ for this purpose, perform the following steps: + +1. Click the `SciTools`_ organisation ``⚙️ Settings`` option. + +.. figure:: assets/scitools-settings.png + :alt: SciTools organisation Settings option + :align: center + :width: 75% + +2. Click the ``GitHub Apps`` option from the ``<> Developer settings`` + section in the left hand sidebar. + +.. figure:: assets/developer-settings-github-apps.png + :alt: Developer settings, GitHub Apps option + :align: center + :width: 25% + +3. Now click the ``New GitHub App`` button to display the ``Register new GitHub App`` + form. + +Within the ``Register new GitHub App`` form, complete the following fields: + +4. Set the **mandatory** ``GitHub App name`` field to be ``iris-actions``. +5. Set the **mandatory** ``Homepage URL`` field to be ``https://github.com/SciTools/iris`` +6. Under the ``Webhook`` section, **uncheck** the ``Active`` checkbox. + Note that, **no** ``Webhook URL`` is required. + +.. figure:: assets/webhook-active.png + :alt: Webhook active checkbox + :align: center + :width: 75% + +7. Under the ``Repository permissions`` section, set the ``Contents`` field to + be ``Access: Read and write``. + +.. figure:: assets/repo-perms-contents.png + :alt: Repository permissions Contents option + :align: center + :width: 75% + +8. Under the ``Repository permissions`` section, set the ``Pull requests`` field + to be ``Access: Read and write``. + +.. figure:: assets/repo-perms-pull-requests.png + :alt: Repository permissions Pull requests option + :align: center + :width: 75% + +9. Under the ``Organization permissions`` section, set the ``Members`` field to + be ``Access: Read-only``. + +.. figure:: assets/org-perms-members.png + :alt: Organization permissions Members + :align: center + :width: 75% + +10. Under the ``User permissions`` section, for the ``Where can this GitHub App be installed?`` + field, **check** the ``Only on this account`` radio-button i.e., only allow + this GitHub App to be installed on the **SciTools** account. + +.. figure:: assets/user-perms.png + :alt: User permissions + :align: center + :width: 75% + +11. Finally, click the ``Create GitHub App`` button. + + +Configure GitHub App +^^^^^^^^^^^^^^^^^^^^ + +Creating the GitHub App will automatically redirect you to the ``SciTools settings / iris-actions`` +form for the newly created app. + +Perform the following GitHub App configuration steps: + +.. _app id: + +1. Under the ``About`` section, note of the GitHub ``App ID`` as this value is + required later. See :ref:`gha secrets`. +2. Under the ``Display information`` section, optionally upload the ``iris`` logo + as a ``png`` image. +3. Under the ``Private keys`` section, click the ``Generate a private key`` button. + +.. figure:: assets/generate-key.png + :alt: Private keys Generate a private key + :align: center + :width: 75% + +.. _private key: + +GitHub will automatically generate a private key to sign access token requests +for the app. Also a separate browser pop-up window will appear with the GitHub +App private key in ``OpenSSL PEM`` format. + +.. figure:: assets/download-pem.png + :alt: Download OpenSSL PEM file + :align: center + :width: 50% + +.. important:: + + Please ensure that you save the ``OpenSSL PEM`` file and **securely** archive + its contents. The private key within this file is required later. + See :ref:`gha secrets`. + + +Install GitHub App +^^^^^^^^^^^^^^^^^^ + +To install the GitHub App: + +1. Select the ``Install App`` option from the top left menu of the + ``Scitools settings / iris-actions`` form, then click the ``Install`` button. + +.. figure:: assets/install-app.png + :alt: Private keys Generate a private key + :align: center + :width: 75% + +2. Select the ``Only select repositories`` radio-button from the ``Install iris-actions`` + form, and choose the ``SciTools/iris`` repository. + +.. figure:: assets/install-iris-actions.png + :alt: Install iris-actions GitHub App + :align: center + :width: 75% + +3. Click the ``Install`` button. + + The successfully installed ``iris-actions`` GitHub App is now available under + the ``GitHub Apps`` option in the ``Integrations`` section of the `SciTools`_ + organisation ``Settings``. Note that, to reconfigure the installed app click + the ``⚙️ App settings`` option. + +.. figure:: assets/installed-app.png + :alt: Installed GitHub App + :align: center + :width: 80% + +4. Finally, confirm that the ``iris-actions`` GitHub App is now available within + the `SciTools/iris`_ repository by clicking the ``GitHub apps`` option in the + ``⚙️ Settings`` section. + +.. figure:: assets/iris-github-apps.png + :alt: Iris installed GitHub App + :align: center + :width: 80% + + +.. _gha secrets: + +Create Repository Secrets +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The GitHub Action that requests an access token from the ``iris-actions`` +GitHub App must be configured with the following information: + +* the ``App ID``, and +* the ``OpenSSL PEM`` private key + +associated with the ``iris-actions`` GitHub App. This **sensitive** information is +made **securely** available by creating `SciTools/iris`_ repository secrets: + +1. Click the `SciTools/iris`_ repository ``⚙️ Settings`` option. + +.. figure:: assets/iris-settings.png + :alt: Iris Settings + :align: center + :width: 75% + +2. Click the ``Actions`` option from the ``Security`` section in the left hand + sidebar. + +.. figure:: assets/iris-security-actions.png + :alt: Iris Settings Security Actions + :align: center + :width: 25% + +3. Click the ``New repository secret`` button. + +.. figure:: assets/iris-actions-secret.png + :alt: Iris Actions Secret + :align: center + :width: 75% + +4. Complete the ``Actions secrets / New secret`` form for the ``App ID``: + + * Set the ``Name`` field to be ``AUTH_APP_ID``. + * Set the ``Value`` field to be the numerical ``iris-actions`` GitHub ``App ID``. + See :ref:`here `. + * Click the ``Add secret`` button. + +5. Click the ``New repository secret`` button again, and complete the form + for the ``OpenSSL PEM``: + + * Set the ``Name`` field to be ``AUTH_APP_PRIVATE_KEY``. + * Set the ``Value`` field to be the entire contents of the ``OpenSSL PEM`` file. + See :ref:`here `. + * Click the ``Add secret`` button. + +A summary of the newly created `SciTools/iris`_ repository secrets is now available: + +.. figure:: assets/iris-secrets-created.png + :alt: Iris Secrets created + :align: center + :width: 75% + + +.. _use gha: + +Use GitHub App +^^^^^^^^^^^^^^ + +The following example workflow shows how to use the `github-app-token`_ GHA +to generate a token for use with the `create-pull-request`_ GHA: + +.. figure:: assets/gha-token-example.png + :alt: GitHub Action token example + :align: center + :width: 50% + + +.. _GitHub App: https://docs.github.com/en/developers/apps/building-github-apps/creating-a-github-app +.. _SciTools/iris: https://github.com/SciTools/iris +.. _conda-lock: https://github.com/conda-incubator/conda-lock +.. _create-pull-request: https://github.com/peter-evans/create-pull-request +.. _github-app-token: https://github.com/tibdex/github-app-token +.. _refresh-lockfiles: https://github.com/SciTools/iris/blob/main/.github/workflows/refresh-lockfiles.yml \ No newline at end of file diff --git a/docs/src/whatsnew/3.2.rst b/docs/src/whatsnew/3.2.rst index 63deb5d459..723f26345e 100644 --- a/docs/src/whatsnew/3.2.rst +++ b/docs/src/whatsnew/3.2.rst @@ -126,7 +126,7 @@ v3.2.1 (11 Mar 2022) of Iris (:issue:`4523`). #. `@pp-mo`_ removed broken tooling for deriving Iris metadata translations - from `Metarelate`_. From now we intend to manage phenonemon translation + from ``Metarelate``. From now we intend to manage phenonemon translation in Iris itself. (:pull:`4484`) #. `@pp-mo`_ improved printout of various cube data component objects : @@ -402,7 +402,6 @@ v3.2.1 (11 Mar 2022) Whatsnew resources in alphabetical order: .. _NEP-29: https://numpy.org/neps/nep-0029-deprecation_policy.html -.. _Metarelate: http://www.metarelate.net/ .. _UGRID: http://ugrid-conventions.github.io/ugrid-conventions/ .. _iris-emsf-regrid: https://github.com/SciTools-incubator/iris-esmf-regrid .. _faster documentation building: https://docs.readthedocs.io/en/stable/guides/conda.html#making-builds-faster-with-mamba diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index a955604c3e..1dca323820 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -226,6 +226,11 @@ This document explains the changes made to Iris for this release bin in the system PATH. (:pull:`4794`) +#. `@trexfeathers`_ and `@pp-mo`_ fixed the CDL headers for + :mod:`iris.tests.stock.netcdf` to allow generation of NetCDF-4 files with an + unlimited time dimension. + (:pull:`4827`) + .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/lib/iris/experimental/MERGENATE.md b/lib/iris/experimental/MERGENATE.md new file mode 100755 index 0000000000..0c1ef675c9 --- /dev/null +++ b/lib/iris/experimental/MERGENATE.md @@ -0,0 +1,63 @@ +# Mergenate + +## Philosophy + +Mergenate is not magic. +Mergenate will tell you why it failed. +Mergenate will make one cube, the way the user asked. + +## What's mergenate doing under the hood? + +* Choose the axis to work on. For no coord provided, that's an anonymous leading + axis. For one coord that's the axis that coord lies on (if it's consistent + across all cubes), or a new anonymous leading one if it's scalar everywhere. + Inconsistent axes causes a failure. Multiple coords are mergenated on one at a + time in order. + +* Validate that that cubes are all the right shape to merge. + +* Reorder the axes of each cube to make the merge axis the leading one + +* Establish the order that the cubes go in to make the given coordinate + monotonic. If all cubes have the coordinate scalar, assume ascending. Sort the + cubes into this order. + +* Concatenate the cube data. + +* Prepare a table of every coordinate in every cube. + +* Use this table to ascertain the correct treatment for each coordinate + (anything dervied from CoordMetadata). + + * All copies of a coordinate lying along the merge axis are concatenated + together in the same order as their source cubes. + + * A coordinate that's not along the merge axis and has the same values in all + cubes is just kept as it is. + + * A coordinate that's not along the merge axis and doesn't have the same + values in all cubes is extended along the merge axis **if + extend_coords=True**. + +* Handle AuxFactories separately. + +* Build a new cube. + +* Restore the order of the output cube's axes to the expected order. + +## What next for mergenate? + +* User testing - putting it in front of users and getting feedback. + +* Additional test coverage - some errors aren't yet tested because spoofing the + scenarios that cause them to fire is hard. + +* Verbose mode - when a merge or concatenate does something odd, it's hard to + trace the reasoning it followed. Mergenate should be easier to follow the + reasoning of, but ideally it would take a kwarg of ``verbose=True`` that made + it describe what it was doing at any decision point. + +* Merge wrapper - in some cases (e.g. pp load) Iris shouldn't need the user to + explain which coordinates they want merging on. In that case there should be a + merge-like wrapper that tries each variable and keeps the merges that work, + discarding the rest. \ No newline at end of file diff --git a/lib/iris/experimental/mergenate.py b/lib/iris/experimental/mergenate.py new file mode 100755 index 0000000000..ea285fd263 --- /dev/null +++ b/lib/iris/experimental/mergenate.py @@ -0,0 +1,640 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. + +""" +Combined merge and concatenate of a cubelist over the given coordinate(s) or on +a new anonymous leading dimension. + +""" + +import dask.array as da +import numpy as np + +from iris.cube import Cube +import iris.exceptions + +COORD_TYPE = 0 +ANC_VAR_TYPE = 1 +CELL_MEAS_TYPE = 2 +AUX_FACT_TYPE = 3 + +dim_func = { + COORD_TYPE: Cube.coord_dims, + ANC_VAR_TYPE: Cube.ancillary_variable_dims, + CELL_MEAS_TYPE: Cube.cell_measure_dims, +} + + +def _get_all_underived_vars(cube, give_types=False): + coords = cube.coords() + # Filter derived coordinates because they'll get recreated + coords = [coord for coord in coords if coord not in cube.derived_coords] + + anc_vars = cube.ancillary_variables() + cm_vars = cube.cell_measures() + af_vars = list(cube.aux_factories) + + vars = coords + anc_vars + cm_vars + af_vars + + if not give_types: + return vars + else: + types = ( + [COORD_TYPE] * len(coords) + + [ANC_VAR_TYPE] * len(anc_vars) + + [CELL_MEAS_TYPE] * len(cm_vars) + + [AUX_FACT_TYPE] * len(af_vars) + ) + return vars, types + + +def _make_coord_table(cubes): + + # Build our prototypes from the first cube + + cube_0_vars, var_types = _get_all_underived_vars(cubes[0], give_types=True) + + coord_table = [cube_0_vars] + + coord_count = len(coord_table[0]) + + # Stick everything else in + for cube_ind, cube in enumerate(cubes[1:]): + coord_table.append([None] * coord_count) + + for cube_coord in _get_all_underived_vars(cube): + for ii, prototype_coord in enumerate(coord_table[0]): + if type(cube_coord) == type( + prototype_coord + ) and cube_coord.metadata.equal(prototype_coord.metadata): + coord_table[-1][ii] = cube_coord + break + else: + # Error if we haven't used any of our coords in this cube + raise iris.exceptions.MergeError( + ( + f"{cube_coord.name()} in cube {cube_ind+1} doesn't match any coord in cube 0", + ) + ) + + missing_coord_inds = [ + ii for ii in range(coord_count) if coord_table[-1][ii] is None + ] + if missing_coord_inds: + error_messages = [ + f"{coord_table[0][ind].name()} in cube 0 doesn't match any coord in cube {cube_ind+1}" + for ind in missing_coord_inds + ] + raise iris.exceptions.MergeError(error_messages) + + return np.array(coord_table), var_types + + +def _merge_metadata(objs, err_name="objects"): + # TODO: Check with Bill that I'm doing this right + + new_metadata = objs[0].metadata + for obj in objs: + if not new_metadata.equal(obj.metadata): + raise iris.exceptions.MergeError( + ( + f"Inconsistent metadata between merging {err_name}.\nDifference is {new_metadata.difference(obj.metadata)}", + ) + ) + new_metadata = new_metadata.combine(obj.metadata) + + return new_metadata + + +def _build_new_coord_pieces(coords, new_dims, cube_depths, extend_coords): + prototype_coord = coords[0] + + if np.all(coords[1:] == coords[:-1]): + # All coordinates are identical + new_points = prototype_coord.points + new_bounds = prototype_coord.bounds + + elif 0 in new_dims: + # Coordinate already spans merge dimension + new_points = [] + has_bounds = coords[0].has_bounds() + if has_bounds: + new_bounds = [] + else: + new_bounds = None + + for coord in coords: + new_points.append(coord.points) + + if has_bounds: + new_bounds.append(coord.bounds) + + new_points = np.concatenate(new_points, axis=0) + if has_bounds: + new_bounds = np.concatenate(new_bounds, axis=0) + + elif not extend_coords: + # The coordinate doesn't span the merge dimension and we shouldn't make + # it + raise iris.exceptions.MergeError( + ( + "Different points or bounds in " + f"{coords[0].name()} coords, " + "but not allowed to extend coords. Consider trying " + "again with extend_coords=True", + ) + ) + + else: + # We need to broadcast the coordinates to span the merge dimension + new_dims = (0,) + new_dims + new_points = [] + has_bounds = coords[0].has_bounds() + if has_bounds: + new_bounds = [] + else: + new_bounds = None + + for cube_ind, coord in enumerate(coords): + cube_depth = cube_depths[cube_ind] + + # If the coord was scalar, it will become 1D + if new_dims == (0,): + broadcast_shape = [cube_depth] + else: + broadcast_shape = [cube_depth] + list(coord.points.shape) + + new_coord_points = np.broadcast_to( + coord.points, broadcast_shape, subok=True + ) + new_points.append(new_coord_points) + + if has_bounds: + + broadcast_shape = [cube_depth] + list(coord.bounds.shape) + + new_coord_bounds = np.broadcast_to( + coord.bounds, broadcast_shape, subok=True + ) + + new_bounds.append(new_coord_bounds) + + new_points = np.concatenate(new_points, axis=0) + if has_bounds: + new_bounds = np.concatenate(new_bounds, axis=0) + + return new_points, new_bounds, new_dims + + +def _build_new_var_pieces(coords, new_dims, cube_depths, extend_coords): + prototype_coord = coords[0] + + if np.all(coords[1:] == coords[:-1]): + # All coordinates are identical + new_points = prototype_coord.data + + elif 0 in new_dims: + # Coordinate already spans merge dimension + new_points = [] + + for coord in coords: + new_points.append(coord.data) + + new_points = np.concatenate(new_points, axis=0) + + elif not extend_coords: + # The coordinate doesn't span the merge dimension and we shouldn't make + # it + raise iris.exceptions.MergeError( + ( + "Different points or bounds in " + f"{coords[0].name()} coords, " + "but not allowed to extend coords. Consider trying " + "again with extend_coords=True", + ) + ) + + else: + # We need to broadcast the coordinates to span the merge dimension + new_dims = (0,) + new_dims + new_points = [] + + for cube_ind, coord in enumerate(coords): + cube_depth = cube_depths[cube_ind] + + # If the coord was scalar, it will become 1D + if new_dims == (0,): + broadcast_shape = [cube_depth] + else: + broadcast_shape = [cube_depth] + list(coord.data.shape) + + new_coord_points = np.broadcast_to( + coord.data, broadcast_shape, subok=True + ) + new_points.append(new_coord_points) + + new_points = np.concatenate(new_points, axis=0) + + return new_points, new_dims + + +def aux_factories_equal(aux_a, aux_b): + if type(aux_a) != type(aux_b): + return False + if aux_a.metadata != aux_b.metadata: + return False + print(aux_a.metadata) + print() + print(aux_b.metadata) + dependencies_a = { + key: coord.metadata for key, coord in aux_a.dependencies.items() + } + dependencies_b = { + key: coord.metadata for key, coord in aux_b.dependencies.items() + } + if len(dependencies_a) != len(dependencies_b): + return False + try: + for key, metadata_a in dependencies_a.items(): + if metadata_a != dependencies_b[key]: + return False + except (TypeError, KeyError): + return False + return True + + +def _mergenate(cubes, extend_coords: bool, merge_coord=None): + """Actually mergenate (on axis 0)""" + + try: + new_data = da.concatenate([cube.core_data() for cube in cubes], axis=0) + except ValueError: + msg = "Failed to concatenate cube datas" + raise iris.exceptions.MergeError((msg,)) + + coord_table, coord_types = _make_coord_table(cubes) + cube_metadata = _merge_metadata(cubes, err_name="cubes") + cube_depths = [cube.shape[0] for cube in cubes] + + # Make the new coords + new_coords_and_dims = [] + current_aux_factories = [] + for coord_col, coord_type in zip(coord_table.T, coord_types): + + if coord_type == AUX_FACT_TYPE: + all_equal = True + for ii in range(len(coord_col) - 1): + aux_a = coord_col[ii] + aux_b = coord_col[ii + 1] + if not aux_factories_equal(aux_a, aux_b): + all_equal = False + break + + if not all_equal: + raise iris.exceptions.MergeError( + ("Inconsistent AuxCoordFactories across cubes",) + ) + current_aux_factories.append(coord_col[0]) + continue + + new_dims = dim_func[coord_type](cubes[0], coord_col[0]) + if coord_type == COORD_TYPE: + new_points, new_bounds, new_dims = _build_new_coord_pieces( + coord_col, new_dims, cube_depths, extend_coords + ) + else: + new_points, new_dims = _build_new_var_pieces( + coord_col, new_dims, cube_depths, extend_coords + ) + new_bounds = None + + new_coord = None + if isinstance(coord_col[0], iris.coords.DimCoord): + try: + new_coord = iris.coords.DimCoord(new_points, bounds=new_bounds) + except ValueError: + new_coord = iris.coords.AuxCoord(new_points, bounds=new_bounds) + if new_coord is None: + if coord_type == COORD_TYPE: + new_coord = coord_col[0].copy( + points=new_points, bounds=new_bounds + ) + else: + new_coord = coord_col[0].copy(values=new_points) + + new_coord.metadata = _merge_metadata(coord_col, err_name="coords") + + new_coords_and_dims.append((new_coord, new_dims)) + + # Type isn't a sufficient indicator of which coords were originally the + # cube's dim coords, so check explicitly + original_dim_coords = cubes[0].coords(dim_coords=True) + if merge_coord is not None: + original_dim_coords.append(cubes[0].coord(merge_coord)) + dim_coord_indices = [] + for ii, coord in enumerate(coord_table[0]): + if coord in original_dim_coords: + dim_coord_indices.append(ii) + + new_dim_coords_and_dims = [] + new_aux_coords_and_dims = [] + new_cell_measures_and_dims = [] + new_ancillary_variables_and_dims = [] + # Order is important here - a CellMeasure is an AncillaryVariable so + # check for instance of CellMeasure first + type_lookup = [ + ( + (iris.coords.DimCoord, iris.coords.AuxCoord), + new_aux_coords_and_dims, + ), + (iris.coords.CellMeasure, new_cell_measures_and_dims), + (iris.coords.AncillaryVariable, new_ancillary_variables_and_dims), + ] + for coord_ind, (new_coord, new_dims) in enumerate(new_coords_and_dims): + if (coord_ind in dim_coord_indices) and isinstance( + new_coord, iris.coords.DimCoord + ): + new_dim_coords_and_dims.append( + ( + new_coord, + new_dims, + ) + ) + continue + for coord_type, coord_list in type_lookup: + if isinstance(new_coord, coord_type): + coord_list.append( + ( + new_coord, + new_dims, + ) + ) + break + else: + raise iris.exceptions.MergeError( + (f"Aux coord of no known type: {new_coord}",) + ) + + merged_cube = Cube( + new_data, + dim_coords_and_dims=new_dim_coords_and_dims, + aux_coords_and_dims=new_aux_coords_and_dims, + cell_measures_and_dims=new_cell_measures_and_dims, + ancillary_variables_and_dims=new_ancillary_variables_and_dims, + ) + + # Deal with aux factories now everything else is set up + nonderived_coords = cubes[0].dim_coords + cubes[0].aux_coords + coord_mapping = { + id(old_co): merged_cube.coord(old_co) for old_co in nonderived_coords + } + for factory in cubes[0].aux_factories: + new_factory = factory.updated(coord_mapping) + merged_cube.add_aux_factory(new_factory) + + merged_cube.metadata = cube_metadata + + return merged_cube + + +def _sort_cubes(cubes, coord): + + merge_coords_points = [cube.coord(coord).points for cube in cubes] + + # Check if the coords are ascending or descending + ascending = None + for ii, merge_coord_points in enumerate(merge_coords_points): + + if len(merge_coord_points) == 1: + pass + elif ascending is None: + ascending = merge_coord_points[1] > merge_coord_points[0] + elif ascending != (merge_coord_points[1] > merge_coord_points[0]): + raise iris.exceptions.MergeError( + ("Mixture of ascending and descending coordinate points",) + ) + + if ascending is None: + ascending = True + + # The order the the cubes should be uesd is based on the first point in each + # of their points arrays. This sorting is reversed if we've established all + # coords are descending + cube_order = sorted( + [*range(len(cubes))], + key=lambda x: merge_coords_points[x][0], + reverse=not ascending, + ) + + # Check that our new coordinate will be monotonic + for ii in range(len(cubes) - 1): + if ( + merge_coords_points[cube_order[ii]][-1] + < merge_coords_points[cube_order[ii + 1]][0] + ) != ascending: + # Question: Here we prevent people from effectively inserting a + # coord within another one - we could allow that, but it seems + # more confusing not to error than it is inconvenient to error. + # Is that sensible? + raise iris.exceptions.MergeError( + ( + "Coordinate points overlap so correct merge order is ambiguous", + ) + ) + + return [cubes[ii] for ii in cube_order] + + +def _calculate_reorders(cubes, coord): + """Work out how cube axis orders should change for mergenate""" + max_ndim = max([cube.ndim for cube in cubes]) + + coord_dims = set([cube.coord_dims(coord) for cube in cubes]) + + coord_dims.discard(()) + + if not coord_dims: + base_order = [*range(max_ndim + 1)] + return base_order, base_order + + try: + (coord_dim_tuple,) = coord_dims + except ValueError: + raise iris.exceptions.MergeError( + ("Coord lies on different axes on different cubes",) + ) + + try: + (coord_dim,) = coord_dim_tuple + except ValueError: + raise iris.exceptions.MergeError( + ("Can only merge on 1D or 0D coordinates",) + ) + + forward_order = [*range(max_ndim)] + forward_order.remove(coord_dim) + forward_order = tuple([coord_dim] + forward_order) + + backward_order = [*range(1, max_ndim)] + backward_order.insert(coord_dim, 0) + backward_order = tuple(backward_order) + + return forward_order, backward_order + + +def _categorise_by_coord(cubes, coord): + category_dict = {} + for cube in cubes: + match_coord = cube.coord(coord) + for seen_coord, categorised_cubes in category_dict.items(): + if np.all(match_coord.points == seen_coord.points) and np.all( + match_coord.bounds == seen_coord.bounds + ): + categorised_cubes.append(cube) + break + else: + category_dict[match_coord] = [cube] + return [*category_dict.values()] + + +def _validate_shapes(cubelist, coord): + + check_shapes = [] + for cube in cubelist: + if coord is None: + check_shapes.append(cube.shape) + else: + coord_dims = cube.coord_dims(coord) + if coord_dims == (): + check_shapes.append(cube.shape) + else: + try: + (coord_dim,) = coord_dims + except ValueError: + msg = "Can't merge on a 2D coordinate" + raise iris.exceptions.MergeError((msg,)) + cube_shape = [*cube.shape] + cube_shape.pop(coord_dim) + check_shapes.append((*cube_shape,)) + + expected_shape = check_shapes[0] + for check_shape in check_shapes: + if check_shape != expected_shape: + msg = "The shapes of cubes to be concatenated can only differ on the affected dimensions" + raise iris.exceptions.MergeError((msg,)) + + +def mergenate(cubelist, coords=None, extend_coords=False): + """ + Mergenate a cubelist into a single cube + + Combine the cubes in `cubelist` into a single cube if possible, along the + merge axis(es) specified by coords. + + Parameters + ---------- + cubelist : iris.cube.Cubelist or list of cubes + The cubes to be mergenated + coords : coord or list of coords or None, optional + Coords can be anything accepted by iris.cube.Cube.coord as + name_or_coord. Identifiers of the DimCoord or DimCoords to merge on. + These coordinates must be dimension coordinates or scalar coordinates of + all of the cubes. + + * If one coord is provided, mergenation is applied along the axis of the + cubes that it lies on, or if it is scalar in every cube then along a + new leading axis. + * If multiple coords are provided, mergenation is applied along the axis + described by each in turn. + * If None, mergenation is applied along a new, anonymous, leading axis. + extend_coords : {False, True}, optional + Whether coordinates that are inconsistent between cubes be extended + along the merge dimension + + Returns + ------- + iris.cube.Cube + The cube formed by mergenating the cubes in `cubelist` into a single + cube. + + Raises + ------ + MergeError + If cubes are incompatible, and as such cannot be merged into a single + cube, an informative exception is raised. + + Examples + -------- + These are written in doctest format, and should illustrate how to use the + function. + + >>> import iris + >>> from iris.cube import CubeList + >>> from iris.experimental.mergenate import mergenate + + >>> filepath = iris.sample_data_path("A1B_north_america.nc") + >>> expected = iris.load_cube(filepath) + + >>> cube_0 = expected[0] + >>> cube_1 = expected[1:] + >>> cubelist = CubeList([cube_0, cube_1]) + >>> result = mergenate(cubelist, "time", extend_coords=True) + + >>> result == expected + True + """ + + if coords is None: + _validate_shapes(cubelist, None) + prepared_cubes = [] + for cube in cubelist: + prepared_cubes.append(iris.util.new_axis(cube)) + out_cube = _mergenate(prepared_cubes, extend_coords=extend_coords) + return out_cube + + elif ( + isinstance(coords, (iris.coords._DimensionalMetadata, str)) + or len(coords) == 1 + ): + try: + (coord,) = coords + except ValueError: + coord = coords + + _validate_shapes(cubelist, coord) + + prepared_cubes = [] + forward_reorder, backward_reorder = _calculate_reorders( + cubelist, coord + ) + for cube in cubelist: + coord_dims = cube.coord_dims(coord) + if coord_dims == (): + prepared_cubes.append( + iris.util.new_axis(cube, scalar_coord=coord) + ) + else: + cube_copy = cube.copy() + cube_copy.transpose(forward_reorder) + prepared_cubes.append(cube_copy) + prepared_cubes = _sort_cubes(prepared_cubes, coord) + out_cube = _mergenate( + prepared_cubes, extend_coords=extend_coords, merge_coord=coord + ) + out_cube.transpose(backward_reorder) + return out_cube + + else: + + prepared_cubes = [] + categorisation = _categorise_by_coord(cubelist, coords[-1]) + for cubelist in categorisation: + prepared_cubes.append( + mergenate(cubelist, coords[:-1], extend_coords=extend_coords) + ) + return mergenate( + prepared_cubes, coords[-1], extend_coords=extend_coords + ) diff --git a/lib/iris/tests/experimental/test_mergenate.py b/lib/iris/tests/experimental/test_mergenate.py new file mode 100644 index 0000000000..e71b2b451d --- /dev/null +++ b/lib/iris/tests/experimental/test_mergenate.py @@ -0,0 +1,670 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Test the mergenation of cubes within iris. + +""" + +# import iris tests first so that some things can be initialised before +# importing anything else +import iris.tests as tests # isort:skip + +import numpy as np + +from iris.coords import AncillaryVariable, AuxCoord, CellMeasure, DimCoord +from iris.cube import CubeList +from iris.exceptions import MergeError +from iris.experimental.mergenate import mergenate +from iris.tests import stock +from iris.util import promote_aux_coord_to_dim_coord + + +class TestBasics(tests.IrisTest): + + # Merge + def test_merge(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, "wibble") + + self.assertEqual(expected, result) + + # Concatenate + def test_concatenate(self): + expected = stock.simple_3d() + + cube_0 = expected[:1] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, "wibble") + + self.assertEqual(expected, result) + + # Combo + def test_combination(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, "wibble") + + self.assertEqual(expected, result) + + # More than 2 pieces + def test_three_pieces(self): + # Invert to make latitude axis ascending, as ascending is default + expected = stock.simple_3d()[:, ::-1] + + cube_0 = expected[:, 0] + cube_1 = expected[:, 1] + cube_2 = expected[:, 2:] + cubelist = CubeList([cube_0, cube_1, cube_2]) + result = mergenate(cubelist, "latitude") + + self.assertEqual(expected, result) + + # Disordered pieces + def test_disordered_pieces(self): + expected = stock.realistic_3d() + + cube_0 = expected[0] + cube_1 = expected[1] + cube_2 = expected[2:] + cubelist = CubeList([cube_0, cube_2, cube_1]) + result = mergenate(cubelist, "time", extend_coords=True) + + self.assertEqual(expected, result) + + # Test descending merge (which assumes ascending as it lacks other info) + def test_descending_merge(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1] + cubelist = CubeList([cube_1, cube_0]) + result = mergenate(cubelist, "wibble") + + self.assertEqual(expected, result) + + # Test descending concatenate + def test_descending_concatenate(self): + expected = stock.realistic_3d()[::-1] + + cube_0 = expected[:3] + cube_1 = expected[3:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, "time") + + self.assertEqual(expected, result) + + # Test descending combo + def test_descending_combo(self): + expected = stock.realistic_3d()[::-1] + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, "time", extend_coords=True) + + self.assertEqual(expected, result) + + # Anonymous dim + def test_anon_dim(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, extend_coords=True) + + promote_aux_coord_to_dim_coord(result, "wibble") + + self.assertEqual(expected, result) + + # Other dim coord + def test_non_leading_dim(self): + expected = stock.simple_3d() + + cube_0 = expected[:, 0] + cube_1 = expected[:, 1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, "latitude") + + self.assertEqual(expected, result) + + # Multiple coords + def test_multiple_coords(self): + expected = stock.simple_3d() + + expected.coord("longitude").circular = False + + cube_0 = expected[:, 0, 0] + cube_1 = expected[:, 0, 1:] + cube_2 = expected[:, 1:, 0] + cube_3 = expected[:, 1:, 1:] + cubelist = CubeList([cube_0, cube_2, cube_3, cube_1]) + + result1 = mergenate(cubelist, ["latitude", "longitude"]) + self.assertEqual(expected, result1) + + result2 = mergenate(cubelist, ["longitude", "latitude"]) + self.assertEqual(expected, result2) + + +class TestArgs(tests.IrisTest): + # Coord object + def test_object(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, expected.coord("wibble")) + + self.assertEqual(expected, result) + + # String list + def test_string_list(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, ["wibble"]) + + self.assertEqual(expected, result) + + # Coord object list + def test_object_list(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, [expected.coord("wibble")]) + + self.assertEqual(expected, result) + + +class TestAuxCoords(tests.IrisTest): + def run_test(self, expected, coord_name="wibble"): + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + result = mergenate(cubelist, coord_name, extend_coords=True) + + print(expected) + for coord in result.coords(): + print(coord) + + self.assertEqual(expected, result) + + # AuxCoords + def test_aux_coord(self): + test_cube = stock.simple_3d() + extra_coord = AuxCoord( + [20, 40], + long_name="foo", + ) + test_cube.add_aux_coord( + extra_coord, + 0, + ) + + self.run_test(test_cube) + + # AuxCoord that's a DimCoord + def test_aux_dim_coord(self): + test_cube = stock.simple_3d() + extra_coord = DimCoord( + [20, 40], + long_name="foo", + ) + test_cube.add_aux_coord( + extra_coord, + 0, + ) + + self.run_test(test_cube) + + # CellMeasures + def test_cell_measure(self): + test_cube = stock.simple_3d() + extra_coord = CellMeasure( + [20, 40], + long_name="foo", + ) + test_cube.add_cell_measure( + extra_coord, + 0, + ) + + self.run_test(test_cube) + + # AncillaryVariables + def test_ancillary_variable(self): + test_cube = stock.simple_3d() + extra_coord = AncillaryVariable( + [20, 40], + long_name="foo", + ) + test_cube.add_ancillary_variable( + extra_coord, + 0, + ) + + self.run_test(test_cube) + + # AuxFactories + def test_aux_factory(self): + test_cube = stock.simple_4d_with_hybrid_height() + + self.run_test(test_cube, "time") + + +class TestCoordExtension(tests.IrisTest): + + # Extending each coord type + # Here the dim coord is demoted to aux coord on extension because it's no + # longer monotonic (or 1D) + def test_extend_dim_coord(self): + test_cube = stock.realistic_3d() + + cube_0 = test_cube[0] + cube_1 = test_cube[1:2] + cube_2 = test_cube[2:] + + coord_data = np.arange(test_cube.shape[1]) + coord_0 = DimCoord( + coord_data, + long_name="foo", + ) + coord_1 = DimCoord( + coord_data + 1, + long_name="foo", + ) + coord_2 = DimCoord( + coord_data + 2, + long_name="foo", + ) + + cube_0.add_aux_coord(coord_0, (0,)) + cube_1.add_aux_coord(coord_1, (1,)) + cube_2.add_aux_coord(coord_2, (1,)) + + result_cube = mergenate( + CubeList([cube_0, cube_1, cube_2]), + coords="time", + extend_coords=True, + ) + + expected_cross_section = np.array([0, 1, 2, 2, 2, 2, 2]) + expected_coord_data = ( + coord_data[np.newaxis, :] + expected_cross_section[:, np.newaxis] + ) + expected_coord = AuxCoord( + expected_coord_data, + long_name="foo", + ) + + self.assertEqual(result_cube.coord("foo"), expected_coord) + + def test_extend_aux_coord(self): + test_cube = stock.realistic_3d() + + cube_0 = test_cube[0] + cube_1 = test_cube[1:2] + cube_2 = test_cube[2:] + + coord_data = np.arange(test_cube.shape[1]) + coord_0 = AuxCoord( + coord_data, + long_name="foo", + ) + coord_1 = AuxCoord( + coord_data + 1, + long_name="foo", + ) + coord_2 = AuxCoord( + coord_data + 2, + long_name="foo", + ) + + cube_0.add_aux_coord(coord_0, (0,)) + cube_1.add_aux_coord(coord_1, (1,)) + cube_2.add_aux_coord(coord_2, (1,)) + + result_cube = mergenate( + CubeList([cube_0, cube_1, cube_2]), + coords="time", + extend_coords=True, + ) + + expected_cross_section = np.array([0, 1, 2, 2, 2, 2, 2]) + expected_coord_data = ( + coord_data[np.newaxis, :] + expected_cross_section[:, np.newaxis] + ) + expected_coord = AuxCoord( + expected_coord_data, + long_name="foo", + ) + + self.assertEqual(result_cube.coord("foo"), expected_coord) + + def test_extend_ancillary_var(self): + test_cube = stock.realistic_3d() + + cube_0 = test_cube[0] + cube_1 = test_cube[1:2] + cube_2 = test_cube[2:] + + coord_data = np.arange(test_cube.shape[1]) + coord_0 = AncillaryVariable( + coord_data, + long_name="foo", + ) + coord_1 = AncillaryVariable( + coord_data + 1, + long_name="foo", + ) + coord_2 = AncillaryVariable( + coord_data + 2, + long_name="foo", + ) + + cube_0.add_ancillary_variable(coord_0, (0,)) + cube_1.add_ancillary_variable(coord_1, (1,)) + cube_2.add_ancillary_variable(coord_2, (1,)) + + result_cube = mergenate( + CubeList([cube_0, cube_1, cube_2]), + coords="time", + extend_coords=True, + ) + + expected_cross_section = np.array([0, 1, 2, 2, 2, 2, 2]) + expected_coord_data = ( + coord_data[np.newaxis, :] + expected_cross_section[:, np.newaxis] + ) + expected_coord = AncillaryVariable( + expected_coord_data, + long_name="foo", + ) + + self.assertEqual(result_cube.ancillary_variable("foo"), expected_coord) + + def test_extend_cell_measure(self): + test_cube = stock.realistic_3d() + + cube_0 = test_cube[0] + cube_1 = test_cube[1:2] + cube_2 = test_cube[2:] + + coord_data = np.arange(test_cube.shape[1]) + coord_0 = CellMeasure( + coord_data, + long_name="foo", + ) + coord_1 = CellMeasure( + coord_data + 1, + long_name="foo", + ) + coord_2 = CellMeasure( + coord_data + 2, + long_name="foo", + ) + + cube_0.add_cell_measure(coord_0, (0,)) + cube_1.add_cell_measure(coord_1, (1,)) + cube_2.add_cell_measure(coord_2, (1,)) + + result_cube = mergenate( + CubeList([cube_0, cube_1, cube_2]), + coords="time", + extend_coords=True, + ) + + expected_cross_section = np.array([0, 1, 2, 2, 2, 2, 2]) + expected_coord_data = ( + coord_data[np.newaxis, :] + expected_cross_section[:, np.newaxis] + ) + expected_coord = CellMeasure( + expected_coord_data, + long_name="foo", + ) + + self.assertEqual(result_cube.cell_measure("foo"), expected_coord) + + # On different dims + def test_extend_lead_dim(self): + test_cube = stock.realistic_3d() + + cube_0 = test_cube[:, 0] + cube_1 = test_cube[:, 1:2] + cube_2 = test_cube[:, 2:] + + coord_data = np.arange(test_cube.shape[0]) + coord_0 = AuxCoord( + coord_data, + long_name="foo", + ) + coord_1 = AuxCoord( + coord_data + 1, + long_name="foo", + ) + coord_2 = AuxCoord( + coord_data + 2, + long_name="foo", + ) + + cube_0.add_aux_coord(coord_0, (0,)) + cube_1.add_aux_coord(coord_1, (0,)) + cube_2.add_aux_coord(coord_2, (0,)) + + result_cube = mergenate( + CubeList([cube_0, cube_1, cube_2]), + coords="grid_latitude", + extend_coords=True, + ) + + expected_cross_section = np.array([0, 1, 2, 2, 2, 2, 2, 2, 2]) + expected_coord_data = ( + coord_data[np.newaxis, :] + expected_cross_section[:, np.newaxis] + ) + expected_coord = AuxCoord( + expected_coord_data, + long_name="foo", + ) + + self.assertEqual(result_cube.coord("foo"), expected_coord) + + # Extend 2D to 3D + def test_extend_2D_to_3D(self): + test_cube = stock.realistic_3d() + + cube_0 = test_cube[0] + cube_1 = test_cube[1:2] + cube_2 = test_cube[2:] + + coord_data = np.arange( + test_cube.shape[1] * test_cube.shape[2] + ).reshape(test_cube.shape[1:3]) + coord_0 = AuxCoord( + coord_data, + long_name="foo", + ) + coord_1 = AuxCoord( + coord_data + 1, + long_name="foo", + ) + coord_2 = AuxCoord( + coord_data + 2, + long_name="foo", + ) + + cube_0.add_aux_coord(coord_0, (0, 1)) + cube_1.add_aux_coord(coord_1, (1, 2)) + cube_2.add_aux_coord(coord_2, (1, 2)) + + result_cube = mergenate( + CubeList([cube_0, cube_1, cube_2]), + coords="time", + extend_coords=True, + ) + + expected_cross_section = np.array([0, 1, 2, 2, 2, 2, 2]) + expected_coord_data = ( + coord_data[np.newaxis, :] + + expected_cross_section[:, np.newaxis, np.newaxis] + ) + expected_coord = AuxCoord( + expected_coord_data, + long_name="foo", + ) + + self.assertEqual(result_cube.coord("foo"), expected_coord) + + +class TestErrors(tests.IrisTest): + def test_extra_coord_in_first(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + + spare_coord = AuxCoord( + [0], + long_name="foo", + ) + cube_0.add_aux_coord(spare_coord, ()) + + msg = "foo in cube 0 doesn't match any coord in cube 1" + with self.assertRaisesRegex(MergeError, msg): + mergenate(cubelist, "wibble") + + def test_extra_coord_in_second(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + + spare_coord = AuxCoord( + [0], + long_name="foo", + ) + cube_1.add_aux_coord(spare_coord, ()) + + msg = "foo in cube 1 doesn't match any coord in cube 0" + with self.assertRaisesRegex(MergeError, msg): + mergenate(cubelist[::-1], "wibble") + + def test_metadata_conflict_cube(self): + expected = stock.simple_3d() + + cube_0 = expected[0] + cube_1 = expected[1:] + cubelist = CubeList([cube_0, cube_1]) + + cube_0.attributes["foo"] = "bar" + cube_1.attributes["foo"] = "barbar" + + msg = "Inconsistent metadata between merging cubes.\nDifference is CubeMetadata\(attributes=\(\{'foo': 'bar'\}, \{'foo': 'barbar'\}\)\)" # noqa: W605 + with self.assertRaisesRegex(MergeError, msg): + mergenate(cubelist, "wibble") + + def test_cant_extend_aux_coord(self): + test_cube = stock.simple_3d() + + cube_0 = test_cube[0] + cube_1 = test_cube[1:] + + coord_data = np.arange(test_cube.shape[1]) + coord_0 = AuxCoord( + coord_data, + long_name="foo", + ) + coord_1 = AuxCoord( + coord_data + 1, + long_name="foo", + ) + + cube_0.add_aux_coord(coord_0, (0,)) + cube_1.add_aux_coord(coord_1, (1,)) + + msg = ( + "Different points or bounds in foo coords, but not allowed to " + "extend coords. Consider trying again with extend_coords=True" + ) + with self.assertRaisesRegex(MergeError, msg): + mergenate(CubeList([cube_0, cube_1]), coords="wibble") + + def test_ascending_descending(self): + test_cube = stock.simple_3d() + + cube_0 = test_cube[:, :, 0:2] + cube_1 = test_cube[:, :, 2:4] + cube_1 = cube_1[:, :, ::-1] + + msg = "Mixture of ascending and descending coordinate points" + with self.assertRaisesRegex(MergeError, msg): + mergenate(CubeList([cube_0, cube_1]), "longitude") + + def test_ambiguous_order(self): + test_cube = stock.simple_3d() + + cube_0 = test_cube[:, :, 0:2] + cube_1 = test_cube[:, :, 1:4] + + msg = "Coordinate points overlap so correct merge order is ambiguous" + with self.assertRaisesRegex(MergeError, msg): + mergenate(CubeList([cube_0, cube_1]), "longitude") + + cube_2 = test_cube[:, :, ::2] + cube_3 = test_cube[:, :, 1::2] + + msg = "Coordinate points overlap so correct merge order is ambiguous" + with self.assertRaisesRegex(MergeError, msg): + mergenate(CubeList([cube_2, cube_3]), "longitude") + + def test_different_axes(self): + test_cube = stock.simple_3d() + + cube_0 = test_cube[0:1] + cube_1 = test_cube[1:] + cube_1.transpose((1, 0, 2)) + + msg = "Coord lies on different axes on different cubes" + with self.assertRaisesRegex(MergeError, msg): + mergenate(CubeList([cube_0, cube_1]), "wibble") + + def test_inconsistent_shapes(self): + test_cube = stock.simple_3d() + + cube_0 = test_cube[0:1] + cube_1 = test_cube[1:, 1:] + + msg = ( + "The shapes of cubes to be concatenated can only differ on the " + "affected dimensions" + ) + with self.assertRaisesRegex(MergeError, msg): + mergenate(CubeList([cube_0, cube_1]), "wibble") + + # TODO: Test for the following errors: + # Inconsistent aux factories (I can't work out how to make these) + # Cube datas that can't be concatenated (but pass shape test) + # Unknown type of non-dimensional coordinate + # Can only merge on 1D or 0D coordinates + + +if __name__ == "__main__": + tests.main() diff --git a/lib/iris/tests/stock/file_headers/xios_2D_face_half_levels.cdl b/lib/iris/tests/stock/file_headers/xios_2D_face_half_levels.cdl index b135546f2d..ba3522b491 100644 --- a/lib/iris/tests/stock/file_headers/xios_2D_face_half_levels.cdl +++ b/lib/iris/tests/stock/file_headers/xios_2D_face_half_levels.cdl @@ -55,4 +55,8 @@ variables: :name = "${DATASET_NAME}" ; // original name = "lfric_ngvat_2D_1t_face_half_levels_main_conv_rain" :Conventions = "UGRID" ; + +// data +data: + time_instant = 0 ; } diff --git a/lib/iris/tests/stock/file_headers/xios_3D_face_full_levels.cdl b/lib/iris/tests/stock/file_headers/xios_3D_face_full_levels.cdl index e4f32de7b7..a87e3055c9 100644 --- a/lib/iris/tests/stock/file_headers/xios_3D_face_full_levels.cdl +++ b/lib/iris/tests/stock/file_headers/xios_3D_face_full_levels.cdl @@ -58,4 +58,8 @@ variables: :name = "${DATASET_NAME}" ; // original name = "lfric_ngvat_3D_1t_full_level_face_grid_main_u3" :Conventions = "UGRID" ; + +// data +data: + time_instant = 0 ; } diff --git a/lib/iris/tests/stock/file_headers/xios_3D_face_half_levels.cdl b/lib/iris/tests/stock/file_headers/xios_3D_face_half_levels.cdl index a193dbe451..f9c9c148dd 100644 --- a/lib/iris/tests/stock/file_headers/xios_3D_face_half_levels.cdl +++ b/lib/iris/tests/stock/file_headers/xios_3D_face_half_levels.cdl @@ -58,4 +58,8 @@ variables: :name = "${DATASET_NAME}" ; // original name = "lfric_ngvat_3D_1t_half_level_face_grid_derived_theta_in_w3" :Conventions = "UGRID" ; + +// data +data: + time_instant = 0 ; } diff --git a/lib/iris/tests/stock/netcdf.py b/lib/iris/tests/stock/netcdf.py index 3e95fce95e..c5ec5ce446 100644 --- a/lib/iris/tests/stock/netcdf.py +++ b/lib/iris/tests/stock/netcdf.py @@ -103,7 +103,7 @@ def _add_standard_data(nc_path, unlimited_dim_size=0): ds = netCDF4.Dataset(nc_path, "r+") unlimited_dim_names = [ - dim for dim in ds.dimensions if ds.dimensions[dim].size == 0 + dim for dim in ds.dimensions if ds.dimensions[dim].isunlimited() ] # Data addition dependent on this assumption: assert len(unlimited_dim_names) < 2 diff --git a/lib/iris/util.py b/lib/iris/util.py index ded72d0f23..492546a44c 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -1155,6 +1155,14 @@ def new_axis(src_cube, scalar_coord=None): coord_dims = np.array(src_cube.coord_dims(coord)) + 1 new_cube.add_dim_coord(coord.copy(), coord_dims) + for coord in src_cube.cell_measures(): + coord_dims = np.array(src_cube.cell_measure_dims(coord)) + 1 + new_cube.add_cell_measure(coord.copy(), coord_dims) + + for coord in src_cube.ancillary_variables(): + coord_dims = np.array(src_cube.ancillary_variable_dims(coord)) + 1 + new_cube.add_ancillary_variable(coord.copy(), coord_dims) + nonderived_coords = src_cube.dim_coords + src_cube.aux_coords coord_mapping = { id(old_co): new_cube.coord(old_co) for old_co in nonderived_coords