diff --git a/.cirrus.yml b/.cirrus.yml index 007bab403e..da425a5691 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -98,6 +98,8 @@ linux_minimal_task: PY_VER: 3.6 env: PY_VER: 3.7 + env: + PY_VER: 3.8 name: "${CIRRUS_OS}: py${PY_VER} tests (minimal)" container: image: gcc:latest @@ -119,6 +121,8 @@ linux_task: PY_VER: 3.6 env: PY_VER: 3.7 + env: + PY_VER: 3.8 name: "${CIRRUS_OS}: py${PY_VER} tests (full)" container: image: gcc:latest @@ -146,9 +150,7 @@ linux_task: gallery_task: matrix: env: - PY_VER: 3.6 - env: - PY_VER: 3.7 + PY_VER: 3.8 name: "${CIRRUS_OS}: py${PY_VER} doc tests (gallery)" container: image: gcc:latest @@ -176,7 +178,7 @@ gallery_task: doctest_task: matrix: env: - PY_VER: 3.7 + PY_VER: 3.8 name: "${CIRRUS_OS}: py${PY_VER} doc tests" container: image: gcc:latest @@ -210,7 +212,7 @@ doctest_task: link_task: matrix: env: - PY_VER: 3.7 + PY_VER: 3.8 name: "${CIRRUS_OS}: py${PY_VER} doc link check" container: image: gcc:latest diff --git a/docs/src/common_links.inc b/docs/src/common_links.inc index 9f6a57f529..157444d65d 100644 --- a/docs/src/common_links.inc +++ b/docs/src/common_links.inc @@ -1,6 +1,7 @@ .. comment Common resources in alphabetical order: +.. _black: https://black.readthedocs.io/en/stable/ .. _.cirrus.yml: https://github.com/SciTools/iris/blob/master/.cirrus.yml .. _.flake8.yml: https://github.com/SciTools/iris/blob/master/.flake8 .. _cirrus-ci: https://cirrus-ci.com/github/SciTools/iris @@ -19,6 +20,7 @@ .. _legacy documentation: https://scitools.org.uk/iris/docs/v2.4.0/ .. _matplotlib: https://matplotlib.org/ .. _napolean: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/sphinxcontrib.napoleon.html +.. _nox: https://nox.thea.codes/en/stable/ .. _New Issue: https://github.com/scitools/iris/issues/new/choose .. _pull request: https://github.com/SciTools/iris/pulls .. _pull requests: https://github.com/SciTools/iris/pulls diff --git a/docs/src/developers_guide/contributing_running_tests.rst b/docs/src/developers_guide/contributing_running_tests.rst index 0fd9fa8486..9bc2d797bd 100644 --- a/docs/src/developers_guide/contributing_running_tests.rst +++ b/docs/src/developers_guide/contributing_running_tests.rst @@ -186,8 +186,6 @@ For further `nox`_ command-line options:: .. note:: `nox`_ will cache its testing environments in the `.nox` root ``iris`` project directory. -.. _black: https://black.readthedocs.io/en/stable/ -.. _nox: https://nox.thea.codes/en/latest/ .. _setuptools: https://setuptools.readthedocs.io/en/latest/ .. _tox: https://tox.readthedocs.io/en/latest/ .. _virtualenv: https://virtualenv.pypa.io/en/latest/ diff --git a/docs/src/further_topics/metadata.rst b/docs/src/further_topics/metadata.rst index e6d6ebc57a..ab6a6450b4 100644 --- a/docs/src/further_topics/metadata.rst +++ b/docs/src/further_topics/metadata.rst @@ -258,12 +258,12 @@ create a **new** instance directly from the metadata class itself, >>> DimCoordMetadata._make(values) DimCoordMetadata(standard_name=1, long_name=2, var_name=3, units=4, attributes=5, coord_system=6, climatological=7, circular=8) -It is also possible to easily convert ``metadata`` to an `OrderedDict`_ +It is also possible to easily convert ``metadata`` to an `dict`_ using the `namedtuple._asdict`_ method. This can be particularly handy when a standard Python built-in container is required to represent your ``metadata``, >>> metadata._asdict() - OrderedDict([('standard_name', 'longitude'), ('long_name', None), ('var_name', 'longitude'), ('units', Unit('degrees')), ('attributes', {'grinning face': '🙃'}), ('coord_system', GeogCS(6371229.0)), ('climatological', False), ('circular', False)]) + {'standard_name': 'longitude', 'long_name': None, 'var_name': 'longitude', 'units': Unit('degrees'), 'attributes': {'grinning face': '🙃'}, 'coord_system': GeogCS(6371229.0), 'climatological': False, 'circular': False} Using the `namedtuple._replace`_ method allows you to create a new metadata class instance, but replacing specified members with **new** associated values, @@ -943,7 +943,7 @@ such as a `dict`_, >>> mapping = latitude.metadata._asdict() >>> mapping - OrderedDict([('standard_name', 'latitude'), ('long_name', None), ('var_name', 'latitude'), ('units', Unit('degrees')), ('attributes', {}), ('coord_system', GeogCS(6371229.0)), ('climatological', False), ('circular', False)]) + {'standard_name': 'latitude', 'long_name': None, 'var_name': 'latitude', 'units': Unit('degrees'), 'attributes': {}, 'coord_system': GeogCS(6371229.0), 'climatological': False, 'circular': False} >>> longitude.metadata = mapping >>> longitude.metadata DimCoordMetadata(standard_name='latitude', long_name=None, var_name='latitude', units=Unit('degrees'), attributes={}, coord_system=GeogCS(6371229.0), climatological=False, circular=False) @@ -1000,7 +1000,6 @@ values. All other metadata members will be left unaltered. .. _NetCDF: https://www.unidata.ucar.edu/software/netcdf/ .. _NetCDF CF Metadata Conventions: https://cfconventions.org/ .. _NumPy: https://github.com/numpy/numpy -.. _OrderedDict: https://docs.python.org/3/library/collections.html#collections.OrderedDict .. _Parametric Vertical Coordinate: https://cfconventions.org/Data/cf-conventions/cf-conventions-1.8/cf-conventions.html#parametric-vertical-coordinate .. _rich comparison: https://www.python.org/dev/peps/pep-0207/ .. _SciTools/iris: https://github.com/SciTools/iris diff --git a/docs/src/installing.rst b/docs/src/installing.rst index 31fc497b85..8deb7043c5 100644 --- a/docs/src/installing.rst +++ b/docs/src/installing.rst @@ -16,8 +16,8 @@ any WSL_ distributions. .. _WSL: https://docs.microsoft.com/en-us/windows/wsl/install-win10 -.. note:: Iris currently supports and is tested against **Python 3.6** and - **Python 3.7**. +.. note:: Iris is currently supported and tested against Python ``3.6``, + ``3.7``, and ``3.8``. .. note:: This documentation was built using Python |python_version|. diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 1efa08874a..c02b61341b 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -12,7 +12,6 @@ This document explains the changes made to Iris for this release :title: text-primary text-center font-weight-bold :body: bg-light :animate: fade-in - :open: The highlights for this major/minor release of Iris include: @@ -96,7 +95,10 @@ This document explains the changes made to Iris for this release #. `@tkknight`_ moved the ``docs/iris`` directory to be in the parent directory ``docs``. (:pull:`3975`) -#. `@jamesp`_ updated a test to the latest numpy version (:pull:`3977`) +#. `@jamesp`_ updated a test for `numpy`_ ``1.20.0``. (:pull:`3977`) + +#. `@bjlittle`_ and `@jamesp`_ extended the `cirrus-ci`_ testing and `nox`_ + testing automation to support `Python 3.8`_. (:pull:`3976`) #. `@bjlittle`_ rationalised the ``noxfile.py``, and added the ability for each ``nox`` session to list its ``conda`` environment packages and @@ -117,3 +119,5 @@ This document explains the changes made to Iris for this release .. _abstract base class: https://docs.python.org/3/library/abc.html .. _GitHub: https://github.com/SciTools/iris/issues/new/choose .. _Met Office: https://www.metoffice.gov.uk/ +.. _numpy: https://numpy.org/doc/stable/release/1.20.0-notes.html +.. _Python 3.8: https://www.python.org/downloads/release/python-380/ diff --git a/lib/iris/coords.py b/lib/iris/coords.py index cfeb24cdcb..6129b35150 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -578,7 +578,21 @@ def shape(self): return self._values_dm.shape def xml_element(self, doc): - """Return a DOM element describing this metadata.""" + """ + Create the :class:`xml.dom.minidom.Element` that describes this + :class:`_DimensionalMetadata`. + + Args: + + * doc: + The parent :class:`xml.dom.minidom.Document`. + + Returns: + The :class:`xml.dom.minidom.Element` that will describe this + :class:`_DimensionalMetadata`, and the dictionary of attributes + that require to be added to this element. + + """ # Create the XML element as the camelCaseEquivalent of the # class name. element_name = type(self).__name__ @@ -881,6 +895,20 @@ def cube_dims(self, cube): return cube.cell_measure_dims(self) def xml_element(self, doc): + """ + Create the :class:`xml.dom.minidom.Element` that describes this + :class:`CellMeasure`. + + Args: + + * doc: + The parent :class:`xml.dom.minidom.Document`. + + Returns: + The :class:`xml.dom.minidom.Element` that describes this + :class:`CellMeasure`. + + """ # Create the XML element as the camelCaseEquivalent of the # class name element = super().xml_element(doc=doc) @@ -2228,14 +2256,26 @@ def nearest_neighbour_index(self, point): return result_index def xml_element(self, doc): - """Return a DOM element describing this Coord.""" + """ + Create the :class:`xml.dom.minidom.Element` that describes this + :class:`Coord`. + + Args: + + * doc: + The parent :class:`xml.dom.minidom.Document`. + + Returns: + The :class:`xml.dom.minidom.Element` that will describe this + :class:`DimCoord`, and the dictionary of attributes that require + to be added to this element. + + """ # Create the XML element as the camelCaseEquivalent of the # class name element = super().xml_element(doc=doc) - element.setAttribute("points", self._xml_array_repr(self.points)) - - # Add bounds handling + # Add bounds, points are handled by the parent class. if self.has_bounds(): element.setAttribute("bounds", self._xml_array_repr(self.bounds)) @@ -2614,7 +2654,20 @@ def is_monotonic(self): return True def xml_element(self, doc): - """Return DOM element describing this :class:`iris.coords.DimCoord`.""" + """ + Create the :class:`xml.dom.minidom.Element` that describes this + :class:`DimCoord`. + + Args: + + * doc: + The parent :class:`xml.dom.minidom.Document`. + + Returns: + The :class:`xml.dom.minidom.Element` that describes this + :class:`DimCoord`. + + """ element = super().xml_element(doc) if self.circular: element.setAttribute("circular", str(self.circular)) @@ -2794,7 +2847,17 @@ def __add__(self, other): def xml_element(self, doc): """ - Return a dom element describing itself + Create the :class:`xml.dom.minidom.Element` that describes this + :class:`CellMethod`. + + Args: + + * doc: + The parent :class:`xml.dom.minidom.Document`. + + Returns: + The :class:`xml.dom.minidom.Element` that describes this + :class:`CellMethod`. """ cellMethod_xml_element = doc.createElement("cellMethod") diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 7c7d6c58e9..5578507d28 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -225,6 +225,7 @@ def __getslice__(self, start, stop): def xml(self, checksum=False, order=True, byteorder=True): """Return a string of the XML that this list of cubes represents.""" + doc = Document() cubes_xml_element = doc.createElement("cubes") cubes_xml_element.setAttribute("xmlns", XML_NAMESPACE_URI) @@ -239,6 +240,7 @@ def xml(self, checksum=False, order=True, byteorder=True): doc.appendChild(cubes_xml_element) # return our newly created XML string + doc = Cube._sort_xml_attrs(doc) return doc.toprettyxml(indent=" ") def extract(self, constraints): @@ -755,6 +757,59 @@ class Cube(CFVariableMixin): #: is similar to Fortran or Matlab, but different than numpy. __orthogonal_indexing__ = True + @classmethod + def _sort_xml_attrs(cls, doc): + """ + Takes an xml document and returns a copy with all element + attributes sorted in alphabetical order. + + This is a private utility method required by iris to maintain + legacy xml behaviour beyond python 3.7. + + Args: + + * doc: + The :class:`xml.dom.minidom.Document`. + + Returns: + The :class:`xml.dom.minidom.Document` with sorted element + attributes. + + """ + from xml.dom.minidom import Document + + def _walk_nodes(node): + """Note: _walk_nodes is called recursively on child elements.""" + + # we don't want to copy the children here, so take a shallow copy + new_node = node.cloneNode(deep=False) + + # Versions of python <3.8 order attributes in alphabetical order. + # Python >=3.8 order attributes in insert order. For consistent behaviour + # across both, we'll go with alphabetical order always. + # Remove all the attribute nodes, then add back in alphabetical order. + attrs = [ + new_node.getAttributeNode(attr_name).cloneNode(deep=True) + for attr_name in sorted(node.attributes.keys()) + ] + for attr in attrs: + new_node.removeAttributeNode(attr) + for attr in attrs: + new_node.setAttributeNode(attr) + + if node.childNodes: + children = [_walk_nodes(x) for x in node.childNodes] + for c in children: + new_node.appendChild(c) + + return new_node + + nodes = _walk_nodes(doc.documentElement) + new_doc = Document() + new_doc.appendChild(nodes) + + return new_doc + def __init__( self, data, @@ -3403,6 +3458,7 @@ def xml(self, checksum=False, order=True, byteorder=True): doc.appendChild(cube_xml_element) # Print our newly created XML + doc = self._sort_xml_attrs(doc) return doc.toprettyxml(indent=" ") def _xml_element(self, doc, checksum=False, order=True, byteorder=True): diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index ac0d313d76..4a85e5cdb2 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -573,6 +573,10 @@ def assertXMLElement(self, obj, reference_filename): """ doc = xml.dom.minidom.Document() doc.appendChild(obj.xml_element(doc)) + # sort the attributes on xml elements before testing against known good state. + # this is to be compatible with stored test output where xml attrs are stored in alphabetical order, + # (which was default behaviour in python <3.8, but changed to insert order in >3.8) + doc = iris.cube.Cube._sort_xml_attrs(doc) pretty_xml = doc.toprettyxml(indent=" ") reference_path = self.get_result_path(reference_filename) self._check_same( diff --git a/noxfile.py b/noxfile.py index b6f9480290..028da099dc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,7 @@ PACKAGE = str("lib" / Path("iris")) #: Cirrus-CI environment variable hook. -PY_VER = os.environ.get("PY_VER", ["3.6", "3.7"]) +PY_VER = os.environ.get("PY_VER", ["3.6", "3.7", "3.8"]) #: Default cartopy cache directory. CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy") diff --git a/requirements/ci/iris.yml b/requirements/ci/iris.yml index e9adb956db..a76932b56e 120000 --- a/requirements/ci/iris.yml +++ b/requirements/ci/iris.yml @@ -1 +1 @@ -py37.yml \ No newline at end of file +py38.yml \ No newline at end of file diff --git a/requirements/ci/py38.yml b/requirements/ci/py38.yml new file mode 100644 index 0000000000..da29d30d71 --- /dev/null +++ b/requirements/ci/py38.yml @@ -0,0 +1,51 @@ +name: iris-dev + +channels: + - conda-forge + +dependencies: + - python=3.8 + +# Setup dependencies. + - setuptools>=40.8.0 + - pyke + +# Core dependencies. + - cartopy>=0.18 + - cf-units>=2 + - cftime<1.3.0 + - dask>=2 + - matplotlib + - netcdf4 + - numpy>=1.14 + - python-xxhash + - scipy + +# Optional dependencies. + - esmpy>=7.0 + - graphviz + - iris-sample-data + - mo_pack + - nc-time-axis + - pandas + - python-stratify + - pyugrid + +# Test dependencies. + - asv + - black=20.8b1 + - filelock + - flake8 + - imagehash>=4.0 + - nose + - pillow<7 + - pre-commit + - requests + +# Documentation dependencies. + - sphinx + - sphinxcontrib-napoleon + - sphinx-copybutton + - sphinx-gallery + - sphinx-panels + - sphinx_rtd_theme