diff --git a/.travis.yml b/.travis.yml index 8f59a9274c..f938791811 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ git: install: - > - export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96"; + export IRIS_TEST_DATA_REF="dba47566a9147645fea586f94a138e0a8d45a48e"; export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//"); # Cut short doctest phase under Python 2 : now only supports Python 3 diff --git a/docs/iris/src/sphinxext/custom_class_autodoc.py b/docs/iris/src/sphinxext/custom_class_autodoc.py index af8309b2d3..25c095cb84 100644 --- a/docs/iris/src/sphinxext/custom_class_autodoc.py +++ b/docs/iris/src/sphinxext/custom_class_autodoc.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2015, Met Office +# (C) British Crown Copyright 2010 - 2019, Met Office # # This file is part of Iris. # @@ -20,6 +20,8 @@ from sphinx.ext import autodoc from sphinx.ext.autodoc import * +from sphinx.util import force_decode +from sphinx.util.docstrings import prepare_docstring import inspect diff --git a/docs/iris/src/userguide/cube_statistics.rst b/docs/iris/src/userguide/cube_statistics.rst index 96e4776cf7..3ca7d9a2e0 100644 --- a/docs/iris/src/userguide/cube_statistics.rst +++ b/docs/iris/src/userguide/cube_statistics.rst @@ -97,10 +97,6 @@ For an example of using this functionality, the in the gallery takes a zonal mean of an ``XYT`` cube by using the ``collapsed`` method with ``latitude`` and ``iris.analysis.MEAN`` as arguments. -You cannot partially collapse a multi-dimensional coordinate. See -:ref:`cube.collapsed ` for more -information. - .. _cube-statistics-collapsing-average: Area averaging diff --git a/docs/iris/src/whatsnew/2.2.rst b/docs/iris/src/whatsnew/2.2.rst index f38bfa1bc3..1eff99ecb4 100644 --- a/docs/iris/src/whatsnew/2.2.rst +++ b/docs/iris/src/whatsnew/2.2.rst @@ -66,6 +66,9 @@ Iris 2.2 Features discontiguous points in coordinates can be explicitly masked using another new feature :func:`iris.util.mask_cube`. +* :func:`iris.util.array_equal` now has a 'withnans' keyword, which provides + a NaN-tolerant array comparison. + Iris 2.2 Dependency updates ============================= @@ -89,8 +92,21 @@ Bugs Fixed * "Gracefully filling..." warnings are now only issued when the coordinate or bound data is actually masked. + +Bugs fixed in v2.2.1 +-------------------- + * Iris can now correctly unpack a column of header objects when saving a pandas DataFrame to a cube. + +* fixed a bug in :meth:`iris.util.new_axis` : copying the resulting cube + resulted in an exception, if it contained an aux-factory. + +* :class:`iris.coords.AuxCoord`'s can now test as 'equal' even when the points + or bounds arrays contain NaN values, if values are the same at all points. + Previously this would fail, as conventionally "NaN != NaN" in normal + floating-point arithmetic. + Documentation Changes diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index cc0ddc5788..0a5265e9ab 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1341,7 +1341,9 @@ def interp_order(length): # Collapse array to its final data shape. slices = [slice(None)] * array.ndim - slices[-1] = 0 + endslice = slice(0, 1) if len(slices) == 1 else 0 + slices[-1] = endslice + slices = tuple(slices) # Numpy>=1.16 : index with tuple, *not* list. if isinstance(array.dtype, np.float): data = array[slices] diff --git a/lib/iris/analysis/cartography.py b/lib/iris/analysis/cartography.py index ab78adedb1..c1cb09e179 100644 --- a/lib/iris/analysis/cartography.py +++ b/lib/iris/analysis/cartography.py @@ -717,6 +717,7 @@ def project(cube, target_proj, nx=None, ny=None): index = list(index) index[xdim] = slice(None, None) index[ydim] = slice(None, None) + index = tuple(index) # Numpy>=1.16 : index with tuple, *not* list. new_data[index] = cartopy.img_transform.regrid(ll_slice.data, source_x, source_y, source_cs, diff --git a/lib/iris/coords.py b/lib/iris/coords.py index 5519630c5d..6487b3f7e6 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2018, Met Office +# (C) British Crown Copyright 2010 - 2019, Met Office # # This file is part of Iris. # @@ -817,11 +817,13 @@ def __eq__(self, other): eq = self._as_defn() == other._as_defn() # points comparison if eq: - eq = iris.util.array_equal(self.points, other.points) + eq = iris.util.array_equal(self.points, other.points, + withnans=True) # bounds comparison if eq: if self.has_bounds() and other.has_bounds(): - eq = iris.util.array_equal(self.bounds, other.bounds) + eq = iris.util.array_equal(self.bounds, other.bounds, + withnans=True) else: eq = self.bounds is None and other.bounds is None diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index 783c1b43e0..2b26b901a2 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -37,6 +37,9 @@ import numpy.ma as ma import cftime +import dask +import dask.array as da + from iris._deprecation import warn_deprecated from iris._lazy_data import as_concrete_data, as_lazy_data, is_lazy_data import iris.config @@ -602,10 +605,10 @@ class PPDataProxy(object): """A reference to the data payload of a single PP field.""" __slots__ = ('shape', 'src_dtype', 'path', 'offset', 'data_len', - '_lbpack', 'boundary_packing', 'mdi', 'mask') + '_lbpack', 'boundary_packing', 'mdi') def __init__(self, shape, src_dtype, path, offset, data_len, - lbpack, boundary_packing, mdi, mask): + lbpack, boundary_packing, mdi): self.shape = shape self.src_dtype = src_dtype self.path = path @@ -614,7 +617,6 @@ def __init__(self, shape, src_dtype, path, offset, data_len, self.lbpack = lbpack self.boundary_packing = boundary_packing self.mdi = mdi - self.mask = mask # lbpack def _lbpack_setter(self, value): @@ -649,14 +651,14 @@ def __getitem__(self, keys): self.lbpack, self.boundary_packing, self.shape, self.src_dtype, - self.mdi, self.mask) + self.mdi) data = data.__getitem__(keys) return np.asanyarray(data, dtype=self.dtype) def __repr__(self): fmt = '<{self.__class__.__name__} shape={self.shape}' \ ' src_dtype={self.dtype!r} path={self.path!r}' \ - ' offset={self.offset} mask={self.mask!r}>' + ' offset={self.offset}>' return fmt.format(self=self) def __getstate__(self): @@ -772,24 +774,29 @@ def _data_bytes_to_shaped_array(data_bytes, lbpack, boundary_packing, elif lbpack.n2 == 2: if mask is None: - raise ValueError('No mask was found to unpack the data. ' - 'Could not load.') - land_mask = mask.data.astype(np.bool) - sea_mask = ~land_mask - new_data = np.ma.masked_all(land_mask.shape) - new_data.fill_value = mdi - if lbpack.n3 == 1: - # Land mask packed data. - # Sometimes the data comes in longer than it should be (i.e. it - # looks like the compressed data is compressed, but the trailing - # data hasn't been clipped off!). - new_data[land_mask] = data[:land_mask.sum()] - elif lbpack.n3 == 2: - # Sea mask packed data. - new_data[sea_mask] = data[:sea_mask.sum()] + # If we are given no mask to apply, then just return raw data, even + # though it does not have the correct shape. + # For dask-delayed loading, this means that mask, data and the + # combination can be properly handled within a dask graph. + # However, we still mask any MDI values in the array (below). + pass else: - raise ValueError('Unsupported mask compression.') - data = new_data + land_mask = mask.data.astype(np.bool) + sea_mask = ~land_mask + new_data = np.ma.masked_all(land_mask.shape) + new_data.fill_value = mdi + if lbpack.n3 == 1: + # Land mask packed data. + # Sometimes the data comes in longer than it should be (i.e. it + # looks like the compressed data is compressed, but the + # trailing data hasn't been clipped off!). + new_data[land_mask] = data[:land_mask.sum()] + elif lbpack.n3 == 2: + # Sea mask packed data. + new_data[sea_mask] = data[:sea_mask.sum()] + else: + raise ValueError('Unsupported mask compression.') + data = new_data else: # Reform in row-column order @@ -1581,51 +1588,68 @@ def _interpret_fields(fields): numpy arrays (via the _create_field_data) function. """ - land_mask = None + land_mask_field = None landmask_compressed_fields = [] for field in fields: # Store the first reference to a land mask, and use this as the # definitive mask for future fields in this generator. - if land_mask is None and field.lbuser[6] == 1 and \ + if land_mask_field is None and field.lbuser[6] == 1 and \ (field.lbuser[3] // 1000) == 0 and \ (field.lbuser[3] % 1000) == 30: - land_mask = field + land_mask_field = field - # Handle land compressed data payloads, - # when lbpack.n2 is 2. + # Handle land compressed data payloads, when lbpack.n2 is 2. if (field.raw_lbpack // 10 % 10) == 2: - if land_mask is None: + # Field uses land-mask packing, so needs a related land-mask field. + if land_mask_field is None: landmask_compressed_fields.append(field) + # Land-masked fields have their size+shape defined by the + # reference landmask field, so we can't yield them if they + # are encountered *before* the landmask. + # In that case, defer them, and process them all afterwards at + # the end of the file. continue - # Land compressed fields don't have a lbrow and lbnpt. - field.lbrow, field.lbnpt = land_mask.lbrow, land_mask.lbnpt + # Land-mask compressed fields don't have an lbrow and lbnpt. + field.lbrow, field.lbnpt = \ + land_mask_field.lbrow, land_mask_field.lbnpt + _create_field_data(field, (field.lbrow, field.lbnpt), + land_mask_field=land_mask_field) + else: + # Field does not use land-mask packing. + _create_field_data(field, (field.lbrow, field.lbnpt)) - data_shape = (field.lbrow, field.lbnpt) - _create_field_data(field, data_shape, land_mask) yield field + # At file end, return any land-masked fields that were deferred because + # they were encountered before the landmask reference field. if landmask_compressed_fields: - if land_mask is None: + if land_mask_field is None: warnings.warn('Landmask compressed fields existed without a ' 'landmask to decompress with. The data will have ' 'a shape of (0, 0) and will not read.') mask_shape = (0, 0) else: - mask_shape = (land_mask.lbrow, land_mask.lbnpt) + mask_shape = (land_mask_field.lbrow, land_mask_field.lbnpt) for field in landmask_compressed_fields: field.lbrow, field.lbnpt = mask_shape - _create_field_data(field, (field.lbrow, field.lbnpt), land_mask) + _create_field_data(field, mask_shape, + land_mask_field=land_mask_field) yield field -def _create_field_data(field, data_shape, land_mask): +def _create_field_data(field, data_shape, land_mask_field=None): """ Modifies a field's ``_data`` attribute either by: - * converting DeferredArrayBytes into a lazy array, + * converting a 'deferred array bytes' tuple into a lazy array, * converting LoadedArrayBytes into an actual numpy array. + If 'land_mask_field' is passed (not None), then it contains the associated + landmask, which is also a field : Its data array is used as a template for + `field`'s data, determining its size, shape and the locations of all the + valid (non-missing) datapoints. + """ if isinstance(field.core_data(), LoadedArrayBytes): loaded_bytes = field.core_data() @@ -1634,7 +1658,8 @@ def _create_field_data(field, data_shape, land_mask): field.boundary_packing, data_shape, loaded_bytes.dtype, - field.bmdi, land_mask) + field.bmdi, + land_mask_field) else: # Wrap the reference to the data payload within a data proxy # in order to support deferred data loading. @@ -1643,9 +1668,64 @@ def _create_field_data(field, data_shape, land_mask): fname, position, n_bytes, field.raw_lbpack, field.boundary_packing, - field.bmdi, land_mask) + field.bmdi) block_shape = data_shape if 0 not in data_shape else (1, 1) - field.data = as_lazy_data(proxy, chunks=block_shape) + if land_mask_field is None: + # For a "normal" (non-landsea-masked) field, the proxy can be + # wrapped directly as a deferred array. + field.data = as_lazy_data(proxy, chunks=block_shape) + else: + # This is a landsea-masked field, and its data must be handled in + # a different way : Because data shape/size is not known in + # advance, the data+mask calculation can't be represented + # as a dask-array operation. Instead, we make that calculation + # 'delayed', and then use 'from_delayed' to make the result back + # into a dask array -- because the final result shape *is* known. + @dask.delayed + def fetch_valid_values_array(): + # Return the data values array (shape+size unknown). + return proxy[:] + + delayed_valid_values = fetch_valid_values_array() + + # Get the mask data-array from the landsea-mask field. + # This is *either* a lazy or a real array, we don't actually care. + # If this is a deferred dependency, the delayed calc can see that. + mask_field_array = land_mask_field.core_data() + + # Check whether this field uses a land or a sea mask. + if field.lbpack.n3 not in (1, 2): + raise ValueError('Unsupported mask compression : ' + 'lbpack.n3 = {}.'.format(field.lbpack.n3)) + if field.lbpack.n3 == 2: + # Sea-mask packing : points are inverse of the land-mask. + mask_field_array = ~mask_field_array + + # Define the mask+data calculation as a deferred operation. + # NOTE: duplicates the operation in _data_bytes_to_shaped_array. + @dask.delayed + def calc_array(mask, values): + # Note: "mask" is True at *valid* points, not missing ones. + # First ensure the mask array is boolean (not int). + mask = mask.astype(bool) + result = ma.masked_all(mask.shape, dtype=dtype) + # Apply the fill-value from the proxy object. + # Note: 'values' is just 'proxy' in a dask wrapper. This arg + # must be a dask type so that 'delayed' can recognise it, but + # that provides no access to the underlying fill value. + result.fill_value = proxy.mdi + n_values = np.sum(mask) + if n_values > 0: + # Note: data field can have excess values, but not fewer. + result[mask] = values[:n_values] + return result + + delayed_result = calc_array(mask_field_array, + delayed_valid_values) + lazy_result_array = da.from_delayed(delayed_result, + shape=block_shape, + dtype=dtype) + field.data = lazy_result_array def _field_gen(filename, read_data_bytes, little_ended=False): @@ -1655,12 +1735,19 @@ def _field_gen(filename, read_data_bytes, little_ended=False): A field returned by the generator is only "half-formed" because its `_data` attribute represents a simple one-dimensional stream of - bytes. (Encoded as an instance of either LoadedArrayBytes or - DeferredArrayBytes, depending on the value of `read_data_bytes`.) + bytes. (Encoded either as an instance of LoadedArrayBytes or as a + 'deferred array bytes' tuple, depending on the value of `read_data_bytes`.) This is because fields encoded with a land/sea mask do not contain sufficient information within the field to determine the final two-dimensional shape of the data. + The callers of this routine are the 'load' routines (both PP and FF). + They both filter the resulting stream of fields with `_interpret_fields`, + which replaces each field.data with an actual array (real or lazy). + This is done separately so that `_interpret_fields` can detect land-mask + fields and use them to construct data arrays for any fields which use + land/sea-mask packing. + """ dtype_endian_char = '<' if little_ended else '>' with open(filename, 'rb') as pp_file: @@ -1738,7 +1825,10 @@ def _field_gen(filename, read_data_bytes, little_ended=False): pp_field.data = LoadedArrayBytes(pp_file.read(data_len), dtype) else: - # Provide enough context to read the data bytes later on. + # Provide enough context to read the data bytes later on, + # as a 'deferred array bytes' tuple. + # N.B. this used to be a namedtuple called DeferredArrayBytes, + # but it no longer is. Possibly for performance reasons? pp_field.data = (filename, pp_file.tell(), data_len, dtype) # Seek over the actual data payload. pp_file_seek(data_len, os.SEEK_CUR) @@ -2155,8 +2245,6 @@ def save_fields(fields, target, append=False): of the file. Only applicable when target is a filename, not a file handle. Default is False. - * callback: - A modifier/filter function. See also :func:`iris.io.save`. @@ -2185,11 +2273,9 @@ def save_fields(fields, target, append=False): if isinstance(target, six.string_types): pp_file = open(target, "ab" if append else "wb") - filename = target elif hasattr(target, "write"): if hasattr(target, "mode") and "b" not in target.mode: raise ValueError("Target not binary") - filename = target.name if hasattr(target, 'name') else None pp_file = target else: raise ValueError("Can only save pp to filename or writable") diff --git a/lib/iris/fileformats/pp_load_rules.py b/lib/iris/fileformats/pp_load_rules.py index 9942670058..83c0c9dfca 100644 --- a/lib/iris/fileformats/pp_load_rules.py +++ b/lib/iris/fileformats/pp_load_rules.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2018, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -30,7 +30,6 @@ from iris.coords import AuxCoord, CellMethod, DimCoord from iris.fileformats.rules import (ConversionMetadata, Factory, Reference, ReferenceTarget) -import iris.fileformats.pp from iris.fileformats._pp_lbproc_pairs import LBPROC_MAP from iris.fileformats.um_cf_map import (LBFC_TO_CF, STASH_TO_CF, STASHCODE_IMPLIED_HEIGHTS) @@ -444,6 +443,81 @@ def _new_coord_and_dims(is_vector_operation, _HOURS_UNIT = cf_units.Unit('hours') +def _epoch_date_hours(epoch_hours_unit, datetime): + """ + Return an 'hours since epoch' number for a date. + + Args: + * epoch_hours_unit (:class:`cf_unit.Unit'): + Unit defining the calendar and zero-time of conversion. + * datetime (:class:`datetime.datetime`-like): + Date object containing year / month / day attributes. + + This routine can also handle dates with a zero year, month or day : such + dates were valid inputs to 'date2num' up to cftime version 1.0.1, but are + now illegal : This routine interprets any zeros as being "1 year/month/day + before a year/month/day of 1". This produces results consistent with the + "old" cftime behaviour. + + """ + days_offset = None + if (datetime.year == 0 or datetime.month == 0 or datetime.day == 0): + # cftime > 1.0.1 no longer allows non-calendar dates. + # Add 1 to year/month/day, to get a valid date, and adjust the result + # according to the actual epoch and calendar. This reproduces 'old' + # results that were produced with cftime <= 1.0.1. + days_offset = 0 + y, m, d = datetime.year, datetime.month, datetime.day + calendar = epoch_hours_unit.calendar + if d == 0: + # Add one day, by changing day=0 to 1. + d = 1 + days_offset += 1 + if m == 0: + # Add a 'January', by changing month=0 to 1. + m = 1 + if calendar == cf_units.CALENDAR_GREGORIAN: + days_offset += 31 + elif calendar == cf_units.CALENDAR_360_DAY: + days_offset += 30 + elif calendar == cf_units.CALENDAR_365_DAY: + days_offset += 31 + else: + msg = 'unrecognised calendar : {}' + raise ValueError(msg.format(calendar)) + + if y == 0: + # Add a 'Year 0', by changing year=0 to 1. + y = 1 + if calendar == cf_units.CALENDAR_GREGORIAN: + days_in_year_0 = 366 + elif calendar == cf_units.CALENDAR_360_DAY: + days_in_year_0 = 360 + elif calendar == cf_units.CALENDAR_365_DAY: + days_in_year_0 = 365 + else: + msg = 'unrecognised calendar : {}' + raise ValueError(msg.format(calendar)) + + days_offset += days_in_year_0 + + # Replace y/m/d with a modified date, that cftime will accept. + datetime = datetime.replace(year=y, month=m, day=d) + + # netcdf4python has changed it's behaviour, at version 1.2, such + # that a date2num calculation returns a python float, not + # numpy.float64. The behaviour of round is to recast this to an + # int, which is not the desired behaviour for PP files. + # So, cast the answer to numpy.float_ to be safe. + epoch_hours = np.float_(epoch_hours_unit.date2num(datetime)) + + if days_offset is not None: + # Correct for any modifications to achieve a valid date. + epoch_hours -= 24.0 * days_offset + + return epoch_hours + + def _convert_time_coords(lbcode, lbtim, epoch_hours_unit, t1, t2, lbft, t1_dims=(), t2_dims=(), lbft_dims=()): @@ -481,12 +555,7 @@ def _convert_time_coords(lbcode, lbtim, epoch_hours_unit, """ def date2hours(t): - # netcdf4python has changed it's behaviour, at version 1.2, such - # that a date2num calculation returns a python float, not - # numpy.float64. The behaviour of round is to recast this to an - # int, which is not the desired behaviour for PP files. - # So, cast the answer to numpy.float_ to be safe. - epoch_hours = np.float_(epoch_hours_unit.date2num(t)) + epoch_hours = _epoch_date_hours(epoch_hours_unit, t) if t.minute == 0 and t.second == 0: epoch_hours = round(epoch_hours) return epoch_hours diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index ba5a2ee1ec..5b760e1447 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2018, Met Office +# (C) British Crown Copyright 2010 - 2019, Met Office # # This file is part of Iris. # @@ -115,7 +115,9 @@ NC_TIME_AXIS_AVAILABLE = False try: - requests.get('https://github.com/SciTools/iris') + # Added a timeout to stop the call to requests.get hanging when running + # on a platform which has restricted/no internet access. + requests.get('https://github.com/SciTools/iris', timeout=10.0) INET_AVAILABLE = True except requests.exceptions.ConnectionError: INET_AVAILABLE = False diff --git a/lib/iris/tests/integration/test_pp.py b/lib/iris/tests/integration/test_pp.py index 0cc0868b1d..e4f3b909a5 100644 --- a/lib/iris/tests/integration/test_pp.py +++ b/lib/iris/tests/integration/test_pp.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2017, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -48,19 +48,27 @@ def _test_coord(self, cube, point, bounds=None, **kwargs): if bounds is not None: self.assertArrayEqual(coords[0].bounds, [bounds]) - def test_soil_level_round_trip(self): - # Use pp.load_cubes() to convert a fake PPField into a Cube. - # NB. Use MagicMock so that SplittableInt header items, such as - # LBCODE, support len(). - soil_level = 1234 + @staticmethod + def _mock_field(**kwargs): mock_data = np.zeros(1) mock_core_data = mock.MagicMock(return_value=mock_data) - field = mock.MagicMock(lbvc=6, lblev=soil_level, - stash=iris.fileformats.pp.STASH(1, 0, 9), - lbuser=[0] * 7, lbrsvd=[0] * 4, + field = mock.MagicMock(lbuser=[0] * 7, lbrsvd=[0] * 4, brsvd=[0] * 4, brlev=0, + t1=mock.MagicMock(year=1990, month=1, day=3), + t2=mock.MagicMock(year=1990, month=1, day=3), core_data=mock_core_data, realised_dtype=mock_data.dtype) + field.configure_mock(**kwargs) + return field + + def test_soil_level_round_trip(self): + # Use pp.load_cubes() to convert a fake PPField into a Cube. + # NB. Use MagicMock so that SplittableInt header items, such as + # LBCODE, support len(). + soil_level = 1234 + field = self._mock_field( + lbvc=6, lblev=soil_level, + stash=iris.fileformats.pp.STASH(1, 0, 9)) load = mock.Mock(return_value=iter([field])) with mock.patch('iris.fileformats.pp.load', new=load) as load: cube = next(iris.fileformats.pp.load_cubes('DUMMY')) @@ -89,14 +97,9 @@ def test_soil_depth_round_trip(self): # LBCODE, support len(). lower, point, upper = 1.2, 3.4, 5.6 brsvd = [lower, 0, 0, 0] - mock_data = np.zeros(1) - mock_core_data = mock.MagicMock(return_value=mock_data) - field = mock.MagicMock(lbvc=6, blev=point, - stash=iris.fileformats.pp.STASH(1, 0, 9), - lbuser=[0] * 7, lbrsvd=[0] * 4, - brsvd=brsvd, brlev=upper, - core_data=mock_core_data, - realised_dtype=mock_data.dtype) + field = self._mock_field( + lbvc=6, blev=point, brsvd=brsvd, brlev=upper, + stash=iris.fileformats.pp.STASH(1, 0, 9)) load = mock.Mock(return_value=iter([field])) with mock.patch('iris.fileformats.pp.load', new=load) as load: cube = next(iris.fileformats.pp.load_cubes('DUMMY')) @@ -126,12 +129,7 @@ def test_potential_temperature_level_round_trip(self): # NB. Use MagicMock so that SplittableInt header items, such as # LBCODE, support len(). potm_value = 22.5 - mock_data = np.zeros(1) - mock_core_data = mock.MagicMock(return_value=mock_data) - field = mock.MagicMock(lbvc=19, blev=potm_value, - lbuser=[0] * 7, lbrsvd=[0] * 4, - core_data=mock_core_data, - realised_dtype=mock_data.dtype) + field = self._mock_field(lbvc=19, blev=potm_value) load = mock.Mock(return_value=iter([field])) with mock.patch('iris.fileformats.pp.load', new=load): cube = next(iris.fileformats.pp.load_cubes('DUMMY')) @@ -149,40 +147,46 @@ def test_potential_temperature_level_round_trip(self): self.assertEqual(field.lbvc, 19) self.assertEqual(field.blev, potm_value) + @staticmethod + def _field_with_data(scale=1, **kwargs): + x, y = 40, 30 + mock_data = np.arange(1200).reshape(y, x) * scale + mock_core_data = mock.MagicMock(return_value=mock_data) + field = mock.MagicMock(core_data=mock_core_data, + realised_dtype=mock_data.dtype, + lbcode=[1], + lbnpt=x, lbrow=y, bzx=350, bdx=1.5, + bzy=40, bdy=1.5, lbuser=[0] * 7, + lbrsvd=[0] * 4, + t1=mock.MagicMock(year=1990, month=1, day=3), + t2=mock.MagicMock(year=1990, month=1, day=3)) + + field._x_coord_name = lambda: 'longitude' + field._y_coord_name = lambda: 'latitude' + field.coord_system = lambda: None + field.configure_mock(**kwargs) + return field + def test_hybrid_pressure_round_trip(self): # Use pp.load_cubes() to convert fake PPFields into Cubes. # NB. Use MagicMock so that SplittableInt header items, such as # LBCODE, support len(). - def field_with_data(scale=1): - x, y = 40, 30 - mock_data = np.arange(1200).reshape(y, x) * scale - mock_core_data = mock.MagicMock(return_value=mock_data) - field = mock.MagicMock(core_data=mock_core_data, - realised_dtype=mock_data.dtype, - lbcode=[1], - lbnpt=x, lbrow=y, bzx=350, bdx=1.5, - bzy=40, bdy=1.5, lbuser=[0] * 7, - lbrsvd=[0] * 4) - - field._x_coord_name = lambda: 'longitude' - field._y_coord_name = lambda: 'latitude' - field.coord_system = lambda: None - return field # Make a fake reference surface field. - pressure_field = field_with_data(10) - pressure_field.stash = iris.fileformats.pp.STASH(1, 0, 409) - pressure_field.lbuser[3] = 409 + pressure_field = self._field_with_data( + 10, + stash=iris.fileformats.pp.STASH(1, 0, 409), + lbuser=[0, 0, 0, 409, 0, 0, 0]) # Make a fake data field which needs the reference surface. model_level = 5678 sigma_lower, sigma, sigma_upper = 0.85, 0.9, 0.95 delta_lower, delta, delta_upper = 0.05, 0.1, 0.15 - data_field = field_with_data() - data_field.configure_mock(lbvc=9, lblev=model_level, - bhlev=delta, bhrlev=delta_lower, - blev=sigma, brlev=sigma_lower, - brsvd=[sigma_upper, delta_upper]) + data_field = self._field_with_data( + lbvc=9, lblev=model_level, + bhlev=delta, bhrlev=delta_lower, + blev=sigma, brlev=sigma_lower, + brsvd=[sigma_upper, delta_upper]) # Convert both fields to cubes. load = mock.Mock(return_value=iter([pressure_field, data_field])) @@ -236,35 +240,21 @@ def field_with_data(scale=1): self.assertEqual(data_field.brsvd, [sigma_upper, delta_upper]) def test_hybrid_pressure_with_duplicate_references(self): - def field_with_data(scale=1): - x, y = 40, 30 - mock_data = np.arange(1200).reshape(y, x) * scale - mock_core_data = mock.MagicMock(return_value=mock_data) - field = mock.MagicMock(core_data=mock_core_data, - realised_dtype=mock_data.dtype, - lbcode=[1], - lbnpt=x, lbrow=y, bzx=350, bdx=1.5, - bzy=40, bdy=1.5, lbuser=[0] * 7, - lbrsvd=[0] * 4) - field._x_coord_name = lambda: 'longitude' - field._y_coord_name = lambda: 'latitude' - field.coord_system = lambda: None - return field - # Make a fake reference surface field. - pressure_field = field_with_data(10) - pressure_field.stash = iris.fileformats.pp.STASH(1, 0, 409) - pressure_field.lbuser[3] = 409 + pressure_field = self._field_with_data( + 10, + stash=iris.fileformats.pp.STASH(1, 0, 409), + lbuser=[0, 0, 0, 409, 0, 0, 0]) # Make a fake data field which needs the reference surface. model_level = 5678 sigma_lower, sigma, sigma_upper = 0.85, 0.9, 0.95 delta_lower, delta, delta_upper = 0.05, 0.1, 0.15 - data_field = field_with_data() - data_field.configure_mock(lbvc=9, lblev=model_level, - bhlev=delta, bhrlev=delta_lower, - blev=sigma, brlev=sigma_lower, - brsvd=[sigma_upper, delta_upper]) + data_field = self._field_with_data( + lbvc=9, lblev=model_level, + bhlev=delta, bhrlev=delta_lower, + blev=sigma, brlev=sigma_lower, + brsvd=[sigma_upper, delta_upper]) # Convert both fields to cubes. load = mock.Mock(return_value=iter([data_field, @@ -351,30 +341,15 @@ def test_hybrid_height_round_trip_no_reference(self): # Use pp.load_cubes() to convert fake PPFields into Cubes. # NB. Use MagicMock so that SplittableInt header items, such as # LBCODE, support len(). - def field_with_data(scale=1): - x, y = 40, 30 - mock_data = np.arange(1200).reshape(y, x) * scale - mock_core_data = mock.MagicMock(return_value=mock_data) - field = mock.MagicMock(core_data=mock_core_data, - realised_dtype=mock_data.dtype, - lbcode=[1], - lbnpt=x, lbrow=y, bzx=350, bdx=1.5, - bzy=40, bdy=1.5, lbuser=[0] * 7, - lbrsvd=[0] * 4) - field._x_coord_name = lambda: 'longitude' - field._y_coord_name = lambda: 'latitude' - field.coord_system = lambda: None - return field - # Make a fake data field which needs the reference surface. model_level = 5678 sigma_lower, sigma, sigma_upper = 0.85, 0.9, 0.95 delta_lower, delta, delta_upper = 0.05, 0.1, 0.15 - data_field = field_with_data() - data_field.configure_mock(lbvc=65, lblev=model_level, - bhlev=sigma, bhrlev=sigma_lower, - blev=delta, brlev=delta_lower, - brsvd=[delta_upper, sigma_upper]) + data_field = self._field_with_data( + lbvc=65, lblev=model_level, + bhlev=sigma, bhrlev=sigma_lower, + blev=delta, brlev=delta_lower, + brsvd=[delta_upper, sigma_upper]) # Convert field to a cube. load = mock.Mock(return_value=iter([data_field])) diff --git a/lib/iris/tests/results/COLPEX/small_colpex_theta_p_alt.cml b/lib/iris/tests/results/COLPEX/small_colpex_theta_p_alt.cml index 0139d1d2fb..5bba278059 100644 --- a/lib/iris/tests/results/COLPEX/small_colpex_theta_p_alt.cml +++ b/lib/iris/tests/results/COLPEX/small_colpex_theta_p_alt.cml @@ -396,8 +396,8 @@ - + @@ -919,8 +919,8 @@ - + diff --git a/lib/iris/tests/results/grib_load/polar_stereo_grib1.cml b/lib/iris/tests/results/grib_load/polar_stereo_grib1.cml index 2ba2e8205b..f8e03e6d18 100644 --- a/lib/iris/tests/results/grib_load/polar_stereo_grib1.cml +++ b/lib/iris/tests/results/grib_load/polar_stereo_grib1.cml @@ -3,7 +3,7 @@ - + diff --git a/lib/iris/tests/results/imagerepo.json b/lib/iris/tests/results/imagerepo.json index 40dd593655..540743366d 100644 --- a/lib/iris/tests/results/imagerepo.json +++ b/lib/iris/tests/results/imagerepo.json @@ -261,7 +261,8 @@ ], "iris.tests.test_plot.Test1dPlotMultiArgs.test_coord.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/83fec2ff7c00a56de9023b52e4143da5d16d7ecad1b76f2094c963929c6471c8.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/8bfec2d77e01a5a5ed013b4ac4521c94817d4e6d91ff63349c6d61991e3278cc.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/8bfec2d77e01a5a5ed013b4ac4521c94817d4e6d91ff63349c6d61991e3278cc.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/8bfec2577e01b5a5ed013b4ac4521c94817d4e4d91ff63369c6d61991e3278cc.png" ], "iris.tests.test_plot.Test1dPlotMultiArgs.test_coord_coord.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/87ff95776a01e1f67801cc36f4075b81c5437668c1167c88d2676d39d6867b68.png", @@ -277,7 +278,8 @@ ], "iris.tests.test_plot.Test1dPlotMultiArgs.test_cube.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/8ffac1547a0792546c179db7f1254f6d945b7392841678e895017e3e91c17a0f.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c1fa7a05b4ea6c059d2ff1494e4b90f26304846d78d1872a6cfc938b2e3e.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c1fa7a05b4ea6c059d2ff1494e4b90f26304846d78d1872a6cfc938b2e3e.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c1fa7a05b4fa6c059d2ef1494e4b90f26304847d78c1872a6cfc938b2e3e.png" ], "iris.tests.test_plot.Test1dPlotMultiArgs.test_cube_coord.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/83fec1ff7e0098757103a71ce4506dc3d11e7b20d2477ec094857db895217f6a.png", @@ -286,7 +288,8 @@ "iris.tests.test_plot.Test1dPlotMultiArgs.test_cube_cube.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c2d73a09b4a76c099d26f14b0e5ad0d643b0d42763e9d51378f895867c39.png", "https://scitools.github.io/test-iris-imagehash/images/v4/8fe8c0173a19b4066d599946f35f0ed5d0b74729d40369d8953678e897877879.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c0567a01b096e4019daff10b464bd4da6391943678e5879f7e3103e67f1c.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c0567a01b096e4019daff10b464bd4da6391943678e5879f7e3103e67f1c.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c0567a01b296e4019d2ff10b464bd4da6391943678e5879f7e3903e63f1c.png" ], "iris.tests.test_plot.Test1dQuickplotPlotMultiArgs.test_coord.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/83fec2777e04256f68023352f6d61da5c109dec8d19bcf089cc9d99a9c85d999.png", @@ -296,7 +299,8 @@ "iris.tests.test_plot.Test1dQuickplotPlotMultiArgs.test_coord_coord.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/83fe9dd77f00e1d73000cc1df707db8184427ef8d1367c88d2667d39d0866b68.png", "https://scitools.github.io/test-iris-imagehash/images/v4/83fe9d977f41e1d73000cc1df707d98184427ef8d1367c88d2667d39d0866b68.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/83ff9d9f7e01e1c2b001c8f8f63e1b1d81cf36e1837e259982ce2f215c9a626c.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/83ff9d9f7e01e1c2b001c8f8f63e1b1d81cf36e1837e259982ce2f215c9a626c.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/83ff9d9f7e01e1c2b001c8f8f63e1b1d81cf36e1837e258982c66f215c9a6a6c.png" ], "iris.tests.test_plot.Test1dQuickplotPlotMultiArgs.test_coord_coord_map.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/fbe0623dc9879d91b41e4b449b6579e78798a49b7872d2644b8c919b39306e6c.png", @@ -318,7 +322,8 @@ "iris.tests.test_plot.Test1dQuickplotPlotMultiArgs.test_cube_cube.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/83ffc8967e0098a6241f9d26e34b8e42f4d20bb4942759e9941f78f8d7867a39.png", "https://scitools.github.io/test-iris-imagehash/images/v4/83f9c8967e009da6245f9946e25f9ed6f0940f29f40749d8853678e8d7857879.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/83ffc9d67e00909624079daef160cf4bd45a439184367ae5979f7e3119e6261c.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/83ffc9d67e00909624079daef160cf4bd45a439184367ae5979f7e3119e6261c.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/83ffc9d67e00909624059daef160cf4bd45a4b9184367ae5979f7e3909e6261c.png" ], "iris.tests.test_plot.Test1dQuickplotScatter.test_coord_coord.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/a3fac1947c99184e62669ca7f65bc96ab81d97b7e248199cc7913662d94ac5a1.png", @@ -394,7 +399,8 @@ "iris.tests.test_plot.TestContour.test_tz.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/8bfe81ff780185fff800955ad4027e00d517d400855f7e0085ff7e8085ff6aed.png", "https://scitools.github.io/test-iris-imagehash/images/v4/8bfe81ff780085fff800855fd4027e00d517d400855f7e0085ff7e8085ff6aed.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/8bfe817ffc00855ef0007e81d4027e80815fd56a03ff7a8085ff3aa883ff6aa5.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/8bfe817ffc00855ef0007e81d4027e80815fd56a03ff7a8085ff3aa883ff6aa5.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/8bff817ffc00857ef0007a81d4027e80815fd56a03ff7a8085ff3aa881ff6aa5.png" ], "iris.tests.test_plot.TestContour.test_yx.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/fa56c3cc34e891b1c9a91c36c5a170e3c71b3e5993a784e492c49b4ecec76393.png", @@ -402,11 +408,13 @@ ], "iris.tests.test_plot.TestContour.test_zx.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/8bfe857f7a01a56afa05854ad015bd00d015d50a90577e80857f7ea0857f7abf.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/affe815ffc008554f8007e01d0027e808557d5ea815f7ea0817f2fea817d2aff.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/affe815ffc008554f8007e01d0027e808557d5ea815f7ea0817f2fea817d2aff.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/affe805ffc008554f8007e01d0027e808557d5ea815f7ea0817f2eea817f2bff.png" ], "iris.tests.test_plot.TestContour.test_zy.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/8bff81ff7a0195fcf8019578d4027e00d550d402857c7e0185fe7a8385fe6aaf.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/abff857ff8018578f8017a80d4027e00855ec42a81fe7a8185fe6a8f85fe6ab7.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/abff857ff8018578f8017a80d4027e00855ec42a81fe7a8185fe6a8f85fe6ab7.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/abff817ff8018578fc017a80d4027e00855ec42a81fe7a8185fe7a8f85fe6ab5.png" ], "iris.tests.test_plot.TestContourf.test_tx.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/faa562ed68569d52857abd12953a8f12951f64e0d30f3ac96a4d6a696ee06a32.png", @@ -627,16 +635,20 @@ ], "iris.tests.test_plot.TestPlot.test_z.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/8ffac1547a0792546c179db7f1254f6d945b7392841678e895017e3e91c17a0f.png", - "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c1fa7a05b4ea6c059d2ff1494e4b90f26304846d78d1872a6cfc938b2e3e.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c1fa7a05b4ea6c059d2ff1494e4b90f26304846d78d1872a6cfc938b2e3e.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/8ff8c1fa7a05b4fa6c059d2ef1494e4b90f26304847d78c1872a6cfc938b2e3e.png" ], "iris.tests.test_plot.TestPlotCitation.test.0": [ - "https://scitools.github.io/test-iris-imagehash/images/v4/abf895067a1d9506f811783585437abd85426ab995067af9f00687f96afe87c8.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/abf895067a1d9506f811783585437abd85426ab995067af9f00687f96afe87c8.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/abf895467a1d9506f811783485437abd85427ab995067ab9f00687f96afe87c8.png" ], "iris.tests.test_plot.TestPlotCitation.test_axes.0": [ - "https://scitools.github.io/test-iris-imagehash/images/v4/abf895067a1d9506f811783585437abd85426ab995067af9f00687f96afe87c8.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/abf895067a1d9506f811783585437abd85426ab995067af9f00687f96afe87c8.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/abf895467a1d9506f811783485437abd85427ab995067ab9f00687f96afe87c8.png" ], "iris.tests.test_plot.TestPlotCitation.test_figure.0": [ - "https://scitools.github.io/test-iris-imagehash/images/v4/abf895067a1d9506f811783585437abd85426ab995067af9f00687f96afe87c8.png" + "https://scitools.github.io/test-iris-imagehash/images/v4/abf895067a1d9506f811783585437abd85426ab995067af9f00687f96afe87c8.png", + "https://scitools.github.io/test-iris-imagehash/images/v4/abf895467a1d9506f811783485437abd85427ab995067ab9f00687f96afe87c8.png" ], "iris.tests.test_plot.TestPlotCoordinatesGiven.test_non_cube_coordinate.0": [ "https://scitools.github.io/test-iris-imagehash/images/v4/fa81857e857e7e81857e7a81857e7a81857e7a818576c02a7e95856a7e81c17a.png", diff --git a/lib/iris/tests/results/netcdf/netcdf_save_load_hybrid_height.cml b/lib/iris/tests/results/netcdf/netcdf_save_load_hybrid_height.cml index 070e7597d3..a89756a77e 100644 --- a/lib/iris/tests/results/netcdf/netcdf_save_load_hybrid_height.cml +++ b/lib/iris/tests/results/netcdf/netcdf_save_load_hybrid_height.cml @@ -414,8 +414,8 @@ - + diff --git a/lib/iris/tests/test_image_json.py b/lib/iris/tests/test_image_json.py index 48690ebe44..57a15f845b 100644 --- a/lib/iris/tests/test_image_json.py +++ b/lib/iris/tests/test_image_json.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2016 - 2018, Met Office +# (C) British Crown Copyright 2016 - 2019, Met Office # # This file is part of Iris. # @@ -29,55 +29,45 @@ import json import os import requests -import unittest -import time @tests.skip_inet -@tests.skip_data class TestImageFile(tests.IrisTest): def test_resolve(self): - # https://developer.github.com/v3/#user-agent-required - headers = {'User-Agent': 'scitools-bot'} - rate_limit_uri = 'https://api.github.com/rate_limit' - rl = requests.get(rate_limit_uri, headers=headers) - some_left = False - if rl.status_code == 200: - rates = rl.json() - remaining = rates.get('rate', {}) - ghapi_remaining = remaining.get('remaining') - else: - ghapi_remaining = 0 + listingfile_uri = ( + 'https://raw.githubusercontent.com/SciTools/test-iris-imagehash' + '/gh-pages/v4_files_listing.txt') + req = requests.get(listingfile_uri) + if req.status_code != 200: + raise ValueError('GET failed on image listings file: {}'.format( + listingfile_uri)) - # Only run this test if there are IP based rate limited calls left. - # 3 is an engineering tolerance, in case of race conditions. - amin = 3 - if ghapi_remaining < amin: - return unittest.skip("Less than {} anonymous calls to " - "GH API left!".format(amin)) - iuri = ('https://api.github.com/repos/scitools/' - 'test-iris-imagehash/contents/images/v4') - r = requests.get(iuri, headers=headers) - if r.status_code != 200: - raise ValueError('Github API get failed: {}'.format(iuri, - r.text)) - rj = r.json() + listings_text = req.content.decode('utf-8') + reference_image_filenames = [line.strip() + for line in listings_text.split('\n')] base = 'https://scitools.github.io/test-iris-imagehash/images/v4' + reference_image_uris = set('{}/{}'.format(base, name) + for name in reference_image_filenames) - known_image_uris = set([os.path.join(base, rji['name']) for rji in rj]) + imagerepo_json_filepath = os.path.join( + os.path.dirname(__file__), 'results', 'imagerepo.json') + with open(imagerepo_json_filepath, 'rb') as fi: + imagerepo = json.load(codecs.getreader('utf-8')(fi)) - repo_fname = os.path.join(os.path.dirname(__file__), 'results', - 'imagerepo.json') - with open(repo_fname, 'rb') as fi: - repo = json.load(codecs.getreader('utf-8')(fi)) - uris = set(itertools.chain.from_iterable(six.itervalues(repo))) + # "imagerepo" maps key: list-of-uris. Put all the uris in one big set. + tests_uris = set(itertools.chain.from_iterable( + six.itervalues(imagerepo))) - amsg = ('Images are referenced in imagerepo.json but not published ' - 'in {}:\n{}') - diffs = list(uris.difference(known_image_uris)) - amsg = amsg.format(base, '\n'.join(diffs)) - - self.assertTrue(uris.issubset(known_image_uris), msg=amsg) + missing_refs = list(tests_uris - reference_image_uris) + n_missing_refs = len(missing_refs) + if n_missing_refs > 0: + amsg = ('Missing images: These {} image uris are referenced in ' + 'imagerepo.json, but not listed in {} : ') + amsg = amsg.format(n_missing_refs, listingfile_uri) + amsg += ''.join('\n {}'.format(uri) + for uri in missing_refs) + # Always fails when we get here: report the problem. + self.assertEqual(n_missing_refs, 0, msg=amsg) if __name__ == "__main__": diff --git a/lib/iris/tests/test_merge.py b/lib/iris/tests/test_merge.py index 7c1c1e2bac..1ca86e6929 100644 --- a/lib/iris/tests/test_merge.py +++ b/lib/iris/tests/test_merge.py @@ -197,16 +197,16 @@ def test__ndarray_ndarray(self): def test__masked_masked(self): for (lazy0, lazy1), (fill0, fill1) in self.combos: cubes = iris.cube.CubeList() - mask = [(0,), (0,)] + mask = ((0,), (0,)) cubes.append(self._make_cube(0, mask=mask, lazy=lazy0, dtype=self.dtype, fill_value=fill0)) - mask = [(1,), (1,)] + mask = ((1,), (1,)) cubes.append(self._make_cube(1, mask=mask, lazy=lazy1, dtype=self.dtype, fill_value=fill1)) result = cubes.merge_cube() - mask = [(0, 1), (0, 1), (0, 1)] + mask = ((0, 1), (0, 1), (0, 1)) expected_fill_value = self._expected_fill_value(fill0, fill1) expected = self._make_data([0, 1], mask=mask, dtype=self.dtype, fill_value=expected_fill_value) diff --git a/lib/iris/tests/test_pp_module.py b/lib/iris/tests/test_pp_module.py index 2c7ac0f1cf..1537c13c29 100644 --- a/lib/iris/tests/test_pp_module.py +++ b/lib/iris/tests/test_pp_module.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2018, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -443,7 +443,7 @@ class TestPPDataProxyEquality(tests.IrisTest): def test_not_implemented(self): class Terry(object): pass pox = pp.PPDataProxy("john", "michael", "eric", "graham", "brian", - "spam", "beans", "eggs", "parrot") + "spam", "beans", "eggs") self.assertIs(pox.__eq__(Terry()), NotImplemented) self.assertIs(pox.__ne__(Terry()), NotImplemented) diff --git a/lib/iris/tests/unit/coords/test_AuxCoord.py b/lib/iris/tests/unit/coords/test_AuxCoord.py index 39e5048b14..3a5cf4151a 100644 --- a/lib/iris/tests/unit/coords/test_AuxCoord.py +++ b/lib/iris/tests/unit/coords/test_AuxCoord.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2017 - 2018, Met Office +# (C) British Crown Copyright 2017 - 2019, Met Office # # This file is part of Iris. # @@ -624,5 +624,20 @@ def test_preserves_lazy(self): self.assertArrayAllClose(coord.bounds, test_bounds_ft) +class TestEquality(tests.IrisTest): + def test_nanpoints_eq_self(self): + co1 = AuxCoord([1., np.nan, 2.]) + self.assertEqual(co1, co1) + + def test_nanpoints_eq_copy(self): + co1 = AuxCoord([1., np.nan, 2.]) + co2 = co1.copy() + self.assertEqual(co1, co2) + + def test_nanbounds_eq_self(self): + co1 = AuxCoord([15., 25.], bounds=[[14., 16.], [24., np.nan]]) + self.assertEqual(co1, co1) + + if __name__ == '__main__': tests.main() diff --git a/lib/iris/tests/unit/fileformats/pp/test_PPDataProxy.py b/lib/iris/tests/unit/fileformats/pp/test_PPDataProxy.py index 0732f296bb..8dc21f288f 100644 --- a/lib/iris/tests/unit/fileformats/pp/test_PPDataProxy.py +++ b/lib/iris/tests/unit/fileformats/pp/test_PPDataProxy.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2019, Met Office # # This file is part of Iris. # @@ -31,14 +31,14 @@ class Test_lbpack(tests.IrisTest): def test_lbpack_SplittableInt(self): lbpack = mock.Mock(spec_set=SplittableInt) proxy = PPDataProxy(None, None, None, None, - None, lbpack, None, None, None) + None, lbpack, None, None) self.assertEqual(proxy.lbpack, lbpack) self.assertIs(proxy.lbpack, lbpack) def test_lnpack_raw(self): lbpack = 4321 proxy = PPDataProxy(None, None, None, None, - None, lbpack, None, None, None) + None, lbpack, None, None) self.assertEqual(proxy.lbpack, lbpack) self.assertIsNot(proxy.lbpack, lbpack) self.assertIsInstance(proxy.lbpack, SplittableInt) diff --git a/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py b/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py index 743d52ea0e..d9816c64b9 100644 --- a/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py +++ b/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2017, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -63,7 +63,6 @@ def test_deferred_bytes(self): core_data = mock.MagicMock(return_value=deferred_bytes) field = mock.Mock(core_data=core_data) data_shape = (100, 120) - land_mask = mock.Mock() proxy = mock.Mock(dtype=np.dtype('f4'), shape=data_shape, spec=pp.PPDataProxy) # We can't directly inspect the concrete data source underlying @@ -71,7 +70,7 @@ def test_deferred_bytes(self): # being created and invoked correctly. with mock.patch('iris.fileformats.pp.PPDataProxy') as PPDataProxy: PPDataProxy.return_value = proxy - pp._create_field_data(field, data_shape, land_mask) + pp._create_field_data(field, data_shape, land_mask_field=None) # The data should be assigned via field.data. As this is a mock object # we can check the attribute directly. self.assertEqual(field.data.shape, data_shape) @@ -84,8 +83,7 @@ def test_deferred_bytes(self): n_bytes, field.raw_lbpack, field.boundary_packing, - field.bmdi, - land_mask) + field.bmdi) if __name__ == "__main__": diff --git a/lib/iris/tests/unit/fileformats/pp/test__data_bytes_to_shaped_array.py b/lib/iris/tests/unit/fileformats/pp/test__data_bytes_to_shaped_array.py index 4870624902..09ca8d99af 100644 --- a/lib/iris/tests/unit/fileformats/pp/test__data_bytes_to_shaped_array.py +++ b/lib/iris/tests/unit/fileformats/pp/test__data_bytes_to_shaped_array.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2017, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -111,16 +111,15 @@ def create_lbpack(self, value): return pp.SplittableInt(value, name_mapping) def test_no_land_mask(self): + # Check that without a mask, it returns the raw (compressed) data. with mock.patch('numpy.frombuffer', return_value=np.arange(3)): - with self.assertRaises(ValueError) as err: - pp._data_bytes_to_shaped_array(mock.Mock(), - self.create_lbpack(120), None, - (3, 4), np.dtype('>f4'), - -999, mask=None) - self.assertEqual(str(err.exception), - ('No mask was found to unpack the data. ' - 'Could not load.')) + result = pp._data_bytes_to_shaped_array( + mock.Mock(), + self.create_lbpack(120), None, + (3, 4), np.dtype('>f4'), + -999, mask=None) + self.assertArrayAllClose(result, np.arange(3)) def test_land_mask(self): # Check basic land unpacking. diff --git a/lib/iris/tests/unit/fileformats/pp/test__interpret_field.py b/lib/iris/tests/unit/fileformats/pp/test__interpret_field.py index 1828f44a87..4273a361da 100644 --- a/lib/iris/tests/unit/fileformats/pp/test__interpret_field.py +++ b/lib/iris/tests/unit/fileformats/pp/test__interpret_field.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2017, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -26,6 +26,7 @@ from copy import deepcopy import numpy as np +import iris import iris.fileformats.pp as pp from iris.tests import mock @@ -37,7 +38,8 @@ def setUp(self): # A field packed using a land/sea mask. self.pp_field = mock.Mock(lblrec=1, lbext=0, lbuser=[0] * 7, lbrow=0, lbnpt=0, - raw_lbpack=20, + raw_lbpack=21, + lbpack=mock.Mock(n1=0, n2=2, n3=1), core_data=core_data) # The field specifying the land/seamask. lbuser = [None, None, None, 30, None, None, 1] # m01s00i030 @@ -91,19 +93,41 @@ def test_deferred_fix_lbrow_lbnpt(self): self.assertEqual(f1.lbrow, 3) self.assertEqual(f1.lbnpt, 4) - def test_shared_land_mask_field(self): - # Check that multiple land masked fields share the - # land mask field instance. - f1 = deepcopy(self.pp_field) - f2 = deepcopy(self.pp_field) - self.assertIsNot(f1, f2) - with mock.patch('iris.fileformats.pp.PPDataProxy') as PPDataProxy: - PPDataProxy.return_value = mock.MagicMock(shape=(3, 4), - dtype=np.float32) - list(pp._interpret_fields([f1, self.land_mask_field, f2])) - for call in PPDataProxy.call_args_list: - positional_args = call[0] - self.assertIs(positional_args[8], self.land_mask_field) + @tests.skip_data + def test_landsea_unpacking_uses_dask(self): + # Ensure that the graph of the (lazy) landsea-masked data contains an + # explicit reference to a (lazy) landsea-mask field. + # Otherwise its compute() will need to invoke another compute(). + # See https://github.com/SciTools/iris/issues/3237 + + # This is too complex to explore in a mock-ist way, so let's load a + # tiny bit of real data ... + testfile_path = tests.get_data_path( + ['FF', 'landsea_masked', 'testdata_mini_lsm.ff']) + landsea_mask, soil_temp = iris.load_cubes( + testfile_path, ('land_binary_mask', 'soil_temperature')) + + # Now check that the soil-temp dask graph correctly references the + # landsea mask, in its dask graph. + lazy_mask_array = landsea_mask.core_data() + lazy_soildata_array = soil_temp.core_data() + + # Work out the main dask key for the mask data, as used by 'compute()'. + mask_toplev_key = (lazy_mask_array.name,) + (0,) * lazy_mask_array.ndim + # Get the 'main' calculation entry. + mask_toplev_item = lazy_mask_array.dask[mask_toplev_key] + # This should be a task (a simple fetch). + self.assertTrue(callable(mask_toplev_item[0])) + # Get the key (name) of the array that it fetches. + mask_data_name = mask_toplev_item[1] + + # Check that the item this refers to is a PPDataProxy. + self.assertIsInstance(lazy_mask_array.dask[mask_data_name], + pp.PPDataProxy) + + # Check that the soil-temp graph references the *same* lazy element, + # showing that the mask+data calculation is handled by dask. + self.assertIn(mask_data_name, lazy_soildata_array.dask.keys()) if __name__ == "__main__": diff --git a/lib/iris/tests/unit/fileformats/pp_load_rules/test__convert_time_coords.py b/lib/iris/tests/unit/fileformats/pp_load_rules/test__convert_time_coords.py index f00b39156f..898df79620 100644 --- a/lib/iris/tests/unit/fileformats/pp_load_rules/test__convert_time_coords.py +++ b/lib/iris/tests/unit/fileformats/pp_load_rules/test__convert_time_coords.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2018, Met Office +# (C) British Crown Copyright 2014 - 2019, Met Office # # This file is part of Iris. # @@ -140,8 +140,8 @@ def test_not_exact_hours(self): lbcode=_lbcode(1), lbtim=lbtim, epoch_hours_unit=_EPOCH_HOURS_UNIT, t1=t1, t2=t2, lbft=None) (fp, _), (t, _), (frt, _) = coords_and_dims - self.assertEqual(fp.points[0], 7.1666666641831398) - self.assertEqual(t.points[0], 394927.16666666418) + self.assertArrayAllClose(fp.points[0], 7.1666666, atol=0.0001, rtol=0) + self.assertArrayAllClose(t.points[0], 394927.166666, atol=0.01, rtol=0) class TestLBTIMx2x_TimePeriod(TestField): diff --git a/lib/iris/tests/unit/fileformats/pp_load_rules/test__epoch_date_hours.py b/lib/iris/tests/unit/fileformats/pp_load_rules/test__epoch_date_hours.py new file mode 100644 index 0000000000..0209df5dd0 --- /dev/null +++ b/lib/iris/tests/unit/fileformats/pp_load_rules/test__epoch_date_hours.py @@ -0,0 +1,150 @@ +# (C) British Crown Copyright 2019, Met Office +# +# This file is part of Iris. +# +# Iris is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Iris is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Iris. If not, see . +""" +Unit tests for +:func:`iris.fileformats.pp_load_rules._epoch_date_hours`. + +""" +from __future__ import (absolute_import, division, print_function) +from six.moves import (filter, input, map, range, zip) # noqa + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import cf_units +from cf_units import Unit +from cftime import datetime as nc_datetime + +from iris.fileformats.pp_load_rules \ + import _epoch_date_hours as epoch_hours_call + + +# +# Run tests for each of the possible calendars from PPfield.calendar(). +# Test year=0 and all=0 cases, plus "normal" dates, for each calendar. +# Result values are the same as from 'date2num' in cftime version <= 1.0.1. +# + +class TestEpochHours__gregorian(tests.IrisTest): + def setUp(self): + self.hrs_unit = Unit('hours since epoch', + calendar=cf_units.CALENDAR_GREGORIAN) + + def test_1970_1_1(self): + test_date = nc_datetime(1970, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, 0.0) + + def test_ymd_1_1_1(self): + test_date = nc_datetime(1, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17259936.0) + + def test_year_0(self): + test_date = nc_datetime(0, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17268720.0) + + def test_ymd_0_0_0(self): + test_date = nc_datetime(0, 0, 0) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17269488.0) + + def test_ymd_0_preserves_timeofday(self): + hrs, mins, secs, usecs = (7, 13, 24, 335772) + hours_in_day = (hrs + + 1./60 * mins + + 1./3600 * secs + + (1.0e-6) / 3600 * usecs) + test_date = nc_datetime(0, 0, 0, + hour=hrs, minute=mins, second=secs, + microsecond=usecs) + result = epoch_hours_call(self.hrs_unit, test_date) + # NOTE: the calculation is only accurate to approx +/- 0.5 seconds + # in such a large number of hours -- even 0.1 seconds is too fine. + absolute_tolerance = 0.5 / 3600 + self.assertArrayAllClose(result, -17269488.0 + hours_in_day, + rtol=0, atol=absolute_tolerance) + + +class TestEpochHours__360day(tests.IrisTest): + def setUp(self): + self.hrs_unit = Unit('hours since epoch', + calendar=cf_units.CALENDAR_360_DAY) + + def test_1970_1_1(self): + test_date = nc_datetime(1970, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, 0.0) + + def test_ymd_1_1_1(self): + test_date = nc_datetime(1, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17012160.0) + + def test_year_0(self): + test_date = nc_datetime(0, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17020800.0) + + def test_ymd_0_0_0(self): + test_date = nc_datetime(0, 0, 0) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17021544.0) + + +class TestEpochHours__365day(tests.IrisTest): + def setUp(self): + self.hrs_unit = Unit('hours since epoch', + calendar=cf_units.CALENDAR_365_DAY) + + def test_1970_1_1(self): + test_date = nc_datetime(1970, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, 0.0) + + def test_ymd_1_1_1(self): + test_date = nc_datetime(1, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17248440.0) + + def test_year_0(self): + test_date = nc_datetime(0, 1, 1) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17257200.0) + + def test_ymd_0_0_0(self): + test_date = nc_datetime(0, 0, 0) + result = epoch_hours_call(self.hrs_unit, test_date) + self.assertEqual(result, -17257968.0) + + +class TestEpochHours__invalid_calendar(tests.IrisTest): + def test_bad_calendar(self): + # Setup a unit with an unrecognised calendar + hrs_unit = Unit('hours since epoch', + calendar=cf_units.CALENDAR_ALL_LEAP) + # Test against a date with year=0, which requires calendar correction. + test_date = nc_datetime(0, 1, 1) + # Check that this causes an error. + with self.assertRaisesRegexp(ValueError, 'unrecognised calendar'): + epoch_hours_call(hrs_unit, test_date) + + +if __name__ == "__main__": + tests.main() diff --git a/lib/iris/tests/unit/fileformats/pp_load_rules/test_convert.py b/lib/iris/tests/unit/fileformats/pp_load_rules/test_convert.py index c1f8bdb891..77761d431b 100644 --- a/lib/iris/tests/unit/fileformats/pp_load_rules/test_convert.py +++ b/lib/iris/tests/unit/fileformats/pp_load_rules/test_convert.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2018, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -39,6 +39,14 @@ import iris.tests.unit.fileformats +def _mock_field(**kwargs): + # Generate a mock field, but ensure T1 and T2 viable for rules. + field = mock.MagicMock(t1=mock.MagicMock(year=1990, month=3, day=7), + t2=mock.MagicMock(year=1990, month=3, day=7)) + field.configure_mock(**kwargs) + return field + + class TestLBCODE(iris.tests.unit.fileformats.TestField): @staticmethod def _is_cross_section_height_coord(coord): @@ -50,7 +58,7 @@ def test_cross_section_height_bdy_zero(self): lbcode = SplittableInt(19902, {'iy': slice(0, 2), 'ix': slice(2, 4)}) points = np.array([10, 20, 30, 40]) bounds = np.array([[0, 15], [15, 25], [25, 35], [35, 45]]) - field = mock.MagicMock(lbcode=lbcode, bdy=0, y=points, y_bounds=bounds) + field = _mock_field(lbcode=lbcode, bdy=0, y=points, y_bounds=bounds) self._test_for_coord(field, convert, TestLBCODE._is_cross_section_height_coord, expected_points=points, @@ -61,8 +69,8 @@ def test_cross_section_height_bdy_bmdi(self): points = np.array([10, 20, 30, 40]) bounds = np.array([[0, 15], [15, 25], [25, 35], [35, 45]]) bmdi = -1.07374e+09 - field = mock.MagicMock(lbcode=lbcode, bdy=bmdi, bmdi=bmdi, - y=points, y_bounds=bounds) + field = _mock_field(lbcode=lbcode, bdy=bmdi, bmdi=bmdi, + y=points, y_bounds=bounds) self._test_for_coord(field, convert, TestLBCODE._is_cross_section_height_coord, expected_points=points, @@ -105,7 +113,7 @@ def _is_soil_depth_coord(coord): def test_soil_levels(self): level = 1234 - field = mock.MagicMock(lbvc=6, lblev=level, brsvd=[0, 0], brlev=0) + field = _mock_field(lbvc=6, lblev=level, brsvd=[0, 0], brlev=0) self._test_for_coord(field, convert, self._is_soil_model_level_number_coord, expected_points=[level], @@ -113,8 +121,7 @@ def test_soil_levels(self): def test_soil_depth(self): lower, point, upper = 1.2, 3.4, 5.6 - field = mock.MagicMock(lbvc=6, blev=point, brsvd=[lower, 0], - brlev=upper) + field = _mock_field(lbvc=6, blev=point, brsvd=[lower, 0], brlev=upper) self._test_for_coord(field, convert, self._is_soil_depth_coord, expected_points=[point], @@ -122,9 +129,9 @@ def test_soil_depth(self): def test_hybrid_pressure_model_level_number(self): level = 5678 - field = mock.MagicMock(lbvc=9, lblev=level, - blev=20, brlev=23, bhlev=42, - bhrlev=45, brsvd=[17, 40]) + field = _mock_field(lbvc=9, lblev=level, + blev=20, brlev=23, bhlev=42, + bhrlev=45, brsvd=[17, 40]) self._test_for_coord(field, convert, TestLBVC._is_model_level_number_coord, expected_points=[level], @@ -134,10 +141,10 @@ def test_hybrid_pressure_delta(self): delta_point = 12.0 delta_lower_bound = 11.0 delta_upper_bound = 13.0 - field = mock.MagicMock(lbvc=9, lblev=5678, - blev=20, brlev=23, bhlev=delta_point, - bhrlev=delta_lower_bound, - brsvd=[17, delta_upper_bound]) + field = _mock_field(lbvc=9, lblev=5678, + blev=20, brlev=23, bhlev=delta_point, + bhrlev=delta_lower_bound, + brsvd=[17, delta_upper_bound]) self._test_for_coord(field, convert, TestLBVC._is_level_pressure_coord, expected_points=[delta_point], @@ -148,10 +155,10 @@ def test_hybrid_pressure_sigma(self): sigma_point = 0.5 sigma_lower_bound = 0.6 sigma_upper_bound = 0.4 - field = mock.MagicMock(lbvc=9, lblev=5678, - blev=sigma_point, brlev=sigma_lower_bound, - bhlev=12, bhrlev=11, - brsvd=[sigma_upper_bound, 13]) + field = _mock_field(lbvc=9, lblev=5678, + blev=sigma_point, brlev=sigma_lower_bound, + bhlev=12, bhrlev=11, + brsvd=[sigma_upper_bound, 13]) self._test_for_coord(field, convert, TestLBVC._is_sigma_coord, expected_points=[sigma_point], expected_bounds=[[sigma_lower_bound, @@ -159,7 +166,7 @@ def test_hybrid_pressure_sigma(self): def test_potential_temperature_levels(self): potm_value = 27.32 - field = mock.MagicMock(lbvc=19, blev=potm_value) + field = _mock_field(lbvc=19, blev=potm_value) self._test_for_coord(field, convert, TestLBVC._is_potm_level_coord, expected_points=np.array([potm_value]), expected_bounds=None) @@ -265,7 +272,7 @@ def test_realization(self): lbrsvd[3] = 71 points = np.array([71]) bounds = None - field = mock.MagicMock(lbrsvd=lbrsvd) + field = _mock_field(lbrsvd=lbrsvd) self._test_for_coord(field, convert, TestLBRSVD._is_realization, expected_points=points, @@ -275,7 +282,7 @@ def test_realization(self): class TestLBSRCE(iris.tests.IrisTest): def check_um_source_attrs(self, lbsrce, source_str=None, um_version_str=None): - field = mock.MagicMock(lbsrce=lbsrce) + field = _mock_field(lbsrce=lbsrce) (factories, references, standard_name, long_name, units, attributes, cell_methods, dim_coords_and_dims, aux_coords_and_dims) = convert(field) @@ -310,7 +317,7 @@ def test_stash_cf_air_temp(self): lbuser = [1, 0, 0, 16203, 0, 0, 1] lbfc = 16 stash = STASH(lbuser[6], lbuser[3] // 1000, lbuser[3] % 1000) - field = mock.MagicMock(lbuser=lbuser, lbfc=lbfc, stash=stash) + field = _mock_field(lbuser=lbuser, lbfc=lbfc, stash=stash) (factories, references, standard_name, long_name, units, attributes, cell_methods, dim_coords_and_dims, aux_coords_and_dims) = convert(field) @@ -321,7 +328,7 @@ def test_no_std_name(self): lbuser = [1, 0, 0, 0, 0, 0, 0] lbfc = 0 stash = STASH(lbuser[6], lbuser[3] // 1000, lbuser[3] % 1000) - field = mock.MagicMock(lbuser=lbuser, lbfc=lbfc, stash=stash) + field = _mock_field(lbuser=lbuser, lbfc=lbfc, stash=stash) (factories, references, standard_name, long_name, units, attributes, cell_methods, dim_coords_and_dims, aux_coords_and_dims) = convert(field) @@ -334,7 +341,7 @@ def test_fc_cf_air_temp(self): lbuser = [1, 0, 0, 0, 0, 0, 0] lbfc = 16 stash = STASH(lbuser[6], lbuser[3] // 1000, lbuser[3] % 1000) - field = mock.MagicMock(lbuser=lbuser, lbfc=lbfc, stash=stash) + field = _mock_field(lbuser=lbuser, lbfc=lbfc, stash=stash) (factories, references, standard_name, long_name, units, attributes, cell_methods, dim_coords_and_dims, aux_coords_and_dims) = convert(field) diff --git a/lib/iris/tests/unit/util/test_array_equal.py b/lib/iris/tests/unit/util/test_array_equal.py index 31fd9dd842..ab5404d43d 100644 --- a/lib/iris/tests/unit/util/test_array_equal.py +++ b/lib/iris/tests/unit/util/test_array_equal.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2019, Met Office # # This file is part of Iris. # @@ -115,6 +115,25 @@ def test_string_arrays_0d_and_scalar(self): self.assertFalse(array_equal(array_a, 'foo')) self.assertFalse(array_equal(array_a, 'foobar.')) + def test_nan_equality_nan_ne_nan(self): + array = np.array([1.0, np.nan, 2.0, np.nan, 3.0]) + self.assertFalse(array_equal(array, array)) + + def test_nan_equality_nan_naneq_nan(self): + array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0]) + array_b = np.array([1.0, np.nan, 2.0, np.nan, 3.0]) + self.assertTrue(array_equal(array_a, array_b, withnans=True)) + + def test_nan_equality_nan_nanne_a(self): + array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0]) + array_b = np.array([1.0, np.nan, 2.0, 0.0, 3.0]) + self.assertFalse(array_equal(array_a, array_b, withnans=True)) + + def test_nan_equality_a_nanne_b(self): + array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0]) + array_b = np.array([1.0, np.nan, 2.0, np.nan, 4.0]) + self.assertFalse(array_equal(array_a, array_b, withnans=True)) + if __name__ == '__main__': tests.main() diff --git a/lib/iris/tests/unit/util/test_new_axis.py b/lib/iris/tests/unit/util/test_new_axis.py index 0f40d92426..78ca24bbd3 100644 --- a/lib/iris/tests/unit/util/test_new_axis.py +++ b/lib/iris/tests/unit/util/test_new_axis.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2017, Met Office +# (C) British Crown Copyright 2013 - 2019, Met Office # # This file is part of Iris. # @@ -25,11 +25,14 @@ import iris.tests.stock as stock import copy -from iris._lazy_data import as_lazy_data import numpy as np import unittest +import six + import iris +from iris._lazy_data import as_lazy_data + from iris.util import new_axis @@ -136,6 +139,17 @@ def test_maint_factory(self): self.assertEqual(res, com) self._assert_cube_notis(res, cube) + # Check that factory dependencies are actual coords within the cube. + # Addresses a former bug : https://github.com/SciTools/iris/pull/3263 + factory, = list(res.aux_factories) + deps = factory.dependencies + for dep_name, dep_coord in six.iteritems(deps): + coord_name = dep_coord.name() + msg = ('Factory dependency {!r} is a coord named {!r}, ' + 'but it is *not* the coord of that name in the new cube.') + self.assertIs(dep_coord, res.coord(coord_name), + msg.format(dep_name, coord_name)) + def test_lazy_data(self): cube = iris.cube.Cube(as_lazy_data(self.data)) cube.add_aux_coord(iris.coords.DimCoord([1], standard_name='time')) diff --git a/lib/iris/util.py b/lib/iris/util.py index 09fd530368..b0f00c3b52 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -332,19 +332,42 @@ def rolling_window(a, window=1, step=1, axis=-1): return rw -def array_equal(array1, array2): +def array_equal(array1, array2, withnans=False): """ Returns whether two arrays have the same shape and elements. - This provides the same functionality as :func:`numpy.array_equal` but with - additional support for arrays of strings. + Args: + + * array1, array2 (arraylike): + args to be compared, after normalising with :func:`np.asarray`. + + Kwargs: + + * withnans (bool): + When unset (default), the result is False if either input contains NaN + points. This is the normal floating-point arithmetic result. + When set, return True if inputs contain the same value in all elements, + _including_ any NaN values. + + This provides much the same functionality as :func:`numpy.array_equal`, but + with additional support for arrays of strings and NaN-tolerant operation. """ array1, array2 = np.asarray(array1), np.asarray(array2) - if array1.shape != array2.shape: - eq = False - else: - eq = bool(np.asarray(array1 == array2).all()) + + eq = (array1.shape == array2.shape) + if eq: + eqs = (array1 == array2) + + if withnans and (array1.dtype.kind == 'f' or array2.dtype.kind == 'f'): + nans1, nans2 = np.isnan(array1), np.isnan(array2) + if not np.all(nans1 == nans2): + eq = False # simply fail + else: + eqs[nans1] = True # fix NaNs; check all the others + + if eq: + eq = np.all(eqs) # check equal at all points return eq @@ -1062,8 +1085,12 @@ def new_axis(src_cube, scalar_coord=None): coord_dims = np.array(src_cube.coord_dims(coord)) + 1 new_cube.add_dim_coord(coord.copy(), coord_dims) + nonderived_coords = src_cube.dim_coords + src_cube.aux_coords + coord_mapping = {id(old_co): new_cube.coord(old_co) + for old_co in nonderived_coords} for factory in src_cube.aux_factories: - new_cube.add_aux_factory(copy.deepcopy(factory)) + new_factory = factory.updated(coord_mapping) + new_cube.add_aux_factory(new_factory) return new_cube diff --git a/requirements/core.txt b/requirements/core.txt index 7e55bd1fd5..e39777edb7 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -5,8 +5,8 @@ cartopy #conda: proj4<5 -cf_units>=2 -cftime==1.0.1 +cf-units>=2 +cftime dask[array] #conda: dask matplotlib>=2,<3 netcdf4