diff --git a/lib/iris/_concatenate.py b/lib/iris/_concatenate.py index 67a5b90a5e..b6c7bc1cff 100644 --- a/lib/iris/_concatenate.py +++ b/lib/iris/_concatenate.py @@ -28,6 +28,7 @@ import dask.array as da import numpy as np +import numpy.ma as ma import iris.coords import iris.cube @@ -329,6 +330,16 @@ def __init__(self, cube): self.defn = cube.metadata self.data_type = cube.dtype + # Concatenation will generate resultant cubes that contain lazy data. + # We need to deal with the special case of preserving the dtype of a + # candidate cube with concrete masked integral data by ensuring that + # the cube metadata has the correct target dtype. + kwargs = self.defn._asdict() + if kwargs['dtype'] is None and not cube.has_lazy_data() and \ + isinstance(cube.data, ma.masked_array) and \ + cube.data.dtype.kind == 'i': + kwargs['dtype'] = self.data_type + self.defn = iris.cube.CubeMetadata(**kwargs) # # Collate the dimension coordinate metadata. @@ -402,6 +413,20 @@ def _coordinate_differences(self, other, attr): ', '.join(diff_names)) return result + def promote_defn(self): + """ + Update the metadata definition to match that of a similar but lazy + cube. A cube with lazy masked integral data must have its + :attr:`metadata.dtype` set appropriately. + + """ + defn = self.defn + kwargs = defn._asdict() + if kwargs['dtype'] is None and self.data_type.kind == 'i': + kwargs['dtype'] = self.data_type + defn = iris.cube.CubeMetadata(**kwargs) + return defn + def match(self, other, error_on_mismatch): """ Return whether this _CubeSignature equals another. @@ -433,10 +458,17 @@ def match(self, other, error_on_mismatch): # Check cube definitions. if self.defn != other.defn: - # Note that the case of different phenomenon names is dealt with - # in :meth:`iris.cube.CubeList.concatenate_cube()`. - msg = 'Cube metadata differs for phenomenon: {}' - msgs.append(msg.format(self.defn.name())) + # Attempt to match metadata for the lazy masked integral + # dtype case. + promoted = self.promote_defn() + if promoted != other.promote_defn(): + # Note that the case of different phenomenon names is dealt + # with in :meth:`iris.cube.CubeList.concatenate_cube()`. + msg = 'Cube metadata differs for phenomenon: {}' + msgs.append(msg.format(self.defn.name())) + else: + # Persist the promoted metadata dtype match case. + self.defn = promoted # Check dim coordinates. if self.dim_metadata != other.dim_metadata: differences = self._coordinate_differences(other, 'dim_metadata') diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index c63cce14b6..9104a96d7c 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -563,7 +563,9 @@ def post_process(self, collapsed_cube, data_result, coords, **kwargs): The collapsed cube with its aggregated data payload. """ + cube_fill_value = collapsed_cube.fill_value collapsed_cube.data = data_result + collapsed_cube.fill_value = cube_fill_value return collapsed_cube def aggregate_shape(self, **kwargs): diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 230162c943..0772467ce3 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -65,8 +65,8 @@ class CubeMetadata(collections.namedtuple('CubeMetadata', 'units', 'attributes', 'cell_methods', - 'fill_value', - 'dtype'])): + 'dtype', + 'fill_value'])): """ Represents the phenomenon metadata for a single :class:`Cube`. @@ -751,10 +751,12 @@ def __init__(self, data, standard_name=None, long_name=None, # Cell Measures self._cell_measures_and_dims = [] + # We need to set the dtype before the fill_value, + # as the fill_value is checked against self.dtype. + self._dtype = None + self.dtype = dtype self.fill_value = fill_value - self._dtype = dtype - identities = set() if dim_coords_and_dims: dims = set() @@ -798,7 +800,7 @@ def metadata(self): """ return CubeMetadata(self.standard_name, self.long_name, self.var_name, self.units, self.attributes, self.cell_methods, - self.fill_value, self._dtype) + self._dtype, self.fill_value) @metadata.setter def metadata(self, value): @@ -813,7 +815,10 @@ def metadata(self, value): if missing_attrs: raise TypeError('Invalid/incomplete metadata') for name in CubeMetadata._fields: - setattr(self, name, getattr(value, name)) + alias = name + if name in ['dtype', 'fill_value']: + alias = '_{}'.format(name) + setattr(self, alias, getattr(value, name)) def is_compatible(self, other, ignore=None): """ @@ -1632,7 +1637,38 @@ def dtype(self): @dtype.setter def dtype(self, dtype): - self._dtype = dtype + if dtype != self.dtype: + if dtype is not None: + if not self.has_lazy_data(): + emsg = 'Cube does not have lazy data, cannot set dtype.' + raise ValueError(emsg) + dtype = np.dtype(dtype) + if dtype.kind != 'i': + emsg = ('Can only cast lazy data to integral dtype, ' + 'got {!r}.') + raise ValueError(emsg.format(dtype)) + self._fill_value = None + self._dtype = dtype + + @property + def fill_value(self): + return self._fill_value + + @fill_value.setter + def fill_value(self, fill_value): + if fill_value is not None: + # Convert the given value to the dtype of the cube. + fill_value = np.asarray([fill_value])[0] + target_dtype = self.dtype + if fill_value.dtype.kind == 'f' and target_dtype.kind == 'i': + # Perform rounding when converting floats to ints. + fill_value = np.rint(fill_value) + try: + [fill_value] = np.asarray([fill_value], dtype=target_dtype) + except OverflowError: + emsg = 'Fill value of {!r} invalid for cube {!r}.' + raise ValueError(emsg.format(fill_value, self.dtype)) + self._fill_value = fill_value @property def ndim(self): @@ -1731,12 +1767,17 @@ def data(self, value): raise ValueError('Require cube data with shape %r, got ' '%r.' % (self.shape, value.shape)) + # Set lazy or real data, and reset the other. if is_lazy_data(value): self._dask_array = value self._numpy_array = None - else: self._numpy_array = value + self._dask_array = None + + # Cancel any 'realisation' datatype conversion, and fill value. + self.dtype = None + self.fill_value = None def has_lazy_data(self): return self._numpy_array is None diff --git a/lib/iris/tests/results/FF/air_temperature_1.cml b/lib/iris/tests/results/FF/air_temperature_1.cml index 2bd0d3f4e2..8d653c28e9 100644 --- a/lib/iris/tests/results/FF/air_temperature_1.cml +++ b/lib/iris/tests/results/FF/air_temperature_1.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/FF/air_temperature_2.cml b/lib/iris/tests/results/FF/air_temperature_2.cml index e06cf23ba6..d06ff4d286 100644 --- a/lib/iris/tests/results/FF/air_temperature_2.cml +++ b/lib/iris/tests/results/FF/air_temperature_2.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/FF/soil_temperature_1.cml b/lib/iris/tests/results/FF/soil_temperature_1.cml index fc221639cf..2549b1b962 100644 --- a/lib/iris/tests/results/FF/soil_temperature_1.cml +++ b/lib/iris/tests/results/FF/soil_temperature_1.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/FF/surface_altitude_1.cml b/lib/iris/tests/results/FF/surface_altitude_1.cml index 4bd5de8cf4..8ec0c305da 100644 --- a/lib/iris/tests/results/FF/surface_altitude_1.cml +++ b/lib/iris/tests/results/FF/surface_altitude_1.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/addition_coord_x.cml b/lib/iris/tests/results/analysis/addition_coord_x.cml index 4898dd6b58..70c95763b9 100644 --- a/lib/iris/tests/results/analysis/addition_coord_x.cml +++ b/lib/iris/tests/results/analysis/addition_coord_x.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/addition_coord_y.cml b/lib/iris/tests/results/analysis/addition_coord_y.cml index 7404a13def..065e034a86 100644 --- a/lib/iris/tests/results/analysis/addition_coord_y.cml +++ b/lib/iris/tests/results/analysis/addition_coord_y.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/apply_ifunc_frompyfunc.cml b/lib/iris/tests/results/analysis/apply_ifunc_frompyfunc.cml index b71ae08a48..73aa603992 100644 --- a/lib/iris/tests/results/analysis/apply_ifunc_frompyfunc.cml +++ b/lib/iris/tests/results/analysis/apply_ifunc_frompyfunc.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/apply_ufunc_frompyfunc.cml b/lib/iris/tests/results/analysis/apply_ufunc_frompyfunc.cml index e33a79366d..fc7cce9134 100644 --- a/lib/iris/tests/results/analysis/apply_ufunc_frompyfunc.cml +++ b/lib/iris/tests/results/analysis/apply_ufunc_frompyfunc.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/division_by_singular_coord.cml b/lib/iris/tests/results/analysis/division_by_singular_coord.cml index 7022d4a6e7..c099becdd7 100644 --- a/lib/iris/tests/results/analysis/division_by_singular_coord.cml +++ b/lib/iris/tests/results/analysis/division_by_singular_coord.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/exponentiate.cml b/lib/iris/tests/results/analysis/exponentiate.cml index caad13a501..62f4bf08ab 100644 --- a/lib/iris/tests/results/analysis/exponentiate.cml +++ b/lib/iris/tests/results/analysis/exponentiate.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/gmean_latitude.cml b/lib/iris/tests/results/analysis/gmean_latitude.cml index 400be996bc..e71bb14e74 100644 --- a/lib/iris/tests/results/analysis/gmean_latitude.cml +++ b/lib/iris/tests/results/analysis/gmean_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/gmean_latitude_longitude.cml b/lib/iris/tests/results/analysis/gmean_latitude_longitude.cml index 9a934998d4..1176acb50c 100644 --- a/lib/iris/tests/results/analysis/gmean_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/gmean_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/gmean_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/gmean_latitude_longitude_1call.cml index b4e8361cd5..ac4ed1681d 100644 --- a/lib/iris/tests/results/analysis/gmean_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/gmean_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/hmean_latitude.cml b/lib/iris/tests/results/analysis/hmean_latitude.cml index 634de70f23..6cc2242648 100644 --- a/lib/iris/tests/results/analysis/hmean_latitude.cml +++ b/lib/iris/tests/results/analysis/hmean_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/hmean_latitude_longitude.cml b/lib/iris/tests/results/analysis/hmean_latitude_longitude.cml index 7c214c8e53..2abf38fef2 100644 --- a/lib/iris/tests/results/analysis/hmean_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/hmean_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/hmean_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/hmean_latitude_longitude_1call.cml index 15c5ed7729..931e8c0a97 100644 --- a/lib/iris/tests/results/analysis/hmean_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/hmean_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/max_latitude.cml b/lib/iris/tests/results/analysis/max_latitude.cml index 415373ecd2..b9acb2e1a0 100644 --- a/lib/iris/tests/results/analysis/max_latitude.cml +++ b/lib/iris/tests/results/analysis/max_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/max_latitude_longitude.cml b/lib/iris/tests/results/analysis/max_latitude_longitude.cml index 988e92e954..623d412a61 100644 --- a/lib/iris/tests/results/analysis/max_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/max_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/max_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/max_latitude_longitude_1call.cml index 6fc4129371..f432a91b37 100644 --- a/lib/iris/tests/results/analysis/max_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/max_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/mean_latitude.cml b/lib/iris/tests/results/analysis/mean_latitude.cml index 58e373ed0c..0317e2b1e2 100644 --- a/lib/iris/tests/results/analysis/mean_latitude.cml +++ b/lib/iris/tests/results/analysis/mean_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/mean_latitude_longitude.cml b/lib/iris/tests/results/analysis/mean_latitude_longitude.cml index ecf1fa35d2..a45e0a610e 100644 --- a/lib/iris/tests/results/analysis/mean_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/mean_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/mean_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/mean_latitude_longitude_1call.cml index 4736766c16..44f9930efd 100644 --- a/lib/iris/tests/results/analysis/mean_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/mean_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/median_latitude.cml b/lib/iris/tests/results/analysis/median_latitude.cml index e9c4b94141..b43235dd05 100644 --- a/lib/iris/tests/results/analysis/median_latitude.cml +++ b/lib/iris/tests/results/analysis/median_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/median_latitude_longitude.cml b/lib/iris/tests/results/analysis/median_latitude_longitude.cml index ef4e733654..4e1335e64b 100644 --- a/lib/iris/tests/results/analysis/median_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/median_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/median_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/median_latitude_longitude_1call.cml index f9850bbfb5..f617c0895f 100644 --- a/lib/iris/tests/results/analysis/median_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/median_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/min_latitude.cml b/lib/iris/tests/results/analysis/min_latitude.cml index 9805d47cc9..7cf5ac7130 100644 --- a/lib/iris/tests/results/analysis/min_latitude.cml +++ b/lib/iris/tests/results/analysis/min_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/min_latitude_longitude.cml b/lib/iris/tests/results/analysis/min_latitude_longitude.cml index d68688ea45..0f4087c17c 100644 --- a/lib/iris/tests/results/analysis/min_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/min_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/min_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/min_latitude_longitude_1call.cml index d6a17ea484..cb6d0f200d 100644 --- a/lib/iris/tests/results/analysis/min_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/min_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/original_common.cml b/lib/iris/tests/results/analysis/original_common.cml index e29c9e5d27..2b7643b909 100644 --- a/lib/iris/tests/results/analysis/original_common.cml +++ b/lib/iris/tests/results/analysis/original_common.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/original_hmean.cml b/lib/iris/tests/results/analysis/original_hmean.cml index 15430608c1..f342ccc1b7 100644 --- a/lib/iris/tests/results/analysis/original_hmean.cml +++ b/lib/iris/tests/results/analysis/original_hmean.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/rms_latitude.cml b/lib/iris/tests/results/analysis/rms_latitude.cml index 9dca51adf3..f134bece0c 100644 --- a/lib/iris/tests/results/analysis/rms_latitude.cml +++ b/lib/iris/tests/results/analysis/rms_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/rms_latitude_longitude.cml b/lib/iris/tests/results/analysis/rms_latitude_longitude.cml index 6a87628f94..16fcffe176 100644 --- a/lib/iris/tests/results/analysis/rms_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/rms_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/rms_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/rms_latitude_longitude_1call.cml index 4e511c53f7..8c18d225bf 100644 --- a/lib/iris/tests/results/analysis/rms_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/rms_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/std_dev_latitude.cml b/lib/iris/tests/results/analysis/std_dev_latitude.cml index 1e04a97d2e..f5c5ee6634 100644 --- a/lib/iris/tests/results/analysis/std_dev_latitude.cml +++ b/lib/iris/tests/results/analysis/std_dev_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/std_dev_latitude_longitude.cml b/lib/iris/tests/results/analysis/std_dev_latitude_longitude.cml index f69b1b4319..cc75ca3b56 100644 --- a/lib/iris/tests/results/analysis/std_dev_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/std_dev_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/std_dev_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/std_dev_latitude_longitude_1call.cml index 45596e7b6c..50f54f537a 100644 --- a/lib/iris/tests/results/analysis/std_dev_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/std_dev_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/subtract_coord_x.cml b/lib/iris/tests/results/analysis/subtract_coord_x.cml index 7e21df3aa5..855c700c3d 100644 --- a/lib/iris/tests/results/analysis/subtract_coord_x.cml +++ b/lib/iris/tests/results/analysis/subtract_coord_x.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/subtract_coord_y.cml b/lib/iris/tests/results/analysis/subtract_coord_y.cml index 0c0e574eb1..c4d695a9da 100644 --- a/lib/iris/tests/results/analysis/subtract_coord_y.cml +++ b/lib/iris/tests/results/analysis/subtract_coord_y.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/sum_latitude.cml b/lib/iris/tests/results/analysis/sum_latitude.cml index 538896df3d..3d1e13a955 100644 --- a/lib/iris/tests/results/analysis/sum_latitude.cml +++ b/lib/iris/tests/results/analysis/sum_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/sum_latitude_longitude.cml b/lib/iris/tests/results/analysis/sum_latitude_longitude.cml index 69994b84dc..4135f70210 100644 --- a/lib/iris/tests/results/analysis/sum_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/sum_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/sum_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/sum_latitude_longitude_1call.cml index 90164b36d9..d7f26a82f8 100644 --- a/lib/iris/tests/results/analysis/sum_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/sum_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/variance_latitude.cml b/lib/iris/tests/results/analysis/variance_latitude.cml index 2759e37577..91921d0e79 100644 --- a/lib/iris/tests/results/analysis/variance_latitude.cml +++ b/lib/iris/tests/results/analysis/variance_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/variance_latitude_longitude.cml b/lib/iris/tests/results/analysis/variance_latitude_longitude.cml index 8122c7ff22..82ef78a4ee 100644 --- a/lib/iris/tests/results/analysis/variance_latitude_longitude.cml +++ b/lib/iris/tests/results/analysis/variance_latitude_longitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/analysis/variance_latitude_longitude_1call.cml b/lib/iris/tests/results/analysis/variance_latitude_longitude_1call.cml index 5485529342..531ec056b7 100644 --- a/lib/iris/tests/results/analysis/variance_latitude_longitude_1call.cml +++ b/lib/iris/tests/results/analysis/variance_latitude_longitude_1call.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml b/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml index d337f92dd5..5cb0de933c 100644 --- a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml +++ b/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml b/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml index 88a433f8f5..62126aa03b 100644 --- a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml +++ b/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/netcdf/netcdf_deferred_index_0.cml b/lib/iris/tests/results/netcdf/netcdf_deferred_index_0.cml index 17455a2ccf..9e03a24a45 100644 --- a/lib/iris/tests/results/netcdf/netcdf_deferred_index_0.cml +++ b/lib/iris/tests/results/netcdf/netcdf_deferred_index_0.cml @@ -1,6 +1,6 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -26,7 +26,7 @@ - + diff --git a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml index 33f1a0571f..8a61b31886 100644 --- a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml +++ b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_0.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml index c03d47a21d..cb0ce6d5fd 100644 --- a/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml +++ b/lib/iris/tests/results/netcdf/netcdf_global_xyzt_gems_iter_1.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/trajectory/constant_latitude.cml b/lib/iris/tests/results/trajectory/constant_latitude.cml index 7d9eb06563..8d6caae0ba 100644 --- a/lib/iris/tests/results/trajectory/constant_latitude.cml +++ b/lib/iris/tests/results/trajectory/constant_latitude.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/trajectory/single_point.cml b/lib/iris/tests/results/trajectory/single_point.cml index 9328b1c586..28e9480e62 100644 --- a/lib/iris/tests/results/trajectory/single_point.cml +++ b/lib/iris/tests/results/trajectory/single_point.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/trajectory/tri_polar_latitude_slice.cml b/lib/iris/tests/results/trajectory/tri_polar_latitude_slice.cml index 0437df1f58..1cfac803d4 100644 --- a/lib/iris/tests/results/trajectory/tri_polar_latitude_slice.cml +++ b/lib/iris/tests/results/trajectory/tri_polar_latitude_slice.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/results/trajectory/zigzag.cml b/lib/iris/tests/results/trajectory/zigzag.cml index 8bf4213781..e78cc1ffe5 100644 --- a/lib/iris/tests/results/trajectory/zigzag.cml +++ b/lib/iris/tests/results/trajectory/zigzag.cml @@ -1,6 +1,6 @@ - + diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index c496fd5774..8f0cdc8204 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -192,10 +192,12 @@ def setUp(self): file = tests.get_data_path(('PP', 'aPProt1', 'rotatedMHtimecube.pp')) cubes = iris.load(file) self.cube = cubes[0] + self.cube_fill_val = self.cube.fill_value self.assertCML(self.cube, ('analysis', 'original.cml')) def _common(self, name, aggregate, original_name='original_common.cml', *args, **kwargs): self.cube.data = self.cube.data.astype(np.float64) + self.cube.fill_value = self.cube_fill_val self.assertCML(self.cube, ('analysis', original_name)) @@ -221,6 +223,7 @@ def test_std_dev(self): def test_hmean(self): # harmonic mean requires data > 0 self.cube.data *= self.cube.data + self.cube.fill_value = self.cube_fill_val self._common('hmean', iris.analysis.HMEAN, 'original_hmean.cml', rtol=1e-05) def test_gmean(self): diff --git a/lib/iris/tests/test_basic_maths.py b/lib/iris/tests/test_basic_maths.py index abf4a175ea..8e2e64f6da 100644 --- a/lib/iris/tests/test_basic_maths.py +++ b/lib/iris/tests/test_basic_maths.py @@ -39,7 +39,11 @@ class TestBasicMaths(tests.IrisTest): def setUp(self): self.cube = iris.tests.stock.global_pp() + # We require to preserve the cube fill_value + # across the cube data setter operation. + fill_value = self.cube.fill_value self.cube.data = self.cube.data - 260 + self.cube.fill_value = fill_value def test_abs(self): a = self.cube @@ -360,7 +364,11 @@ def test_type_error(self): class TestDivideAndMultiply(tests.IrisTest): def setUp(self): self.cube = iris.tests.stock.global_pp() + # We require to preserve the cube fill_value + # across the cube data setter operation. + fill_value = self.cube.fill_value self.cube.data = self.cube.data - 260 + self.cube.fill_value = fill_value def test_divide(self): a = self.cube @@ -428,7 +436,7 @@ def test_divide_by_coordinate_dim2(self): # Check that the division has had no effect on the original self.assertCML(a, ('analysis', 'maths_original.cml')) - def test_divide_by_singluar_coordinate(self): + def test_divide_by_singular_coordinate(self): a = self.cube coord = iris.coords.DimCoord(points=2, long_name='foo', units='1') @@ -503,11 +511,17 @@ def test_type_error(self): class TestExponentiate(tests.IrisTest): def setUp(self): self.cube = iris.tests.stock.global_pp() + # We require to preserve the cube fill_value + # across the cube data setter operation. + self.fill_value = self.cube.fill_value self.cube.data = self.cube.data - 260 + self.cube.fill_value = self.fill_value def test_exponentiate(self): a = self.cube a.data = a.data.astype(np.float64) + # We require to preserve the cube fill_value after setting the data. + a.fill_value = self.fill_value e = pow(a, 4) self.assertCMLApproxData(e, ('analysis', 'exponentiate.cml')) @@ -515,6 +529,8 @@ def test_square_root(self): # Make sure we have something which we can take the root of. a = self.cube a.data = abs(a.data) + # We require to preserve the cube fill_value after setting the data. + a.fill_value = self.fill_value a.units **= 2 e = a ** 0.5 diff --git a/lib/iris/tests/test_cdm.py b/lib/iris/tests/test_cdm.py index 19b59f294a..db0b0b9696 100644 --- a/lib/iris/tests/test_cdm.py +++ b/lib/iris/tests/test_cdm.py @@ -791,7 +791,7 @@ def test_metadata_nop(self): def test_metadata_tuple(self): metadata = ('air_pressure', 'foo', 'bar', '', {'random': '12'}, (), - -99, np.dtype('f8')) + np.dtype('f8'), -99) self.t.metadata = metadata self.assertEqual(self.t.standard_name, 'air_pressure') self.assertEqual(self.t.long_name, 'foo') diff --git a/lib/iris/tests/test_concatenate.py b/lib/iris/tests/test_concatenate.py index 260261236f..e644870065 100644 --- a/lib/iris/tests/test_concatenate.py +++ b/lib/iris/tests/test_concatenate.py @@ -35,7 +35,8 @@ def _make_cube(x, y, data, aux=None, offset=0, scalar=None, - dtype=np.dtype('float32'), fill_value=None): + dtype=np.dtype('float32'), fill_value=None, + mask=None): """ A convenience test function that creates a custom 2D cube. @@ -67,6 +68,10 @@ def _make_cube(x, y, data, aux=None, offset=0, scalar=None, * scalar: Create a 'height' scalar coordinate with the given value. + * mask: + Create cube masked data with the specified mask. Boolean or + array indicies of points to be masked. + Returns: The newly created 2D :class:`iris.cube.Cube`. @@ -75,10 +80,19 @@ def _make_cube(x, y, data, aux=None, offset=0, scalar=None, y_range = np.arange(*y, dtype=dtype) x_size = len(x_range) y_size = len(y_range) + shape = (y_size, x_size) - cube_data = np.empty((y_size, x_size), dtype=dtype) - cube_data[:] = data - cube = iris.cube.Cube(cube_data, fill_value=fill_value, dtype=dtype) + if mask is not None: + cube_data = ma.empty(shape, dtype=dtype) + cube_data.data[:] = data + if isinstance(mask, bool): + cube_data.mask = mask + else: + cube_data[mask] = ma.masked + else: + cube_data = np.empty(shape, dtype=dtype) + cube_data[:] = data + cube = iris.cube.Cube(cube_data, fill_value=fill_value) coord = DimCoord(y_range, long_name='y') coord.guess_bounds() cube.add_dim_coord(coord, 0) @@ -217,15 +231,18 @@ def concatenate(cubes, order=None): result = cubelist.concatenate() for cube in result: + # Setting the cube.data clears the cube.dtype and cube.fill_value. + # We want to maintain the fill_value for testing purposes, even + # though we have lost the lazy data in order to pin down the array + # data order to overcome testing on different architectures. + fill_value = cube.fill_value if ma.isMaskedArray(cube.data): - # cube.data = ma.copy(cube.data, order=order) data = np.array(cube.data.data, copy=True, order=order) mask = np.array(cube.data.mask, copy=True, order=order) - fill_value = cube.data.fill_value - cube.data = ma.array(data, mask=mask, fill_value=fill_value) + cube.data = ma.array(data, mask=mask) else: - # cube.data = np.copy(cube.data, order=order) cube.data = np.array(cube.data, copy=True, order=order) + cube.fill_value = fill_value return result @@ -310,7 +327,7 @@ def test_uncommon(self): result = concatenate(cubes) self.assertEqual(len(result), 2) - def test_order(self): + def test_order_difference(self): cubes = [] y = (0, 2) cubes.append(_make_cube((0, 2), y, 1)) @@ -318,42 +335,78 @@ def test_order(self): result = concatenate(cubes) self.assertEqual(len(result), 2) + def test_masked_fill_value_difference(self): + cubes = [] + y = (0, 2) + cube = _make_cube((0, 2), y, 1, mask=True) + cube.fill_value = 10 + cubes.append(cube) + cube = _make_cube((2, 4), y, 1, mask=True) + cube.fill_value = 20 + cubes.append(cube) + result = concatenate(cubes) + self.assertEqual(len(result), 2) + + +class Test2D(tests.IrisTest): def test_masked_and_unmasked(self): + cubes = [] + y = (0, 2) + cube = _make_cube((0, 2), y, 1, mask=True) + cubes.append(cube) + cubes.append(_make_cube((2, 4), y, 2)) + result = concatenate(cubes) + self.assertEqual(len(result), 1) + + def test_masked_and_unmasked_int16(self): + cubes = [] + y = (0, 2) + dtype = np.dtype('int16') + cube = _make_cube((0, 2), y, 1, dtype=dtype, mask=True) + cubes.append(cube) + cubes.append(_make_cube((2, 4), y, 2, dtype=dtype)) + result = concatenate(cubes) + self.assertEqual(len(result), 1) + + def test_unmasked_and_masked(self): cubes = [] y = (0, 2) cubes.append(_make_cube((0, 2), y, 1)) - cube = _make_cube((2, 4), y, 2) - cube.data = ma.asarray(cube.data) + cube = _make_cube((2, 4), y, 2, mask=True) + cubes.append(cube) + result = concatenate(cubes) + self.assertEqual(len(result), 1) + + def test_unmasked_and_masked_int16(self): + cubes = [] + y = (0, 2) + dtype = np.dtype('int16') + cubes.append(_make_cube((0, 2), y, 1, dtype=dtype)) + cube = _make_cube((2, 4), y, 2, dtype=dtype, mask=True) cubes.append(cube) result = concatenate(cubes) self.assertEqual(len(result), 1) - def test_masked_fill_value(self): + def test_masked_with_data_fill_value_difference(self): cubes = [] y = (0, 2) - cube = _make_cube((0, 2), y, 1) - cube.data = ma.asarray(cube.data) + cube = _make_cube((0, 2), y, 1, mask=True) cube.data.fill_value = 10 cubes.append(cube) - cube = _make_cube((2, 4), y, 1) - cube.data = ma.asarray(cube.data) + cube = _make_cube((2, 4), y, 1, mask=True) cube.data.fill_value = 20 cubes.append(cube) result = concatenate(cubes) self.assertEqual(len(result), 1) - -class Test2D(tests.IrisTest): def test_concat_masked_2x2d(self): cubes = [] y = (0, 2) - cube = _make_cube((0, 2), y, 1) - cube.data = ma.asarray(cube.data) - cube.data[(0, 1), (0, 1)] = ma.masked + mask = [(0, 1), (0, 1)] + cube = _make_cube((0, 2), y, 1, mask=mask) cubes.append(cube) - cube = _make_cube((2, 4), y, 2) - cube.data = ma.asarray(cube.data) - cube.data[(0, 1), (1, 0)] = ma.masked + mask = [(0, 1), (1, 0)] + cube = _make_cube((2, 4), y, 2, mask=mask) cubes.append(cube) result = concatenate(cubes) self.assertCML(result, ('concatenate', 'concat_masked_2x2d.cml')) @@ -366,13 +419,11 @@ def test_concat_masked_2x2d(self): def test_concat_masked_2y2d(self): cubes = [] x = (0, 2) - cube = _make_cube(x, (0, 2), 1) - cube.data = np.ma.asarray(cube.data) - cube.data[(0, 1), (0, 1)] = ma.masked + mask = [(0, 1), (0, 1)] + cube = _make_cube(x, (0, 2), 1, mask=mask) cubes.append(cube) - cube = _make_cube(x, (2, 4), 2) - cube.data = ma.asarray(cube.data) - cube.data[(0, 1), (1, 0)] = ma.masked + mask = [(0, 1), (1, 0)] + cube = _make_cube(x, (2, 4), 2, mask=mask) cubes.append(cube) result = concatenate(cubes) self.assertCML(result, ('concatenate', 'concat_masked_2y2d.cml')) @@ -387,15 +438,65 @@ def test_concat_masked_2y2d(self): def test_concat_masked_2y2d_int16(self): cubes = [] x = (0, 2) - cube = _make_cube(x, (0, 2), 1, dtype=np.dtype('int16'), - fill_value=-37) - cube.data = np.ma.asarray(cube.data) - cube.data[(0, 1), (0, 1)] = ma.masked + dtype = np.dtype('int16') + fill_value = -37 + mask = [(0, 1), (0, 1)] + cube = _make_cube(x, (0, 2), 1, dtype=dtype, fill_value=fill_value, + mask=mask) + cubes.append(cube) + mask = [(0, 1), (1, 0)] + cube = _make_cube(x, (2, 4), 2, dtype=dtype, fill_value=fill_value, + mask=mask) + cubes.append(cube) + result = concatenate(cubes) + self.assertCML(result, ('concatenate', 'concat_masked_2y2d_int16.cml')) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].shape, (4, 2)) + mask = np.array([[True, False], + [False, True], + [False, True], + [True, False]], dtype=np.bool) + self.assertArrayEqual(result[0].data.mask, mask) + + def test_concat_masked_2y2d_int16_with_concrete_and_lazy(self): + cubes = [] + x = (0, 2) + dtype = np.dtype('int16') + fill_value = -37 + mask = [(0, 1), (0, 1)] + cube = _make_cube(x, (0, 2), 1, dtype=dtype, fill_value=fill_value, + mask=mask) + cubes.append(cube) + mask = [(0, 1), (1, 0)] + cube = _make_cube(x, (2, 4), 2, dtype=dtype, mask=mask) + cube.data = cube.lazy_data() + cube.dtype = dtype + cube.fill_value = fill_value + cubes.append(cube) + result = concatenate(cubes) + self.assertCML(result, ('concatenate', 'concat_masked_2y2d_int16.cml')) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].shape, (4, 2)) + mask = np.array([[True, False], + [False, True], + [False, True], + [True, False]], dtype=np.bool) + self.assertArrayEqual(result[0].data.mask, mask) + + def test_concat_masked_2y2d_int16_with_lazy_and_concrete(self): + cubes = [] + x = (0, 2) + dtype = np.dtype('int16') + fill_value = -37 + mask = [(0, 1), (1, 0)] + cube = _make_cube(x, (2, 4), 2, dtype=dtype, mask=mask) + cube.data = cube.lazy_data() + cube.dtype = dtype + cube.fill_value = fill_value cubes.append(cube) - cube = _make_cube(x, (2, 4), 2, dtype=np.dtype('int16'), - fill_value=-37) - cube.data = ma.asarray(cube.data) - cube.data[(0, 1), (1, 0)] = ma.masked + mask = [(0, 1), (0, 1)] + cube = _make_cube(x, (0, 2), 1, dtype=dtype, fill_value=fill_value, + mask=mask) cubes.append(cube) result = concatenate(cubes) self.assertCML(result, ('concatenate', 'concat_masked_2y2d_int16.cml')) diff --git a/lib/iris/tests/test_interpolation.py b/lib/iris/tests/test_interpolation.py index f3648e3297..bf537477bc 100644 --- a/lib/iris/tests/test_interpolation.py +++ b/lib/iris/tests/test_interpolation.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2016, Met Office +# (C) British Crown Copyright 2010 - 2017, Met Office # # This file is part of Iris. # @@ -26,7 +26,6 @@ import iris.tests as tests import numpy as np -import numpy.ma as ma from scipy.interpolate import interp1d import iris @@ -43,7 +42,9 @@ def normalise_order(cube): # function when the circular flag is true. # * scipy.interpolate.interp1d in 0.11.0 which is used in # `Linear1dExtrapolator`. + cube_fill_val = cube.fill_value cube.data = np.ascontiguousarray(cube.data) + cube.fill_value = cube_fill_val class TestLinearExtrapolator(tests.IrisTest): diff --git a/lib/iris/tests/test_pp_cf.py b/lib/iris/tests/test_pp_cf.py index 430f01123d..39227fb82e 100644 --- a/lib/iris/tests/test_pp_cf.py +++ b/lib/iris/tests/test_pp_cf.py @@ -80,6 +80,8 @@ def callback_aaxzc_n10r13xy_b_pp(cube, field, filename): cube.add_aux_coord(height_coord) +# XXX: Issue with integer.b.pp and invalid bmdi fill_value for int32 dtype +@tests.skip_biggus @tests.skip_data class TestAll(tests.IrisTest, pp.PPTest): _ref_dir = ('usecases', 'pp_to_cf_conversion') diff --git a/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py b/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py index 243a1579fb..b181676e01 100644 --- a/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py +++ b/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py @@ -1262,7 +1262,9 @@ def test_circular_src__masked_missingmask(self): # instead of being an array. src = self.src src.coord('longitude').circular = True + src_fill_value = src.fill_value src.data = np.ma.MaskedArray(src.data) + src.fill_value = src_fill_value self.assertEqual(src.data.mask, False) method_results = self._check_circular_results(src, 'missingmask') for method_result in method_results: diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index 849491fc19..9260d8dc86 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -1240,6 +1240,175 @@ def test_lazy(self): self.assertTrue(cube.has_lazy_data()) +class Test_data_dtype_fillvalue(tests.IrisTest): + def _sample_data(self, dtype='f4', masked=False, lazy=False): + data = np.arange(6).reshape((2, 3)) + dtype = np.dtype(dtype) + data = data.astype(dtype) + if masked: + data = np.ma.masked_array(data, mask=[[0, 1, 0], [0, 0, 0]]) + if lazy: + data = as_lazy_data(data) + return data + + def _sample_cube(self, dtype='f4', masked=False, lazy=False, + cube_dtype=None, cube_fill_value=None): + data = self._sample_data(dtype=dtype, masked=masked, lazy=lazy) + if cube_dtype is None: + cube_dtype = dtype + # NOTE: with masked lazy integers, the resulting data dtype will + # be changed, so we must set the cube to the 'original' dtype. + else: + cube_dtype = np.dtype(cube_dtype) + cube = Cube(data, dtype=cube_dtype, fill_value=cube_fill_value) + return cube + + def test_realdata_change(self): + # Check that re-assigning real data resets dtype and fill_value. + cube = self._sample_cube(cube_fill_value=27.3) + self.assertEqual(cube.dtype, np.dtype('f4')) + self.assertArrayAllClose(cube.fill_value, 27.3) + new_data = self._sample_data(dtype=np.int32) + cube.data = new_data + self.assertIs(cube.core_data, new_data) + self.assertEqual(cube.dtype, np.dtype('i4')) + self.assertIsNone(cube.fill_value) + + def test_realise_unmasked(self): + # Check that touching unmasked lazy data retains the fill_value. + cube = self._sample_cube(cube_fill_value=27.3, lazy=True) + data = cube.data + self.assertIs(cube.core_data, data) + self.assertArrayAllClose(cube.fill_value, 27.3) + + def test_realise_masked(self): + # Check that touching masked lazy data retains the fill_value. + cube = self._sample_cube(cube_fill_value=27.3, masked=True, lazy=True) + data = cube.data + self.assertIs(cube.core_data, data) + self.assertArrayAllClose(cube.fill_value, 27.3) + + def test_realise_maskedints(self): + # Check that touching masked integer lazy data retains the fill_value. + cube = self._sample_cube(dtype=np.int16, masked=True, lazy=True, + cube_fill_value=999) + self.assertIs(cube.core_data.dtype, np.dtype('f8')) + data = cube.data + self.assertIs(cube.core_data.dtype, np.dtype('i2')) + self.assertEqual(cube.fill_value.dtype, np.dtype('i2')) + self.assertArrayAllClose(cube.fill_value, 999) + + def test_lazydata_change(self): + # Check that re-assigning lazy data resets dtype and fill_value. + cube = self._sample_cube(dtype=np.float32, lazy=True, + cube_dtype=np.int16, + cube_fill_value=27.3) + # Set a modified dtype to check it gets reset. + self.assertEqual(cube.dtype, np.dtype('i2')) + self.assertEqual(cube.core_data.dtype, np.dtype('f4')) + self.assertArrayAllClose(cube.fill_value, 27) + new_data = self._sample_data(np.float64, lazy=True) + cube.data = new_data + self.assertEqual(cube.dtype, np.dtype('f8')) + self.assertIsNone(cube.fill_value) + + def test_realdata_dtype_change(self): + # Check that cannot change real data dtype. + cube = self._sample_cube() + with self.assertRaisesRegexp(ValueError, "cannot set dtype"): + cube.dtype = np.dtype('i8') + + def test_lazydata_nonmaskedints(self): + # Check that lazy ints retain their dtype. + data_original = self._sample_data(dtype=np.int32) + data_lazy = as_lazy_data(data_original) + cube = Cube(data_lazy) + self.assertEqual(cube.dtype, np.dtype('i4')) + self.assertEqual(cube.core_data.dtype, np.dtype('i4')) + data = cube.data + self.assertArrayEqual(data, data_original) + + def test_lazydata_maskedints(self): + # Check that lazy masked ints have a modified dtype. + masked_data_original = self._sample_data(dtype=np.int32, masked=True) + masked_data_lazy = as_lazy_data(masked_data_original) + cube = Cube(masked_data_lazy) + cube.dtype = masked_data_original.dtype + self.assertEqual(cube.dtype, np.dtype('i4')) + self.assertEqual(cube.core_data.dtype, np.dtype('f8')) + data = cube.data + self.assertTrue(np.ma.is_masked(data)) + self.assertMaskedArrayEqual(data, masked_data_original) + + def test_lazydata_floating_dtype_change(self): + # Check that re-assigning dtype won't allow a floating type. + cube = self._sample_cube(dtype=np.float32, lazy=True, + cube_fill_value=23.7) + self.assertArrayAllClose(cube.fill_value, 23.7) + msg = "Can only cast lazy data to integral dtype" + with self.assertRaisesRegexp(ValueError, msg): + cube.dtype = np.float64 + + def test_lazydata_dtype_change(self): + # Check that re-assigning dtype resets fill_value. + cube = self._sample_cube(dtype=np.float32, lazy=True, + cube_fill_value=23.7) + self.assertArrayAllClose(cube.fill_value, 23.7) + cube.dtype = np.int16 + self.assertEqual(cube.dtype, np.dtype('i2')) + self.assertEqual(cube.core_data.dtype, np.dtype('f4')) + self.assertIsNone(cube.fill_value) + + def test_lazydata_maskedints_dtype_change(self): + # Check that re-assigning dtype resets fill_value. + cube = self._sample_cube(dtype=np.int16, masked=True, lazy=True, + cube_fill_value=199) + self.assertEqual(cube.dtype, np.dtype('i2')) + self.assertEqual(cube.core_data.dtype, np.dtype('f8')) + self.assertArrayAllClose(cube.fill_value, 199) + cube.dtype = np.int64 + self.assertEqual(cube.dtype, np.dtype('i8')) + self.assertEqual(cube.core_data.dtype, np.dtype('f8')) + self.assertIsNone(cube.fill_value) + + def test_fill_value__int_dtype_to_float(self): + # Check that fill_value is cast to dtype, e.g. float --> int. + cube = self._sample_cube(dtype=np.float32, cube_fill_value=1735) + self.assertEqual(cube.fill_value.dtype, np.dtype('f4')) + self.assertArrayAllClose(cube.fill_value, 1735.0) + + def test_fill_value__float_dtype_to_int(self): + # Check that fill_value is rounded to dtype, e.g. float --> int. + cube = self._sample_cube(dtype=np.int16, cube_fill_value=1734.99999) + self.assertEqual(cube.fill_value.dtype, np.dtype('i2')) + self.assertArrayAllClose(cube.fill_value, 1735) + + def test_set_fill_value(self): + cube = self._sample_cube() + cube.fill_value = -74.6 + self.assertEqual(cube.fill_value.dtype, np.dtype('f4')) + self.assertArrayAllClose(cube.fill_value, -74.6) + + def test_set_fill_value__typecast(self): + cube = self._sample_cube(dtype=np.int16) + cube.fill_value = -74.6 + self.assertEqual(cube.fill_value.dtype, np.dtype('i2')) + self.assertArrayAllClose(cube.fill_value, -75) + + def test_set_fill_value__casterror(self): + cube = self._sample_cube(dtype=np.int16) + msg = "invalid for cube dtype\('int16'\)" + with self.assertRaisesRegexp(ValueError, msg): + # NOTE: this doesn't actually work properly in most cases. + # E.G. it will happily assign 1e12 to an int16 and gets 4096. + cube.fill_value = -1.0e23 + + def test_clear_fill_value(self): + cube = self._sample_cube(cube_fill_value=123.768) + cube.fill_value = None + self.assertIsNone(cube.fill_value) + + class TestSubset(tests.IrisTest): def test_scalar_coordinate(self): cube = Cube(0, long_name='apricot', units='1') diff --git a/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py b/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py index 56480de793..8666c39cab 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py +++ b/lib/iris/tests/unit/fileformats/netcdf/test__load_cube.py @@ -139,7 +139,7 @@ def _make(self, names, attrs): cf_var = mock.MagicMock(spec=iris.fileformats.cf.CFVariable, dtype=np.dtype('i4'), - cf_data=mock.Mock(), + cf_data=mock.Mock(_FillValue=None), cf_name='DUMMY_VAR', cf_group=coords, shape=(1,))