diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index 0e6670533f..74889e5066 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -142,7 +142,9 @@ def callback(cube, field, filename): class Future(threading.local): """Run-time configuration controller.""" - def __init__(self, datum_support=False, pandas_ndim=False): + def __init__( + self, datum_support=False, pandas_ndim=False, save_split_attrs=False + ): """ A container for run-time options controls. @@ -164,6 +166,11 @@ def __init__(self, datum_support=False, pandas_ndim=False): pandas_ndim : bool, default=False See :func:`iris.pandas.as_data_frame` for details - opts in to the newer n-dimensional behaviour. + save_split_attrs : bool, default=False + Save "global" and "local" cube attributes to netcdf in appropriately + different ways : "global" ones are saved as dataset attributes, where + possible, while "local" ones are saved as data-variable attributes. + See :func:`iris.fileformats.netcdf.saver.save`. """ # The flag 'example_future_flag' is provided as a reference for the @@ -175,12 +182,15 @@ def __init__(self, datum_support=False, pandas_ndim=False): # self.__dict__['example_future_flag'] = example_future_flag self.__dict__["datum_support"] = datum_support self.__dict__["pandas_ndim"] = pandas_ndim + self.__dict__["save_split_attrs"] = save_split_attrs def __repr__(self): # msg = ('Future(example_future_flag={})') # return msg.format(self.example_future_flag) - msg = "Future(datum_support={}, pandas_ndim={})" - return msg.format(self.datum_support, self.pandas_ndim) + msg = "Future(datum_support={}, pandas_ndim={}, save_split_attrs={})" + return msg.format( + self.datum_support, self.pandas_ndim, self.save_split_attrs + ) # deprecated_options = {'example_future_flag': 'warning',} deprecated_options = {} diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 8bb9d7c00e..0d1f531bf9 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -930,19 +930,19 @@ def _normalise_attrs( return attributes @property - def locals(self): + def locals(self) -> LimitedAttributeDict: return self._locals @locals.setter - def locals(self, attributes): + def locals(self, attributes: Optional[Mapping]): self._locals = self._normalise_attrs(attributes) @property - def globals(self): + def globals(self) -> LimitedAttributeDict: return self._globals @globals.setter - def globals(self, attributes): + def globals(self, attributes: Optional[Mapping]): self._globals = self._normalise_attrs(attributes) # @@ -1335,8 +1335,12 @@ def _names(self): # # Ensure that .attributes is always a :class:`CubeAttrsDict`. # - @CFVariableMixin.attributes.setter - def attributes(self, attributes): + @property + def attributes(self) -> CubeAttrsDict: + return super().attributes + + @attributes.setter + def attributes(self, attributes: Optional[Mapping]): """ An override to CfVariableMixin.attributes.setter, which ensures that Cube attributes are stored in a way which distinguishes global + local ones. diff --git a/lib/iris/fileformats/netcdf/saver.py b/lib/iris/fileformats/netcdf/saver.py index 312eea9c43..1c6aefb9a0 100644 --- a/lib/iris/fileformats/netcdf/saver.py +++ b/lib/iris/fileformats/netcdf/saver.py @@ -541,6 +541,10 @@ def write( matching keys will become attributes on the data variable rather than global attributes. + .. Note:: + + Has no effect if :attr:`iris.FUTURE.save_split_attrs` is ``True``. + * unlimited_dimensions (iterable of strings and/or :class:`iris.coords.Coord` objects): List of coordinate names (or coordinate objects) @@ -709,20 +713,23 @@ def write( # aux factory in the cube. self._add_aux_factories(cube, cf_var_cube, cube_dimensions) - # Add data variable-only attribute names to local_keys. - if local_keys is None: - local_keys = set() - else: - local_keys = set(local_keys) - local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) - - # Add global attributes taking into account local_keys. - global_attributes = { - k: v - for k, v in cube.attributes.items() - if (k not in local_keys and k.lower() != "conventions") - } - self.update_global_attributes(global_attributes) + if not iris.FUTURE.save_split_attrs: + # In the "old" way, we update global attributes as we go. + # Add data variable-only attribute names to local_keys. + if local_keys is None: + local_keys = set() + else: + local_keys = set(local_keys) + local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) + + # Add global attributes taking into account local_keys. + cube_attributes = cube.attributes + global_attributes = { + k: v + for k, v in cube_attributes.items() + if (k not in local_keys and k.lower() != "conventions") + } + self.update_global_attributes(global_attributes) if cf_profile_available: cf_patch = iris.site_configuration.get("cf_patch") @@ -778,6 +785,9 @@ def update_global_attributes(self, attributes=None, **kwargs): CF global attributes to be updated. """ + # TODO: when we no longer support combined attribute saving, this routine will + # only be called once: it can reasonably be renamed "_set_global_attributes", + # and the 'kwargs' argument can be removed. if attributes is not None: # Handle sequence e.g. [('fruit', 'apple'), ...]. if not hasattr(attributes, "keys"): @@ -2219,6 +2229,8 @@ def _create_cf_data_variable( The newly created CF-netCDF data variable. """ + # TODO: when iris.FUTURE.save_split_attrs is removed, the 'local_keys' arg can + # be removed. # Get the values in a form which is valid for the file format. data = self._ensure_valid_dtype(cube.core_data(), "cube", cube) @@ -2307,16 +2319,20 @@ def set_packing_ncattrs(cfvar): if cube.units.calendar: _setncattr(cf_var, "calendar", cube.units.calendar) - # Add data variable-only attribute names to local_keys. - if local_keys is None: - local_keys = set() + if iris.FUTURE.save_split_attrs: + attr_names = cube.attributes.locals.keys() else: - local_keys = set(local_keys) - local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) + # Add data variable-only attribute names to local_keys. + if local_keys is None: + local_keys = set() + else: + local_keys = set(local_keys) + local_keys.update(_CF_DATA_ATTRS, _UKMO_DATA_ATTRS) + + # Add any cube attributes whose keys are in local_keys as + # CF-netCDF data variable attributes. + attr_names = set(cube.attributes).intersection(local_keys) - # Add any cube attributes whose keys are in local_keys as - # CF-netCDF data variable attributes. - attr_names = set(cube.attributes).intersection(local_keys) for attr_name in sorted(attr_names): # Do not output 'conventions' attribute. if attr_name.lower() == "conventions": @@ -2600,9 +2616,15 @@ def save( Save cube(s) to a netCDF file, given the cube and the filename. * Iris will write CF 1.7 compliant NetCDF files. - * The attributes dictionaries on each cube in the saved cube list - will be compared and common attributes saved as NetCDF global - attributes where appropriate. + * If **split-attribute saving is disabled**, i.e. + :attr:`iris.FUTURE.save_split_attrs` is ``False``, then attributes dictionaries + on each cube in the saved cube list will be compared and common attributes saved + as NetCDF global attributes where appropriate. + + Or, **when **split-attribute saving is enabled**, then `cube.attributes.locals`` + are always saved as attributes of data-variables, and ``cube.attributes.globals`` + are saved as global (dataset) attributes, where possible. + Since the 2 types are now distinguished : see :class:`~iris.cube.CubeAttrsDict`. * Keyword arguments specifying how to save the data are applied to each cube. To use different settings for different cubes, use the NetCDF Context manager (:class:`~Saver`) directly. @@ -2635,6 +2657,8 @@ def save( An interable of cube attribute keys. Any cube attributes with matching keys will become attributes on the data variable rather than global attributes. + **NOTE:** this is *ignored* if 'split-attribute saving' is **enabled**, + i.e. when ``iris.FUTURE.save_split_attrs`` is ``True``. * unlimited_dimensions (iterable of strings and/or :class:`iris.coords.Coord` objects): @@ -2773,26 +2797,118 @@ def save( else: cubes = cube - if local_keys is None: + if iris.FUTURE.save_split_attrs: + # We don't actually use 'local_keys' in this case. + # TODO: can remove this when the iris.FUTURE.save_split_attrs is removed. local_keys = set() + + # Find any collisions in the cube global attributes and "demote" all those to + # local attributes (where possible, else warn they are lost). + # N.B. "collision" includes when not all cubes *have* that attribute. + global_names = set() + for cube in cubes: + global_names |= set(cube.attributes.globals.keys()) + + # Fnd any global attributes which are not the same on *all* cubes. + def attr_values_equal(val1, val2): + # An equality test which also works when some values are numpy arrays (!) + # As done in :meth:`iris.common.mixin.LimitedAttributeDict.__eq__`. + match = val1 == val2 + try: + match = bool(match) + except ValueError: + match = match.all() + return match + + cube0 = cubes[0] + invalid_globals = [ + attrname + for attrname in global_names + if not all( + attr_values_equal( + cube.attributes.globals.get(attrname), + cube0.attributes.globals.get(attrname), + ) + for cube in cubes[1:] + ) + ] + + # Establish all the global attributes which we will write to the file (at end). + global_attributes = { + attr: cube0.attributes.globals.get(attr) + for attr in global_names + if attr not in invalid_globals + } + if invalid_globals: + # Some cubes have different global attributes: modify cubes as required. + warnings.warn( + f"Saving the cube global attributes {invalid_globals} as local" + "(i.e. data-variable) attributes, where possible, since they are not " + "the same on all input cubes." + ) + cubes = list(cubes) # avoiding modifying the actual input arg. + for i_cube in range(len(cubes)): + # We iterate over cube *index*, so we can replace the list entries with + # with cube *copies* -- just to avoid changing our call args. + cube = cubes[i_cube] + demote_attrs = [ + attr + for attr in cube.attributes.globals + if attr in invalid_globals + ] + if any(demote_attrs): + # This cube contains some 'demoted' global attributes. + # Replace the input cube with a copy, so we can modify attributes. + cube = cube.copy() + cubes[i_cube] = cube + # Catch any demoted attrs where there is already a local version + blocked_attrs = [ + attrname + for attrname in demote_attrs + if attrname in cube.attributes.locals + ] + if blocked_attrs: + warnings.warn( + f"Global cube attributes {blocked_attrs} " + f'of cube "{cube.name()}" have been lost, overlaid ' + "by existing local attributes with the same names." + ) + for attr in demote_attrs: + if attr not in blocked_attrs: + cube.attributes.locals[ + attr + ] = cube.attributes.globals[attr] + cube.attributes.globals.pop(attr) + else: - local_keys = set(local_keys) - - # Determine the attribute keys that are common across all cubes and - # thereby extend the collection of local_keys for attributes - # that should be attributes on data variables. - attributes = cubes[0].attributes - common_keys = set(attributes) - for cube in cubes[1:]: - keys = set(cube.attributes) - local_keys.update(keys.symmetric_difference(common_keys)) - common_keys.intersection_update(keys) - different_value_keys = [] - for key in common_keys: - if np.any(attributes[key] != cube.attributes[key]): - different_value_keys.append(key) - common_keys.difference_update(different_value_keys) - local_keys.update(different_value_keys) + # Determine the attribute keys that are common across all cubes and + # thereby extend the collection of local_keys for attributes + # that should be attributes on data variables. + # NOTE: in 'legacy' mode, this code derives a common value for 'local_keys', which + # is employed in saving each cube. + # However, in `split_attrs` mode, this considers ONLY global attributes, and the + # resulting 'common_keys' is the fixed result : each cube is then saved like ... + # "sman.write(... localkeys=list(cube.attributes) - common_keys, ...)" + if local_keys is None: + local_keys = set() + else: + local_keys = set(local_keys) + + common_attr_values = None + for cube in cubes: + cube_attributes = cube.attributes + keys = set(cube_attributes) + if common_attr_values is None: + common_attr_values = cube_attributes.copy() + common_keys = keys.copy() + local_keys.update(keys.symmetric_difference(common_keys)) + common_keys.intersection_update(keys) + different_value_keys = [] + for key in common_keys: + if np.any(common_attr_values[key] != cube_attributes[key]): + different_value_keys.append(key) + common_keys.difference_update(different_value_keys) + local_keys.update(different_value_keys) def is_valid_packspec(p): """Only checks that the datatype is valid.""" @@ -2894,7 +3010,12 @@ def is_valid_packspec(p): warnings.warn(msg) # Add conventions attribute. - sman.update_global_attributes(Conventions=conventions) + if iris.FUTURE.save_split_attrs: + # In the "new way", we just create all the global attributes at once. + global_attributes["Conventions"] = conventions + sman.update_global_attributes(global_attributes) + else: + sman.update_global_attributes(Conventions=conventions) if compute: # No more to do, since we used Saver(compute=True). diff --git a/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py b/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py index a1cad53336..56bfd2ff71 100644 --- a/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py +++ b/lib/iris/tests/integration/test_netcdf__loadsaveattrs.py @@ -19,13 +19,17 @@ """ import inspect -from typing import Iterable, Optional, Union +import re +from typing import Iterable, List, Optional, Union +import warnings +import numpy as np import pytest import iris import iris.coord_systems -from iris.cube import Cube, CubeAttrsDict +from iris.coords import DimCoord +from iris.cube import Cube import iris.fileformats.netcdf import iris.fileformats.netcdf._thread_safe_nc as threadsafe_nc4 @@ -39,6 +43,7 @@ # A list of "global-style" attribute names : those which should be global attributes by # default (i.e. file- or group-level, *not* attached to a variable). + _GLOBAL_TEST_ATTRS = set(iris.fileformats.netcdf.saver._CF_GLOBAL_ATTRS) # Remove this one, which has peculiar behaviour + is tested separately # N.B. this is not the same as 'Conventions', but is caught in the crossfire when that @@ -70,6 +75,30 @@ def local_attr(request): return request.param # Return the name of the attribute to test. +def check_captured_warnings( + expected_keys: List[str], captured_warnings: List[warnings] +): + if expected_keys is None: + expected_keys = [] + elif hasattr(expected_keys, "upper"): + # Handle a single string + expected_keys = [expected_keys] + expected_keys = [re.compile(key) for key in expected_keys] + found_results = [str(warning.message) for warning in captured_warnings] + remaining_keys = expected_keys.copy() + for i_message, message in enumerate(found_results.copy()): + i_found = None + for i_key, key in enumerate(remaining_keys): + if key.search(message): + # Hit : remove one + only one matching warning from the list + i_found = i_message + break + if i_found is not None: + found_results[i_found] = key + remaining_keys.remove(key) + assert found_results == expected_keys + + class MixinAttrsTesting: @staticmethod def _calling_testname(): @@ -77,7 +106,7 @@ def _calling_testname(): Search up the callstack for a function named "test_*", and return the name for use as a test identifier. - Idea borrowed from :meth:`iris.tests.IrisTest_nometa.result_path`. + Idea borrowed from :meth:`iris.tests.IrisTest.result_path`. Returns ------- @@ -128,28 +157,34 @@ def _default_vars_and_attrvalues(vars_and_attrvalues): vars_and_attrvalues = {"var": vars_and_attrvalues} return vars_and_attrvalues - def create_testcase_files( + def create_testcase_files_or_cubes( self, attr_name: str, global_value_file1: Optional[str] = None, var_values_file1: Union[None, str, dict] = None, global_value_file2: Optional[str] = None, var_values_file2: Union[None, str, dict] = None, + cubes: bool = False, ): """ - Create temporary input netcdf files with specific content. + Create temporary input netcdf files, or cubes, with specific content. Creates a temporary netcdf test file (or two) with the given global and - variable-local attributes. - The file(s) are used to test the behaviour of the attribute. + variable-local attributes. Or build cubes, similarly. + If ``cubes`` is ``True``, save cubes in ``self.input_cubes``. + Else save filepaths in ``self.input_filepaths``. Note: 'var_values_file' args are dictionaries. The named variables are created, with an attribute = the dictionary value, *except* that a dictionary value of None means that a local attribute is _not_ created on the variable. """ - # Make some input file paths. - filepath1 = self._testfile_path("testfile") - filepath2 = self._testfile_path("testfile2") + # save attribute on the instance + self.attrname = attr_name + + if not cubes: + # Make some input file paths. + filepath1 = self._testfile_path("testfile") + filepath2 = self._testfile_path("testfile2") def make_file( filepath: str, global_value=None, var_values=None @@ -169,24 +204,187 @@ def make_file( ds.close() return filepath - # Create one input file (always). - filepaths = [ - make_file( - filepath1, - global_value=global_value_file1, - var_values=var_values_file1, - ) - ] - if global_value_file2 is not None or var_values_file2 is not None: - # Make a second testfile and add it to files-to-be-loaded. - filepaths.append( - make_file( - filepath2, - global_value=global_value_file2, - var_values=var_values_file2, - ), + def make_cubes(var_name, global_value=None, var_values=None): + cubes = [] + var_values = self._default_vars_and_attrvalues(var_values) + for varname, local_value in var_values.items(): + cube = Cube(np.arange(3.0), var_name=var_name) + cubes.append(cube) + dimco = DimCoord(np.arange(3.0), var_name="x") + cube.add_dim_coord(dimco, 0) + if global_value is not None: + cube.attributes.globals[attr_name] = global_value + if local_value is not None: + cube.attributes.locals[attr_name] = local_value + return cubes + + if cubes: + results = make_cubes("v1", global_value_file1, var_values_file1) + if global_value_file2 is not None or var_values_file2 is not None: + results.extend( + make_cubes("v2", global_value_file2, var_values_file2) + ) + else: + results = [ + make_file(filepath1, global_value_file1, var_values_file1) + ] + if global_value_file2 is not None or var_values_file2 is not None: + # Make a second testfile and add it to files-to-be-loaded. + results.append( + make_file(filepath2, global_value_file2, var_values_file2) + ) + + # Save results on the instance + if cubes: + self.input_cubes = results + else: + self.input_filepaths = results + return results + + def run_testcase( + self, + attr_name: str, + values: Union[List, List[List]], + create_cubes_or_files: str = "files", + ) -> None: + """ + Create testcase inputs (files or cubes) with specified attributes. + + Parameters + ---------- + attr_name : str + name for all attributes created in this testcase. + Also saved as ``self.attrname``, as used by ``fetch_results``. + values : list + list, or lists, of values for created attributes, each containing one global + and one-or-more local attribute values as [global, local1, local2...] + create_cubes_or_files : str, default "files" + create either cubes or testfiles. + + If ``create_cubes_or_files`` == "files", create one temporary netCDF file per + values-list, and record in ``self.input_filepaths``. + Else if ``create_cubes_or_files`` == "cubes", create sets of cubes with common + global values and store all of them to ``self.input_cubes``. + + """ + # Save common attribute-name on the instance + self.attrname = attr_name + + # Standardise input to a list-of-lists, each inner list = [global, *locals] + assert isinstance(values, list) + if not isinstance(values[0], list): + values = [values] + assert len(values) in (1, 2) + assert len(values[0]) > 1 + + # Decode into global1, *locals1, and optionally global2, *locals2 + global1 = values[0][0] + vars1 = {} + i_var = 0 + for value in values[0][1:]: + vars1[f"var_{i_var}"] = value + i_var += 1 + if len(values) == 1: + global2 = None + vars2 = None + else: + assert len(values) == 2 + global2 = values[1][0] + vars2 = {} + for value in values[1][1:]: + vars2[f"var_{i_var}"] = value + i_var += 1 + + # Create test files or cubes (and store data on the instance) + assert create_cubes_or_files in ("cubes", "files") + make_cubes = create_cubes_or_files == "cubes" + self.create_testcase_files_or_cubes( + attr_name=attr_name, + global_value_file1=global1, + var_values_file1=vars1, + global_value_file2=global2, + var_values_file2=vars2, + cubes=make_cubes, + ) + + def fetch_results( + self, + filepath: str = None, + cubes: Iterable[Cube] = None, + oldstyle_combined: bool = False, + ): + """ + Return testcase results from an output file or cubes in a standardised form. + + Unpick the global+local values of the attribute ``self.attrname``, resulting + from a test operation. + A file result is always [global_value, *local_values] + A cubes result is [*[global_value, *local_values]] (over different global vals) + + When ``oldstyle_combined`` is ``True``, simulate the "legacy" style results, + that is when each cube had a single combined attribute dictionary. + This enables us to check against former behaviour, by combining results into a + single dictionary. N.B. per-cube single results are then returned in the form: + [None, cube1, cube2...]. + N.B. if results are from a *file*, this key has **no effect**. + + """ + attr_name = self.attrname + if filepath is not None: + # Fetch global and local values from a file + try: + ds = threadsafe_nc4.DatasetWrapper(filepath) + global_result = ( + ds.getncattr(attr_name) + if attr_name in ds.ncattrs() + else None + ) + # Fetch local attr value from all data variables (except dimcoord vars) + local_vars_results = [ + ( + var.name, + ( + var.getncattr(attr_name) + if attr_name in var.ncattrs() + else None + ), + ) + for var in ds.variables.values() + if var.name not in ds.dimensions + ] + finally: + ds.close() + # This version always returns a single result set [global, local1[, local2]] + # Return global, plus locals sorted by varname + local_vars_results = sorted(local_vars_results, key=lambda x: x[0]) + results = [global_result] + [val for _, val in local_vars_results] + else: + assert cubes is not None + # Sort result cubes according to a standard ordering. + cubes = sorted(cubes, key=lambda cube: cube.name()) + # Fetch globals and locals from cubes. + if oldstyle_combined: + # Replace cubes attributes with all-combined dictionaries + cubes = [cube.copy() for cube in cubes] + for cube in cubes: + combined = dict(cube.attributes) + cube.attributes.clear() + cube.attributes.locals = combined + global_values = set( + cube.attributes.globals.get(attr_name, None) for cube in cubes ) - return filepaths + # This way returns *multiple* result 'sets', one for each global value + results = [ + [globalval] + + [ + cube.attributes.locals.get(attr_name, None) + for cube in cubes + if cube.attributes.globals.get(attr_name, None) + == globalval + ] + for globalval in sorted(global_values) + ] + return results class TestRoundtrip(MixinAttrsTesting): @@ -205,24 +403,16 @@ class TestRoundtrip(MixinAttrsTesting): """ - def _roundtrip_load_and_save( - self, input_filepaths: Union[str, Iterable[str]], output_filepath: str - ) -> None: - """ - Load netcdf input file(s) and re-write all to a given output file. - """ - # Do a load+save to produce a testable output result in a new file. - cubes = iris.load(input_filepaths) - iris.save(cubes, output_filepath) + # Parametrise all tests over split/unsplit saving. + @pytest.fixture( + params=[False, True], ids=["nosplit", "split"], autouse=True + ) + def do_split(self, request): + do_split = request.param + self.save_split_attrs = do_split + return do_split - def create_roundtrip_testcase( - self, - attr_name, - global_value_file1=None, - vars_values_file1=None, - global_value_file2=None, - vars_values_file2=None, - ): + def run_roundtrip_testcase(self, attr_name, values, expect_warnings=None): """ Initialise the testcase from the passed-in controls, configure the input files and run a save-load roundtrip to produce the output file. @@ -231,48 +421,36 @@ def create_roundtrip_testcase( stored on the instance, where "self.check_roundtrip_results()" can get them. """ - self.attrname = attr_name - self.input_filepaths = self.create_testcase_files( - attr_name=attr_name, - global_value_file1=global_value_file1, - var_values_file1=vars_values_file1, - global_value_file2=global_value_file2, - var_values_file2=vars_values_file2, + self.run_testcase( + attr_name=attr_name, values=values, create_cubes_or_files="files" ) self.result_filepath = self._testfile_path("result") - self._roundtrip_load_and_save( - self.input_filepaths, self.result_filepath - ) - def check_roundtrip_results( - self, global_attr_value=None, var_attr_vals=None - ): + with warnings.catch_warnings(record=True) as captured_warnings: + # Do a load+save to produce a testable output result in a new file. + cubes = iris.load(self.input_filepaths) + # Ensure stable result order. + cubes = sorted(cubes, key=lambda cube: cube.name()) + do_split = getattr(self, "save_split_attrs", False) + with iris.FUTURE.context(save_split_attrs=do_split): + iris.save(cubes, self.result_filepath) + + check_captured_warnings(expect_warnings, captured_warnings) + + def check_roundtrip_results(self, expected): """ Run checks on the generated output file. - The counterpart to create_testcase, with similar control arguments. - Check existence (or not) of : a global attribute, named variables, and their - local attributes. Values of 'None' mean to check that the relevant global/local - attribute does *not* exist. + The counterpart to :meth:`run_roundtrip_testcase`, with similar arguments. + Check existence (or not) of a global attribute, and a number of local + (variable) attributes. + Values of 'None' mean to check that the relevant global/local attribute does + *not* exist. """ # N.B. there is only ever one result-file, but it can contain various variables # which came from different input files. - ds = threadsafe_nc4.DatasetWrapper(self.result_filepath) - if global_attr_value is None: - assert self.attrname not in ds.ncattrs() - else: - assert self.attrname in ds.ncattrs() - assert ds.getncattr(self.attrname) == global_attr_value - if var_attr_vals: - var_attr_vals = self._default_vars_and_attrvalues(var_attr_vals) - for var_name, value in var_attr_vals.items(): - assert var_name in ds.variables - v = ds.variables[var_name] - if value is None: - assert self.attrname not in v.ncattrs() - else: - assert self.attrname in v.ncattrs() - assert v.getncattr(self.attrname) == value + results = self.fetch_results(filepath=self.result_filepath) + assert results == expected ####################################################### # Tests on "user-style" attributes. @@ -281,94 +459,102 @@ def check_roundtrip_results( # def test_01_userstyle_single_global(self): - self.create_roundtrip_testcase( - attr_name="myname", # A generic "user" attribute with no special handling - global_value_file1="single-value", - vars_values_file1={ - "myvar": None - }, # the variable has no such attribute + self.run_roundtrip_testcase( + attr_name="myname", values=["single-value", None] ) # Default behaviour for a general global user-attribute. # It simply remains global. - self.check_roundtrip_results( - global_attr_value="single-value", # local values eclipse the global ones - var_attr_vals={ - "myvar": None - }, # the variable has no such attribute - ) + self.check_roundtrip_results(["single-value", None]) - def test_02_userstyle_single_local(self): + def test_02_userstyle_single_local(self, do_split): # Default behaviour for a general local user-attribute. # It results in a "promoted" global attribute. - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="myname", # A generic "user" attribute with no special handling - vars_values_file1={"myvar": "single-value"}, - ) - self.check_roundtrip_results( - global_attr_value="single-value", # local values eclipse the global ones - # N.B. the output var has NO such attribute + values=[None, "single-value"], ) + if do_split: + expected = [None, "single-value"] + else: + expected = ["single-value", None] + self.check_roundtrip_results(expected) - def test_03_userstyle_multiple_different(self): + def test_03_userstyle_multiple_different(self, do_split): # Default behaviour for general user-attributes. # The global attribute is lost because there are local ones. - vars1 = {"f1_v1": "f1v1", "f1_v2": "f2v2"} - vars2 = {"f2_v1": "x1", "f2_v2": "x2"} - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="random", # A generic "user" attribute with no special handling - global_value_file1="global_file1", - vars_values_file1=vars1, - global_value_file2="global_file2", - vars_values_file2=vars2, - ) - # combine all 4 vars in one dict - all_vars_and_attrs = vars1.copy() - all_vars_and_attrs.update(vars2) - # TODO: replace with "|", when we drop Python 3.8 - # see: https://peps.python.org/pep-0584/ - # just check they are all there and distinct - assert len(all_vars_and_attrs) == len(vars1) + len(vars2) - self.check_roundtrip_results( - global_attr_value=None, # local values eclipse the global ones - var_attr_vals=all_vars_and_attrs, + values=[ + ["common_global", "f1v1", "f1v2"], + ["common_global", "x1", "x2"], + ], ) + expected_result = ["common_global", "f1v1", "f1v2", "x1", "x2"] + if not do_split: + # in legacy mode, global is lost + expected_result[0] = None + # just check they are all there and distinct + self.check_roundtrip_results(expected_result) - def test_04_userstyle_matching_promoted(self): + def test_04_userstyle_matching_promoted(self, do_split): # matching local user-attributes are "promoted" to a global one. - self.create_roundtrip_testcase( + # (but not when saving split attributes) + input_values = ["global_file1", "same-value", "same-value"] + self.run_roundtrip_testcase( attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "same-value", "v2": "same-value"}, - ) - self.check_roundtrip_results( - global_attr_value="same-value", - var_attr_vals={"v1": None, "v2": None}, + values=input_values, ) + if do_split: + expected = input_values + else: + expected = ["same-value", None, None] + self.check_roundtrip_results(expected) - def test_05_userstyle_matching_crossfile_promoted(self): + def test_05_userstyle_matching_crossfile_promoted(self, do_split): # matching user-attributes are promoted, even across input files. - self.create_roundtrip_testcase( + # (but not when saving split attributes) + input_values = [ + ["global_file1", "same-value", "same-value"], + [None, "same-value", "same-value"], + ] + if do_split: + # newstyle saves: locals are preserved, mismathced global is *lost* + expected_result = [ + None, + "same-value", + "same-value", + "same-value", + "same-value", + ] + # warnings about the clash + expected_warnings = [ + "Saving.* global attributes.* as local", + 'attributes.* of cube "var_0" have been lost', + 'attributes.* of cube "var_1" have been lost', + ] + else: + # oldstyle saves: matching locals promoted, override original global + expected_result = ["same-value", None, None, None, None] + expected_warnings = None + + self.run_roundtrip_testcase( attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "same-value", "v2": "same-value"}, - vars_values_file2={"f2_v1": "same-value", "f2_v2": "same-value"}, - ) - self.check_roundtrip_results( - global_attr_value="same-value", - var_attr_vals={x: None for x in ("v1", "v2", "f2_v1", "f2_v2")}, + values=input_values, + expect_warnings=expected_warnings, ) + self.check_roundtrip_results(expected_result) - def test_06_userstyle_nonmatching_remainlocal(self): + def test_06_userstyle_nonmatching_remainlocal(self, do_split): # Non-matching user attributes remain 'local' to the individual variables. - self.create_roundtrip_testcase( - attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "value-1", "v2": "value-2"}, - ) - self.check_roundtrip_results( - global_attr_value=None, # NB it still destroys the global one !! - var_attr_vals={"v1": "value-1", "v2": "value-2"}, - ) + input_values = ["global_file1", "value-1", "value-2"] + if do_split: + # originals are preserved + expected_result = input_values + else: + # global is lost + expected_result = [None, "value-1", "value-2"] + self.run_roundtrip_testcase(attr_name="random", values=input_values) + self.check_roundtrip_results(expected_result) ####################################################### # Tests on "Conventions" attribute. @@ -383,129 +569,131 @@ def test_06_userstyle_nonmatching_remainlocal(self): def test_07_conventions_var_local(self): # What happens if 'Conventions' appears as a variable-local attribute. # N.B. this is not good CF, but we'll see what happens anyway. - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="Conventions", - global_value_file1=None, - vars_values_file1="user_set", - ) - self.check_roundtrip_results( - global_attr_value="CF-1.7", # standard content from Iris save - var_attr_vals=None, + values=[None, "user_set"], ) + self.check_roundtrip_results(["CF-1.7", None]) def test_08_conventions_var_both(self): # What happens if 'Conventions' appears as both global + local attribute. - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name="Conventions", - global_value_file1="global-setting", - vars_values_file1="local-setting", - ) - self.check_roundtrip_results( - global_attr_value="CF-1.7", # standard content from Iris save - var_attr_vals=None, + values=["global-setting", "local-setting"], ) + # standard content from Iris save + self.check_roundtrip_results(["CF-1.7", None]) ####################################################### # Tests on "global" style attributes # = those specific ones which 'ought' only to be global (except on collisions) # - def test_09_globalstyle__global(self, global_attr): attr_content = f"Global tracked {global_attr}" - self.create_roundtrip_testcase( + self.run_roundtrip_testcase( attr_name=global_attr, - global_value_file1=attr_content, + values=[attr_content, None], ) - self.check_roundtrip_results(global_attr_value=attr_content) + self.check_roundtrip_results([attr_content, None]) - def test_10_globalstyle__local(self, global_attr): + def test_10_globalstyle__local(self, global_attr, do_split): # Strictly, not correct CF, but let's see what it does with it. attr_content = f"Local tracked {global_attr}" - self.create_roundtrip_testcase( + input_values = [None, attr_content] + if do_split: + # remains local as supplied, but there is a warning + expected_result = input_values + expected_warning = f"'{global_attr}'.* should only be a CF global" + else: + # promoted to global + expected_result = [attr_content, None] + expected_warning = None + self.run_roundtrip_testcase( attr_name=global_attr, - vars_values_file1=attr_content, + values=input_values, + expect_warnings=expected_warning, ) - self.check_roundtrip_results( - global_attr_value=attr_content - ) # "promoted" + self.check_roundtrip_results(expected_result) - def test_11_globalstyle__both(self, global_attr): + def test_11_globalstyle__both(self, global_attr, do_split): attr_global = f"Global-{global_attr}" attr_local = f"Local-{global_attr}" - self.create_roundtrip_testcase( + input_values = [attr_global, attr_local] + if do_split: + # remains local as supplied, but there is a warning + expected_result = input_values + expected_warning = "should only be a CF global" + else: + # promoted to global, no local value, original global lost + expected_result = [attr_local, None] + expected_warning = None + self.run_roundtrip_testcase( attr_name=global_attr, - global_value_file1=attr_global, - vars_values_file1=attr_local, - ) - self.check_roundtrip_results( - global_attr_value=attr_local # promoted local setting "wins" + values=input_values, + expect_warnings=expected_warning, ) + self.check_roundtrip_results(expected_result) def test_12_globalstyle__multivar_different(self, global_attr): # Multiple *different* local settings are retained, not promoted attr_1 = f"Local-{global_attr}-1" attr_2 = f"Local-{global_attr}-2" - with pytest.warns( - UserWarning, match="should only be a CF global attribute" - ): - # A warning should be raised when writing the result. - self.create_roundtrip_testcase( - attr_name=global_attr, - vars_values_file1={"v1": attr_1, "v2": attr_2}, - ) - self.check_roundtrip_results( - global_attr_value=None, - var_attr_vals={"v1": attr_1, "v2": attr_2}, + expect_warning = "should only be a CF global attribute" + # A warning should be raised when writing the result. + self.run_roundtrip_testcase( + attr_name=global_attr, + values=[None, attr_1, attr_2], + expect_warnings=expect_warning, ) + self.check_roundtrip_results([None, attr_1, attr_2]) - def test_13_globalstyle__multivar_same(self, global_attr): + def test_13_globalstyle__multivar_same(self, global_attr, do_split): # Multiple *same* local settings are promoted to a common global one attrval = f"Locally-defined-{global_attr}" - self.create_roundtrip_testcase( + input_values = [None, attrval, attrval] + if do_split: + # remains local, but with a warning + expected_warning = "should only be a CF global" + expected_result = input_values + else: + # promoted to global + expected_warning = None + expected_result = [attrval, None, None] + self.run_roundtrip_testcase( attr_name=global_attr, - vars_values_file1={"v1": attrval, "v2": attrval}, - ) - self.check_roundtrip_results( - global_attr_value=attrval, - var_attr_vals={"v1": None, "v2": None}, + values=input_values, + expect_warnings=expected_warning, ) + self.check_roundtrip_results(expected_result) - def test_14_globalstyle__multifile_different(self, global_attr): + def test_14_globalstyle__multifile_different(self, global_attr, do_split): # Different global attributes from multiple files are retained as local ones attr_1 = f"Global-{global_attr}-1" attr_2 = f"Global-{global_attr}-2" - with pytest.warns( - UserWarning, match="should only be a CF global attribute" - ): - # A warning should be raised when writing the result. - self.create_roundtrip_testcase( - attr_name=global_attr, - global_value_file1=attr_1, - vars_values_file1={"v1": None}, - global_value_file2=attr_2, - vars_values_file2={"v2": None}, - ) - self.check_roundtrip_results( - # Combining them "demotes" the common global attributes to local ones - var_attr_vals={"v1": attr_1, "v2": attr_2} + # A warning should be raised when writing the result. + expect_warnings = ["should only be a CF global attribute"] + if do_split: + # An extra warning, only when saving with split-attributes. + expect_warnings = ["Saving.* as local"] + expect_warnings + self.run_roundtrip_testcase( + attr_name=global_attr, + values=[[attr_1, None], [attr_2, None]], + expect_warnings=expect_warnings, ) + self.check_roundtrip_results([None, attr_1, attr_2]) def test_15_globalstyle__multifile_same(self, global_attr): # Matching global-type attributes in multiple files are retained as global attrval = f"Global-{global_attr}" - self.create_roundtrip_testcase( - attr_name=global_attr, - global_value_file1=attrval, - vars_values_file1={"v1": None}, - global_value_file2=attrval, - vars_values_file2={"v2": None}, - ) - self.check_roundtrip_results( - # The attribute remains as a common global setting - global_attr_value=attrval, - # The individual variables do *not* have an attribute of this name - var_attr_vals={"v1": None, "v2": None}, + self.run_roundtrip_testcase( + attr_name=global_attr, values=[[attrval, None], [attrval, None]] ) + # # The attribute remains as a common global setting + # global_attr_value=attrval, + # # The individual variables do *not* have an attribute of this name + # var_attr_vals={"v1": None, "v2": None}, + # ) + self.check_roundtrip_results([attrval, None, None]) ####################################################### # Tests on "local" style attributes @@ -514,7 +702,7 @@ def test_15_globalstyle__multifile_same(self, global_attr): # @pytest.mark.parametrize("origin_style", ["input_global", "input_local"]) - def test_16_localstyle(self, local_attr, origin_style): + def test_16_localstyle(self, local_attr, origin_style, do_split): # local-style attributes should *not* get 'promoted' to global ones # Set the name extension to avoid tests with different 'style' params having # collisions over identical testfile names @@ -535,20 +723,19 @@ def test_16_localstyle(self, local_attr, origin_style): # as global or a variable attribute if origin_style == "input_global": # Record in source as a global attribute - self.create_roundtrip_testcase( - attr_name=local_attr, global_value_file1=attrval - ) + values = [attrval, None] else: assert origin_style == "input_local" # Record in source as a variable-local attribute - self.create_roundtrip_testcase( - attr_name=local_attr, vars_values_file1=attrval - ) + values = [None, attrval] + self.run_roundtrip_testcase(attr_name=local_attr, values=values) - if local_attr in iris.fileformats.netcdf.saver._CF_DATA_ATTRS: - # These ones are simply discarded on loading. - # By experiment, this overlap between _CF_ATTRS and _CF_DATA_ATTRS - # currently contains only 'missing_value' and 'standard_error_multiplier'. + if ( + local_attr in ("missing_value", "standard_error_multiplier") + and origin_style == "input_local" + ): + # These ones are actually discarded by roundtrip. + # Not clear why, but for now this captures the facts. expect_global = None expect_var = None else: @@ -556,6 +743,7 @@ def test_16_localstyle(self, local_attr, origin_style): if ( local_attr == "ukmo__process_flags" and origin_style == "input_global" + and not do_split ): # This is very odd behaviour + surely unintended. # It's supposed to handle vector values (which we are not checking). @@ -564,14 +752,17 @@ def test_16_localstyle(self, local_attr, origin_style): attrval = "p r o c e s s" expect_var = attrval - if local_attr == "STASH": + if local_attr == "STASH" and ( + origin_style == "input_local" or not do_split + ): # A special case, output translates this to a different attribute name. self.attrname = "um_stash_source" - self.check_roundtrip_results( - global_attr_value=expect_global, - var_attr_vals=expect_var, - ) + expected_result = [expect_global, expect_var] + if do_split and origin_style == "input_global": + # The result is simply the "other way around" + expected_result = expected_result[::-1] + self.check_roundtrip_results(expected_result) class TestLoad(MixinAttrsTesting): @@ -588,33 +779,21 @@ class TestLoad(MixinAttrsTesting): """ - def create_load_testcase( - self, - attr_name, - global_value_file1=None, - vars_values_file1=None, - global_value_file2=None, - vars_values_file2=None, - ) -> iris.cube.CubeList: - """ - Initialise the testcase from the passed-in controls, configure the input - files and run a save-load roundtrip to produce the output file. - - The name of the tested attribute and all the temporary filepaths are stored - on the instance, from where "self.check_load_results()" can get them. - - """ - self.attrname = attr_name - self.input_filepaths = self.create_testcase_files( - attr_name=attr_name, - global_value_file1=global_value_file1, - var_values_file1=vars_values_file1, - global_value_file2=global_value_file2, - var_values_file2=vars_values_file2, + def run_load_testcase(self, attr_name, values): + self.run_testcase( + attr_name=attr_name, values=values, create_cubes_or_files="files" ) + + def check_load_results(self, expected, oldstyle_combined=False): result_cubes = iris.load(self.input_filepaths) - result_cubes = sorted(result_cubes, key=lambda cube: cube.name()) - return result_cubes + results = self.fetch_results( + cubes=result_cubes, oldstyle_combined=oldstyle_combined + ) + # Standardise expected form to list(lists). + assert isinstance(expected, list) + if not isinstance(expected[0], list): + expected = [expected] + assert results == expected ####################################################### # Tests on "user-style" attributes. @@ -623,83 +802,58 @@ def create_load_testcase( # def test_01_userstyle_single_global(self): - cube1, cube2 = self.create_load_testcase( - attr_name="myname", # A generic "user" attribute with no special handling - global_value_file1="single-value", - vars_values_file1={ - "myvar": None, - "myvar2": None, - }, # the variable has no such attribute + self.run_load_testcase( + attr_name="myname", values=["single_value", None, None] ) - # Default behaviour for a general global user-attribute. - # It is attached to all loaded cubes. - - expected_dict = {"myname": "single-value"} - for cube in (cube1, cube2): - # #1 : legacy results, for cube.attributes **viewed as a plain dictionary**. - assert dict(cube1.attributes) == expected_dict - # #2 : exact expected result, viewed as newstyle split-attributes - assert cube1.attributes == CubeAttrsDict(globals=expected_dict) + # Legacy-equivalent result check (single attributes dict per cube) + self.check_load_results( + [None, "single_value", "single_value"], + oldstyle_combined=True, + ) + # Full new-style results check + self.check_load_results(["single_value", None, None]) def test_02_userstyle_single_local(self): # Default behaviour for a general local user-attribute. # It is attached to only the specific cube. - cube1, cube2 = self.create_load_testcase( + self.run_load_testcase( attr_name="myname", # A generic "user" attribute with no special handling - vars_values_file1={"myvar1": "single-value", "myvar2": None}, + values=[None, "single-value", None], + ) + self.check_load_results( + [None, "single-value", None], oldstyle_combined=True ) - assert cube1.attributes == {"myname": "single-value"} - assert cube2.attributes == {} + self.check_load_results([None, "single-value", None]) def test_03_userstyle_multiple_different(self): # Default behaviour for differing local user-attributes. # The global attribute is simply lost, because there are local ones. - vars1 = {"f1_v1": "f1v1", "f1_v2": "f1v2"} - vars2 = {"f2_v1": "x1", "f2_v2": "x2"} - cube1, cube2, cube3, cube4 = self.create_load_testcase( + self.run_load_testcase( attr_name="random", # A generic "user" attribute with no special handling - global_value_file1="global_file1", - vars_values_file1=vars1, - global_value_file2="global_file2", - vars_values_file2=vars2, - ) - - # (#1) : legacy equivalence : for cube.attributes viewed as a plain 'dict' - assert dict(cube1.attributes) == {"random": "f1v1"} - assert dict(cube2.attributes) == {"random": "f1v2"} - assert dict(cube3.attributes) == {"random": "x1"} - assert dict(cube4.attributes) == {"random": "x2"} - - # (#1) : exact results check, for newstyle "split" cube attrs - assert cube1.attributes == CubeAttrsDict( - globals={"random": "global_file1"}, locals={"random": "f1v1"} - ) - assert cube2.attributes == CubeAttrsDict( - globals={"random": "global_file1"}, locals={"random": "f1v2"} + values=[ + ["global_file1", "f1v1", "f1v2"], + ["global_file2", "x1", "x2"], + ], ) - assert cube3.attributes == CubeAttrsDict( - globals={"random": "global_file2"}, locals={"random": "x1"} + self.check_load_results( + [None, "f1v1", "f1v2", "x1", "x2"], + oldstyle_combined=True, ) - assert cube4.attributes == CubeAttrsDict( - globals={"random": "global_file2"}, locals={"random": "x2"} + self.check_load_results( + [["global_file1", "f1v1", "f1v2"], ["global_file2", "x1", "x2"]] ) def test_04_userstyle_multiple_same(self): # Nothing special to note in this case # TODO: ??remove?? - cube1, cube2 = self.create_load_testcase( + self.run_load_testcase( attr_name="random", - global_value_file1="global_file1", - vars_values_file1={"v1": "same-value", "v2": "same-value"}, - ) - for cube in (cube1, cube2): - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {"random": "same-value"} - # (#2): exact results, with newstyle "split" cube attrs - assert cube2.attributes == CubeAttrsDict( - globals={"random": "global_file1"}, - locals={"random": "same-value"}, - ) + values=["global_file1", "same-value", "same-value"], + ) + self.check_load_results( + oldstyle_combined=True, expected=[None, "same-value", "same-value"] + ) + self.check_load_results(["global_file1", "same-value", "same-value"]) ####################################################### # Tests on "Conventions" attribute. @@ -714,28 +868,27 @@ def test_04_userstyle_multiple_same(self): def test_07_conventions_var_local(self): # What happens if 'Conventions' appears as a variable-local attribute. # N.B. this is not good CF, but we'll see what happens anyway. - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name="Conventions", - global_value_file1=None, - vars_values_file1="user_set", + values=[None, "user_set"], ) - assert cube.attributes == {"Conventions": "user_set"} + # Legacy result + self.check_load_results([None, "user_set"], oldstyle_combined=True) + # Newstyle result + self.check_load_results([None, "user_set"]) def test_08_conventions_var_both(self): # What happens if 'Conventions' appears as both global + local attribute. - # = the global version gets lost. - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name="Conventions", - global_value_file1="global-setting", - vars_values_file1="local-setting", + values=["global-setting", "local-setting"], ) - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {"Conventions": "local-setting"} - # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == CubeAttrsDict( - globals={"Conventions": "global-setting"}, - locals={"Conventions": "local-setting"}, + # (#1): legacy result : the global version gets lost. + self.check_load_results( + [None, "local-setting"], oldstyle_combined=True ) + # (#2): newstyle results : retain both. + self.check_load_results(["global-setting", "local-setting"]) ####################################################### # Tests on "global" style attributes @@ -744,74 +897,67 @@ def test_08_conventions_var_both(self): def test_09_globalstyle__global(self, global_attr): attr_content = f"Global tracked {global_attr}" - (cube,) = self.create_load_testcase( - attr_name=global_attr, - global_value_file1=attr_content, + self.run_load_testcase( + attr_name=global_attr, values=[attr_content, None] ) - assert cube.attributes == {global_attr: attr_content} + # (#1) legacy + self.check_load_results([None, attr_content], oldstyle_combined=True) + # (#2) newstyle : global status preserved. + self.check_load_results([attr_content, None]) def test_10_globalstyle__local(self, global_attr): # Strictly, not correct CF, but let's see what it does with it. - # = treated the same as a global setting attr_content = f"Local tracked {global_attr}" - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - vars_values_file1=attr_content, + values=[None, attr_content], ) - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {global_attr: attr_content} - # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == CubeAttrsDict( - locals={global_attr: attr_content} + # (#1): legacy result = treated the same as a global setting + self.check_load_results([None, attr_content], oldstyle_combined=True) + # (#2): newstyle result : remains local + self.check_load_results( + [None, attr_content], ) def test_11_globalstyle__both(self, global_attr): attr_global = f"Global-{global_attr}" attr_local = f"Local-{global_attr}" - (cube,) = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - global_value_file1=attr_global, - vars_values_file1=attr_local, - ) - # promoted local setting "wins" - # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == {global_attr: attr_local} - # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == CubeAttrsDict( - globals={global_attr: attr_global}, - locals={global_attr: attr_local}, + values=[attr_global, attr_local], ) + # (#1) legacy result : promoted local setting "wins" + self.check_load_results([None, attr_local], oldstyle_combined=True) + # (#2) newstyle result : both retained + self.check_load_results([attr_global, attr_local]) def test_12_globalstyle__multivar_different(self, global_attr): # Multiple *different* local settings are retained attr_1 = f"Local-{global_attr}-1" attr_2 = f"Local-{global_attr}-2" - cube1, cube2 = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - vars_values_file1={"v1": attr_1, "v2": attr_2}, + values=[None, attr_1, attr_2], ) # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube1.attributes) == {global_attr: attr_1} - assert dict(cube2.attributes) == {global_attr: attr_2} + self.check_load_results([None, attr_1, attr_2], oldstyle_combined=True) # (#2): exact results, with newstyle "split" cube attrs - assert cube1.attributes == CubeAttrsDict(locals={global_attr: attr_1}) - assert cube2.attributes == CubeAttrsDict(locals={global_attr: attr_2}) + self.check_load_results([None, attr_1, attr_2]) def test_14_globalstyle__multifile_different(self, global_attr): - # Different global attributes from multiple files are retained as local ones + # Different global attributes from multiple files attr_1 = f"Global-{global_attr}-1" attr_2 = f"Global-{global_attr}-2" - cube1, cube2, cube3, cube4 = self.create_load_testcase( + self.run_load_testcase( attr_name=global_attr, - global_value_file1=attr_1, - vars_values_file1={"f1v1": None, "f1v2": None}, - global_value_file2=attr_2, - vars_values_file2={"f2v1": None, "f2v2": None}, + values=[[attr_1, None, None], [attr_2, None, None]], + ) + # (#1) legacy : multiple globals retained as local ones + self.check_load_results( + [None, attr_1, attr_1, attr_2, attr_2], oldstyle_combined=True ) - assert cube1.attributes == {global_attr: attr_1} - assert cube2.attributes == {global_attr: attr_1} - assert cube3.attributes == {global_attr: attr_2} - assert cube4.attributes == {global_attr: attr_2} + # (#1) newstyle : result same as input + self.check_load_results([[attr_1, None, None], [attr_2, None, None]]) ####################################################### # Tests on "local" style attributes @@ -837,38 +983,39 @@ def test_16_localstyle(self, local_attr, origin_style): # Create testfiles and load them, which should always produce a single cube. if origin_style == "input_global": # Record in source as a global attribute - (cube,) = self.create_load_testcase( - attr_name=local_attr, global_value_file1=attrval - ) + values = [attrval, None] else: assert origin_style == "input_local" # Record in source as a variable-local attribute - (cube,) = self.create_load_testcase( - attr_name=local_attr, vars_values_file1=attrval - ) + values = [None, attrval] + + self.run_load_testcase(attr_name=local_attr, values=values) # Work out the expected result. - # NOTE: generally, result will be the same whether the original attribute is - # provided as a global or variable attribute ... - expected_result = {local_attr: attrval} - # ... but there are some special cases + result_value = attrval + # ... there are some special cases if origin_style == "input_local": if local_attr == "ukmo__process_flags": # Some odd special behaviour here. - expected_result = {local_attr: ("process",)} + result_value = (result_value,) elif local_attr in ("standard_error_multiplier", "missing_value"): # For some reason, these ones never appear on the cube - expected_result = {} + result_value = None + # NOTE: **legacy** result is the same, whether the original attribute was + # provided as a global or local attribute ... + expected_result_legacy = [None, result_value] + + # While 'newstyle' results preserve the input type local/global. if origin_style == "input_local": - expected_result_newstyle = CubeAttrsDict(expected_result) + expected_result_newstyle = [None, result_value] else: - expected_result_newstyle = CubeAttrsDict(globals=expected_result) + expected_result_newstyle = [result_value, None] # (#1): legacy values, for cube.attributes viewed as a single dict - assert dict(cube.attributes) == expected_result + self.check_load_results(expected_result_legacy, oldstyle_combined=True) # (#2): exact results, with newstyle "split" cube attrs - assert cube.attributes == expected_result_newstyle + self.check_load_results(expected_result_newstyle) class TestSave(MixinAttrsTesting): @@ -877,120 +1024,183 @@ class TestSave(MixinAttrsTesting): """ - def create_save_testcase(self, attr_name, value1, value2=None): + # Parametrise all tests over split/unsplit saving. + @pytest.fixture( + params=[False, True], ids=["nosplit", "split"], autouse=True + ) + def do_split(self, request): + do_split = request.param + self.save_split_attrs = do_split + return do_split + + def run_save_testcase( + self, attr_name: str, values: list, expect_warnings: List[str] = None + ): + # Create input cubes. + self.run_testcase( + attr_name=attr_name, + values=values, + create_cubes_or_files="cubes", + ) + + # Save input cubes to a temporary result file. + with warnings.catch_warnings(record=True) as captured_warnings: + self.result_filepath = self._testfile_path("result") + do_split = getattr(self, "save_split_attrs", False) + with iris.FUTURE.context(save_split_attrs=do_split): + iris.save(self.input_cubes, self.result_filepath) + + check_captured_warnings(expect_warnings, captured_warnings) + + def run_save_testcase_legacytype( + self, attr_name: str, values: list, expect_warnings: List[str] = None + ): """ - Test attribute saving for cube(s) with given value(s). + Legacy-type means : before cubes had split attributes. - Create cubes(s) and save to temporary file, then return the global and all - variable-local attributes of that name (or None-s) from the file. + This just means we have only one "set" of cubes, with ***no*** distinct global + attribute. """ - self.attrname = ( - attr_name # Required for common testfile-naming function. - ) - if value2 is None: - n_cubes = 1 - values = [value1] - else: - n_cubes = 2 - values = [value1, value2] - cube_names = [f"cube_{i_cube}" for i_cube in range(n_cubes)] - cubes = [ - Cube([0], long_name=cube_name, attributes={attr_name: attr_value}) - for cube_name, attr_value in zip(cube_names, values) - ] - self.result_filepath = self._testfile_path("result") - iris.save(cubes, self.result_filepath) - # Get the global+local attribute values directly from the file with netCDF4 - if attr_name == "STASH": - # A special case : the stored name is different - attr_name = "um_stash_source" - try: - ds = threadsafe_nc4.DatasetWrapper(self.result_filepath) - global_result = ( - ds.getncattr(attr_name) if attr_name in ds.ncattrs() else None - ) - local_results = [ - ( - var.getncattr(attr_name) - if attr_name in var.ncattrs() - else None - ) - for var in ds.variables.values() - ] - finally: - ds.close() - return [global_result] + local_results + if not isinstance(values, list): + # Translate single input value to list-of-1 + values = [values] + + self.run_save_testcase(attr_name, [None] + values, expect_warnings) - def test_01_userstyle__single(self): - results = self.create_save_testcase("random", "value-x") - # It is stored as a *global* by default. - assert results == ["value-x", None] + def check_save_results(self, expected: list): + results = self.fetch_results(filepath=self.result_filepath) + assert results == expected - def test_02_userstyle__multiple_same(self): - results = self.create_save_testcase("random", "value-x", "value-x") - # As above. - assert results == ["value-x", None, None] + def test_userstyle__single(self, do_split): + self.run_save_testcase_legacytype("random", "value-x") + if do_split: + # result as input values + expected_result = [None, "value-x"] + else: + # in legacy mode, promoted = stored as a *global* by default. + expected_result = ["value-x", None] + self.check_save_results(expected_result) + + def test_userstyle__multiple_same(self, do_split): + self.run_save_testcase_legacytype("random", ["value-x", "value-x"]) + if do_split: + # result as input values + expected_result = [None, "value-x", "value-x"] + else: + # in legacy mode, promoted = stored as a *global* by default. + expected_result = ["value-x", None, None] + self.check_save_results(expected_result) - def test_03_userstyle__multiple_different(self): - results = self.create_save_testcase("random", "value-A", "value-B") + def test_userstyle__multiple_different(self): # Clashing values are stored as locals on the individual variables. - assert results == [None, "value-A", "value-B"] + self.run_save_testcase_legacytype("random", ["value-A", "value-B"]) + self.check_save_results([None, "value-A", "value-B"]) + + def test_userstyle__multiple_onemissing(self, global_attr): + # Multiple user-type, with one missing, behave like different values. + self.run_save_testcase_legacytype( + global_attr, + ["value", None], + expect_warnings="should only be a CF global attribute", + ) + # Stored as locals when there are differing values. + self.check_save_results([None, "value", None]) - def test_04_Conventions__single(self): - results = self.create_save_testcase("Conventions", "x") + def test_Conventions__single(self): + self.run_save_testcase_legacytype("Conventions", "x") # Always discarded + replaced by a single global setting. - assert results == ["CF-1.7", None] + self.check_save_results(["CF-1.7", None]) - def test_05_Conventions__multiple_same(self): - results = self.create_save_testcase( - "Conventions", "same-value", "same-value" + def test_Conventions__multiple_same(self): + self.run_save_testcase_legacytype( + "Conventions", ["same-value", "same-value"] ) # Always discarded + replaced by a single global setting. - assert results == ["CF-1.7", None, None] + self.check_save_results(["CF-1.7", None, None]) - def test_06_Conventions__multiple_different(self): - results = self.create_save_testcase( - "Conventions", "value-A", "value-B" + def test_Conventions__multiple_different(self): + self.run_save_testcase_legacytype( + "Conventions", ["value-A", "value-B"] ) # Always discarded + replaced by a single global setting. - assert results == ["CF-1.7", None, None] + self.check_save_results(["CF-1.7", None, None]) - def test_07_globalstyle__single(self, global_attr): - results = self.create_save_testcase(global_attr, "value") - # Defaults to global - assert results == ["value", None] + def test_globalstyle__single(self, global_attr, do_split): + if do_split: + # result as input values + expected_warning = "should only be a CF global" + expected_result = [None, "value"] + else: + # in legacy mode, promoted + expected_warning = None + expected_result = ["value", None] - def test_08_globalstyle__multiple_same(self, global_attr): - results = self.create_save_testcase( - global_attr, "value-same", "value-same" + self.run_save_testcase_legacytype( + global_attr, ["value"], expect_warnings=expected_warning ) - assert results == ["value-same", None, None] + self.check_save_results(expected_result) + + def test_globalstyle__multiple_same(self, global_attr, do_split): + # Multiple global-type with same values are made global. + if do_split: + # result as input values + expected_warning = "should only be a CF global attribute" + expected_result = [None, "value-same", "value-same"] + else: + # in legacy mode, promoted + expected_warning = None + expected_result = ["value-same", None, None] + self.run_save_testcase_legacytype( + global_attr, + ["value-same", "value-same"], + expect_warnings=expected_warning, + ) + self.check_save_results(expected_result) - def test_09_globalstyle__multiple_different(self, global_attr): + def test_globalstyle__multiple_different(self, global_attr): + # Multiple global-type with different values become local, with warning. msg_regexp = ( f"'{global_attr}' is being added as CF data variable attribute," f".* should only be a CF global attribute." ) - with pytest.warns(UserWarning, match=msg_regexp): - results = self.create_save_testcase( - global_attr, "value-A", "value-B" - ) + self.run_save_testcase_legacytype( + global_attr, ["value-A", "value-B"], expect_warnings=msg_regexp + ) # *Only* stored as locals when there are differing values. - assert results == [None, "value-A", "value-B"] + self.check_save_results([None, "value-A", "value-B"]) + + def test_globalstyle__multiple_onemissing(self, global_attr): + # Multiple global-type, with one missing, behave like different values. + msg_regexp = ( + f"'{global_attr}' is being added as CF data variable attribute," + f".* should only be a CF global attribute." + ) + self.run_save_testcase_legacytype( + global_attr, ["value", "value", None], expect_warnings=msg_regexp + ) + # Stored as locals when there are differing values. + self.check_save_results([None, "value", "value", None]) + + def test_localstyle__single(self, local_attr): + self.run_save_testcase_legacytype(local_attr, ["value"]) - def test_10_localstyle__single(self, local_attr): - results = self.create_save_testcase(local_attr, "value") # Defaults to local expected_results = [None, "value"] + # .. but a couple of special cases if local_attr == "ukmo__process_flags": # A particular, really weird case expected_results = [None, "v a l u e"] - assert results == expected_results + elif local_attr == "STASH": + # A special case : the stored name is different + self.attrname = "um_stash_source" + + self.check_save_results(expected_results) - def test_11_localstyle__multiple_same(self, local_attr): - results = self.create_save_testcase( - local_attr, "value-same", "value-same" + def test_localstyle__multiple_same(self, local_attr): + self.run_save_testcase_legacytype( + local_attr, ["value-same", "value-same"] ) + # They remain separate + local expected_results = [None, "value-same", "value-same"] if local_attr == "ukmo__process_flags": @@ -1000,10 +1210,14 @@ def test_11_localstyle__multiple_same(self, local_attr): "v a l u e - s a m e", "v a l u e - s a m e", ] - assert results == expected_results + elif local_attr == "STASH": + # A special case : the stored name is different + self.attrname = "um_stash_source" + + self.check_save_results(expected_results) - def test_12_localstyle__multiple_different(self, local_attr): - results = self.create_save_testcase(local_attr, "value-A", "value-B") + def test_localstyle__multiple_different(self, local_attr): + self.run_save_testcase_legacytype(local_attr, ["value-A", "value-B"]) # Different values are treated just the same as matching ones. expected_results = [None, "value-A", "value-B"] if local_attr == "ukmo__process_flags": @@ -1013,4 +1227,60 @@ def test_12_localstyle__multiple_different(self, local_attr): "v a l u e - A", "v a l u e - B", ] - assert results == expected_results + elif local_attr == "STASH": + # A special case : the stored name is different + self.attrname = "um_stash_source" + self.check_save_results(expected_results) + + # + # Test handling of newstyle independent global+local cube attributes. + # + def test_globallocal_clashing(self, do_split): + # A cube has clashing local + global attrs. + original_values = ["valueA", "valueB"] + self.run_save_testcase("userattr", original_values, expect_warnings=[]) + expected_result = original_values.copy() + if not do_split: + # in legacy mode, "promote" = lose the local one + expected_result[0] = expected_result[1] + expected_result[1] = None + # N.B. legacy code sees only the local value, and promotes it. + self.check_save_results(expected_result) + + def test_globallocal_oneeach_same(self): + # One cube with global attr, another with identical local one. + self.run_save_testcase( + "userattr", [[None, "value"], ["value", None]], expect_warnings=[] + ) + # N.B. legacy code sees only two equal values (and promotes). + self.check_save_results(["value", None, None]) + + def test_globallocal_oneeach_different(self): + # One cube with global attr, another with different local one. + self.run_save_testcase( + "userattr", + [[None, "valueA"], ["valueB", None]], + expect_warnings=[], + ) + # N.B. legacy code does not warn of global-to-local "demotion". + self.check_save_results([None, "valueA", "valueB"]) + + def test_globallocal_one_other_clashingglobals(self): + # Two cubes with both, second cube has a clashing global attribute. + self.run_save_testcase( + "userattr", + [["valueA", "valueB"], ["valueXXX", "valueB"]], + expect_warnings=[], + ) + # N.B. legacy code sees only the locals, and promotes them. + self.check_save_results(["valueB", None, None]) + + def test_globallocal_one_other_clashinglocals(self): + # Two cubes with both, second cube has a clashing local attribute. + self.run_save_testcase( + "userattr", + [["valueA", "valueB"], ["valueA", "valueXXX"]], + expect_warnings=[], + ) + # N.B. legacy code sees only the locals. + self.check_save_results([None, "valueB", "valueXXX"])