Splitattrs ncsave redo commonmeta

docs/src/further_topics/metadata.rst

@@ -736,7 +736,7 @@ Let's reinforce this behaviour, but this time by combining metadata where the
     >>> metadata != cube.metadata
     True
     >>> metadata.combine(cube.metadata).attributes
-    {'Model scenario': 'A1B'}
+    CubeAttrsDict(globals={}, locals={'Model scenario': 'A1B'})
 
 The combined result for the ``attributes`` member only contains those
 **common keys** with **common values**.

lib/iris/common/metadata.py

@@ -140,6 +140,88 @@ def __new__(mcs, name, bases, namespace):
         return super().__new__(mcs, name, bases, namespace)
 
 
+#
+# Dictionary operations for dealing with the CubeAttrsDict "split"-style attribute
+# dictionaries.
+#
+# The idea here is to convert a split-dictionary into a "plain" one for calculations,
+# whose keys are all pairs of the form ('global', <keyname>) or ('local', <keyname>).
+# And to convert back again after the operation, if the result is a dictionary.
+#
+# For "strict" operations this clearly does all that is needed.  For lenient ones,
+# we _might_ want for local+global attributes of the same name to interact.
+# However, on careful consideration, it seems that this is not actually desirable for
+# any of the common-metadata operations.
+# So, we simply treat "global" and "local" attributes of the same name as entirely
+# independent. Which happily is also the easiest to code, and to explain.
+#
+def xd_is_split(dic):
+    """Detect whether a dictionary is a "split-attribute" type."""
+    return hasattr(dic, "globals") and hasattr(dic, "locals")
+
+
+def xd_to_normal(dic):
+    """
+    Convert the input to a 'normal' dict with paired keys, if it is split-attrs type
+    """
+
+    def _global_then_local_items(dic):
+        # Routine to produce global, then local 'items' in order, and with all keys
+        # "labelled" as local or global type, to ensure they are all unique.
+        for key, value in dic.globals.items():
+            yield ("global", key), value
+        for key, value in dic.locals.items():
+            yield ("local", key), value
+
+    return dict(_global_then_local_items(dic))
+
+
+def xd_from_normal(dic):
+    """
+    Convert an input with global/local paired keys back into a split-attrs dict.
+
+    For now, this is always+only a CubeAttrsDict.
+    """
+    from iris.cube import CubeAttrsDict
+
+    result = CubeAttrsDict()
+    for key, value in dic.items():
+        keytype, keyname = key
+        if keytype == "global":
+            result.globals[keyname] = value
+        else:
+            assert keytype == "local"
+            result.locals[keyname] = value
+    return result
+
+
+def xd_normalise_input_pair(left, right):
+    """Work out whether inputs are "split" type, and convert if so."""
+    from iris.cube import CubeAttrsDict
+
+    left_split, right_split = xd_is_split(left), xd_is_split(right)
+    is_split = left_split or right_split
+    if is_split:
+        # Convert any "normal" dicts to split equivalents first
+        # - this divides contents into global+local according to default rules
+        if not left_split:
+            left = CubeAttrsDict(left)
+        if not right_split:
+            right = CubeAttrsDict(right)
+        # convert both to paired-key form for calculations
+        left = xd_to_normal(left)
+        right = xd_to_normal(right)
+
+    return is_split, left, right
+
+
+def xd_reconvert_output(is_split, result):
+    """Re-form a 'split dict' result from a dict with paired keys, if needed."""
+    if is_split:
+        result = xd_from_normal(result)
+    return result
+
+
 class BaseMetadata(metaclass=_NamedTupleMeta):
     """
     Container for common metadata.
@@ -373,6 +455,8 @@ def _combine_lenient_attributes(left, right):
         # Copy the dictionaries.
         left = deepcopy(left)
         right = deepcopy(right)
+        # convert from split form if required
+        is_split, left, right = xd_normalise_input_pair(left, right)
         # Use xxhash to perform an extremely fast non-cryptographic hash of
         # each dictionary key rvalue, thus ensuring that the dictionary is
         # completely hashable, as required by a set.
@@ -393,7 +477,8 @@ def _combine_lenient_attributes(left, right):
         result = {k: left[k] for k, _ in common}
         result.update({k: left[k] for k in dsleft.keys()})
         result.update({k: right[k] for k in dsright.keys()})
-
+        # Convert result back to split-attrs dict, if original inputs were
+        result = xd_reconvert_output(is_split, result)
         return result
 
     @staticmethod
@@ -402,6 +487,8 @@ def _combine_strict_attributes(left, right):
         # Copy the dictionaries.
         left = deepcopy(left)
         right = deepcopy(right)
+        # convert from split form if required
+        is_split, left, right = xd_normalise_input_pair(left, right)
         # Use xxhash to perform an extremely fast non-cryptographic hash of
         # each dictionary key rvalue, thus ensuring that the dictionary is
         # completely hashable, as required by a set.
@@ -411,7 +498,8 @@ def _combine_strict_attributes(left, right):
         common = sleft & sright
         # Now bring the result together.
         result = {k: left[k] for k, _ in common}
-
+        # Convert result back to split-attrs dict, if original inputs were
+        result = xd_reconvert_output(is_split, result)
         return result
 
     def _compare_lenient(self, other):
@@ -464,6 +552,10 @@ def _compare_lenient_attributes(left, right):
         # Use xxhash to perform an extremely fast non-cryptographic hash of
         # each dictionary key rvalue, thus ensuring that the dictionary is
         # completely hashable, as required by a set.
+
+        # Convert from split if required --> i.e. all distinct keys (global+local)
+        _, left, right = xd_normalise_input_pair(left, right)
+
         sleft = {(k, hexdigest(v)) for k, v in left.items()}
         sright = {(k, hexdigest(v)) for k, v in right.items()}
         # Items in sleft different from sright.
@@ -481,6 +573,10 @@ def _compare_strict_attributes(left, right):
         # Use xxhash to perform an extremely fast non-cryptographic hash of
         # each dictionary key rvalue, thus ensuring that the dictionary is
         # completely hashable, as required by a set.
+
+        # Convert from split if required --> i.e. all distinct keys (global+local)
+        _, left, right = xd_normalise_input_pair(left, right)
+
         sleft = {(k, hexdigest(v)) for k, v in left.items()}
         sright = {(k, hexdigest(v)) for k, v in right.items()}
 
@@ -550,6 +646,12 @@ def _difference_lenient_attributes(left, right):
         # Use xxhash to perform an extremely fast non-cryptographic hash of
         # each dictionary key rvalue, thus ensuring that the dictionary is
         # completely hashable, as required by a set.
+
+        # Convert from split if required --> i.e. all distinct keys (global+local)
+        is_split, left, right = xd_normalise_input_pair(left, right)
+        # TODO: ?maybe? consider if we flag different global+local values of a
+        #  given attr (name).  BUT not clear how we would report that, anyway.
+
         sleft = {(k, hexdigest(v)) for k, v in left.items()}
         sright = {(k, hexdigest(v)) for k, v in right.items()}
         # Items in sleft different from sright.
@@ -568,6 +670,11 @@ def _difference_lenient_attributes(left, right):
             # Replace hash-rvalue with original rvalue.
             dsleft = {k: left[k] for k in dsleft.keys()}
             dsright = {k: right[k] for k in dsright.keys()}
+            if is_split:
+                # Convert results back to split-attrs dicts, if originals were.
+                dsleft, dsright = (
+                    xd_from_normal(dic) for dic in (dsleft, dsright)
+                )
             result = (dsleft, dsright)
 
         return result
@@ -578,6 +685,10 @@ def _difference_strict_attributes(left, right):
         # Use xxhash to perform an extremely fast non-cryptographic hash of
         # each dictionary key rvalue, thus ensuring that the dictionary is
         # completely hashable, as required by a set.
+
+        # Convert from split if required --> i.e. all distinct keys (global+local)
+        is_split, left, right = xd_normalise_input_pair(left, right)
+
         sleft = {(k, hexdigest(v)) for k, v in left.items()}
         sright = {(k, hexdigest(v)) for k, v in right.items()}
         # Items in sleft different from sright.
@@ -591,6 +702,11 @@ def _difference_strict_attributes(left, right):
             # Replace hash-rvalue with original rvalue.
             dsleft = {k: left[k] for k in dsleft.keys()}
             dsright = {k: right[k] for k in dsright.keys()}
+            if is_split:
+                # Convert results back to split-attrs dicts, if originals were.
+                dsleft, dsright = (
+                    xd_from_normal(dic) for dic in (dsleft, dsright)
+                )
             result = (dsleft, dsright)
 
         return result

lib/iris/tests/integration/test_netcdf__loadsaveattrs.py

@@ -565,7 +565,8 @@ def encode_matrix_result(results: List[List[str]]) -> List[str]:
     if not isinstance(results[0], list):
         results = [results]
     assert all(
-        all(val is None or len(val) == 1 for val in vals) for vals in results
+        all(val is None or isinstance(val, str) for val in vals)
+        for vals in results
     )
 
     # Translate "None" values to "-"