PP constraint

lib/iris/__init__.py

@@ -220,7 +220,7 @@ def context(self, **kwargs):
     _update(site_configuration)
 
 
-def _generate_cubes(uris, callback):
+def _generate_cubes(uris, callback, constraints):
     """Returns a generator of cubes given the URIs and a callback."""
     if isinstance(uris, basestring):
         uris = [uris]
@@ -233,7 +233,7 @@ def _generate_cubes(uris, callback):
         # Call each scheme handler with the appropriate URIs
         if scheme == 'file':
             part_names = [x[1] for x in groups]
-            for cube in iris.io.load_files(part_names, callback):
+            for cube in iris.io.load_files(part_names, callback, constraints):
                 yield cube
         elif scheme in ['http', 'https']:
             urls = [':'.join(x) for x in groups]
@@ -245,7 +245,7 @@ def _generate_cubes(uris, callback):
 
 def _load_collection(uris, constraints=None, callback=None):
     try:
-        cubes = _generate_cubes(uris, callback)
+        cubes = _generate_cubes(uris, callback, constraints)
         result = iris.cube._CubeFilterCollection.from_cubes(cubes, constraints)
     except EOFError as e:
         raise iris.exceptions.TranslationError(

lib/iris/fileformats/__init__.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2010 - 2013, Met Office
+# (C) British Crown Copyright 2010 - 2014, Met Office
 #
 # This file is part of Iris.
 #
@@ -53,15 +53,17 @@ def _pp_little_endian(filename, *args, **kwargs):
                                           MagicNumber(4),
                                           0x00000100,
                                           pp.load_cubes,
-                                          priority=5))
+                                          priority=5,
+                                          constraint_aware_handler=True))
 
 
 FORMAT_AGENT.add_spec(
     FormatSpecification('UM Post Processing file (PP) little-endian',
                         MagicNumber(4),
                         0x00010000,
                         _pp_little_endian,
-                        priority=3))
+                        priority=3,
+                        constraint_aware_handler=True))
 
 
 #
@@ -115,37 +117,42 @@ def _pp_little_endian(filename, *args, **kwargs):
                                           MagicNumber(8),
                                           0x000000000000000F,
                                           ff.load_cubes,
-                                          priority=3))
+                                          priority=3,
+                                          constraint_aware_handler=True))
 
 
 FORMAT_AGENT.add_spec(FormatSpecification('UM Fieldsfile (FF) post v5.2',
                                           MagicNumber(8),
                                           0x0000000000000014,
                                           ff.load_cubes,
-                                          priority=4))
+                                          priority=4,
+                                          constraint_aware_handler=True))
 
 
 FORMAT_AGENT.add_spec(FormatSpecification('UM Fieldsfile (FF) ancillary',
                                           MagicNumber(8),
                                           0xFFFFFFFFFFFF8000,
                                           ff.load_cubes,
-                                          priority=3))
+                                          priority=3,
+                                          constraint_aware_handler=True))
 
 
 FORMAT_AGENT.add_spec(FormatSpecification('UM Fieldsfile (FF) converted '
                                           'with ieee to 32 bit',
                                           MagicNumber(4),
                                           0x00000014,
                                           ff.load_cubes_32bit_ieee,
-                                          priority=3))
+                                          priority=3,
+                                          constraint_aware_handler=True))
 
 
 FORMAT_AGENT.add_spec(FormatSpecification('UM Fieldsfile (FF) ancillary '
                                           'converted with ieee to 32 bit',
                                           MagicNumber(4),
                                           0xFFFF8000,
                                           ff.load_cubes_32bit_ieee,
-                                          priority=3))
+                                          priority=3,
+                                          constraint_aware_handler=True))
 
 
 #

lib/iris/fileformats/ff.py

@@ -647,7 +647,7 @@ def __iter__(self):
         return pp._interpret_fields(self._extract_field())
 
 
-def load_cubes(filenames, callback):
+def load_cubes(filenames, callback, constraints=None):
     """
     Loads cubes from a list of fields files filenames.
 
@@ -667,10 +667,11 @@ def load_cubes(filenames, callback):
         orography references).
 
     """
-    return pp._load_cubes_variable_loader(filenames, callback, FF2PP)
+    return pp._load_cubes_variable_loader(filenames, callback, FF2PP,
+                                          constraints=constraints)
 
 
-def load_cubes_32bit_ieee(filenames, callback):
+def load_cubes_32bit_ieee(filenames, callback, constraints=None):
     """
     Loads cubes from a list of 32bit ieee converted fieldsfiles filenames.
 
@@ -680,4 +681,5 @@ def load_cubes_32bit_ieee(filenames, callback):
 
     """
     return pp._load_cubes_variable_loader(filenames, callback, FF2PP,
-                                          {'word_depth': 4})
+                                          {'word_depth': 4},
+                                          constraints=constraints)

lib/iris/fileformats/pp.py

@@ -1677,8 +1677,35 @@ def reset_save_rules():
 
     _save_rules = None
 
+def _convert_constraints(constraints):
+    """
+    Converts known constraints from Iris semantics to PP semantics
+    ignoring all unknown constraints.
 
-def load_cubes(filenames, callback=None):
+    """
+    constraints = iris._constraints.list_of_constraints(constraints)
+    pp_constraints = {}
+    for con in constraints:
+        if hasattr(con, '_attributes') and con._attributes.keys() == ['STASH']:
+            stashobj = STASH.from_msi(con._attributes['STASH'])
+            if not pp_constraints.has_key('stash'):
+                pp_constraints['stash'] = [stashobj]
+            else:
+                pp_constraints['stash'].append(stashobj)
+    def pp_filter(field):
+        """
+        return True if field is to be kept,
+        False if field does not match filter
+
+        """
+        res = True
+        if pp_constraints.get('stash'):
+            if field.stash not in pp_constraints['stash']:
+                res = False
+        return res
+    return pp_filter
+
+def load_cubes(filenames, callback=None, constraints=None):
     """
     Loads cubes from a list of pp filenames.
 
@@ -1688,6 +1715,8 @@ def load_cubes(filenames, callback=None):
 
     Kwargs:
 
+    * constraints - a list of Iris constraints
+
     * callback - a function which can be passed on to :func:`iris.io.run_callback`
 
     .. note::
@@ -1696,15 +1725,21 @@ def load_cubes(filenames, callback=None):
         is not preserved when there is a field with orography references)
 
     """
-    return _load_cubes_variable_loader(filenames, callback, load)
+    return _load_cubes_variable_loader(filenames, callback, load,
+                                       constraints=constraints)
 
 
 def _load_cubes_variable_loader(filenames, callback, loading_function,
-                                loading_function_kwargs=None):
+                                loading_function_kwargs=None,
+                                constraints=None):
+    pp_filter = None
+    if constraints is not None:
+        pp_filter = _convert_constraints(constraints)
     pp_loader = iris.fileformats.rules.Loader(
         loading_function, loading_function_kwargs or {},
         iris.fileformats.pp_rules.convert, _load_rules)
-    return iris.fileformats.rules.load_cubes(filenames, callback, pp_loader)
+    return iris.fileformats.rules.load_cubes(filenames, callback, pp_loader,
+                                             pp_filter)
 
 
 def save(cube, target, append=False, field_coords=None):

lib/iris/fileformats/rules.py

@@ -773,7 +773,7 @@ def _make_cube(field, converter):
     return cube, factories, references
 
 
-def load_cubes(filenames, user_callback, loader):
+def load_cubes(filenames, user_callback, loader, filter_function=None):
     concrete_reference_targets = {}
     results_needing_reference = []
 
@@ -782,6 +782,10 @@ def load_cubes(filenames, user_callback, loader):
 
     for filename in filenames:
         for field in loader.field_generator(filename, **loader.field_generator_kwargs):
+            # evaluate field against format specific desired attributes
+            # load if no format specific desired attributes are violated
+            if filter_function is not None and not filter_function(field):
+                continue
             # Convert the field to a Cube.
             cube, factories, references = _make_cube(field, loader.converter)
 

lib/iris/io/__init__.py

@@ -165,10 +165,11 @@ def expand_filespecs(file_specs):
     return sum(value_lists, [])
 
 
-def load_files(filenames, callback):
+def load_files(filenames, callback, constraints=None):
     """
     Takes a list of filenames which may also be globs, and optionally a
-    callback function, and returns a generator of Cubes from the given files.
+    constraint set and a callback function, and returns a
+    generator of Cubes from the given files.
 
     .. note::
 
@@ -187,8 +188,13 @@ def load_files(filenames, callback):
 
     # Call each iris format handler with the approriate filenames
     for handling_format_spec, fnames in handler_map.iteritems():
-        for cube in handling_format_spec.handler(fnames, callback):
-            yield cube
+        if handling_format_spec.constraint_aware_handler:
+            for cube in handling_format_spec.handler(fnames, callback,
+                                                     constraints):
+                yield cube
+        else:
+            for cube in handling_format_spec.handler(fnames, callback):
+                yield cube
 
 
 def load_http(urls, callback):

lib/iris/io/format_picker.py

@@ -153,7 +153,8 @@ class FormatSpecification(object):
     a FileElement, such as filename extension or 32-bit magic number, with an associated value for format identification.
 
     """
-    def __init__(self, format_name, file_element, file_element_value, handler=None, priority=0):
+    def __init__(self, format_name, file_element, file_element_value,
+                 handler=None, priority=0, constraint_aware_handler=False):
         """
         Constructs a new FormatSpecification given the format_name and particular FileElements
 
@@ -179,6 +180,7 @@ def __init__(self, format_name, file_element, file_element_value, handler=None,
         self._format_name = format_name
         self._handler = handler
         self.priority = priority
+        self.constraint_aware_handler = constraint_aware_handler
 
     def __hash__(self):
         # Hashed by specification for consistent ordering in FormatAgent (including self._handler in this hash

lib/iris/tests/unit/fileformats/pp/test__convert_constraints.py

@@ -0,0 +1,53 @@
+# (C) British Crown Copyright 2014, Met Office
+#
+# This file is part of Iris.
+#
+# Iris is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Iris is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Iris.  If not, see <http://www.gnu.org/licenses/>.
+"""Unit tests for the `iris.fileformats.pp.load` function."""
+
+# Import iris.tests first so that some things can be initialised before
+# importing anything else.
+import iris.tests as tests
+
+import mock
+
+import iris
+from iris.fileformats.pp import _convert_constraints
+from iris.fileformats.pp import STASH
+
+
+class Test_convert_constraints(tests.IrisTest):
+    def test_single_stash(self):
+        stcube = mock.Mock(stash=STASH.from_msi('m01s03i236'))
+        constraint = iris.AttributeConstraint(STASH='m01s03i236')
+        pp_filter = _convert_constraints(constraint)
+        self.assertTrue(pp_filter(stcube))
+
+    def test_multiple_with_stash(self):
+        constraints = [iris.Constraint('air_potential_temperature'),
+                       iris.AttributeConstraint(STASH='m01s00i004')]
+        pp_filter = _convert_constraints(constraints)
+        stcube = mock.Mock(stash=STASH.from_msi('m01s00i004'))
+        self.assertTrue(pp_filter(stcube))
+
+    def test_no_stash(self):
+        constraints = [iris.Constraint('air_potential_temperature'),
+                       iris.AttributeConstraint(source='asource')]
+        pp_filter = _convert_constraints(constraints)
+        stcube = mock.Mock(stash=STASH.from_msi('m01s00i004'))
+        self.assertTrue(pp_filter(stcube))
+
+
+if __name__ == "__main__":
+    tests.main()

lib/iris/tests/unit/fileformats/rules/test_load_cubes.py

@@ -0,0 +1,61 @@
+# (C) British Crown Copyright 2014, Met Office
+#
+# This file is part of Iris.
+#
+# Iris is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Iris is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Iris.  If not, see <http://www.gnu.org/licenses/>.
+"""Unit tests for :func:`iris.fileformats.rules.load_cubes`."""
+
+# Import iris.tests first so that some things can be initialised before
+# importing anything else.
+import iris.tests as tests
+
+import mock
+
+import iris
+from iris.fileformats import pp
+from iris.fileformats.pp_rules import convert
+from iris.fileformats.rules import load_cubes
+
+
+class Test(tests.IrisTest):
+    def test_pp_with_stash_constraint(self):
+        filenames = [tests.get_data_path(('PP', 'globClim1', 'dec_subset.pp'))]
+        stcon = iris.AttributeConstraint(STASH='m01s00i004')
+        pp_constraints = pp._convert_constraints(stcon)
+        pp_loader = iris.fileformats.rules.Loader(pp.load, {},
+                                                  convert, pp._load_rules)
+        cubes = list(load_cubes(filenames, None, pp_loader, pp_constraints))
+        self.assertEqual(len(cubes), 38)
+
+    def test_pp_with_stash_constraints(self):
+        filenames = [tests.get_data_path(('PP', 'globClim1', 'dec_subset.pp'))]
+        stcon1 = iris.AttributeConstraint(STASH='m01s00i004')
+        stcon2 = iris.AttributeConstraint(STASH='m01s00i010')
+        pp_constraints = pp._convert_constraints([stcon1, stcon2])
+        pp_loader = iris.fileformats.rules.Loader(pp.load, {},
+                                                  convert, pp._load_rules)
+        cubes = list(load_cubes(filenames, None, pp_loader, pp_constraints))
+        self.assertEqual(len(cubes), 76)
+
+    def test_pp_no_constraint(self):
+        filenames = [tests.get_data_path(('PP', 'globClim1', 'dec_subset.pp'))]
+        pp_constraints = pp._convert_constraints(None)
+        pp_loader = iris.fileformats.rules.Loader(pp.load, {},
+                                                  convert, pp._load_rules)
+        cubes = list(load_cubes(filenames, None, pp_loader, pp_constraints))
+        self.assertEqual(len(cubes), 152)
+
+
+if __name__ == "__main__":
+    tests.main()