Provide utility function "file_is_newer_than" for results caching.

lib/iris/io/__init__.py

@@ -143,27 +143,48 @@ def decode_uri(uri, default='file'):
     return scheme, part
 
 
-def load_files(filenames, callback):
+def expand_filespecs(file_specs):
     """
-    Takes a list of filenames which may also be globs, and optionally a
-    callback function, and returns a generator of Cubes from the given files.
+    Work out matching file paths from a list of file-specs.
 
-    .. note::
+    Args:
 
-        Typically, this function should not be called directly; instead, the
-        intended interface for loading is :func:`iris.load`.
+    * file_specs (iterable of string):
+        File paths which may contain '~' elements or wildcards.
+
+    Returns:
+        A dictionary of {globspec: file-paths-list}.  The 'globspec's retain
+        any wildcards but have any '~' elements expanded.
 
     """
     # Remove any hostname component - currently unused
-    filenames = [os.path.expanduser(fn[2:] if fn.startswith('//') else fn) for fn in filenames]
+    filenames = [os.path.expanduser(fn[2:] if fn.startswith('//') else fn)
+                 for fn in file_specs]
 
     # Try to expand all filenames as globs
     glob_expanded = {fn : sorted(glob.glob(fn)) for fn in filenames}
 
-    # If any of the filenames or globs expanded to an empty list then raise an error
+    # If any of the specs expanded to an empty list then raise an error
     if not all(glob_expanded.viewvalues()):
         raise IOError("One or more of the files specified did not exist %s." %
-        ["%s expanded to %s" % (pattern, expanded if expanded else "empty") for pattern, expanded in glob_expanded.iteritems()])
+        ["%s expanded to %s" % (pattern, expanded if expanded else "empty")
+         for pattern, expanded in glob_expanded.iteritems()])
+
+    return glob_expanded
+
+
+def load_files(filenames, callback):
+    """
+    Takes a list of filenames which may also be globs, and optionally a
+    callback function, and returns a generator of Cubes from the given files.
+
+    .. note::
+
+        Typically, this function should not be called directly; instead, the
+        intended interface for loading is :func:`iris.load`.
+
+    """
+    glob_expanded = expand_filespecs(filenames)
 
     # Create default dict mapping iris format handler to its associated filenames
     handler_map = collections.defaultdict(list)

lib/iris/tests/test_util.py

@@ -24,7 +24,10 @@
 
 import inspect
 import os
+import shutil
 import StringIO
+import tempfile
+import time
 import unittest
 
 import numpy as np
@@ -350,5 +353,107 @@ def dim_to_aux(cube, coord_name):
         self.assertEqual(res, expected)
 
 
+class TestFileIsNewer(tests.IrisTest):
+    """Test the :meth:`iris.util.file_is_newer_than` function."""
+
+    def _name2path(self, filename):
+        """Add the temporary dirpath to a filename to make a full path."""
+        return os.path.join(self.temp_dir, filename)
+
+    def setUp(self):
+        # make a temporary directory with testfiles of known timestamp order.
+        self.temp_dir = tempfile.mkdtemp('_testfiles_tempdir')
+        # define the names of some files to create
+        create_file_names = ['older_source_1', 'older_source_2',
+                             'example_result',
+                             'newer_source_1', 'newer_source_2']
+        # create test files in given name order (!important!)
+        for file_name in create_file_names:
+            file_path = self._name2path(file_name)
+            with open(file_path, 'w') as file:
+                file.write('..content..')
+            # Needs a tiny pause to prevent possibly equal timestamps
+            time.sleep(0.002)
+
+    def tearDown(self):
+        # destroy whole contents of temporary directory
+        shutil.rmtree(self.temp_dir)
+
+    def _test(self, boolean_result, result_name, source_names):
+        """Test expected result of executing with given args."""
+        # Make args into full paths
+        result_path = self._name2path(result_name)
+        if isinstance(source_names, basestring):
+            source_paths = self._name2path(source_names)
+        else:
+            source_paths = [self._name2path(name)
+                            for name in source_names]
+        # Check result is as expected.
+        self.assertEqual(
+            boolean_result,
+            iris.util.file_is_newer_than(result_path, source_paths))
+
+    def test_no_sources(self):
+        self._test(True, 'example_result', [])
+
+    def test_string_ok(self):
+        self._test(True, 'example_result', 'older_source_1')
+
+    def test_string_fail(self):
+        self._test(False, 'example_result', 'newer_source_1')
+
+    def test_self_result(self):
+        # This fails, because same-timestamp is *not* acceptable.
+        self._test(False, 'example_result', 'example_result')
+
+    def test_single_ok(self):
+        self._test(True, 'example_result', ['older_source_2'])
+
+    def test_single_fail(self):
+        self._test(False, 'example_result', ['newer_source_2'])
+
+    def test_multiple_ok(self):
+        self._test(True, 'example_result', ['older_source_1',
+                                            'older_source_2'])
+
+    def test_multiple_fail(self):
+        self._test(False, 'example_result', ['older_source_1',
+                                             'older_source_2',
+                                             'newer_source_1'])
+
+    def test_wild_ok(self):
+        self._test(True, 'example_result', ['older_sour*_*'])
+
+    def test_wild_fail(self):
+        self._test(False, 'example_result', ['older_sour*', 'newer_sour*'])
+
+    def test_error_missing_result(self):
+        try:
+            self._test(False, 'non_exist', ['older_sour*'])
+        except Exception as error:
+            pass
+        self.assertIsInstance(error, OSError)
+        self.assertEqual(error.strerror, 'No such file or directory')
+        self.assertEqual(error.filename, self._name2path('non_exist'))
+
+    def test_error_missing_source(self):
+        try:
+            self._test(False, 'example_result', ['older_sour*', 'non_exist'])
+        except Exception as error:
+            pass
+        self.assertIsInstance(error, IOError)
+        self.assertTrue(error.message.startswith(
+            'One or more of the files specified did not exist'))
+
+    def test_error_missing_wild(self):
+        try:
+            self._test(False, 'example_result', ['older_sour*', 'unknown_*'])
+        except Exception as error:
+            pass
+        self.assertIsInstance(error, IOError)
+        self.assertTrue(error.message.startswith(
+            'One or more of the files specified did not exist'))
+
+
 if __name__ == '__main__':
     unittest.main()

lib/iris/util.py

@@ -24,6 +24,7 @@
 import copy
 import inspect
 import os
+import os.path
 import sys
 import tempfile
 import time
@@ -1039,3 +1040,60 @@ def add_coord(coord):
         new_cube.add_aux_factory(factory.updated(coord_mapping))
 
     return new_cube
+
+
+def file_is_newer_than(result_path, source_paths):
+    """
+    Check that source files have not changed since a saved result was stored.
+
+    If a stored result depends entirely on known 'sources', it need only be
+    re-built when one of them changes.  This function can be used to test that
+    by comparing file timestamps.
+
+    Args:
+
+    * result_path (string):
+        The filepath of a file containing some derived result data.
+    * source_paths (string or iterable of strings):
+        The path(s) to the original datafiles used to make the result.  May
+        include wildcards and '~' expansions (like Iris load paths), but not
+        URIs.
+
+    Returns:
+        True if all the sources are older than the result, else False.
+
+        If any of the file paths describes no existing files, an exception will
+        be raised.
+
+    .. note::
+        There are obvious caveats to using file timestamps for this, as correct
+        usage depends on how the sources might change.  For example, a file
+        could be replaced by one of the same name, but an older timestamp.
+
+        If wildcards and '~' expansions are used, this introduces even more
+        uncertainty, as then you cannot even be sure that the resulting list of
+        file names is the same as the originals.  For example, some files may
+        have been deleted or others added.
+
+    .. note::
+        The result file may often be a :mod:`pickle` file.  In that case, it
+        also depends on the relevant module sources, so extra caution is
+        required.  Ideally, an additional check on iris.__version__ is advised.
+
+    """
+    # Accept a string as a single source path
+    if isinstance(source_paths, basestring):
+        source_paths = [source_paths]
+    # Fix our chosen timestamp function
+    file_date = os.path.getmtime
+    # Get the 'result file' time
+    result_timestamp = file_date(result_path)
+    # Get all source filepaths, with normal Iris.io load helper function
+    possibles = iris.io.expand_filespecs(source_paths)
+    # Compare each filetime, for each spec, with the 'result time'
+    for paths in possibles.itervalues():
+        for path in paths:
+            source_timestamp = file_date(path)
+            if source_timestamp >= result_timestamp:
+                return False
+    return True