Filespecs sorted absolute

lib/iris/io/__init__.py

@@ -23,6 +23,7 @@
 from six.moves import (filter, input, map, range, zip)  # noqa
 import six
 
+from collections import OrderedDict
 import glob
 import os.path
 import re
@@ -147,25 +148,35 @@ def expand_filespecs(file_specs):
         File paths which may contain '~' elements or wildcards.
 
     Returns:
-        A list of matching file paths.  If any of the file-specs matches no
-        existing files, an exception is raised.
+        A well-ordered list of matching absolute file paths.
+        If any of the file-specs match no existing files, an
+        exception is raised.
 
     """
     # Remove any hostname component - currently unused
-    filenames = [os.path.expanduser(fn[2:] if fn.startswith('//') else fn)
+    filenames = [os.path.abspath(os.path.expanduser(
+                    fn[2:] if fn.startswith('//') else fn))
                  for fn in file_specs]
 
     # Try to expand all filenames as globs
-    glob_expanded = {fn : sorted(glob.glob(fn)) for fn in filenames}
+    glob_expanded = OrderedDict([[fn, sorted(glob.glob(fn))]
+                                 for fn in filenames])
 
     # If any of the specs expanded to an empty list then raise an error
-    value_lists = glob_expanded.values()
-    if not all(value_lists):
-        raise IOError("One or more of the files specified did not exist %s." %
-        ["%s expanded to %s" % (pattern, expanded if expanded else "empty")
-         for pattern, expanded in six.iteritems(glob_expanded)])
-
-    return sum(value_lists, [])
+    all_expanded = glob_expanded.values()
+
+    if not all(all_expanded):
+        msg = "One or more of the files specified did not exist:"
+        for pattern, expanded in six.iteritems(glob_expanded):
+            if expanded:
+                file_list = '\n       - {}'.format(', '.join(expanded))
+            else:
+                file_list = ''
+            msg += '\n    - "{}" matched {} file(s){}'.format(
+                    pattern, len(expanded), file_list)
+        raise IOError(msg)
+
+    return [fname for fnames in all_expanded for fname in fnames]
 
 
 def load_files(filenames, callback, constraints=None):

lib/iris/tests/__init__.py

@@ -249,6 +249,16 @@ def get_result_path(relative_path):
             relative_path = os.path.join(*relative_path)
         return os.path.abspath(os.path.join(_RESULT_PATH, relative_path))
 
+    def assertStringEqual(self, reference_str, test_str,
+                          type_comparison_name='strings'):
+        if reference_str != test_str:
+            diff = '\n'.join(difflib.unified_diff(reference_str.splitlines(),
+                                                test_str.splitlines(),
+                                                'Reference', 'Test result',
+                                                '', '', 0))
+            self.fail("{} do not match:\n{}".format(type_comparison_name,
+                                                    diff))
+
     def result_path(self, basename=None, ext=''):
         """
         Return the full path to a test result, generated from the \

lib/iris/tests/unit/io/test_expand_filespecs.py

@@ -0,0 +1,103 @@
+# (C) British Crown Copyright 2017, Met Office
+#
+# This file is part of Iris.
+#
+# Iris is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Iris is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Iris.  If not, see <http://www.gnu.org/licenses/>.
+"""Unit tests for the `iris.io.expand_filespecs` function."""
+
+from __future__ import (absolute_import, division, print_function)
+from six.moves import (filter, input, map, range, zip)  # noqa
+
+# Import iris.tests first so that some things can be initialised before
+# importing anything else.
+import iris.tests as tests
+
+import os
+import tempfile
+import shutil
+import textwrap
+
+import iris.io as iio
+
+
+class TestExpandFilespecs(tests.IrisTest):
+    def setUp(self):
+        tests.IrisTest.setUp(self)
+        self.tmpdir = os.path.realpath(tempfile.mkdtemp())
+        self.fnames = ['a.foo', 'b.txt']
+        for fname in self.fnames:
+            with open(os.path.join(self.tmpdir, fname), 'w') as fh:
+                fh.write('anything')
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+    def test_absolute_path(self):
+        result = iio.expand_filespecs([os.path.join(self.tmpdir, '*')])
+        expected = [os.path.join(self.tmpdir, fname) for fname in self.fnames]
+        self.assertEqual(result, expected)
+
+    def test_double_slash(self):
+        product = iio.expand_filespecs(['//' + os.path.join(self.tmpdir, '*')])
+        predicted = [os.path.join(self.tmpdir, fname) for fname in self.fnames]
+        self.assertEqual(product, predicted)
+
+    def test_relative_path(self):
+        cwd = os.getcwd()
+        try:
+            os.chdir(self.tmpdir)
+            item_out = iio.expand_filespecs(['*'])
+            item_in = [os.path.join(self.tmpdir, fname)
+                       for fname in self.fnames]
+            self.assertEqual(item_out, item_in)
+        finally:
+            os.chdir(cwd)
+
+    def test_return_order(self):
+        # It is really quite important what order we return the
+        # files. They should be in the order that was provided,
+        # so that we can control the order of load (for instance,
+        # this can be used with PP files to ensure that there is
+        # a surface reference).
+        patterns = [os.path.join(self.tmpdir, 'a.*'),
+                    os.path.join(self.tmpdir, 'b.*')]
+        expected = [os.path.join(self.tmpdir, fname)
+                    for fname in ['a.foo', 'b.txt']]
+        result = iio.expand_filespecs(patterns)
+        self.assertEqual(result, expected)
+        result = iio.expand_filespecs(patterns[::-1])
+        self.assertEqual(result, expected[::-1])
+
+    def test_no_files_found(self):
+        msg = r'\/no_exist.txt\" matched 0 file\(s\)'
+        with self.assertRaisesRegexp(IOError, msg):
+            iio.expand_filespecs([os.path.join(self.tmpdir, 'no_exist.txt')])
+
+    def test_files_and_none(self):
+        with self.assertRaises(IOError) as err:
+            iio.expand_filespecs(
+                [os.path.join(self.tmpdir, 'does_not_exist.txt'),
+                 os.path.join(self.tmpdir, '*')])
+        expected = textwrap.dedent("""
+            One or more of the files specified did not exist:
+                - "{0}/does_not_exist.txt" matched 0 file(s)
+                - "{0}/*" matched 2 file(s)
+                   - {0}/a.foo, {0}/b.txt
+            """).strip().format(self.tmpdir)
+
+        self.assertStringEqual(str(err.exception), expected)
+
+
+if __name__ == "__main__":
+    tests.main()