diff --git a/lib/iris/io/__init__.py b/lib/iris/io/__init__.py index b431e814c1..6eeaf8060f 100644 --- a/lib/iris/io/__init__.py +++ b/lib/iris/io/__init__.py @@ -23,6 +23,7 @@ from six.moves import (filter, input, map, range, zip) # noqa import six +from collections import OrderedDict import glob import os.path import re @@ -147,25 +148,35 @@ def expand_filespecs(file_specs): File paths which may contain '~' elements or wildcards. Returns: - A list of matching file paths. If any of the file-specs matches no - existing files, an exception is raised. + A well-ordered list of matching absolute file paths. + If any of the file-specs match no existing files, an + exception is raised. """ # Remove any hostname component - currently unused - filenames = [os.path.expanduser(fn[2:] if fn.startswith('//') else fn) + filenames = [os.path.abspath(os.path.expanduser( + fn[2:] if fn.startswith('//') else fn)) for fn in file_specs] # Try to expand all filenames as globs - glob_expanded = {fn : sorted(glob.glob(fn)) for fn in filenames} + glob_expanded = OrderedDict([[fn, sorted(glob.glob(fn))] + for fn in filenames]) # If any of the specs expanded to an empty list then raise an error - value_lists = glob_expanded.values() - if not all(value_lists): - raise IOError("One or more of the files specified did not exist %s." % - ["%s expanded to %s" % (pattern, expanded if expanded else "empty") - for pattern, expanded in six.iteritems(glob_expanded)]) - - return sum(value_lists, []) + all_expanded = glob_expanded.values() + + if not all(all_expanded): + msg = "One or more of the files specified did not exist:" + for pattern, expanded in six.iteritems(glob_expanded): + if expanded: + file_list = '\n - {}'.format(', '.join(expanded)) + else: + file_list = '' + msg += '\n - "{}" matched {} file(s){}'.format( + pattern, len(expanded), file_list) + raise IOError(msg) + + return [fname for fnames in all_expanded for fname in fnames] def load_files(filenames, callback, constraints=None): diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index 480239c295..a7890832e6 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -249,6 +249,16 @@ def get_result_path(relative_path): relative_path = os.path.join(*relative_path) return os.path.abspath(os.path.join(_RESULT_PATH, relative_path)) + def assertStringEqual(self, reference_str, test_str, + type_comparison_name='strings'): + if reference_str != test_str: + diff = '\n'.join(difflib.unified_diff(reference_str.splitlines(), + test_str.splitlines(), + 'Reference', 'Test result', + '', '', 0)) + self.fail("{} do not match:\n{}".format(type_comparison_name, + diff)) + def result_path(self, basename=None, ext=''): """ Return the full path to a test result, generated from the \ diff --git a/lib/iris/tests/unit/io/test_expand_filespecs.py b/lib/iris/tests/unit/io/test_expand_filespecs.py new file mode 100644 index 0000000000..387291241d --- /dev/null +++ b/lib/iris/tests/unit/io/test_expand_filespecs.py @@ -0,0 +1,103 @@ +# (C) British Crown Copyright 2017, Met Office +# +# This file is part of Iris. +# +# Iris is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Iris is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Iris. If not, see . +"""Unit tests for the `iris.io.expand_filespecs` function.""" + +from __future__ import (absolute_import, division, print_function) +from six.moves import (filter, input, map, range, zip) # noqa + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import os +import tempfile +import shutil +import textwrap + +import iris.io as iio + + +class TestExpandFilespecs(tests.IrisTest): + def setUp(self): + tests.IrisTest.setUp(self) + self.tmpdir = os.path.realpath(tempfile.mkdtemp()) + self.fnames = ['a.foo', 'b.txt'] + for fname in self.fnames: + with open(os.path.join(self.tmpdir, fname), 'w') as fh: + fh.write('anything') + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def test_absolute_path(self): + result = iio.expand_filespecs([os.path.join(self.tmpdir, '*')]) + expected = [os.path.join(self.tmpdir, fname) for fname in self.fnames] + self.assertEqual(result, expected) + + def test_double_slash(self): + product = iio.expand_filespecs(['//' + os.path.join(self.tmpdir, '*')]) + predicted = [os.path.join(self.tmpdir, fname) for fname in self.fnames] + self.assertEqual(product, predicted) + + def test_relative_path(self): + cwd = os.getcwd() + try: + os.chdir(self.tmpdir) + item_out = iio.expand_filespecs(['*']) + item_in = [os.path.join(self.tmpdir, fname) + for fname in self.fnames] + self.assertEqual(item_out, item_in) + finally: + os.chdir(cwd) + + def test_return_order(self): + # It is really quite important what order we return the + # files. They should be in the order that was provided, + # so that we can control the order of load (for instance, + # this can be used with PP files to ensure that there is + # a surface reference). + patterns = [os.path.join(self.tmpdir, 'a.*'), + os.path.join(self.tmpdir, 'b.*')] + expected = [os.path.join(self.tmpdir, fname) + for fname in ['a.foo', 'b.txt']] + result = iio.expand_filespecs(patterns) + self.assertEqual(result, expected) + result = iio.expand_filespecs(patterns[::-1]) + self.assertEqual(result, expected[::-1]) + + def test_no_files_found(self): + msg = r'\/no_exist.txt\" matched 0 file\(s\)' + with self.assertRaisesRegexp(IOError, msg): + iio.expand_filespecs([os.path.join(self.tmpdir, 'no_exist.txt')]) + + def test_files_and_none(self): + with self.assertRaises(IOError) as err: + iio.expand_filespecs( + [os.path.join(self.tmpdir, 'does_not_exist.txt'), + os.path.join(self.tmpdir, '*')]) + expected = textwrap.dedent(""" + One or more of the files specified did not exist: + - "{0}/does_not_exist.txt" matched 0 file(s) + - "{0}/*" matched 2 file(s) + - {0}/a.foo, {0}/b.txt + """).strip().format(self.tmpdir) + + self.assertStringEqual(str(err.exception), expected) + + +if __name__ == "__main__": + tests.main()