Check graphic hash

.travis.yml

@@ -105,10 +105,10 @@ install:
 
 script:
   - if [[ $TEST_TARGET == 'default' ]]; then
-      python -m iris.tests.runner --default-tests --system-tests --print-failed-images;
+      python -m iris.tests.runner --default-tests --system-tests;
     fi
   - if [[ $TEST_TARGET == 'example' ]]; then
-      python -m iris.tests.runner --example-tests --print-failed-images;
+      python -m iris.tests.runner --example-tests;
     fi
   # "make html" produces an error when run on Travis that does not
   # affect any downstream functionality but causes the build to fail

lib/iris/tests/__init__.py

@@ -34,13 +34,16 @@
 from six.moves import (filter, input, map, range, zip)  # noqa
 import six
 
+import codecs
 import collections
 import contextlib
 import difflib
 import filecmp
 import functools
 import gzip
+import hashlib
 import inspect
+import json
 import io
 import logging
 import os
@@ -620,16 +623,20 @@ def _unique_id(self):
 
         """
         # Obtain a consistent ID for the current test.
-
         # NB. unittest.TestCase.id() returns different values depending on
         # whether the test has been run explicitly, or via test discovery.
         # For example:
         #   python tests/test_plot.py => '__main__.TestContourf.test_tx'
         #   ird -t => 'iris.tests.test_plot.TestContourf.test_tx'
-        bits = self.id().split('.')[-3:]
+        bits = self.id().split('.')
         if bits[0] == '__main__':
-            file_name = os.path.basename(sys.modules['__main__'].__file__)
+            floc = sys.modules['__main__'].__file__
+            path, file_name = os.path.split(os.path.abspath(floc))
             bits[0] = os.path.splitext(file_name)[0]
+            folder, location = os.path.split(path)
+            while location != 'iris':
+                folder, location = os.path.split(folder)
+                bits = [location] + bits
         test_id = '.'.join(bits)
 
         # Derive the sequential assertion ID within the test
@@ -652,21 +659,22 @@ def _ensure_folder(self, path):
             logger.warning('Creating folder: %s', dir_path)
             os.makedirs(dir_path)
 
-    def check_graphic(self, tol=_DEFAULT_IMAGE_TOLERANCE):
-        """Checks the CRC matches for the current matplotlib.pyplot figure, and closes the figure."""
+    def check_graphic(self, tol=None):
+        """
+        Checks the CRC matches for the current matplotlib.pyplot figure,
+        and closes the figure.
+
+        """
 
         unique_id = self._unique_id()
 
         figure = plt.gcf()
+
+        repo_fname = os.path.join(os.path.dirname(__file__), 'results', 'imagerepo.json')
+        with open(repo_fname, 'rb') as fi:
+            repo = json.load(codecs.getreader('utf-8')(fi))
 
         try:
-            expected_fname = os.path.join(os.path.dirname(__file__),
-                                          'results', 'visual_tests',
-                                          unique_id + '.png')
-
-            if not os.path.isdir(os.path.dirname(expected_fname)):
-                os.makedirs(os.path.dirname(expected_fname))
-
             #: The path where the images generated by the tests should go.
             image_output_directory = os.path.join(os.path.dirname(__file__),
                                                   'result_image_comparison')
@@ -695,18 +703,50 @@ def check_graphic(self, tol=_DEFAULT_IMAGE_TOLERANCE):
 
             figure.savefig(result_fname)
 
-            if not os.path.exists(expected_fname):
-                warnings.warn('Created image for test %s' % unique_id)
+            # XXX: Deal with a new test result i.e. it's not in the repo
+
+            # hash the created image using sha1
+            with open(result_fname, 'rb') as res_file:
+                sha1 = hashlib.sha1(res_file.read())
+
+            emsg = None
+
+            try:
+                uris = repo[unique_id]
+            except KeyError:
+                # This test is not registered in the image json store,
+                # therefore treat it as a *new* test to be registered.
+                dname = os.path.join(os.path.dirname(__file__),
+                                     'results', 'visual_tests')
+                expected_fname = os.path.join(dname, unique_id+'.png')
+                if not os.path.isdir(dname):
+                    os.makedirs(dname)
+                wmsg = 'Created image for test {}'
+                warnings.warn(wmsg.format(unique_id))
                 shutil.copy2(result_fname, expected_fname)
-
-            err = mcompare.compare_images(expected_fname, result_fname, tol=tol)
+            else:
+                # Cherry-pick the registered expected hashes from the
+                # test case uri/s.
+                expected = []
+                for uri in uris:
+                    ehash = os.path.splitext(os.path.basename(uri))[0]
+                    expected.append(ehash)
+
+                if sha1.hexdigest() not in expected:
+                    emsg = 'Actual SHA1 {} not in expected {} for test {}.'
+                    emsg = emsg.format(sha1.hexdigest(), expected, unique_id)
+                else:
+                    # There is no difference between the actual and expected
+                    # result, so remove the actual result file.
+                    os.remove(result_fname)
 
             if _DISPLAY_FIGURES:
-                if err:
-                    print('Image comparison would have failed. Message: %s' % err)
+                if emsg is not None:
+                    print('Image comparison would have failed. '
+                          'Message: %s' % emsg)
                 plt.show()
             else:
-                assert not err, 'Image comparison failed. Message: %s' % err
+                assert not emsg, 'Image comparison failed. Message: %s' % emsg
 
         finally:
             plt.close()