Skip to content

Check graphic create missing #2192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
marqh merged 3 commits into from
Oct 14, 2016
Merged

Check graphic create missing #2192

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d6e63e9
Check graphic creates missing expected images.
bjlittle Oct 13, 2016
f7f4357
Bump up the filelock timeout
bjlittle Oct 13, 2016
915be32
Include additional create missing case.
bjlittle Oct 14, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 67 additions & 27 deletions lib/iris/tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
except ImportError:
import mock

import filelock
import numpy as np
import numpy.ma as ma

Expand Down Expand Up @@ -660,16 +661,21 @@ def _ensure_folder(self, path):
logger.warning('Creating folder: %s', dir_path)
os.makedirs(dir_path)

def check_graphic(self, tol=None):
def _assert_graphic(self):
"""
Checks the CRC matches for the current matplotlib.pyplot figure,
and closes the figure.
Check the hash of the current matplotlib figure matches the expected
image hash for the current graphic test.

"""
To create missing image test results, set the IRIS_TEST_CREATE_MISSING
environment variable before running the tests. This will result in new
and appropriately "<hash>.png" image files being generated in the image
output directory, and the imagerepo.json file being updated.

"""
dev_mode = os.environ.get('IRIS_TEST_CREATE_MISSING')
unique_id = self._unique_id()

repo_fname = os.path.join(os.path.dirname(__file__), 'results', 'imagerepo.json')
repo_fname = os.path.join(os.path.dirname(__file__),
'results', 'imagerepo.json')
with open(repo_fname, 'rb') as fi:
repo = json.load(codecs.getreader('utf-8')(fi))

Expand Down Expand Up @@ -700,20 +706,37 @@ def check_graphic(self, tol=None):
if err.errno != 17:
raise

def save_figure_hash():
figure = plt.gcf()
figure.savefig(result_fname)

# hash the created image using sha1
with open(result_fname, 'rb') as res_file:
sha1 = hashlib.sha1(res_file.read())
def _save_figure_hash():
plt.gcf().savefig(result_fname)
# Determine the test result image hash using sha1.
with open(result_fname, 'rb') as fi:
sha1 = hashlib.sha1(fi.read())
return sha1

sha1 = save_figure_hash()
def _create_missing():
fname = sha1.hexdigest() + '.png'
base_uri = ('https://scitools.github.io/test-images-scitools/'
'image_files/{}')
uri = base_uri.format(fname)
hash_fname = os.path.join(image_output_directory, fname)
uris = repo.setdefault(unique_id, [])
uris.append(uri)
print('Creating image file: {}'.format(hash_fname))
os.rename(result_fname, hash_fname)
msg = 'Creating imagerepo entry: {} -> {}'
print(msg.format(unique_id, uri))
with open(repo_fname, 'wb') as fo:
json.dump(repo, codecs.getwriter('utf-8')(fo), indent=4,
sort_keys=True)

sha1 = _save_figure_hash()

if unique_id not in repo:
msg = 'Image comparison failed: Created image {} for test {}.'
raise ValueError(msg.format(result_fname, unique_id))
if dev_mode:
_create_missing()
else:
emsg = 'Missing image test result: {}.'
raise ValueError(emsg.format(unique_id))
else:
uris = repo[unique_id]
# Cherry-pick the registered expected hashes from the
Expand All @@ -725,29 +748,46 @@ def save_figure_hash():
# This can be an accidental failure, unusual, but it occurs
# https://github.com/SciTools/iris/issues/2195
# retry once, in case it passes second time round.
sha1 = save_figure_hash()
sha1 = _save_figure_hash()

if sha1.hexdigest() not in expected:
emsg = 'Actual SHA1 {} not in expected {} for test {}.'
emsg = emsg.format(sha1.hexdigest(), expected, unique_id)
if _DISPLAY_FIGURES:
print('Image comparison would have failed. '
'Message: %s' % emsg)
if dev_mode:
_create_missing()
else:
raise ValueError('Image comparison failed.'
' Message: {}'.format(emsg))

emsg = 'Actual SHA1 {} not in expected {} for test {}.'
emsg = emsg.format(sha1.hexdigest(), expected,
unique_id)
if _DISPLAY_FIGURES:
print('Image comparison would have failed. '
'Message: %s' % emsg)
else:
raise ValueError('Image comparison failed. '
'Message: {}'.format(emsg))
else:
# There is no difference between the actual and expected
# result, so remove the actual result file.
os.remove(result_fname)

if _DISPLAY_FIGURES:
plt.show()

finally:
plt.close()

def check_graphic(self, tol=None):
"""
Checks that the image hash for the current matplotlib figure matches
the expected image hash for the current test.

"""
fname = os.path.join(os.path.dirname(__file__),
'results', 'imagerepo.lock')
lock = filelock.FileLock(fname)
# The imagerepo.json file is a critical resource, so ensure thread
# safe read/write behaviour via platform independent file locking.
with lock.acquire(timeout=600):
Copy link
Member Author

@bjlittle bjlittle Oct 13, 2016 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marqh A suitably high timeout ...

Having the graphics tests fighting over the imagerepo.lock doesn't seem to impact the overall running time of the tests in travis ... but it's hard to tell due to the variable loads on travis.

I don't think this is a problem, but we could choose to only acquire a lock if the IRIS_TEST_CREATE_MISSING environment variable is set - as this means there could be potential writing to the imagerepo.json.

In travis the IRIS_TEST_CREATE_MISSING environment variable should never be set. So there is the opportunity to avoid any potential lock contention overhead that may be incurred.

My expectation is that the IRIS_TEST_CREATE_MISSING environment variable will only be set by the developer on their desktop when needed.

self._assert_graphic()

def _remove_testcase_patches(self):
"""Helper to remove per-testcase patches installed by :meth:`patch`."""
# Remove all patches made, ignoring errors.
Expand Down Expand Up @@ -798,7 +838,7 @@ def assertArrayShapeStats(self, result, shape, mean, std_dev):
mean and standard deviation of the data array are also as provided.
Thus build confidence that a cube processing operation, such as a
cube.regrid, has maintained its behaviour.

"""
self.assertEqual(result.shape, shape)
self.assertAlmostEqual(result.data.mean(), mean, places=5)
Expand Down