From 4c61fb3b2af73202db1b8a1a435c7f67be8513d0 Mon Sep 17 00:00:00 2001 From: TomekTrzeciak Date: Mon, 17 Feb 2020 21:22:15 +0000 Subject: [PATCH 1/5] nimrod monkey patch --- improver/utilities/load.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/improver/utilities/load.py b/improver/utilities/load.py index f1023f5aff..a530d9bd32 100644 --- a/improver/utilities/load.py +++ b/improver/utilities/load.py @@ -65,6 +65,22 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, Cube that has been loaded from the input filepath given the constraints provided. """ + # FIXME: monkey patched nimrod loading in iris, so it works for radar files + try: + iris.fileformats.nimrod_load_rules.DEFAULT_UNITS + except AttributeError: + try: + from iris_nimrod_patch import nimrod, nimrod_load_rules + except ImportError: + pass + else: + for attr in ['general_header_int16s', 'general_header_float32s', + 'data_header_int16s', 'data_header_float32s']: + setattr(iris.fileformats.nimrod, attr, getattr(nimrod, attr)) + iris.fileformats.nimrod_load_rules = nimrod_load_rules + else: + raise RuntimeError('FIXME: nimrod monkey patch is no longer needed') + if filepath is None and allow_none: return None # Remove metadata prefix cube if present From be015cba251d23360befdf48f8dfb69526433355 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 12:49:05 +0000 Subject: [PATCH 2/5] Makes monkey-patch a temporary addition so that repeated calls from the acceptance tests do not falsely trigger the FIXME message. --- improver/utilities/load.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/improver/utilities/load.py b/improver/utilities/load.py index a530d9bd32..b8f84b8a03 100644 --- a/improver/utilities/load.py +++ b/improver/utilities/load.py @@ -30,6 +30,7 @@ # POSSIBILITY OF SUCH DAMAGE. """Module for loading cubes.""" +import contextlib import glob import iris @@ -38,6 +39,16 @@ enforce_coordinate_ordering, merge_cubes) +@contextlib.contextmanager +def monkeypatched(object, name, patch): + """ Temporarily monkeypatches an object. """ + + pre_patched_value = getattr(object, name) + setattr(object, name, patch) + yield object + setattr(object, name, pre_patched_value) + + def load_cube(filepath, constraints=None, no_lazy_load=False, allow_none=False): """Load the filepath provided using Iris into a cube. @@ -77,7 +88,6 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, for attr in ['general_header_int16s', 'general_header_float32s', 'data_header_int16s', 'data_header_float32s']: setattr(iris.fileformats.nimrod, attr, getattr(nimrod, attr)) - iris.fileformats.nimrod_load_rules = nimrod_load_rules else: raise RuntimeError('FIXME: nimrod monkey patch is no longer needed') @@ -89,12 +99,14 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, # Load each file individually to avoid partial merging (not used # iris.load_raw() due to issues with time representation) - if isinstance(filepath, str): - cubes = iris.load(filepath, constraints=constraints) - else: - cubes = iris.cube.CubeList([]) - for item in filepath: - cubes.extend(iris.load(item, constraints=constraints)) + with monkeypatched(iris.fileformats, 'nimrod_load_rules', + nimrod_load_rules): + if isinstance(filepath, str): + cubes = iris.load(filepath, constraints=constraints) + else: + cubes = iris.cube.CubeList([]) + for item in filepath: + cubes.extend(iris.load(item, constraints=constraints)) # Merge loaded cubes if not cubes: From 6abbe8dc86d9aba9e84ea2600a922b4ef8c70048 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 12:50:02 +0000 Subject: [PATCH 3/5] Adds unit-tests for Nimrod-format Radarnet data. These are only triggered if the iris_nimrod_patch is available. --- improver_tests/acceptance/acceptance.py | 13 ++++++++++ improver_tests/acceptance/test_standardise.py | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/improver_tests/acceptance/acceptance.py b/improver_tests/acceptance/acceptance.py index e4bfcc3f69..a8f22bb78b 100644 --- a/improver_tests/acceptance/acceptance.py +++ b/improver_tests/acceptance/acceptance.py @@ -158,6 +158,13 @@ def statsmodels_available(): return False +def iris_nimrod_patch_available(): + """True if iris_nimrod_patch library is importable""" + if importlib.util.find_spec('iris_nimrod_patch'): + return True + return False + + def compare(output_path, kgo_path, recreate=True, atol=DEFAULT_TOLERANCE, rtol=DEFAULT_TOLERANCE, exclude_vars=None): """ @@ -215,3 +222,9 @@ def message_recorder(exception_message): # pylint: disable=invalid-name skip_if_no_statsmodels = pytest.mark.skipif( not statsmodels_available(), reason="statsmodels library is not available") + +# Pytest decorator to skip tests if iris_nimrod_patch is not available +# pylint: disable=invalid-name +skip_if_no_iris_nimrod_patch = pytest.mark.skipif( + not iris_nimrod_patch_available(), + reason="iris_nimrod_patch library is not available") diff --git a/improver_tests/acceptance/test_standardise.py b/improver_tests/acceptance/test_standardise.py index ca56231f41..c44b8c4125 100644 --- a/improver_tests/acceptance/test_standardise.py +++ b/improver_tests/acceptance/test_standardise.py @@ -223,3 +223,29 @@ def test_stage_v110_basic(tmp_path): "--output", output_path] run_cli(args) acc.compare(output_path, kgo_path) + + +@acc.skip_if_no_iris_nimrod_patch +def test_nimrod_radarrate_basic(tmp_path): + """Test updating a file with Nimrod-format Radarnet data""" + kgo_dir = acc.kgo_root() / "standardise/radarnet" + kgo_path = kgo_dir / "kgo_preciprate.nc" + input_path = kgo_dir / "input_preciprate.nimrod" + output_path = tmp_path / "output.nc" + args = [input_path, + "--output", output_path] + run_cli(args) + acc.compare(output_path, kgo_path) + + +@acc.skip_if_no_iris_nimrod_patch +def test_nimrod_radarcoverage_basic(tmp_path): + """Test updating a file with Nimrod-format Radarnet data""" + kgo_dir = acc.kgo_root() / "standardise/radarnet" + kgo_path = kgo_dir / "kgo_coverage.nc" + input_path = kgo_dir / "input_coverage.nimrod" + output_path = tmp_path / "output.nc" + args = [input_path, + "--output", output_path] + run_cli(args) + acc.compare(output_path, kgo_path) From a216083da449cbd3563c0ced1a49c12d6e164e9d Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 13:05:41 +0000 Subject: [PATCH 4/5] Fixes problems with the monkey-patch context manager. --- improver/utilities/load.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/improver/utilities/load.py b/improver/utilities/load.py index b8f84b8a03..6a59722b63 100644 --- a/improver/utilities/load.py +++ b/improver/utilities/load.py @@ -40,13 +40,13 @@ @contextlib.contextmanager -def monkeypatched(object, name, patch): +def monkeypatched(obj, name, patch): """ Temporarily monkeypatches an object. """ - pre_patched_value = getattr(object, name) - setattr(object, name, patch) - yield object - setattr(object, name, pre_patched_value) + pre_patched_value = getattr(obj, name) + setattr(obj, name, patch) + yield obj + setattr(obj, name, pre_patched_value) def load_cube(filepath, constraints=None, no_lazy_load=False, @@ -77,6 +77,7 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, constraints provided. """ # FIXME: monkey patched nimrod loading in iris, so it works for radar files + patcher = contextlib.suppress() try: iris.fileformats.nimrod_load_rules.DEFAULT_UNITS except AttributeError: @@ -88,6 +89,8 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, for attr in ['general_header_int16s', 'general_header_float32s', 'data_header_int16s', 'data_header_float32s']: setattr(iris.fileformats.nimrod, attr, getattr(nimrod, attr)) + patcher = monkeypatched(iris.fileformats, 'nimrod_load_rules', + nimrod_load_rules) else: raise RuntimeError('FIXME: nimrod monkey patch is no longer needed') @@ -99,8 +102,7 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, # Load each file individually to avoid partial merging (not used # iris.load_raw() due to issues with time representation) - with monkeypatched(iris.fileformats, 'nimrod_load_rules', - nimrod_load_rules): + with patcher: if isinstance(filepath, str): cubes = iris.load(filepath, constraints=constraints) else: From ec7464afd33d112d52cbbef9ff539fb0b8c5d999 Mon Sep 17 00:00:00 2001 From: Stephen Moseley Date: Tue, 18 Feb 2020 13:25:03 +0000 Subject: [PATCH 5/5] Pycodestyle fix --- improver/utilities/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/improver/utilities/load.py b/improver/utilities/load.py index 6a59722b63..9c20e3d007 100644 --- a/improver/utilities/load.py +++ b/improver/utilities/load.py @@ -90,7 +90,7 @@ def load_cube(filepath, constraints=None, no_lazy_load=False, 'data_header_int16s', 'data_header_float32s']: setattr(iris.fileformats.nimrod, attr, getattr(nimrod, attr)) patcher = monkeypatched(iris.fileformats, 'nimrod_load_rules', - nimrod_load_rules) + nimrod_load_rules) else: raise RuntimeError('FIXME: nimrod monkey patch is no longer needed')