From 1fa2dcc2416481003bade6b87c695c63c56f9867 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 22 Jan 2019 12:14:39 +0200 Subject: [PATCH 1/6] Make it possible to disable data reduction with a kwarg --- satpy/scene.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 8f06d9d81a..2ac8793d06 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -933,13 +933,16 @@ def _resampled_scene(self, new_scn, destination_area, **resample_kwargs): LOG.debug("Resampling %s", ds_id) source_area = dataset.attrs['area'] try: - slice_x, slice_y = source_area.get_area_slices( - destination_area) - source_area = source_area[slice_y, slice_x] - dataset = dataset.isel(x=slice_x, y=slice_y) - assert ('x', source_area.x_size) in dataset.sizes.items() - assert ('y', source_area.y_size) in dataset.sizes.items() - dataset.attrs['area'] = source_area + if resample_kwargs.get('reduce_data', True): + slice_x, slice_y = source_area.get_area_slices( + destination_area) + source_area = source_area[slice_y, slice_x] + dataset = dataset.isel(x=slice_x, y=slice_y) + assert ('x', source_area.x_size) in dataset.sizes.items() + assert ('y', source_area.y_size) in dataset.sizes.items() + dataset.attrs['area'] = source_area + else: + LOG.info("Data reduction disabled by the user") except NotImplementedError: LOG.info("Not reducing data before resampling.") if source_area not in resamplers: From 51a982365a0c7016e5d649f73809c4f6f8a6bf6d Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 22 Jan 2019 15:38:29 +0200 Subject: [PATCH 2/6] Make 'reduce_data' explicit kwarg --- satpy/scene.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 2ac8793d06..5b91a11bed 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -900,7 +900,8 @@ def load(self, wishlist, calibration=None, resolution=None, if unload: self.unload(keepables=keepables) - def _resampled_scene(self, new_scn, destination_area, **resample_kwargs): + def _resampled_scene(self, new_scn, destination_area, reduce_data=True, + **resample_kwargs): """Resample `datasets` to the `destination` area.""" new_datasets = {} datasets = list(new_scn.datasets.values()) @@ -933,7 +934,7 @@ def _resampled_scene(self, new_scn, destination_area, **resample_kwargs): LOG.debug("Resampling %s", ds_id) source_area = dataset.attrs['area'] try: - if resample_kwargs.get('reduce_data', True): + if reduce_data: slice_x, slice_y = source_area.get_area_slices( destination_area) source_area = source_area[slice_y, slice_x] From 569cae33ff328af7f59b3a2e71185ab5a08768c9 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 22 Jan 2019 15:38:59 +0200 Subject: [PATCH 3/6] Change log level of data reduction disabling from info to debug --- satpy/scene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/scene.py b/satpy/scene.py index 5b91a11bed..1c65c7a701 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -943,7 +943,7 @@ def _resampled_scene(self, new_scn, destination_area, reduce_data=True, assert ('y', source_area.y_size) in dataset.sizes.items() dataset.attrs['area'] = source_area else: - LOG.info("Data reduction disabled by the user") + LOG.debug("Data reduction disabled by the user") except NotImplementedError: LOG.info("Not reducing data before resampling.") if source_area not in resamplers: From 83e27760abb0692fea0373d8c9c20171d551079f Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 23 Jan 2019 10:46:22 +0200 Subject: [PATCH 4/6] Move data reduction to a separate method --- satpy/scene.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 1c65c7a701..c76ef7d632 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -900,6 +900,17 @@ def load(self, wishlist, calibration=None, resolution=None, if unload: self.unload(keepables=keepables) + def _reduce_data(self, source_area, destination_area, dataset): + """Reduce data by slicing it.""" + slice_x, slice_y = source_area.get_area_slices(destination_area) + source_area = source_area[slice_y, slice_x] + dataset = dataset.isel(x=slice_x, y=slice_y) + assert ('x', source_area.x_size) in dataset.sizes.items() + assert ('y', source_area.y_size) in dataset.sizes.items() + dataset.attrs['area'] = source_area + + return dataset + def _resampled_scene(self, new_scn, destination_area, reduce_data=True, **resample_kwargs): """Resample `datasets` to the `destination` area.""" @@ -935,13 +946,8 @@ def _resampled_scene(self, new_scn, destination_area, reduce_data=True, source_area = dataset.attrs['area'] try: if reduce_data: - slice_x, slice_y = source_area.get_area_slices( - destination_area) - source_area = source_area[slice_y, slice_x] - dataset = dataset.isel(x=slice_x, y=slice_y) - assert ('x', source_area.x_size) in dataset.sizes.items() - assert ('y', source_area.y_size) in dataset.sizes.items() - dataset.attrs['area'] = source_area + dataset = self._reduce_data(source_area, destination_area, + dataset) else: LOG.debug("Data reduction disabled by the user") except NotImplementedError: From 4716b4e80f10af47007aa38e91f4147b594baf4d Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 23 Jan 2019 10:50:12 +0200 Subject: [PATCH 5/6] Add tests for disabling data reduction --- satpy/tests/test_scene.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/satpy/tests/test_scene.py b/satpy/tests/test_scene.py index 6c395c943d..d15ba390ae 100644 --- a/satpy/tests/test_scene.py +++ b/satpy/tests/test_scene.py @@ -1488,10 +1488,11 @@ def _fake_resample_dataset(self, dataset, dest_area, **kwargs): """Return copy of dataset pretending it was resampled.""" return dataset.copy() + @mock.patch('satpy.scene.Scene._reduce_data') @mock.patch('satpy.scene.resample_dataset') @mock.patch('satpy.composites.CompositorLoader.load_compositors') @mock.patch('satpy.scene.Scene.create_reader_instances') - def test_resample_scene_copy(self, cri, cl, rs): + def test_resample_scene_copy(self, cri, cl, rs, reduce_data): """Test that the Scene is properly copied during resampled. The Scene that is created as a copy of the original Scene should not @@ -1555,6 +1556,20 @@ def test_resample_scene_copy(self, cri, cl, rs): self.assertTupleEqual( tuple(loaded_ids[1]), tuple(DatasetID(name='new_ds'))) + # Test that data reduction can be disabled + scene = satpy.scene.Scene(filenames=['bla'], + base_dir='bli', + reader='fake_reader') + + scene.load(['comp19']) + scene['comp19'].attrs['area'] = area_def + new_scene = scene.resample(area_def, reduce_data=False) + self.assertFalse(reduce_data.called) + new_scene = scene.resample(area_def) + self.assertTrue(reduce_data.called_once) + new_scene = scene.resample(area_def, reduce_data=True) + self.assertEqual(reduce_data.call_count, 2) + @mock.patch('satpy.scene.resample_dataset') @mock.patch('satpy.composites.CompositorLoader.load_compositors') @mock.patch('satpy.scene.Scene.create_reader_instances') From c3bb7315e7db956cebe55b1192a4ba189a744605 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Sat, 26 Jan 2019 10:24:51 +0200 Subject: [PATCH 6/6] Add 'reduce_data' kwarg to scn.resample() --- satpy/scene.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index c76ef7d632..ab1759b581 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -968,7 +968,8 @@ def _resampled_scene(self, new_scn, destination_area, reduce_data=True, replace_anc(res, pres) def resample(self, destination=None, datasets=None, generate=True, - unload=True, resampler=None, **resample_kwargs): + unload=True, resampler=None, reduce_data=True, + **resample_kwargs): """Resample datasets and return a new scene. Args: @@ -987,6 +988,8 @@ def resample(self, destination=None, datasets=None, generate=True, ('nearest'). Other possible values include 'native', 'ewa', etc. See the :mod:`~satpy.resample` documentation for more information. + reduce_data (bool): Reduce data by matching the input and output + areas and slicing the data arrays (default: True) resample_kwargs: Remaining keyword arguments to pass to individual resampler classes. See the individual resampler class documentation :mod:`here ` for available @@ -1002,7 +1005,7 @@ def resample(self, destination=None, datasets=None, generate=True, # we may have some datasets we asked for but don't exist yet new_scn.wishlist = self.wishlist.copy() self._resampled_scene(new_scn, destination, resampler=resampler, - **resample_kwargs) + reduce_data=reduce_data, **resample_kwargs) # regenerate anything from the wishlist that needs it (combining # multiple resolutions, etc.)