diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f2e872125..9b4f9da50 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,16 @@ +DIALS 3.1.3 (2020-09-28) +======================== + +Bugfixes +-------- + +- ``dxtbx.image_average``: Better use of MPI to avoid errors and increase + performance (#207) +- Update DLS I23 bad pixel mask after detector has been cleaned, fixing + previously bad modules. (#220) +- Change default bit depth for DLS eigers where header information is missing + + DIALS 3.1.1 (2020-09-01) ======================== diff --git a/command_line/image_average.py b/command_line/image_average.py index b1bbe0ba2..7bec46548 100644 --- a/command_line/image_average.py +++ b/command_line/image_average.py @@ -10,6 +10,7 @@ import copy import sys from builtins import range +import numpy as np import libtbx.load_env from libtbx import easy_mp, option_parser @@ -284,7 +285,7 @@ def run(argv=None): iterable = iterable[command_line.options.skip_images :] if ( command_line.options.num_images_max is not None - and command_line.options.num_images_max < iterable + and command_line.options.num_images_max < len(iterable) ): iterable = iterable[: command_line.options.num_images_max] assert len(iterable) >= 2, "Need more than one image to average" @@ -308,14 +309,39 @@ def run(argv=None): # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes iterable = [i for n, i in enumerate(iterable) if (n + rank) % size == 0] - results = [worker(iterable)] - results = comm.gather(results, root=0) + r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img, r_sum_wavelength = worker( + iterable + ) + + nfail = np.array([0]) + nmemb = np.array([0]) + sum_distance = np.array([0.0]) + sum_wavelength = np.array([0.0]) + comm.Reduce(np.array([r_nfail]), nfail) + comm.Reduce(np.array([r_nmemb]), nmemb) + comm.Reduce(np.array([r_sum_distance]), sum_distance) + comm.Reduce(np.array([r_sum_wavelength]), sum_wavelength) + nfail = int(nfail[0]) + nmemb = int(nmemb) + sum_distance = float(sum_distance[0]) + sum_wavelength = float(sum_wavelength[0]) + + def reduce_image(data, op=MPI.SUM): + result = [] + for panel_data in data: + panel_data = panel_data.as_numpy_array() + reduced_data = np.zeros(panel_data.shape).astype(panel_data.dtype) + comm.Reduce(panel_data, reduced_data, op=op) + result.append(flex.double(reduced_data)) + return result + + max_img = reduce_image(r_max_img, MPI.MAX) + sum_img = reduce_image(r_sum_img) + ssq_img = reduce_image(r_ssq_img) + if rank != 0: return - results_set = [] - for r in results: - results_set.extend(r) - results = results_set + avg_img = tuple(s / nmemb for s in sum_img) else: if command_line.options.nproc == 1: results = [worker(iterable)] @@ -325,38 +351,38 @@ def run(argv=None): func=worker, iterable=iterable, processes=command_line.options.nproc ) - nfail = 0 - nmemb = 0 - for ( - i, - ( - r_nfail, - r_nmemb, - r_max_img, - r_sum_distance, - r_sum_img, - r_ssq_img, - r_sum_wavelength, - ), - ) in enumerate(results): - nfail += r_nfail - nmemb += r_nmemb - if i == 0: - max_img = r_max_img - sum_distance = r_sum_distance - sum_img = r_sum_img - ssq_img = r_ssq_img - sum_wavelength = r_sum_wavelength - else: - for p in range(len(sum_img)): - sel = (r_max_img[p] > max_img[p]).as_1d() - max_img[p].set_selected(sel, r_max_img[p].select(sel)) + nfail = 0 + nmemb = 0 + for ( + i, + ( + r_nfail, + r_nmemb, + r_max_img, + r_sum_distance, + r_sum_img, + r_ssq_img, + r_sum_wavelength, + ), + ) in enumerate(results): + nfail += r_nfail + nmemb += r_nmemb + if i == 0: + max_img = r_max_img + sum_distance = r_sum_distance + sum_img = r_sum_img + ssq_img = r_ssq_img + sum_wavelength = r_sum_wavelength + else: + for p in range(len(sum_img)): + sel = (r_max_img[p] > max_img[p]).as_1d() + max_img[p].set_selected(sel, r_max_img[p].select(sel)) - sum_img[p] += r_sum_img[p] - ssq_img[p] += r_ssq_img[p] + sum_img[p] += r_sum_img[p] + ssq_img[p] += r_ssq_img[p] - sum_distance += r_sum_distance - sum_wavelength += r_sum_wavelength + sum_distance += r_sum_distance + sum_wavelength += r_sum_wavelength # Early exit if no statistics were accumulated. if command_line.options.verbose: diff --git a/format/FormatCBFMiniPilatusDLS12M.py b/format/FormatCBFMiniPilatusDLS12M.py index 5978a62c2..3e91af62c 100644 --- a/format/FormatCBFMiniPilatusDLS12M.py +++ b/format/FormatCBFMiniPilatusDLS12M.py @@ -170,7 +170,11 @@ def _mask_bad_modules(self, detector): cy = 97 # chip pixels y dx = 7 # module gap size - if timestamp > calendar.timegm((2020, 2, 21, 0, 0, 0)): + if timestamp > calendar.timegm((2020, 9, 8, 0, 0, 0)): + # 2020 run 4 + # Detector serviced by Dectris, no bad modules + pass + elif timestamp > calendar.timegm((2020, 2, 21, 0, 0, 0)): # 2020 run 1 # module @ row 1 column 4 # blank chip @ row 15 column 4 (chip row 0, column 1) diff --git a/format/FormatNexusEigerDLS.py b/format/FormatNexusEigerDLS.py index 80813c3fb..52ecccf98 100644 --- a/format/FormatNexusEigerDLS.py +++ b/format/FormatNexusEigerDLS.py @@ -67,7 +67,7 @@ def __init__(self, image_file, **kwargs): self._meta = find_meta_filename(image_file) self._bit_depth_image = get_bit_depth_from_meta(self._meta) except Exception: - self._bit_depth_image = 0 + self._bit_depth_image = 16 def get_detector(self, index=None): # workaround for https://jira.diamond.ac.uk/browse/I03-365 diff --git a/tests/command_line/test_average.py b/tests/command_line/test_average.py index 2900875d6..1e60488e8 100644 --- a/tests/command_line/test_average.py +++ b/tests/command_line/test_average.py @@ -6,18 +6,29 @@ import dxtbx +import pytest + + +@pytest.mark.parametrize("use_mpi", [True, False]) +def test_average(dials_regression, tmpdir, use_mpi): + # Only allow MPI tests if we've got MPI capabilities + if use_mpi: + pytest.importorskip("mpi4py") -def test_average(dials_regression, tmpdir): data = os.path.join( dials_regression, "image_examples", "SACLA_MPCCD_Cheetah", "run266702-0-subset.h5", ) + if use_mpi: + command = "mpirun" + mpargs = "-n 2 dxtbx.image_average".split() + else: + command = "dxtbx.image_average" + mpargs = "-n 2".split() result = procrunner.run( - ["dxtbx.image_average"] - + "-v -a avg.cbf -s stddev.cbf -m max.cbf".split() - + [data], + [command] + mpargs + "-v -a avg.cbf -s stddev.cbf -m max.cbf".split() + [data], working_directory=tmpdir, ) assert not result.returncode and not result.stderr diff --git a/tests/format/test_FormatCBFMiniPilatusDLS12M.py b/tests/format/test_FormatCBFMiniPilatusDLS12M.py index 197448544..50ce87222 100644 --- a/tests/format/test_FormatCBFMiniPilatusDLS12M.py +++ b/tests/format/test_FormatCBFMiniPilatusDLS12M.py @@ -33,6 +33,8 @@ False, 293699 + n_pixels_vertical_gaps, ), + (calendar.timegm((2020, 9, 8, 0, 0, 1)), True, 3053), + (calendar.timegm((2020, 9, 8, 0, 0, 1)), False, 3053 + n_pixels_vertical_gaps), ), ) def test_bad_pixel_mask(