Format.get_mask() -> get_static_mask() (#73)

- Add an optional Format.get_static_mask() method to allow format classes to define a static mask to be used on all images.
- Add tests for static masks in FormatHDF5SaclaMPCCD and FormatPYunspecifiedStill.
- Remove superfluous Format.get_mask() functions - these are no longer called and the functionality they used to provide are provided elsewhere.

Resolves #70 (see also #65).

.travis.yml

@@ -73,7 +73,7 @@ jobs:
       script:
          # Finally. Do what we are here for. Run tests. Yay.
          - cd $HOME/build_dials/modules/dxtbx
-         - pytest -ra -n 2 --regression
+         - pytest -ra -n 2
 
     - name: "Flake8 - Python 2.7"
       addons: false
@@ -141,7 +141,7 @@ jobs:
       script:
          # Finally. Do what we are here for. Run tests. Yay.
          - cd $HOME/build_dials/modules/dxtbx
-         - pytest -ra -n 2 --regression
+         - pytest -ra -n 2
 
 before_cache:
   - cd $HOME

format/Format.py

@@ -299,6 +299,7 @@ def get_imageset(
         Factory method to create an imageset
 
         """
+        from dxtbx.format.image import ImageBool
         from dxtbx.imageset import ImageSetData
         from dxtbx.imageset import ImageSet
         from dxtbx.imageset import ImageSweep
@@ -446,6 +447,16 @@ def get_imageset(
                 scan=scan,
             )
 
+        if format_instance is not None:
+            static_mask = format_instance.get_static_mask()
+            if static_mask is not None:
+                if not iset.external_lookup.mask.data.empty():
+                    for m1, m2 in zip(static_mask, iset.external_lookup.mask.data):
+                        m1 &= m2.data()
+                    iset.external_lookup.mask.data = ImageBool(static_mask)
+                else:
+                    iset.external_lookup.mask.data = ImageBool(static_mask)
+
         # Return the imageset
         return iset
 
@@ -491,10 +502,8 @@ def _scan(self):
         long as the result is an scan."""
         return None
 
-    def get_mask(self, index=None, goniometer=None):
-        """Overload this method to provide dynamic masks to be used during
-        spotfinding or integration."""
-
+    def get_static_mask(self):
+        """Overload this method to override the static mask."""
         return None
 
     def get_goniometer_shadow_masker(self, goniometer=None):

format/FormatCBFFullPilatus.py

@@ -149,33 +149,6 @@ def get_raw_data(self):
 
         return self._raw_data
 
-    def get_mask(self, goniometer=None):
-        from scitbx.array_family import flex
-
-        detector = self.get_detector()
-        mask = [
-            flex.bool(flex.grid(reversed(p.get_image_size())), True) for p in detector
-        ]
-        for i, p in enumerate(detector):
-            untrusted_regions = p.get_mask()
-            for j, (f0, s0, f1, s1) in enumerate(untrusted_regions):
-                sub_array = flex.bool(flex.grid(s1 - s0, f1 - f0), False)
-                mask[i].matrix_paste_block_in_place(sub_array, s0, f0)
-
-        if len(detector) == 1:
-            raw_data = [self.get_raw_data()]
-        else:
-            raw_data = self.get_raw_data()
-            assert len(raw_data) == len(detector)
-        trusted_mask = [
-            p.get_trusted_range_mask(im) for im, p in zip(raw_data, detector)
-        ]
-
-        # returns merged untrusted pixels and active areas using bitwise AND
-        # (pixels are accepted if they are inside of the active areas AND inside
-        # of the trusted range)
-        return tuple(m & tm for m, tm in zip(mask, trusted_mask))
-
     def get_vendortype(self):
         from dxtbx.format.FormatPilatusHelpers import get_vendortype as gv
 

format/FormatCBFFullPilatusDLS300KSN104.py

@@ -55,19 +55,6 @@ def __init__(self, image_file, **kwargs):
         self._dynamic_shadowing = self.has_dynamic_shadowing(**kwargs)
         FormatCBFFullPilatus.__init__(self, image_file, **kwargs)
 
-    def get_mask(self, goniometer=None):
-        mask = super(FormatCBFFullPilatusDLS300KSN104, self).get_mask()
-        if self._dynamic_shadowing:
-            gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
-            scan = self.get_scan()
-            detector = self.get_detector()
-            shadow_mask = gonio_masker.get_mask(detector, scan.get_oscillation()[0])
-            assert len(mask) == len(shadow_mask)
-            for m, sm in zip(mask, shadow_mask):
-                if sm is not None:
-                    m &= ~sm
-        return mask
-
     def get_goniometer_shadow_masker(self, goniometer=None):
         if goniometer is None:
             goniometer = self.get_goniometer()

format/FormatCBFFullPilatusDLS6MSN100.py

@@ -51,19 +51,6 @@ def __init__(self, image_file, **kwargs):
         self._dynamic_shadowing = self.has_dynamic_shadowing(**kwargs)
         FormatCBFFullPilatus.__init__(self, image_file, **kwargs)
 
-    def get_mask(self, goniometer=None):
-        mask = super(FormatCBFFullPilatusDLS6MSN100, self).get_mask()
-        if self._dynamic_shadowing:
-            gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
-            scan = self.get_scan()
-            detector = self.get_detector()
-            shadow_mask = gonio_masker.get_mask(detector, scan.get_oscillation()[0])
-            assert len(mask) == len(shadow_mask)
-            for m, sm in zip(mask, shadow_mask):
-                if sm is not None:
-                    m &= sm
-        return mask
-
     def get_goniometer_shadow_masker(self, goniometer=None):
         if goniometer is None:
             goniometer = self.get_goniometer()

format/FormatCBFFullPilatusDLS6MSN126.py

@@ -56,23 +56,6 @@ def __init__(self, image_file, **kwargs):
         self._dynamic_shadowing = self.has_dynamic_shadowing(**kwargs)
         FormatCBFFullPilatus.__init__(self, image_file, **kwargs)
 
-    def get_mask(self, goniometer=None):
-        mask = super(FormatCBFFullPilatusDLS6MSN126, self).get_mask()
-
-        # when device was installed with SmarGon on I03 this was the signature
-        # now it lives on i02-1 => detect - at the moment device has single
-        # axis in new home
-        if len(self.get_goniometer().get_names()) == 3 and self._dynamic_shadowing:
-            gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
-            scan = self.get_scan()
-            detector = self.get_detector()
-            shadow_mask = gonio_masker.get_mask(detector, scan.get_oscillation()[0])
-            assert len(mask) == len(shadow_mask)
-            for m, sm in zip(mask, shadow_mask):
-                if sm is not None:
-                    m &= sm
-        return mask
-
     def get_goniometer_shadow_masker(self, goniometer=None):
         if goniometer is None:
             goniometer = self.get_goniometer()

format/FormatCBFMiniEiger.py

@@ -201,33 +201,6 @@ def get_raw_data(self):
 
         return self._raw_data
 
-    def get_mask(self, goniometer=None):
-        from scitbx.array_family import flex
-
-        detector = self.get_detector()
-        mask = [
-            flex.bool(flex.grid(reversed(p.get_image_size())), True) for p in detector
-        ]
-        for i, p in enumerate(detector):
-            untrusted_regions = p.get_mask()
-            for j, (f0, s0, f1, s1) in enumerate(untrusted_regions):
-                sub_array = flex.bool(flex.grid(s1 - s0, f1 - f0), False)
-                mask[i].matrix_paste_block_in_place(sub_array, s0, f0)
-
-        if len(detector) == 1:
-            raw_data = [self.get_raw_data()]
-        else:
-            raw_data = self.get_raw_data()
-            assert len(raw_data) == len(detector)
-        trusted_mask = [
-            p.get_trusted_range_mask(im) for im, p in zip(raw_data, detector)
-        ]
-
-        # returns merged untrusted pixels and active areas using bitwise AND
-        # (pixels are accepted if they are inside of the active areas AND
-        # inside of the trusted range)
-        return tuple([m & tm for m, tm in zip(mask, trusted_mask)])
-
     def detectorbase_start(self):
         from iotbx.detectors.eiger_minicbf import EigerCBFImage
 

format/FormatCBFMiniEigerDLS16MSN160.py

@@ -90,19 +90,6 @@ def _goniometer(self):
             axes, angles, names, scan_axis=2
         )
 
-    def get_mask(self, goniometer=None):
-        mask = super(FormatCBFMiniEigerDLS16MSN160, self).get_mask()
-        if self._dynamic_shadowing:
-            gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
-            scan = self.get_scan()
-            detector = self.get_detector()
-            shadow_mask = gonio_masker.get_mask(detector, scan.get_oscillation()[0])
-            assert len(mask) == len(shadow_mask)
-            for m, sm in zip(mask, shadow_mask):
-                if sm is not None:
-                    m &= sm
-        return mask
-
     def get_goniometer_shadow_masker(self, goniometer=None):
         if goniometer is None:
             goniometer = self.get_goniometer()

format/FormatCBFMiniPilatus.py

@@ -93,32 +93,6 @@ def _scan(self):
             self._image_file, format, exposure_time, osc_start, osc_range, timestamp
         )
 
-    def get_mask(self, goniometer=None):
-        from scitbx.array_family import flex
-
-        detector = self.get_detector()
-        mask = [
-            flex.bool(flex.grid(reversed(p.get_image_size())), True) for p in detector
-        ]
-        for i, p in enumerate(detector):
-            untrusted_regions = p.get_mask()
-            for j, (f0, s0, f1, s1) in enumerate(untrusted_regions):
-                sub_array = flex.bool(flex.grid(s1 - s0, f1 - f0), False)
-                mask[i].matrix_paste_block_in_place(sub_array, s0, f0)
-
-        if len(detector) == 1:
-            raw_data = [self.get_raw_data()]
-        else:
-            raw_data = self.get_raw_data()
-            assert len(raw_data) == len(detector)
-        trusted_mask = [
-            p.get_trusted_range_mask(im) for im, p in zip(raw_data, detector)
-        ]
-
-        # returns merged untrusted pixels and active areas using bitwise AND (pixels are accepted
-        # if they are inside of the active areas AND inside of the trusted range)
-        return tuple([m & tm for m, tm in zip(mask, trusted_mask)])
-
     def get_vendortype(self):
         from dxtbx.format.FormatPilatusHelpers import get_vendortype as gv
 

format/FormatCBFMiniPilatusDLS12M.py

@@ -211,24 +211,6 @@ def get_goniometer_shadow_masker(self, goniometer=None):
             goniometer = self.get_goniometer()
         return GoniometerMaskerFactory.dls_i23_kappa(goniometer)
 
-    def get_mask(self, goniometer=None):
-        from dxtbx.model import MultiAxisGoniometer
-
-        mask = super(FormatCBFMiniPilatusDLS12M, self).get_mask()
-        if (
-            isinstance(self.get_goniometer(), MultiAxisGoniometer)
-            and self._dynamic_shadowing
-        ):
-            gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
-            scan = self.get_scan()
-            detector = self.get_detector()
-            shadow_mask = gonio_masker.get_mask(detector, scan.get_oscillation()[0])
-            assert len(mask) == len(shadow_mask)
-            for m, sm in zip(mask, shadow_mask):
-                if sm is not None:
-                    m &= sm
-        return mask
-
     def _goniometer(self):
         """Return a model for a simple single-axis goniometer. This should
         probably be checked against the image header."""

format/FormatCBFMiniPilatusDLS6MSN100.py

@@ -357,19 +357,6 @@ def _detector(self):
     def get_goniometer_shadow_masker(self, goniometer=None):
         return GoniometerMaskerFactory.mini_kappa(goniometer)
 
-    def get_mask(self, goniometer=None):
-        mask = super(FormatCBFMiniPilatusDLS6MSN100, self).get_mask()
-        if self._dynamic_shadowing:
-            gonio_masker = self.get_goniometer_shadow_masker(goniometer=goniometer)
-            scan = self.get_scan()
-            detector = self.get_detector()
-            shadow_mask = gonio_masker.get_mask(detector, scan.get_oscillation()[0])
-            assert len(mask) == len(shadow_mask)
-            for m, sm in zip(mask, shadow_mask):
-                if sm is not None:
-                    m &= sm
-        return mask
-
     def _read_cbf_image(self):
         from cbflib_adaptbx import uncompress
         import binascii

format/FormatHDF5SaclaMPCCD.py

@@ -143,7 +143,7 @@ def read_metadata(self):
             panel_origins = list(zip(posx, posy, posz))
             sensor = h5_handle["metadata/sensor_id"][0]
             thickness = 0.050
-            if sensor.startswith("MPCCD-8B"):
+            if sensor.startswith(b"MPCCD-8B"):
                 thickness = 0.300  # Phase 3 sensor
             orig_mask = numpy.logical_not(h5_handle["metadata/pixelmask"][()])
             mask = self.split_panels(orig_mask, bool=True)
@@ -323,11 +323,11 @@ def reconst_image(self):
                         )
                         if abs(round(self.panel_rotations[i]) + 90) < 1:
                             det[
-                                round(subpanel_origin[1]) : round(
-                                    subpanel_origin[1] + size_slow
+                                int(round(subpanel_origin[1])) : int(
+                                    round(subpanel_origin[1] + size_slow)
                                 ),
-                                round(subpanel_origin[0]) : round(
-                                    subpanel_origin[0] + size_fast
+                                int(round(subpanel_origin[0])) : int(
+                                    round(subpanel_origin[0] + size_fast)
                                 ),
                             ] = source
                             # TODO: Is the border inclusive?
@@ -341,11 +341,11 @@ def reconst_image(self):
                             )
                         elif abs(round(self.panel_rotations[i]) - 90) < 1:
                             det[
-                                round(subpanel_origin[1]) : round(
-                                    subpanel_origin[1] - size_slow
+                                int(round(subpanel_origin[1])) : int(
+                                    round(subpanel_origin[1] - size_slow)
                                 ) : -1,
-                                round(subpanel_origin[0]) : round(
-                                    subpanel_origin[0] - size_fast
+                                int(round(subpanel_origin[0])) : int(
+                                    round(subpanel_origin[0] - size_fast)
                                 ) : -1,
                             ] = source
                             self.active_areas.extend(
@@ -397,7 +397,7 @@ def get_detector(self, index=None):
 
         return self._detector_instance
 
-    def get_mask(self, index=None, goniometer=None):
+    def get_static_mask(self):
         # This means when the pixel mask is present, trusted region is ignored.
         # The used provided masks (if any) will be automatically merged.
         # see https://github.com/dials/dials/issues/236

format/FormatHDF5SaclaRayonix.py

@@ -130,7 +130,7 @@ def get_detector(self, index=None):
 
         return self._detector_instance
 
-    def get_mask(self, index=None, goniometer=None):
+    def get_static_mask(self):
         # This means when the pixel mask is present, trusted region is ignored.
         # The used provided masks (if any) will be automatically merged.
         # see https://github.com/dials/dials/issues/236

format/FormatMultiImage.py

@@ -80,9 +80,6 @@ def get_scan(self, index=None):
     def get_raw_data(self, index=None):
         raise NotImplementedError
 
-    def get_mask(self, index=None, goniometer=None):
-        return None
-
     def get_detectorbase(self, index=None):
         raise NotImplementedError
 
@@ -131,6 +128,7 @@ def get_imageset(
         Factory method to create an imageset
 
         """
+        from dxtbx.format.image import ImageBool
         from dxtbx.imageset import ImageSetData
         from dxtbx.imageset import ImageSweep
 
@@ -320,5 +318,15 @@ def get_imageset(
                 indices=single_file_indices,
             )
 
+        if format_instance is not None:
+            static_mask = format_instance.get_static_mask()
+            if static_mask is not None:
+                if not iset.external_lookup.mask.data.empty():
+                    for m1, m2 in zip(static_mask, iset.external_lookup.mask.data):
+                        m1 &= m2.data()
+                    iset.external_lookup.mask.data = ImageBool(static_mask)
+                else:
+                    iset.external_lookup.mask.data = ImageBool(static_mask)
+
         # Return the imageset
         return iset