Fix long evaluation time on microscopy images (ivadomed#1081)

* config_default.json: Add object_detection_metrics in evaluation params

* config_microscopy.json: Set eval param "object_detection_metrics": false

* evaluate.py: Add object_detection_metrics attribute to Evaluation3DMetrics

* configuration_file.rst: Add object_detection_metrics evaluation param

* Add object detection metrics param to microscopy tuto

* Update configuration_file.rst

* Typo and wording

* evaluation.py: painted not saved when object_detection_metrics false

* Skip Hausdorff metrics in testing when object_detection_metrics false

* Revert changes to keep hausdorff_score with seg task

* Don't compute Hausdorff score if object_detection_metrics is false at
evaluation time

* Update doc and microscopy tutorial

docs/source/configuration_file.rst

@@ -2278,8 +2278,28 @@ Postprocessing
 
 Evaluation Parameters
 ---------------------
-Dict. Parameters to get object detection metrics (true positive and false detection rates), and this, for defined
-object sizes.
+Dict. Parameters to get object detection metrics (lesions true positive rate, lesions false detection rate
+and Hausdorff score), and this, for defined object sizes.
+
+.. jsonschema::
+
+    {
+        "$schema": "http://json-schema.org/draft-04/schema#",
+        "title": "object_detection_metrics",
+        "$$description": [
+            "Indicate if object detection metrics (lesions true positive rate, lesions false detection rate\n",
+            "and Hausdorff score) are computed or not at evaluation time. Default: ``true``",
+        ],
+        "type": "boolean"
+    }
+
+.. code-block:: JSON
+
+    {
+        "evaluation_parameters": {
+            "object_detection_metrics": true
+        }
+    }
 
 
 .. jsonschema::
@@ -2295,7 +2315,8 @@ object sizes.
                     "These values will create several consecutive target size bins. For instance\n",
                     "with a list of two values, we will have three target size bins: minimal size\n",
                     "to first list element, first list element to second list element, and second\n",
-                    "list element to infinity. Default: ``[20, 100]``."
+                    "list element to infinity. Default: ``[20, 100]``.\n",
+                    "``object_detection_metrics`` must be ``true`` for the target_size to apply."
                 ]
             },
             "unit": {
@@ -2329,7 +2350,10 @@ object sizes.
         "options": {
             "thr": {
                 "type": "int",
-                "description": "Minimal object size overlapping to be considered a TP, FP, or FN. Default: ``3``."
+                "$$description": [
+                    "Minimal object size overlapping to be considered a TP, FP, or FN. Default: ``3``.\n",
+                    "``object_detection_metrics`` must be ``true`` for the overlap to apply."
+                ]
             },
             "unit": {
                 "type": "string",

docs/source/tutorials/two_class_microscopy_seg_2d_unet.rst

@@ -114,9 +114,9 @@ microscopy segmentation training.
       "train_fraction": 0.6
       "test_fraction": 0.1
 
-- ``training_parameters:training_time:num_epochs``: The maximum number of epochs that will be run during training. Each epoch is composed
-  of a training part and a validation part. It should be a strictly positive integer. In our case, we will use
-  50 epochs.
+- ``training_parameters:training_time:num_epochs``: The maximum number of epochs that will be run during training.
+  Each epoch is composed of a training part and a validation part. It should be a strictly positive integer.
+  In our case, we will use 50 epochs.
 
   .. code-block:: xml
 
@@ -132,12 +132,21 @@ microscopy segmentation training.
      "length_2D": [256, 256]
      "stride_2D": [244, 244]
 
-- ``postprocessing:binarize_maxpooling``: Used to binarize predictions across all classes in multiclass models. For each pixel, the class, including the background class, with the highest output probability will be segmented.
+- ``postprocessing:binarize_maxpooling``: Used to binarize predictions across all classes in multiclass models.
+  For each pixel, the class, including the background class, with the highest output probability will be segmented.
 
   .. code-block:: xml
 
       "binarize_maxpooling": {}
 
+- ``evaluation_parameters:object_detection_metrics``: Used to indicate if object detection metrics
+  (lesions true positive rate, lesions false detection rate and Hausdorff score) are computed or
+  not at evaluation time. For the axons and myelin segmentation task, we set this parameter to ``false``.
+
+  .. code-block:: xml
+
+      "object_detection_metrics": false
+
 - ``transformation:Resample``: Used to resample images to a common resolution (in mm) before splitting into patches,
   according to each image real pixel size. In our case, we resample the images to a common resolution of 0.0001 mm
   (0.1 μm) in both dimensions.

ivadomed/config/config_default.json

@@ -95,6 +95,7 @@
     },
     "postprocessing": {},
     "evaluation_parameters": {
+        "object_detection_metrics": true
     },
     "transformation": {
     }

ivadomed/config/config_microscopy.json

@@ -91,6 +91,9 @@
     "postprocessing": {
         "binarize_maxpooling": {}
     },
+    "evaluation_parameters": {
+        "object_detection_metrics": false
+    },
     "transformation": {
         "Resample":
         {

ivadomed/evaluation.py

@@ -86,24 +86,25 @@ def evaluate(bids_df, path_output, target_suffix, eval_params):
                                    params=eval_params)
         results_pred, data_painted = eval.run_eval()
 
-        # SAVE PAINTED DATA, TP FP FN
-        fname_paint = str(fname_pred).split('.nii.gz')[0] + '_painted.nii.gz'
-        nib_painted = nib.Nifti1Image(
-            dataobj=data_painted,
-            affine=nib_pred.header.get_best_affine(),
-            header=nib_pred.header.copy()
-        )
-        nib.save(nib_painted, fname_paint)
-
-        # For Microscopy PNG/TIF files (TODO: implement OMETIFF behavior)
-        if "nii" not in extension:
-            painted_list = imed_inference.split_classes(nib_painted)
-            # Reformat target list to include class index and be compatible with multiple raters
-            target_list = ["_class-%d" % i for i in range(len(target_suffix))]
-            imed_inference.pred_to_png(painted_list,
-                                       target_list,
-                                       str(path_preds.joinpath(subj_acq)),
-                                       suffix="_pred_painted.png")
+        if eval_params['object_detection_metrics']:
+            # SAVE PAINTED DATA, TP FP FN
+            fname_paint = str(fname_pred).split('.nii.gz')[0] + '_painted.nii.gz'
+            nib_painted = nib.Nifti1Image(
+                dataobj=data_painted,
+                affine=nib_pred.header.get_best_affine(),
+                header=nib_pred.header.copy()
+            )
+            nib.save(nib_painted, fname_paint)
+
+            # For Microscopy PNG/TIF files (TODO: implement OMETIFF behavior)
+            if "nii" not in extension:
+                painted_list = imed_inference.split_classes(nib_painted)
+                # Reformat target list to include class index and be compatible with multiple raters
+                target_list = ["_class-%d" % i for i in range(len(target_suffix))]
+                imed_inference.pred_to_png(painted_list,
+                                           target_list,
+                                           str(path_preds.joinpath(subj_acq)),
+                                           suffix="_pred_painted.png")
 
         # SAVE RESULTS FOR THIS PRED
         results_pred['image_id'] = subj_acq
@@ -135,6 +136,8 @@ class Evaluation3DMetrics(object):
         bin_struct (ndarray): Binary structure.
         size_min (int): Minimum size of objects. Objects that are smaller than this limit can be removed if
             "removeSmall" is in params.
+        object_detection_metrics (bool): Indicate if object detection metrics (lesions true positive and false detection
+            rates) are computed or not.
         overlap_vox (int): A prediction and ground-truth are considered as overlapping if they overlap for at least this
             amount of voxels.
         overlap_ratio (float): A prediction and ground-truth are considered as overlapping if they overlap for at least
@@ -167,6 +170,8 @@ def __init__(self, data_pred, data_gt, dim_lst, params=None):
         self.postprocessing_dict = {}
         self.size_min = 0
 
+        self.object_detection_metrics = params["object_detection_metrics"]
+
         if "target_size" in params:
             self.size_rng_lst, self.size_suffix_lst = \
                 self._get_size_ranges(thr_lst=params["target_size"]["thr"],
@@ -389,8 +394,12 @@ def get_ltpr(self, label_size=None, class_idx=0):
             label_size (int): Size of label.
             class_idx (int): Label index. If monolabel 0, else ranges from 0 to number of output channels - 1.
 
-        Note: computed only if n_obj >= 1.
+        Note: computed only if n_obj >= 1 and "object_detection_metrics" evaluation parameter is True.
         """
+        if not self.object_detection_metrics:
+            n_obj = 0
+            return np.nan, n_obj
+
         ltp, lfn, n_obj = self._get_ltp_lfn(label_size, class_idx)
 
         denom = ltp + lfn
@@ -406,8 +415,12 @@ def get_lfdr(self, label_size=None, class_idx=0):
             label_size (int): Size of label.
             class_idx (int): Label index. If monolabel 0, else ranges from 0 to number of output channels - 1.
 
-        Note: computed only if n_obj >= 1.
+        Note: computed only if n_obj >= 1 and "object_detection_metrics" evaluation parameter is True.
         """
+
+        if not self.object_detection_metrics:
+            return np.nan
+
         ltp, _, n_obj = self._get_ltp_lfn(label_size, class_idx)
         lfp = self._get_lfp(label_size, class_idx)
 

ivadomed/main.py

@@ -517,7 +517,8 @@ def run_command(context, n_gif=0, thr_increment=None, resume_training=False, no_
                                                                            'requires_undo': True}}, device=device,
                                            cuda_available=cuda_available)
 
-        metric_fns = imed_metrics.get_metric_fns(ds_test.task)
+        eval_params = context[ConfigKW.EVALUATION_PARAMETERS]
+        metric_fns = imed_metrics.get_metric_fns(ds_test.task, eval_params)
 
         if ModelParamsKW.FILM_LAYERS in model_params and any(model_params[ModelParamsKW.FILM_LAYERS]):
             ds_test, model_params = update_film_model_params(context, ds_test, model_params, path_output)
@@ -535,7 +536,7 @@ def run_command(context, n_gif=0, thr_increment=None, resume_training=False, no_
         # RUN EVALUATION
         df_results = imed_evaluation.evaluate(bids_df, path_output=path_output,
                                               target_suffix=loader_params[LoaderParamsKW.TARGET_SUFFIX],
-                                              eval_params=context[ConfigKW.EVALUATION_PARAMETERS])
+                                              eval_params=eval_params)
         return df_results, pred_metrics
 
 

ivadomed/metrics.py

@@ -6,15 +6,19 @@
 
 
 # METRICS
-def get_metric_fns(task):
+def get_metric_fns(task, eval_params=None):
     metric_fns = [dice_score,
                   multi_class_dice_score,
                   precision_score,
                   recall_score,
                   specificity_score,
                   intersection_over_union,
                   accuracy_score]
-    if task == "segmentation":
+    if eval_params:
+        object_detection_metrics = eval_params['object_detection_metrics']
+    else:
+        object_detection_metrics = True
+    if task == "segmentation" and object_detection_metrics:
         metric_fns = metric_fns + [hausdorff_score]
 
     return metric_fns