Inclusion of references to definitions in documentation

MetricsReloaded/metrics/pairwise_measures.py

@@ -127,6 +127,9 @@ def matthews_correlation_coefficient(self):
         """
         Calculates the multiclass Matthews Correlation Coefficient defined as
 
+        Brian W Matthews. 1975. Comparison of the predicted and observed secondary structure of T4 phage lysozyme.
+        Biochimica et Biophysica Acta (BBA)-Protein Structure 405, 2 (1975), 442–451.
+
         .. math::
 
             R_k = \dfrac{cov_k(Pred,Ref)}{\sqrt{cov_k(Pred,Pred)*cov_k(Ref,Ref)}}
@@ -475,6 +478,9 @@ def balanced_accuracy(self):
         Calculates and returns the balanced accuracy defined for the
         binary case as the average between sensitivity and specificity
 
+        Margherita Grandini, Enrico Bagli, and Giorgio Visani. 2020. Metrics for multi-class classification: an overview. arXiv
+        preprint arXiv:2008.05756 (2020).
+
         :return: balanced accuracy
         """
         return 0.5 * self.sensitivity() + 0.5 * self.specificity()
@@ -483,6 +489,9 @@ def accuracy(self):
         """
         Calculate and returns the accuracy defined as
 
+        Margherita Grandini, Enrico Bagli, and Giorgio Visani. 2020. Metrics for multi-class classification: an overview. arXiv
+        preprint arXiv:2008.05756 (2020).
+
         .. math::
 
             Acc = \dfrac{TN+TP}{TN+TP+FN+FP}
@@ -504,6 +513,14 @@ def false_positive_rate(self):
         return self.fp() / self.n_neg_ref()
 
     def normalised_expected_cost(self):
+        """
+        Calculates and returns the normalised expected cost
+
+        Luciana Ferrer. 2022. Analysis and Comparison of Classification Metrics. arXiv preprint arXiv:2209.05355 (2022).
+
+        :return: normalised expected cost
+        """
+
         prior_background = (self.tn() + self.fp()) / (np.size(self.ref))
         prior_foreground = (self.tp() + self.fn()) / np.size(self.ref)
 
@@ -588,6 +605,9 @@ def positive_likelihood_ratio(self):
         """
         Calculates the positive likelihood ratio
 
+        John Attia. 2003. Moving beyond sensitivity and specificity: using likelihood ratios to help interpret diagnostic tests.
+        Australian prescriber 26, 5 (2003), 111–113.
+
         .. math::
 
             LR+ = \dfrac{Sensitivity}{1-Specificity}
@@ -652,6 +672,8 @@ def dsc(self):
         """
         Calculates the Dice Similarity Coefficient defined as
 
+        Lee R Dice. 1945. Measures of the amount of ecologic association between species. Ecology 26, 3 (1945), 297–302.
+
         ..math::
 
             DSC = \dfrac{2TP}{2TP+FP+FN}
@@ -671,6 +693,10 @@ def fbeta(self):
         """
         Calculates FBeta score defined as
 
+        Nancy Chinchor. 1992. MUC-4 Evaluation Metrics. In Proceedings of the 4th Conference on Message Understanding
+        (McLean, Virginia) (MUC4 ’92). Association for Computational Linguistics, USA, 22–29. https://doi.org/10.3115/
+        1072064.1072067
+
         .. math::
 
             F_{\\beta} = (1+{\\beta}^2) \dfrac{Precision  * Recall}{{\\beta}^2 * Precision + recall}
@@ -709,6 +735,9 @@ def net_benefit_treated(self):
         """
         This functions calculates the net benefit treated according to a specified exchange rate
 
+        Andrew J Vickers, Ben Van Calster, and Ewout W Steyerberg. 2016. Net benefit approaches to the evaluation of
+        prediction models, molecular markers, and diagnostic tests. bmj 352 (2016).
+
         .. math::
 
             NB = \dfrac{TP}{N} - \dfrac{FP}{N} * ER
@@ -915,6 +944,10 @@ def centreline_dsc(self):
         """
         Calculates the centre line dice score defined as
 
+        Suprosanna Shit, Johannes C Paetzold, Anjany Sekuboyina, Ivan Ezhov, Alexander Unger, Andrey Zhylka, Josien PW
+        Pluim, Ulrich Bauer, and Bjoern H Menze. 2021. clDice-a novel topology-preserving loss function for tubular structure
+        segmentation. In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 16560–16569
+
         .. math::
 
             cDSC = 2\dfrac{Sens_{Top} * Prec_{Top}}{Sens_{Top} + Prec_{Top}}
@@ -1005,6 +1038,11 @@ def normalised_surface_distance(self):
         Calculates the normalised surface distance (NSD) between prediction and reference
         using the distance parameter :math:`{\\tau}`
 
+        Stanislav Nikolov, Sam Blackwell, Alexei Zverovitch, Ruheena Mendes, Michelle Livne, Jeffrey De Fauw, Yojan Patel,
+        Clemens Meyer, Harry Askham, Bernadino Romera-Paredes, et al. 2021. Clinically applicable segmentation of head
+        and neck anatomy for radiotherapy: deep learning algorithm development and validation study. Journal of Medical
+        Internet Research 23, 7 (2021), e26151.
+
         .. math::
 
             NSD(A,B)^{(\\tau)} = \dfrac{|S_{A} \cap Bord_{B,\\tau}| + |S_{B} \cup Bord_{A,\\tau}|}{|S_{A}| + S_{B}}
@@ -1095,6 +1133,9 @@ def measured_average_distance(self):
     def measured_masd(self):
         """
         This function returns only the mean average surface distance defined as
+        
+        Miroslav Beneš and Barbara Zitová. 2015. Performance evaluation of image segmentation algorithms on microscopic
+        image data. Journal of microscopy 257, 1 (2015), 65–85.
 
         .. math::
 
@@ -1109,6 +1150,9 @@ def measured_hausdorff_distance(self):
         This function returns only the hausdorff distance when calculated the
         distances between prediction and reference
 
+        Daniel P Huttenlocher, Gregory A. Klanderman, and William J Rucklidge. 1993. Comparing images using the Hausdorff
+        distance. IEEE Transactions on pattern analysis and machine intelligence 15, 9 (1993), 850–863.
+
         :return: hausdorff_distance
         """
         return self.measured_distance()[0]
@@ -1117,6 +1161,9 @@ def measured_hausdorff_distance_perc(self):
         """
         This function returns the xth percentile hausdorff distance
 
+        Daniel P Huttenlocher, Gregory A. Klanderman, and William J Rucklidge. 1993. Comparing images using the Hausdorff
+        distance. IEEE Transactions on pattern analysis and machine intelligence 15, 9 (1993), 850–863.
+        
         :return: hausdorff_distance_perc
         """
         return self.measured_distance()[2]

MetricsReloaded/metrics/prob_pairwise_measures.py

@@ -240,6 +240,9 @@ def auroc(self):
         Calculation of AUROC using trapezoidal integration based
          on the threshold and values list obtained from the all_multi_threshold_values method
 
+        James A Hanley and Barbara J McNeil. 1982. The meaning and use of the area under a receiver operating characteristic
+        (ROC) curve. Radiology 143, 1 (1982), 29–36.
+
         :return: AUC
         """
         (
@@ -263,6 +266,11 @@ def auroc(self):
     def froc(self):
         """
         Calculation of FROC score
+
+        Bram Van Ginneken, Samuel G Armato III, Bartjan de Hoop, Saskia van Amelsvoort-van de Vorst, Thomas Duindam,
+        Meindert Niemeijer, Keelin Murphy, Arnold Schilham, Alessandra Retico, Maria Evelina Fantacci, et al. 2010.
+        Comparing and combining algorithms for computer-aided detection of pulmonary nodules in computed tomography
+        scans: the ANODE09 study. Medical image analysis 14, 6 (2010), 707–722.
         """
         (
             unique_thresh,
@@ -309,6 +317,10 @@ def average_precision(self):
         Average precision calculation using trapezoidal integration. This integrates
         the precision as function of recall curve
 
+        Tsung-Yi Lin, Michael Maire, Serge Belongie, James Hays, Pietro Perona, Deva Ramanan, Piotr Dollár, and C Lawrence
+        Zitnick. 2014. Microsoft coco: Common objects in context. In European conference on computer vision. Springer,
+        740–755.
+
         :return: AP
 
         """

MetricsReloaded/processes/mixed_measures_processes.py

@@ -109,6 +109,9 @@ def __init__(
         ]
 
     def segmentation_quality(self):
+        """
+        Calculates the segmentation quality in an instance segmentation case with SQ defined as the average IoU over TP instances
+        """
         list_iou = []
         for (p, r) in zip(self.predimg, self.refimg):
             PE = BinaryPairwiseMeasures(p, r)
@@ -117,23 +120,35 @@ def segmentation_quality(self):
         return np.mean(np.asarray(list_iou))
 
     def detection_quality(self):
+        """
+        Calculates the detection quality for a case of instance segmentation defined as the F1 score
+        """
         PE = BinaryPairwiseMeasures(self.pred, self.ref)
         #print("pred is ", self.pred, "ref is ", self.ref)
         return PE.fbeta()
 
     def panoptic_quality(self):
-        print("RQ ", self.detection_quality())
+        """
+        Calculates the panopitic quality defined as the production between
+        detection quality and segmentation quality
+
+        Alexander Kirillov, Kaiming He, Ross Girshick, Carsten Rother, and Piotr Dollár. 2019. Panoptic segmentation. In
+        Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 9404–9413.
+
+        :return: PQ
+        """
+        print("DQ ", self.detection_quality())
         print("SQ ", self.segmentation_quality())
-        RQ = self.detection_quality()
+        DQ = self.detection_quality()
         SQ = self.segmentation_quality()
         if np.isnan(SQ):
-            if RQ == 0:
+            if DQ == 0:
                 SQ = 0
             else:
                 SQ = 1
                 # TODO modify to nan if this is the value adopted for empty situations
-        print("PQ is ", RQ * SQ, RQ, SQ)
-        return RQ * SQ
+        print("PQ is ", DQ * SQ, DQ, SQ)
+        return DQ * SQ
 
     def to_dict_mt(self):
         dict_output = self.prob_res.to_dict_meas()