diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index 9240d6e..2e781f2 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -127,6 +127,9 @@ def matthews_correlation_coefficient(self): """ Calculates the multiclass Matthews Correlation Coefficient defined as + Brian W Matthews. 1975. Comparison of the predicted and observed secondary structure of T4 phage lysozyme. + Biochimica et Biophysica Acta (BBA)-Protein Structure 405, 2 (1975), 442–451. + .. math:: R_k = \dfrac{cov_k(Pred,Ref)}{\sqrt{cov_k(Pred,Pred)*cov_k(Ref,Ref)}} @@ -475,6 +478,9 @@ def balanced_accuracy(self): Calculates and returns the balanced accuracy defined for the binary case as the average between sensitivity and specificity + Margherita Grandini, Enrico Bagli, and Giorgio Visani. 2020. Metrics for multi-class classification: an overview. arXiv + preprint arXiv:2008.05756 (2020). + :return: balanced accuracy """ return 0.5 * self.sensitivity() + 0.5 * self.specificity() @@ -483,6 +489,9 @@ def accuracy(self): """ Calculate and returns the accuracy defined as + Margherita Grandini, Enrico Bagli, and Giorgio Visani. 2020. Metrics for multi-class classification: an overview. arXiv + preprint arXiv:2008.05756 (2020). + .. math:: Acc = \dfrac{TN+TP}{TN+TP+FN+FP} @@ -504,6 +513,14 @@ def false_positive_rate(self): return self.fp() / self.n_neg_ref() def normalised_expected_cost(self): + """ + Calculates and returns the normalised expected cost + + Luciana Ferrer. 2022. Analysis and Comparison of Classification Metrics. arXiv preprint arXiv:2209.05355 (2022). + + :return: normalised expected cost + """ + prior_background = (self.tn() + self.fp()) / (np.size(self.ref)) prior_foreground = (self.tp() + self.fn()) / np.size(self.ref) @@ -588,6 +605,9 @@ def positive_likelihood_ratio(self): """ Calculates the positive likelihood ratio + John Attia. 2003. Moving beyond sensitivity and specificity: using likelihood ratios to help interpret diagnostic tests. + Australian prescriber 26, 5 (2003), 111–113. + .. math:: LR+ = \dfrac{Sensitivity}{1-Specificity} @@ -652,6 +672,8 @@ def dsc(self): """ Calculates the Dice Similarity Coefficient defined as + Lee R Dice. 1945. Measures of the amount of ecologic association between species. Ecology 26, 3 (1945), 297–302. + ..math:: DSC = \dfrac{2TP}{2TP+FP+FN} @@ -671,6 +693,10 @@ def fbeta(self): """ Calculates FBeta score defined as + Nancy Chinchor. 1992. MUC-4 Evaluation Metrics. In Proceedings of the 4th Conference on Message Understanding + (McLean, Virginia) (MUC4 ’92). Association for Computational Linguistics, USA, 22–29. https://doi.org/10.3115/ + 1072064.1072067 + .. math:: F_{\\beta} = (1+{\\beta}^2) \dfrac{Precision * Recall}{{\\beta}^2 * Precision + recall} @@ -709,6 +735,9 @@ def net_benefit_treated(self): """ This functions calculates the net benefit treated according to a specified exchange rate + Andrew J Vickers, Ben Van Calster, and Ewout W Steyerberg. 2016. Net benefit approaches to the evaluation of + prediction models, molecular markers, and diagnostic tests. bmj 352 (2016). + .. math:: NB = \dfrac{TP}{N} - \dfrac{FP}{N} * ER @@ -915,6 +944,10 @@ def centreline_dsc(self): """ Calculates the centre line dice score defined as + Suprosanna Shit, Johannes C Paetzold, Anjany Sekuboyina, Ivan Ezhov, Alexander Unger, Andrey Zhylka, Josien PW + Pluim, Ulrich Bauer, and Bjoern H Menze. 2021. clDice-a novel topology-preserving loss function for tubular structure + segmentation. In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 16560–16569 + .. math:: cDSC = 2\dfrac{Sens_{Top} * Prec_{Top}}{Sens_{Top} + Prec_{Top}} @@ -1005,6 +1038,11 @@ def normalised_surface_distance(self): Calculates the normalised surface distance (NSD) between prediction and reference using the distance parameter :math:`{\\tau}` + Stanislav Nikolov, Sam Blackwell, Alexei Zverovitch, Ruheena Mendes, Michelle Livne, Jeffrey De Fauw, Yojan Patel, + Clemens Meyer, Harry Askham, Bernadino Romera-Paredes, et al. 2021. Clinically applicable segmentation of head + and neck anatomy for radiotherapy: deep learning algorithm development and validation study. Journal of Medical + Internet Research 23, 7 (2021), e26151. + .. math:: NSD(A,B)^{(\\tau)} = \dfrac{|S_{A} \cap Bord_{B,\\tau}| + |S_{B} \cup Bord_{A,\\tau}|}{|S_{A}| + S_{B}} @@ -1095,6 +1133,9 @@ def measured_average_distance(self): def measured_masd(self): """ This function returns only the mean average surface distance defined as + + Miroslav Beneš and Barbara Zitová. 2015. Performance evaluation of image segmentation algorithms on microscopic + image data. Journal of microscopy 257, 1 (2015), 65–85. .. math:: @@ -1109,6 +1150,9 @@ def measured_hausdorff_distance(self): This function returns only the hausdorff distance when calculated the distances between prediction and reference + Daniel P Huttenlocher, Gregory A. Klanderman, and William J Rucklidge. 1993. Comparing images using the Hausdorff + distance. IEEE Transactions on pattern analysis and machine intelligence 15, 9 (1993), 850–863. + :return: hausdorff_distance """ return self.measured_distance()[0] @@ -1117,6 +1161,9 @@ def measured_hausdorff_distance_perc(self): """ This function returns the xth percentile hausdorff distance + Daniel P Huttenlocher, Gregory A. Klanderman, and William J Rucklidge. 1993. Comparing images using the Hausdorff + distance. IEEE Transactions on pattern analysis and machine intelligence 15, 9 (1993), 850–863. + :return: hausdorff_distance_perc """ return self.measured_distance()[2] diff --git a/MetricsReloaded/metrics/prob_pairwise_measures.py b/MetricsReloaded/metrics/prob_pairwise_measures.py index c745a78..fbcd991 100644 --- a/MetricsReloaded/metrics/prob_pairwise_measures.py +++ b/MetricsReloaded/metrics/prob_pairwise_measures.py @@ -240,6 +240,9 @@ def auroc(self): Calculation of AUROC using trapezoidal integration based on the threshold and values list obtained from the all_multi_threshold_values method + James A Hanley and Barbara J McNeil. 1982. The meaning and use of the area under a receiver operating characteristic + (ROC) curve. Radiology 143, 1 (1982), 29–36. + :return: AUC """ ( @@ -263,6 +266,11 @@ def auroc(self): def froc(self): """ Calculation of FROC score + + Bram Van Ginneken, Samuel G Armato III, Bartjan de Hoop, Saskia van Amelsvoort-van de Vorst, Thomas Duindam, + Meindert Niemeijer, Keelin Murphy, Arnold Schilham, Alessandra Retico, Maria Evelina Fantacci, et al. 2010. + Comparing and combining algorithms for computer-aided detection of pulmonary nodules in computed tomography + scans: the ANODE09 study. Medical image analysis 14, 6 (2010), 707–722. """ ( unique_thresh, @@ -309,6 +317,10 @@ def average_precision(self): Average precision calculation using trapezoidal integration. This integrates the precision as function of recall curve + Tsung-Yi Lin, Michael Maire, Serge Belongie, James Hays, Pietro Perona, Deva Ramanan, Piotr Dollár, and C Lawrence + Zitnick. 2014. Microsoft coco: Common objects in context. In European conference on computer vision. Springer, + 740–755. + :return: AP """ diff --git a/MetricsReloaded/processes/mixed_measures_processes.py b/MetricsReloaded/processes/mixed_measures_processes.py index fad9c79..c90f8be 100644 --- a/MetricsReloaded/processes/mixed_measures_processes.py +++ b/MetricsReloaded/processes/mixed_measures_processes.py @@ -109,6 +109,9 @@ def __init__( ] def segmentation_quality(self): + """ + Calculates the segmentation quality in an instance segmentation case with SQ defined as the average IoU over TP instances + """ list_iou = [] for (p, r) in zip(self.predimg, self.refimg): PE = BinaryPairwiseMeasures(p, r) @@ -117,23 +120,35 @@ def segmentation_quality(self): return np.mean(np.asarray(list_iou)) def detection_quality(self): + """ + Calculates the detection quality for a case of instance segmentation defined as the F1 score + """ PE = BinaryPairwiseMeasures(self.pred, self.ref) #print("pred is ", self.pred, "ref is ", self.ref) return PE.fbeta() def panoptic_quality(self): - print("RQ ", self.detection_quality()) + """ + Calculates the panopitic quality defined as the production between + detection quality and segmentation quality + + Alexander Kirillov, Kaiming He, Ross Girshick, Carsten Rother, and Piotr Dollár. 2019. Panoptic segmentation. In + Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 9404–9413. + + :return: PQ + """ + print("DQ ", self.detection_quality()) print("SQ ", self.segmentation_quality()) - RQ = self.detection_quality() + DQ = self.detection_quality() SQ = self.segmentation_quality() if np.isnan(SQ): - if RQ == 0: + if DQ == 0: SQ = 0 else: SQ = 1 # TODO modify to nan if this is the value adopted for empty situations - print("PQ is ", RQ * SQ, RQ, SQ) - return RQ * SQ + print("PQ is ", DQ * SQ, DQ, SQ) + return DQ * SQ def to_dict_mt(self): dict_output = self.prob_res.to_dict_meas()