From 23a5fada7f05d9fb89a9dcdd3991e8dcdaed7305 Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Tue, 10 Mar 2026 15:00:58 -0400 Subject: [PATCH 1/7] fixed the changing points calculations, added changing column calculation, and updated the log within the comparision calculations of the translate test --- ndsl/testing/comparison.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index c1a5b1ef..3144cf96 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -261,8 +261,12 @@ def __init__( # all the way down, we bail out if we can measure input vs reference if input_values is not None and input_values.shape == reference_values.shape: self.number_changing_values = ( - (input_values != reference_values).flatten().shape[0] + (input_values != reference_values).sum() ) + if len(input_values.shape) == 3: + self.changing_column_map = (input_values != reference_values).any(axis=2) + else: + self.changing_column_map = np.nan else: self.number_changing_values = None @@ -338,6 +342,14 @@ def report(self, file_path: str | None = None) -> list[str]: failed_indices = np.logical_not(self.success).nonzero() # List all errors to terminal and file bad_indices_count = len(failed_indices[0]) + if self.success.ndim == 3: + bad_column_count = (np.logical_not(self.success).any(axis=2) & self.changing_column_map).sum() + total_column_count = self.changing_column_map.sum() + bad_column_pct = round(bad_column_count/total_column_count * 100, 2) + else: + bad_column_count = np.nan + total_column_count = np.nan + bad_column_pct = np.nan full_count = len(self.references.flatten()) failures_of_all_grid_points_pct = round( 100.0 * (bad_indices_count / full_count), 2 @@ -346,12 +358,11 @@ def report(self, file_path: str | None = None) -> list[str]: failures_of_changing_gridpoint_pct = round( 100.0 * (bad_indices_count / self.number_changing_values), 2 ) - report_local_failures = f"Failures (changing grid points) ({bad_indices_count}/{self.number_changing_values}) ({failures_of_changing_gridpoint_pct}%)\n" + report_local_failures = f"changing grid points: {bad_indices_count}/{self.number_changing_values} - {failures_of_changing_gridpoint_pct}%; changing columns: {bad_column_count}/{total_column_count} - {bad_column_pct}%; all grid points: {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" else: report_local_failures = "" report = [ f"{report_local_failures}" - f"Failures (all grid points) ({bad_indices_count}/{full_count}) ({failures_of_all_grid_points_pct}%)\n", f"Index Computed Reference " f"{'🔶 ' if not self.absolute_eps.is_default else ''}Absolute E(<{self.absolute_eps.value:.2e}) " f"{'🔶 ' if not self.relative_fraction.is_default else ''}Relative E(<{self.relative_fraction.value * 100:.2e}%) " From 2bec98fd250ddb37d9ccc3538b67cf9957340788 Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Tue, 10 Mar 2026 15:05:36 -0400 Subject: [PATCH 2/7] linting --- ndsl/testing/comparison.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index 3144cf96..ed63539c 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -260,11 +260,11 @@ def __init__( # We might have sliced outputs in the translate test. Rather than funnel the slicing # all the way down, we bail out if we can measure input vs reference if input_values is not None and input_values.shape == reference_values.shape: - self.number_changing_values = ( - (input_values != reference_values).sum() - ) + self.number_changing_values = (input_values != reference_values).sum() if len(input_values.shape) == 3: - self.changing_column_map = (input_values != reference_values).any(axis=2) + self.changing_column_map = (input_values != reference_values).any( + axis=2 + ) else: self.changing_column_map = np.nan else: @@ -343,9 +343,11 @@ def report(self, file_path: str | None = None) -> list[str]: # List all errors to terminal and file bad_indices_count = len(failed_indices[0]) if self.success.ndim == 3: - bad_column_count = (np.logical_not(self.success).any(axis=2) & self.changing_column_map).sum() + bad_column_count = ( + np.logical_not(self.success).any(axis=2) & self.changing_column_map + ).sum() total_column_count = self.changing_column_map.sum() - bad_column_pct = round(bad_column_count/total_column_count * 100, 2) + bad_column_pct = round(bad_column_count / total_column_count * 100, 2) else: bad_column_count = np.nan total_column_count = np.nan From 84c1d70bd90c3c6f74769475562ac7fbe1dd5662 Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Wed, 11 Mar 2026 15:33:48 -0400 Subject: [PATCH 3/7] better error handling in cases where bad columns cannot be computed --- ndsl/testing/comparison.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index ed63539c..c0dc38d7 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -269,6 +269,7 @@ def __init__( self.changing_column_map = np.nan else: self.number_changing_values = None + self.changing_column_map = None def _compute_all_metrics( self, @@ -342,12 +343,17 @@ def report(self, file_path: str | None = None) -> list[str]: failed_indices = np.logical_not(self.success).nonzero() # List all errors to terminal and file bad_indices_count = len(failed_indices[0]) - if self.success.ndim == 3: - bad_column_count = ( - np.logical_not(self.success).any(axis=2) & self.changing_column_map - ).sum() - total_column_count = self.changing_column_map.sum() - bad_column_pct = round(bad_column_count / total_column_count * 100, 2) + if self.changing_column_map is not None: + if self.success.ndim == 3: + bad_column_count = ( + np.logical_not(self.success).any(axis=2) & self.changing_column_map + ).sum() + total_column_count = self.changing_column_map.sum() + bad_column_pct = round(bad_column_count / total_column_count * 100, 2) + else: + bad_column_count = np.nan + total_column_count = np.nan + bad_column_pct = np.nan else: bad_column_count = np.nan total_column_count = np.nan @@ -362,7 +368,7 @@ def report(self, file_path: str | None = None) -> list[str]: ) report_local_failures = f"changing grid points: {bad_indices_count}/{self.number_changing_values} - {failures_of_changing_gridpoint_pct}%; changing columns: {bad_column_count}/{total_column_count} - {bad_column_pct}%; all grid points: {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" else: - report_local_failures = "" + report_local_failures = "Something went wrong when calculating statistics one line print, see log for raw data.\n" report = [ f"{report_local_failures}" f"Index Computed Reference " From 4c6340632f75c9871638a29781dc4eae7c2c9e15 Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Fri, 13 Mar 2026 09:41:28 -0400 Subject: [PATCH 4/7] Better messaging when no input data is available --- ndsl/testing/comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index c0dc38d7..d34620ed 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -368,7 +368,7 @@ def report(self, file_path: str | None = None) -> list[str]: ) report_local_failures = f"changing grid points: {bad_indices_count}/{self.number_changing_values} - {failures_of_changing_gridpoint_pct}%; changing columns: {bad_column_count}/{total_column_count} - {bad_column_pct}%; all grid points: {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" else: - report_local_failures = "Something went wrong when calculating statistics one line print, see log for raw data.\n" + report_local_failures = f"all grid points: {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" report = [ f"{report_local_failures}" f"Index Computed Reference " From 11807915f12324c8b69652b328d2202e30cbabeb Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Thu, 19 Mar 2026 16:37:48 -0400 Subject: [PATCH 5/7] new print statement and other minor changes based on PR feedback --- ndsl/testing/comparison.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index d34620ed..6f6df78f 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -266,7 +266,7 @@ def __init__( axis=2 ) else: - self.changing_column_map = np.nan + self.changing_column_map = None else: self.number_changing_values = None self.changing_column_map = None @@ -351,13 +351,13 @@ def report(self, file_path: str | None = None) -> list[str]: total_column_count = self.changing_column_map.sum() bad_column_pct = round(bad_column_count / total_column_count * 100, 2) else: - bad_column_count = np.nan - total_column_count = np.nan - bad_column_pct = np.nan + bad_column_count = None + total_column_count = None + bad_column_pct = None else: - bad_column_count = np.nan - total_column_count = np.nan - bad_column_pct = np.nan + bad_column_count = None + total_column_count = None + bad_column_pct = None full_count = len(self.references.flatten()) failures_of_all_grid_points_pct = round( 100.0 * (bad_indices_count / full_count), 2 @@ -366,7 +366,7 @@ def report(self, file_path: str | None = None) -> list[str]: failures_of_changing_gridpoint_pct = round( 100.0 * (bad_indices_count / self.number_changing_values), 2 ) - report_local_failures = f"changing grid points: {bad_indices_count}/{self.number_changing_values} - {failures_of_changing_gridpoint_pct}%; changing columns: {bad_column_count}/{total_column_count} - {bad_column_pct}%; all grid points: {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" + report_local_failures = f"Failures: (changing columns, chainging points, all points) | {bad_column_count}/{total_column_count} - {bad_column_pct}%, {bad_indices_count}/{self.number_changing_values} - {failures_of_changing_gridpoint_pct}%, {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" else: report_local_failures = f"all grid points: {bad_indices_count}/{full_count} - {failures_of_all_grid_points_pct}%\n" report = [ From abc2ddf161f83ae366896670d44eb7e36b98a2a5 Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Fri, 20 Mar 2026 14:54:03 -0400 Subject: [PATCH 6/7] checking if column count and changing point count worked before printing one line result --- ndsl/testing/comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index 6f6df78f..0aa2830e 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -362,7 +362,7 @@ def report(self, file_path: str | None = None) -> list[str]: failures_of_all_grid_points_pct = round( 100.0 * (bad_indices_count / full_count), 2 ) - if self.number_changing_values is not None: + if self.number_changing_values is not None and bad_indices_count is not None and bad_column_count is not None: failures_of_changing_gridpoint_pct = round( 100.0 * (bad_indices_count / self.number_changing_values), 2 ) From efff68e020a7e5a71005c19df65c6cd52894b68e Mon Sep 17 00:00:00 2001 From: Charles Kropiewnicki Date: Wed, 25 Mar 2026 11:55:17 -0400 Subject: [PATCH 7/7] addded a comming + linting --- ndsl/testing/comparison.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ndsl/testing/comparison.py b/ndsl/testing/comparison.py index 0aa2830e..b5c1ac0d 100644 --- a/ndsl/testing/comparison.py +++ b/ndsl/testing/comparison.py @@ -261,6 +261,7 @@ def __init__( # all the way down, we bail out if we can measure input vs reference if input_values is not None and input_values.shape == reference_values.shape: self.number_changing_values = (input_values != reference_values).sum() + # column information is only relevant if data is three-dimensional if len(input_values.shape) == 3: self.changing_column_map = (input_values != reference_values).any( axis=2 @@ -362,7 +363,11 @@ def report(self, file_path: str | None = None) -> list[str]: failures_of_all_grid_points_pct = round( 100.0 * (bad_indices_count / full_count), 2 ) - if self.number_changing_values is not None and bad_indices_count is not None and bad_column_count is not None: + if ( + self.number_changing_values is not None + and bad_indices_count is not None + and bad_column_count is not None + ): failures_of_changing_gridpoint_pct = round( 100.0 * (bad_indices_count / self.number_changing_values), 2 )