diff --git a/improver/calibration/dz_rescaling.py b/improver/calibration/dz_rescaling.py index aafa21d3fa..4f047c70df 100644 --- a/improver/calibration/dz_rescaling.py +++ b/improver/calibration/dz_rescaling.py @@ -326,6 +326,7 @@ def _check_mismatched_sites(self, forecast: Cube, scaled_dz: Cube) -> None: mismatched_sites = set( forecast.coord(self.site_id_coord).points ).symmetric_difference(scaled_dz.coord(self.site_id_coord).points) + mismatched_sites = set(map(str, mismatched_sites)) n_forecast_sites = len(forecast.coord(self.site_id_coord).points) n_scaled_dz_sites = len(scaled_dz.coord(self.site_id_coord).points) msg = ( diff --git a/improver/calibration/reliability_calibration.py b/improver/calibration/reliability_calibration.py index 85418470ac..9e2ce88f8b 100644 --- a/improver/calibration/reliability_calibration.py +++ b/improver/calibration/reliability_calibration.py @@ -251,9 +251,10 @@ def _get_coords_and_dims(coord_names: List[str]) -> List[Tuple[DimCoord, int]]: # Define reliability table specific coordinates probability_bins_coord = self._create_probability_bins_coord() - (reliability_index_coord, reliability_name_coord) = ( - self._create_reliability_table_coords() - ) + ( + reliability_index_coord, + reliability_name_coord, + ) = self._create_reliability_table_coords() frt_coord = create_unified_frt_coord(forecast.coord("forecast_reference_time")) # List of required non-spatial coordinates from the forecast @@ -1306,9 +1307,10 @@ def _apply_calibration( reliability_threshold = self._extract_matching_reliability_table( forecast_threshold, reliability_table ) - (reliability_probabilities, observation_frequencies) = ( - self._calculate_reliability_probabilities(reliability_threshold) - ) + ( + reliability_probabilities, + observation_frequencies, + ) = self._calculate_reliability_probabilities(reliability_threshold) if reliability_probabilities is None: calibrated_cubes.append(forecast_threshold) @@ -1329,6 +1331,7 @@ def _apply_calibration( self._ensure_monotonicity_across_thresholds(calibrated_forecast) if uncalibrated_thresholds: + uncalibrated_thresholds = list(map(float, uncalibrated_thresholds)) msg = ( "The following thresholds were not calibrated due to " "insufficient forecast counts in reliability table bins: " diff --git a/improver/calibration/utilities.py b/improver/calibration/utilities.py index 6e804e3a59..68eec73f6d 100644 --- a/improver/calibration/utilities.py +++ b/improver/calibration/utilities.py @@ -363,6 +363,7 @@ def check_forecast_consistency(forecasts: Cube) -> None: frt_hours = get_frt_hours(forecasts.coord("forecast_reference_time")) if len(frt_hours) != 1: + frt_hours = set(map(int, frt_hours)) msg = ( "Forecasts have been provided with differing hours for the " "forecast reference time {}" diff --git a/improver/utilities/indexing_operations.py b/improver/utilities/indexing_operations.py index 7a6ce8cf30..eed210ccd9 100644 --- a/improver/utilities/indexing_operations.py +++ b/improver/utilities/indexing_operations.py @@ -76,10 +76,7 @@ def choose(index_array: ndarray, array_set: ndarray) -> ndarray: raise IndexError(msg) result = np.array( - [ - array_set[index_array[i]][i[1:]] - for i in np.lib.index_tricks.ndindex(index_array.shape) - ] + [array_set[index_array[i]][i[1:]] for i in np.ndindex(index_array.shape)] ).reshape(index_array.shape) return result diff --git a/improver_tests/calibration/dz_rescaling/test_apply_dz_rescaling.py b/improver_tests/calibration/dz_rescaling/test_apply_dz_rescaling.py index 9dd7f6d924..d1f7f33e4b 100644 --- a/improver_tests/calibration/dz_rescaling/test_apply_dz_rescaling.py +++ b/improver_tests/calibration/dz_rescaling/test_apply_dz_rescaling.py @@ -211,7 +211,10 @@ def test_mismatching_sites(): forecast = _create_forecasts(forecast_reference_time, validity_time, [10, 20, 30]) scaling_factor = _create_scaling_factor_cube(3, forecast_period, 1.0) - with pytest.raises(ValueError, match="The mismatched sites are: {'00002'}"): + with pytest.raises( + ValueError, + match="The mismatched sites are: {'00002'}.", + ): ApplyDzRescaling()(forecast, scaling_factor[..., :1]) diff --git a/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py b/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py index 91d8bb953c..a7ad9bea2a 100644 --- a/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py +++ b/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py @@ -150,7 +150,7 @@ def setUp(self): dtype=np.float32, ) self.expected_loc_param_realizations_sites = np.array( - [277.7531, 277.4529, 277.553, 277.2528], dtype=np.float32 + [277.7437, 277.4434, 277.5435, 277.2432], dtype=np.float32 ) self.expected_scale_param_realizations_sites = np.array( @@ -159,18 +159,18 @@ def setUp(self): self.expected_loc_param_mean_alt = np.array( [ - [275.18134, 276.18134, 277.01465], - [278.58133, 279.44797, 280.2813], - [281.48132, 281.91464, 283.11465], + [275.1603, 276.1604, 276.9938], + [278.5606, 279.4273, 280.2607], + [281.4609, 281.8943, 283.0944], ], dtype=np.float32, ) self.expected_scale_param_mean_alt = np.array( [ - [0.6593, 0.663, 0.1756], - [0.2242, 0.2093, 0.1756], - [0.3441, 0.4645, 0.1452], + [1.0636, 1.0695, 0.2832], + [0.3617, 0.3376, 0.2832], + [0.5551, 0.7493, 0.2343], ], dtype=np.float32, ) @@ -371,10 +371,11 @@ def setUp(self): def test_basic(self): """Test that the cubes created containing the location and scale parameter are formatted as expected.""" - (location_parameter_cube, scale_parameter_cube) = ( - self.plugin._create_output_cubes( - self.expected_loc_param_mean, self.expected_scale_param_mean - ) + ( + location_parameter_cube, + scale_parameter_cube, + ) = self.plugin._create_output_cubes( + self.expected_loc_param_mean, self.expected_scale_param_mean ) self.assertEqual(location_parameter_cube, self.expected_loc_param_mean_cube) self.assertEqual(scale_parameter_cube, self.expected_scale_param_mean_cube) diff --git a/improver_tests/calibration/ensemble_calibration/test_EstimateCoefficientsForEnsembleCalibration.py b/improver_tests/calibration/ensemble_calibration/test_EstimateCoefficientsForEnsembleCalibration.py index 60b9fb5561..325c9307c2 100644 --- a/improver_tests/calibration/ensemble_calibration/test_EstimateCoefficientsForEnsembleCalibration.py +++ b/improver_tests/calibration/ensemble_calibration/test_EstimateCoefficientsForEnsembleCalibration.py @@ -55,7 +55,7 @@ def setUp(self): [-0.6989, -0.4923, 0.3968, 0.7732, 0.0001, 1.0787], dtype=np.float32 ) self.expected_mean_pred_norm_alt = np.array( - [0.9980, 1.0, 0.0, 0.0, 0.6976], dtype=np.float32 + [0.9495, 1.0001, 0.0, 0.0, 1.1253], dtype=np.float32 ) self.expected_mean_pred_each_grid_point = { @@ -143,18 +143,18 @@ def setUp(self): ), } self.expected_mean_pred_each_site = { - "emos_coefficient_alpha": np.array([0.993, 0.993, 1.0014, 0.993]), - "emos_coefficient_beta": np.array([1.0, 1.0, 1.0, 1.0]), - "emos_coefficient_gamma": np.array([0.0, 0.0, 0.0, 0.0]), - "emos_coefficient_delta": np.array([1.1899, 1.1899, 1.1897, 1.1899]), + "emos_coefficient_alpha": np.array([0.9387, 0.9387, 0.9469, 0.9387]), + "emos_coefficient_beta": np.array([1.0002, 1.0002, 1.0002, 1.0002]), + "emos_coefficient_gamma": np.array([0.0001, 0.0001, 0.0001, 0.0001]), + "emos_coefficient_delta": np.array([1.0793, 1.0793, 1.0788, 1.0792]), } self.expected_mean_pred_each_site_alt = { - "emos_coefficient_alpha": np.array([0.0092, 0.0023, 0.0011, 0.0006]), + "emos_coefficient_alpha": np.array([0.0100, 0.0025, 0.0011, 0.0006]), "emos_coefficient_beta": np.array( - [[1.0, 1.0, 1.0, 1.0], [0.0987, 0.0497, 0.0334, 0.0249]] + [[1.0, 1.0, 1.0, 0.9999], [0.0986, 0.0497, 0.0334, 0.0258]] ), "emos_coefficient_gamma": np.array([0.0, 0.0, 0.0, 0.0]), - "emos_coefficient_delta": np.array([0.9906, 0.5446, 0.5805, 0.6700]), + "emos_coefficient_delta": np.array([1.0248, 1.0819, 1.0868, 1.1720]), } self.expected_realizations_each_site_alt = { "emos_coefficient_alpha": np.array([0.0045, 0.0011, 0.0005, 0.0003]), @@ -174,12 +174,12 @@ def setUp(self): "emos_coefficient_delta": np.array([0.6324, 0.6335, 0.6354, 0.6331]), } self.expected_realizations_each_site = { - "emos_coefficient_alpha": np.array([0.8126, 0.8126, 0.8126, 0.8126]), + "emos_coefficient_alpha": np.array([0.7417, 0.7417, 0.7417, 0.7417]), "emos_coefficient_beta": np.array( - [[0.8166, 0.8166, 0.8166, 0.8166], [0.5778, 0.5778, 0.5778, 0.5778]] + [[0.6317, 0.6317, 0.6317, 0.6317], [0.7758, 0.7758, 0.7758, 0.7758]] ), - "emos_coefficient_gamma": np.array([0.0005, 0.0005, 0.0005, 0.0005]), - "emos_coefficient_delta": np.array([0.2673, 0.2673, 0.2673, 0.2673]), + "emos_coefficient_gamma": np.array([0.0004, 0.0004, 0.0004, 0.0004]), + "emos_coefficient_delta": np.array([0.5045, 0.5045, 0.5045, 0.5045]), } @@ -1189,6 +1189,7 @@ def test_point_by_point_sites(self): expected_dim_coords = ["spot_index"] plugin = self.plugin(self.distribution, point_by_point=True) result = plugin.process(self.historic_forecast_spot_cube, self.truth_spot_cube) + for cube in result: self.assertEMOSCoefficientsAlmostEqual( cube.data, self.expected_mean_pred_each_site[cube.name()] diff --git a/improver_tests/calibration/reliability_calibration/test_ApplyReliabilityCalibration.py b/improver_tests/calibration/reliability_calibration/test_ApplyReliabilityCalibration.py index c363f7fc34..510d3e8dcc 100644 --- a/improver_tests/calibration/reliability_calibration/test_ApplyReliabilityCalibration.py +++ b/improver_tests/calibration/reliability_calibration/test_ApplyReliabilityCalibration.py @@ -476,8 +476,7 @@ def test_one_threshold_uncalibrated(self): ) warning_msg = ( "The following thresholds were not calibrated due to " - "insufficient forecast counts in reliability table " - "bins: \\[275.0\\]" + "insufficient forecast counts in reliability table bins: \\[275.0\\]" ) with pytest.warns(UserWarning, match=warning_msg): result = self.plugin.process(self.forecast, reliability_cubelist) diff --git a/improver_tests/calibration/test_init.py b/improver_tests/calibration/test_init.py index e65b3392e0..49a7be7268 100644 --- a/improver_tests/calibration/test_init.py +++ b/improver_tests/calibration/test_init.py @@ -296,11 +296,15 @@ def setUp(self): def test_realization_forecast_and_coefficients(self): """Test a realization forecast input.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [self.realization_forecast, self.coefficient_cubelist], - self.land_sea_mask_name, - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [self.realization_forecast, self.coefficient_cubelist], + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.realization_forecast[0]) @@ -311,11 +315,15 @@ def test_realization_forecast_and_coefficients(self): def test_percentile_forecast_and_coefficients(self): """Test a percentile forecast input.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [self.percentile_forecast, self.coefficient_cubelist], - self.land_sea_mask_name, - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [self.percentile_forecast, self.coefficient_cubelist], + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.percentile_forecast[0]) self.assertCubeListEqual(coeffs, self.coefficient_cubelist) @@ -325,10 +333,14 @@ def test_percentile_forecast_and_coefficients(self): def test_probability_forecast_and_coefficients(self): """Test a probability forecast input.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [self.probability_forecast, self.coefficient_cubelist] - ), + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [self.probability_forecast, self.coefficient_cubelist], self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.probability_forecast[0]) @@ -339,15 +351,19 @@ def test_probability_forecast_and_coefficients(self): def test_forecast_coefficients_additional_predictor(self): """Test the addition of a static additional predictor.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.additional_predictors, - ], - self.land_sea_mask_name, - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [ + self.realization_forecast, + self.coefficient_cubelist, + self.additional_predictors, + ], + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.realization_forecast[0]) self.assertCubeListEqual(coeffs, self.coefficient_cubelist) @@ -357,15 +373,19 @@ def test_forecast_coefficients_additional_predictor(self): def test_forecast_coefficients_and_land_sea_mask(self): """Test the addition of a land-sea mask.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.land_sea_mask, - ], - self.land_sea_mask_name, - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [ + self.realization_forecast, + self.coefficient_cubelist, + self.land_sea_mask, + ], + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.realization_forecast[0]) @@ -376,10 +396,14 @@ def test_forecast_coefficients_and_land_sea_mask(self): def test_no_land_sea_mask_name(self): """Test when not providing the land_sea_mask_name option.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [self.realization_forecast, self.coefficient_cubelist] - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [self.realization_forecast, self.coefficient_cubelist] ) self.assertCubeEqual(forecast, self.realization_forecast[0]) @@ -390,15 +414,19 @@ def test_no_land_sea_mask_name(self): def test_forecast_coefficients_prob_template(self): """Test the addition of a probability template cube.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.probability_forecast, - ], - self.land_sea_mask_name, - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [ + self.realization_forecast, + self.coefficient_cubelist, + self.probability_forecast, + ], + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.realization_forecast[0]) self.assertCubeListEqual(coeffs, self.coefficient_cubelist) @@ -409,17 +437,21 @@ def test_forecast_coefficients_prob_template(self): def test_all_options(self): """Test providing a forecast, coefficients, additional predictor, land-sea mask and a probability template.""" - (forecast, coeffs, additional_predictors, land_sea_mask, template) = ( - split_forecasts_and_coeffs( - [ - self.realization_forecast, - self.coefficient_cubelist, - self.additional_predictors, - self.land_sea_mask, - self.probability_forecast, - ], - self.land_sea_mask_name, - ) + ( + forecast, + coeffs, + additional_predictors, + land_sea_mask, + template, + ) = split_forecasts_and_coeffs( + [ + self.realization_forecast, + self.coefficient_cubelist, + self.additional_predictors, + self.land_sea_mask, + self.probability_forecast, + ], + self.land_sea_mask_name, ) self.assertCubeEqual(forecast, self.realization_forecast[0]) self.assertCubeListEqual(coeffs, self.coefficient_cubelist) diff --git a/improver_tests/calibration/utilities/test_utilities.py b/improver_tests/calibration/utilities/test_utilities.py index 8b3f42f11c..c8864f62aa 100644 --- a/improver_tests/calibration/utilities/test_utilities.py +++ b/improver_tests/calibration/utilities/test_utilities.py @@ -710,8 +710,8 @@ def test_mismatching_frt_hours(self): forecasts = iris.cube.CubeList([self.forecast1, forecast2]).merge_cube() msg = ( - "Forecasts have been provided with differing hours for the " - "forecast reference time {1, 2}" + "Forecasts have been provided with differing hours for the forecast " + "reference time {1, 2}" ) with self.assertRaisesRegex(ValueError, msg):