From 61bab0cb428c4afad35a19561cf3ee85e083ea95 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:02:36 +0000 Subject: [PATCH 1/7] add a test for interval_coverage_quantile to catch insufficent quantiles for requested range --- tests/testthat/test-metrics-quantile.R | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index b941a95aa..cb4b23c19 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -601,6 +601,17 @@ test_that("interval_coverage_quantile rejects wrong inputs", { ) }) +test_that("interval_coverage_quantile throws a warning when a required quantile is not available", { + dropped_quantile_pred <- predicted[, -4] + dropped_quantiles <- quantile[-4] + expect_warning( + interval_coverage_quantile( + observed, dropped_quantile_pred, dropped_quantiles, range = 50 + ), + "Median not available, computing coverage as mean of the two innermost quantiles in order to compute coverage." + ) +}) + # ============================================================================ # # `interval_coverage_dev_quantile` ===================================== # From d4f779e3f113e02a1e19e973f06c0a28d8eb3651 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:04:09 +0000 Subject: [PATCH 2/7] update interval_coverage_quantile to test the new warning message --- tests/testthat/test-metrics-quantile.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index cb4b23c19..f0274484c 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -608,7 +608,7 @@ test_that("interval_coverage_quantile throws a warning when a required quantile interval_coverage_quantile( observed, dropped_quantile_pred, dropped_quantiles, range = 50 ), - "Median not available, computing coverage as mean of the two innermost quantiles in order to compute coverage." + "To compute the interval coverage for a range of 50%, the quantiles `0.25, 0.75` are required. Returning `NA`" ) }) From 8a9915ac24697dc2bad4be4ecd8495286867a6fc Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:06:39 +0000 Subject: [PATCH 3/7] make interval_coverage_dev_quantile part of mainline tests (i.e. drop _dev) --- tests/testthat/test-metrics-quantile.R | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index f0274484c..1eb8810de 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -612,16 +612,16 @@ test_that("interval_coverage_quantile throws a warning when a required quantile ) }) - -# ============================================================================ # -# `interval_coverage_dev_quantile` ===================================== # -# ============================================================================ # -test_that("interval_coverage_dev_quantile works", { +test_that("interval_coverage_quantile works", { existing_ranges <- unique(get_range_from_quantile(quantile)) expect_equal(existing_ranges, c(80, 50, 0)) - cov_50 <- interval_coverage_quantile(observed, predicted, quantile, range = c(50)) - cov_80 <- interval_coverage_quantile(observed, predicted, quantile, range = c(80)) + cov_50 <- interval_coverage_quantile( + observed, predicted, quantile, range = 50 + ) + cov_80 <- interval_coverage_quantile( + observed, predicted, quantile, range = 80 + ) manual <- 0.5 * (cov_50 - 0.5) + 0.5 * (cov_80 - 0.8) expect_equal( From 866c9d05b85cf4ed8461f44847ed0bdbe4012ce1 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:12:08 +0000 Subject: [PATCH 4/7] Revert "make interval_coverage_dev_quantile part of mainline tests (i.e. drop _dev)" This reverts commit 8a9915ac24697dc2bad4be4ecd8495286867a6fc. --- tests/testthat/test-metrics-quantile.R | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index 1eb8810de..f0274484c 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -612,16 +612,16 @@ test_that("interval_coverage_quantile throws a warning when a required quantile ) }) -test_that("interval_coverage_quantile works", { + +# ============================================================================ # +# `interval_coverage_dev_quantile` ===================================== # +# ============================================================================ # +test_that("interval_coverage_dev_quantile works", { existing_ranges <- unique(get_range_from_quantile(quantile)) expect_equal(existing_ranges, c(80, 50, 0)) - cov_50 <- interval_coverage_quantile( - observed, predicted, quantile, range = 50 - ) - cov_80 <- interval_coverage_quantile( - observed, predicted, quantile, range = 80 - ) + cov_50 <- interval_coverage_quantile(observed, predicted, quantile, range = c(50)) + cov_80 <- interval_coverage_quantile(observed, predicted, quantile, range = c(80)) manual <- 0.5 * (cov_50 - 0.5) + 0.5 * (cov_80 - 0.8) expect_equal( From 060eaee6ac3b447d0fef52133d828c2228018b66 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:15:30 +0000 Subject: [PATCH 5/7] add a test for interval_coverage_dev_quantile --- tests/testthat/test-metrics-quantile.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/testthat/test-metrics-quantile.R b/tests/testthat/test-metrics-quantile.R index f0274484c..a6faf051c 100644 --- a/tests/testthat/test-metrics-quantile.R +++ b/tests/testthat/test-metrics-quantile.R @@ -628,6 +628,12 @@ test_that("interval_coverage_dev_quantile works", { interval_coverage_dev_quantile(observed, predicted, quantile), manual ) + expect_warning( + interval_coverage_dev_quantile( + observed, predicted, c(quantile[-4], 0.76) + ), + "To compute inteval coverage deviation, all quantiles must form central symmetric prediction intervals. Missing quantiles: 0.24, 0.75. Returning `NA`." + ) }) From 4fe7a4a1ebcd13842a9df0146f58869ed8d4ab82 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:17:58 +0000 Subject: [PATCH 6/7] add news item --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 24bf6832f..7bfcf0743 100644 --- a/NEWS.md +++ b/NEWS.md @@ -38,7 +38,7 @@ The update introduces breaking changes. If you want to keep using the older vers - Files ending in ".Rda" were renamed to ".rds" where appropriate when used together with `saveRDS()` or `readRDS()`. - `score()` now calls `na.omit()` on the data, instead of only removing rows with missing values in the columns `observed` and `predicted`. This is because `NA` values in other columns can also mess up e.g. grouping of forecasts according to the unit of a single forecast. - added documentation for the return value of `summarise_scores()`. - +- Added unit tests for `interval_coverage_quantile()` and `interval_coverage_dev_quantile()` in order to make sure that the functions provide the correct warnings when insufficient quantiles are provided. # scoringutils 1.2.1 From 85733bf61c0f5517b8ecaf5bc8bcc3e1c131e4de Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 5 Jan 2024 12:23:37 +0000 Subject: [PATCH 7/7] ping CI