From 9563d8fbf4ffe751e1a7c7c5c3c6c37830b98d57 Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Mon, 6 Oct 2025 14:29:29 -0700 Subject: [PATCH 1/5] add required_pkgs methods to all int_conformal methods --- NAMESPACE | 4 ++ R/conformal_infer.R | 10 ++++ R/conformal_infer_cv.R | 13 +++++ R/conformal_infer_quantile.R | 10 ++++ R/conformal_infer_split.R | 10 ++++ man/int_conformal_full.Rd | 71 +++++++++++++++++++++++++++ man/required_pkgs.int_conformal_cv.Rd | 29 +++++++++++ 7 files changed, 147 insertions(+) create mode 100644 man/required_pkgs.int_conformal_cv.Rd diff --git a/NAMESPACE b/NAMESPACE index 23ed2dd..de28cd6 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -126,6 +126,10 @@ S3method(required_pkgs,cal_estimate_multinomial) S3method(required_pkgs,cal_estimate_multinomial_spline) S3method(required_pkgs,cal_estimate_none) S3method(required_pkgs,cal_object) +S3method(required_pkgs,int_conformal_cv) +S3method(required_pkgs,int_conformal_full) +S3method(required_pkgs,int_conformal_quantile) +S3method(required_pkgs,int_conformal_split) S3method(threshold_perf,data.frame) S3method(type_sum,cal_object) S3method(vec_cast,character.class_pred) diff --git a/R/conformal_infer.R b/R/conformal_infer.R index 10fadfa..8ade6b0 100644 --- a/R/conformal_infer.R +++ b/R/conformal_infer.R @@ -113,6 +113,16 @@ print.int_conformal_full <- function(x, ...) { invisible(x) } +#' @rdname required_pkgs.int_conformal_cv +#' @keywords internal +#' @export +required_pkgs.int_conformal_full <- function(x, ...) { + model_pkgs <- required_pkgs(x$wflow) + model_pkgs <- c(model_pkgs, "probably") + model_pkgs <- unique(model_pkgs) + model_pkgs +} + # ------------------------------------------------------------------------------ #' Prediction intervals from conformal methods diff --git a/R/conformal_infer_cv.R b/R/conformal_infer_cv.R index 41a774c..21f56db 100644 --- a/R/conformal_infer_cv.R +++ b/R/conformal_infer_cv.R @@ -148,6 +148,19 @@ print.int_conformal_cv <- function(x, ...) { invisible(x) } +#' S3 methods to track which additional packages are needed for prediction +#' intervals via conformal inference +#' @param x a conformal interval object +#' @inheritParams generics::required_pkgs +#' @export +required_pkgs.int_conformal_cv <- function(x, ...) { + model_pkgs <- map(x$models, required_pkgs) + model_pkgs <- unlist(model_pkgs) + model_pkgs <- c(model_pkgs, "probably") + model_pkgs <- unique(model_pkgs) + model_pkgs +} + # ------------------------------------------------------------------------------ # helpers diff --git a/R/conformal_infer_quantile.R b/R/conformal_infer_quantile.R index c2227b5..c0a680e 100644 --- a/R/conformal_infer_quantile.R +++ b/R/conformal_infer_quantile.R @@ -115,6 +115,16 @@ print.int_conformal_quantile <- function(x, ...) { invisible(x) } +#' @rdname required_pkgs.int_conformal_cv +#' @keywords internal +#' @export +required_pkgs.int_conformal_quantile <- function(x, ...) { + model_pkgs <- required_pkgs(x$wflow) + model_pkgs <- c(model_pkgs, "probably") + model_pkgs <- unique(model_pkgs) + model_pkgs +} + #' @export #' @rdname predict.int_conformal_full predict.int_conformal_quantile <- function(object, new_data, ...) { diff --git a/R/conformal_infer_split.R b/R/conformal_infer_split.R index ddcfb4a..a8b5d7c 100644 --- a/R/conformal_infer_split.R +++ b/R/conformal_infer_split.R @@ -94,6 +94,16 @@ print.int_conformal_split <- function(x, ...) { invisible(x) } +#' @rdname required_pkgs.int_conformal_cv +#' @keywords internal +#' @export +required_pkgs.int_conformal_split <- function(x, ...) { + model_pkgs <- required_pkgs(x$wflow) + model_pkgs <- c(model_pkgs, "probably") + model_pkgs <- unique(model_pkgs) + model_pkgs +} + #' @export #' @rdname predict.int_conformal_full predict.int_conformal_split <- function(object, new_data, level = 0.95, ...) { diff --git a/man/int_conformal_full.Rd b/man/int_conformal_full.Rd index 09faf20..e950fe7 100644 --- a/man/int_conformal_full.Rd +++ b/man/int_conformal_full.Rd @@ -68,6 +68,77 @@ the initial set of training set residuals are modeled using a Gamma generalized linear model with a log link (see the reference by Aitkin below). For a new sample, the absolute size of the residual is estimated and a multiple of this value is computed as an initial guess of the search boundaries. + +\subsection{Speed}{ + +The time it takes to compute the intervals depends on the training set +size, search parameters (i.e., convergence criterion, number of +iterations), the grid size, and the number of worker processes that are +used. For the last item, the computations can be parallelized using the +future and furrr packages. + +To use parallelism, the \code{\link[future:plan]{future::plan()}} function can +be invoked to create a parallel backend. For example, let’s make an +initial workflow: + +\if{html}{\out{
}}\preformatted{library(tidymodels) +library(probably) +library(future) + +tidymodels_prefer() + +## Make a fitted workflow from some simulated data: +set.seed(121) +train_dat <- sim_regression(200) +new_dat <- sim_regression( 5) |> select(-outcome) + +lm_fit <- + workflow() |> + add_model(linear_reg()) |> + add_formula(outcome ~ .) |> + fit(data = train_dat) + +# Create the object to be used to make prediction intervals +lm_conform <- int_conformal_full(lm_fit, train_dat) +}\if{html}{\out{
}} + +We’ll use a \code{"multisession"} parallel processing plan to compute the +intervals for the five new samples in parallel: + +\if{html}{\out{
}}\preformatted{plan("multisession") + +# This is run in parallel: +predict(lm_conform, new_dat) +}\if{html}{\out{
}} + +\if{html}{\out{
}}\preformatted{## # A tibble: 5 x 2 +## .pred_lower .pred_upper +## +## 1 -17.9 59.6 +## 2 -33.7 51.1 +## 3 -30.6 48.2 +## 4 -17.3 59.6 +## 5 -23.3 55.2 +}\if{html}{\out{
}} + +Using simulations, there are slightly sub-linear speed-ups when using +parallel processing to compute the row-wise intervals. + +In comparison with parametric intervals: + +\if{html}{\out{
}}\preformatted{predict(lm_fit, new_dat, type = "pred_int") +}\if{html}{\out{
}} + +\if{html}{\out{
}}\preformatted{## # A tibble: 5 x 2 +## .pred_lower .pred_upper +## +## 1 -19.2 59.1 +## 2 -31.8 49.7 +## 3 -31.0 47.6 +## 4 -17.8 60.1 +## 5 -23.6 54.3 +}\if{html}{\out{
}} +} } \references{ Jing Lei, Max G'Sell, Alessandro Rinaldo, Ryan J. Tibshirani and Larry diff --git a/man/required_pkgs.int_conformal_cv.Rd b/man/required_pkgs.int_conformal_cv.Rd new file mode 100644 index 0000000..890922a --- /dev/null +++ b/man/required_pkgs.int_conformal_cv.Rd @@ -0,0 +1,29 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/conformal_infer.R, R/conformal_infer_cv.R, +% R/conformal_infer_quantile.R, R/conformal_infer_split.R +\name{required_pkgs.int_conformal_full} +\alias{required_pkgs.int_conformal_full} +\alias{required_pkgs.int_conformal_cv} +\alias{required_pkgs.int_conformal_quantile} +\alias{required_pkgs.int_conformal_split} +\title{S3 methods to track which additional packages are needed for prediction +intervals via conformal inference} +\usage{ +\method{required_pkgs}{int_conformal_full}(x, ...) + +\method{required_pkgs}{int_conformal_cv}(x, ...) + +\method{required_pkgs}{int_conformal_quantile}(x, ...) + +\method{required_pkgs}{int_conformal_split}(x, ...) +} +\arguments{ +\item{x}{a conformal interval object} + +\item{...}{Other arguments passed to methods} +} +\description{ +S3 methods to track which additional packages are needed for prediction +intervals via conformal inference +} +\keyword{internal} From c1348a8cdf6886485abad8fc0b79acc0396915ff Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Mon, 6 Oct 2025 14:29:36 -0700 Subject: [PATCH 2/5] add tests --- tests/testthat/test-conformal-intervals-quantile.R | 5 +++++ tests/testthat/test-conformal-intervals-split.R | 5 +++++ tests/testthat/test-conformal-intervals.R | 10 ++++++++++ 3 files changed, 20 insertions(+) diff --git a/tests/testthat/test-conformal-intervals-quantile.R b/tests/testthat/test-conformal-intervals-quantile.R index c11102f..51b880e 100644 --- a/tests/testthat/test-conformal-intervals-quantile.R +++ b/tests/testthat/test-conformal-intervals-quantile.R @@ -63,4 +63,9 @@ test_that("split conformal quantile intervals", { nrow(new_int), nrow(sim_new) ) + + expect_identical( + required_pkgs(lm_int), + c(required_pkgs(lm_int$wflow), "probably") + ) }) diff --git a/tests/testthat/test-conformal-intervals-split.R b/tests/testthat/test-conformal-intervals-split.R index 280440f..67d7da3 100644 --- a/tests/testthat/test-conformal-intervals-split.R +++ b/tests/testthat/test-conformal-intervals-split.R @@ -64,4 +64,9 @@ test_that("split conformal intervals", { nrow(new_int), nrow(sim_new) ) + + expect_identical( + required_pkgs(lm_int), + c(required_pkgs(lm_int$wflow), "probably") + ) }) diff --git a/tests/testthat/test-conformal-intervals.R b/tests/testthat/test-conformal-intervals.R index 467fada..48eebd5 100644 --- a/tests/testthat/test-conformal-intervals.R +++ b/tests/testthat/test-conformal-intervals.R @@ -229,6 +229,16 @@ test_that("conformal intervals", { expect_true( all(grid_bounds$.pred_lower < grid_bounds_90$.pred_lower) ) + + expect_identical( + required_pkgs(smol_obj), + c(required_pkgs(smol_obj$wflow), "probably") + ) + + expect_identical( + required_pkgs(grid_int), + c(unique(unlist(map(grid_int$models, required_pkgs))), "probably") + ) }) test_that("conformal control", { From c599d03144278045e109a13e443d33ed0b690da2 Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Mon, 6 Oct 2025 14:29:45 -0700 Subject: [PATCH 3/5] add news --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index e788116..7be95a0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # probably (development version) +* Add `required_pkgs()` methods to `int_conformal_cv()`, `int_conformal_full()`, `int_conformal_quantile()`, and `int_conformal_split()`. (#190) + # probably 1.1.1 * Updated unit tests for new ggplot2 release (#180). From 148c65a25d4a51d00258ab8fb538f90cd01a65ed Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Mon, 6 Oct 2025 16:10:48 -0700 Subject: [PATCH 4/5] update version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 4e59523..08959d6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: probably Title: Tools for Post-Processing Predicted Values -Version: 1.1.1.9000 +Version: 1.1.1.9001 Authors@R: c( person("Max", "Kuhn", , "max@posit.co", role = c("aut", "cre")), person("Davis", "Vaughan", , "davis@posit.co", role = "aut"), From 3b86f4038c23e9e177af252273ebaa6e63d790d8 Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Tue, 7 Oct 2025 09:43:51 -0700 Subject: [PATCH 5/5] add infra argument --- R/conformal_infer.R | 10 +++++++--- R/conformal_infer_cv.R | 10 +++++++--- R/conformal_infer_quantile.R | 10 +++++++--- R/conformal_infer_split.R | 10 +++++++--- man/required_pkgs.int_conformal_cv.Rd | 8 ++++---- tests/testthat/test-conformal-intervals-quantile.R | 4 ++++ tests/testthat/test-conformal-intervals-split.R | 4 ++++ tests/testthat/test-conformal-intervals.R | 8 ++++++++ 8 files changed, 48 insertions(+), 16 deletions(-) diff --git a/R/conformal_infer.R b/R/conformal_infer.R index 8ade6b0..6511193 100644 --- a/R/conformal_infer.R +++ b/R/conformal_infer.R @@ -116,9 +116,13 @@ print.int_conformal_full <- function(x, ...) { #' @rdname required_pkgs.int_conformal_cv #' @keywords internal #' @export -required_pkgs.int_conformal_full <- function(x, ...) { - model_pkgs <- required_pkgs(x$wflow) - model_pkgs <- c(model_pkgs, "probably") +required_pkgs.int_conformal_full <- function(x, infra = TRUE, ...) { + model_pkgs <- required_pkgs(x$wflow, infra = infra) + + if (infra) { + model_pkgs <- c(model_pkgs, "probably") + } + model_pkgs <- unique(model_pkgs) model_pkgs } diff --git a/R/conformal_infer_cv.R b/R/conformal_infer_cv.R index 21f56db..79c2ebb 100644 --- a/R/conformal_infer_cv.R +++ b/R/conformal_infer_cv.R @@ -153,10 +153,14 @@ print.int_conformal_cv <- function(x, ...) { #' @param x a conformal interval object #' @inheritParams generics::required_pkgs #' @export -required_pkgs.int_conformal_cv <- function(x, ...) { - model_pkgs <- map(x$models, required_pkgs) +required_pkgs.int_conformal_cv <- function(x, infra = TRUE, ...) { + model_pkgs <- map(x$models, required_pkgs, infra = infra) model_pkgs <- unlist(model_pkgs) - model_pkgs <- c(model_pkgs, "probably") + + if (infra) { + model_pkgs <- c(model_pkgs, "probably") + } + model_pkgs <- unique(model_pkgs) model_pkgs } diff --git a/R/conformal_infer_quantile.R b/R/conformal_infer_quantile.R index c0a680e..3f7803c 100644 --- a/R/conformal_infer_quantile.R +++ b/R/conformal_infer_quantile.R @@ -118,9 +118,13 @@ print.int_conformal_quantile <- function(x, ...) { #' @rdname required_pkgs.int_conformal_cv #' @keywords internal #' @export -required_pkgs.int_conformal_quantile <- function(x, ...) { - model_pkgs <- required_pkgs(x$wflow) - model_pkgs <- c(model_pkgs, "probably") +required_pkgs.int_conformal_quantile <- function(x, infra = TRUE, ...) { + model_pkgs <- required_pkgs(x$wflow, infra = infra) + + if (infra) { + model_pkgs <- c(model_pkgs, "probably") + } + model_pkgs <- unique(model_pkgs) model_pkgs } diff --git a/R/conformal_infer_split.R b/R/conformal_infer_split.R index a8b5d7c..a8f38c3 100644 --- a/R/conformal_infer_split.R +++ b/R/conformal_infer_split.R @@ -97,9 +97,13 @@ print.int_conformal_split <- function(x, ...) { #' @rdname required_pkgs.int_conformal_cv #' @keywords internal #' @export -required_pkgs.int_conformal_split <- function(x, ...) { - model_pkgs <- required_pkgs(x$wflow) - model_pkgs <- c(model_pkgs, "probably") +required_pkgs.int_conformal_split <- function(x, infra = TRUE, ...) { + model_pkgs <- required_pkgs(x$wflow, infra = infra) + + if (infra) { + model_pkgs <- c(model_pkgs, "probably") + } + model_pkgs <- unique(model_pkgs) model_pkgs } diff --git a/man/required_pkgs.int_conformal_cv.Rd b/man/required_pkgs.int_conformal_cv.Rd index 890922a..e5d2d84 100644 --- a/man/required_pkgs.int_conformal_cv.Rd +++ b/man/required_pkgs.int_conformal_cv.Rd @@ -9,13 +9,13 @@ \title{S3 methods to track which additional packages are needed for prediction intervals via conformal inference} \usage{ -\method{required_pkgs}{int_conformal_full}(x, ...) +\method{required_pkgs}{int_conformal_full}(x, infra = TRUE, ...) -\method{required_pkgs}{int_conformal_cv}(x, ...) +\method{required_pkgs}{int_conformal_cv}(x, infra = TRUE, ...) -\method{required_pkgs}{int_conformal_quantile}(x, ...) +\method{required_pkgs}{int_conformal_quantile}(x, infra = TRUE, ...) -\method{required_pkgs}{int_conformal_split}(x, ...) +\method{required_pkgs}{int_conformal_split}(x, infra = TRUE, ...) } \arguments{ \item{x}{a conformal interval object} diff --git a/tests/testthat/test-conformal-intervals-quantile.R b/tests/testthat/test-conformal-intervals-quantile.R index 51b880e..293c773 100644 --- a/tests/testthat/test-conformal-intervals-quantile.R +++ b/tests/testthat/test-conformal-intervals-quantile.R @@ -68,4 +68,8 @@ test_that("split conformal quantile intervals", { required_pkgs(lm_int), c(required_pkgs(lm_int$wflow), "probably") ) + expect_identical( + required_pkgs(lm_int, infra = FALSE), + required_pkgs(lm_int$wflow, infra = FALSE) + ) }) diff --git a/tests/testthat/test-conformal-intervals-split.R b/tests/testthat/test-conformal-intervals-split.R index 67d7da3..6824e4e 100644 --- a/tests/testthat/test-conformal-intervals-split.R +++ b/tests/testthat/test-conformal-intervals-split.R @@ -69,4 +69,8 @@ test_that("split conformal intervals", { required_pkgs(lm_int), c(required_pkgs(lm_int$wflow), "probably") ) + expect_identical( + required_pkgs(lm_int, infra = FALSE), + required_pkgs(lm_int$wflow, infra = FALSE) + ) }) diff --git a/tests/testthat/test-conformal-intervals.R b/tests/testthat/test-conformal-intervals.R index 48eebd5..5233d2e 100644 --- a/tests/testthat/test-conformal-intervals.R +++ b/tests/testthat/test-conformal-intervals.R @@ -234,11 +234,19 @@ test_that("conformal intervals", { required_pkgs(smol_obj), c(required_pkgs(smol_obj$wflow), "probably") ) + expect_identical( + required_pkgs(smol_obj, infra = FALSE), + required_pkgs(smol_obj$wflow, infra = FALSE) + ) expect_identical( required_pkgs(grid_int), c(unique(unlist(map(grid_int$models, required_pkgs))), "probably") ) + expect_identical( + required_pkgs(grid_int, infra = FALSE), + unique(unlist(map(grid_int$models, required_pkgs, infra = FALSE))) + ) }) test_that("conformal control", {