diff --git a/DESCRIPTION b/DESCRIPTION index 1032875..9ca8580 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: funcml Type: Package Title: Functional Machine Learning Framework -Version: 0.9.12 +Version: 0.9.13 Depends: R (>= 3.5.0) Authors@R: c( person(given = "Imad", family = "El Badisy", role = c("aut", "cre"), email = "elbadisyimad@gmail.com") diff --git a/NAMESPACE b/NAMESPACE index 8ff59f1..c0ff1a5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -66,6 +66,8 @@ export(brier) export(calibration_curve) export(calibrationcurve) export(calibrationfit) +export(calslope) +export(citl) export(compare) export(cv) export(dca) diff --git a/NEWS.md b/NEWS.md index 9a0ddeb..97d27be 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# funcml 0.9.13 + +* New calibration metrics `calslope()` (calibration slope) and `citl()` (calibration-in-the-large), the `slope` and `intercept` rows of `calibrationfit()`. As metric names `"calslope"` and `"citl"` they work in `bootvalidate()` and `evaluate()` and are listed by `listmetrics()` (direction `"target"`: ideal values 1 and 0). In `bootvalidate()` the optimism-corrected slope equals `rms::validate()`'s corrected `Slope` when both are seeded identically; `gamma` and `est632plus` are `NA` for these metrics. `tune()` refuses them as tuning criteria; `compare()` ranks them by distance to the ideal value. + # funcml 0.9.12 * `auc()` overflowed integer arithmetic in the pair count (positives times diff --git a/R/bootvalidate.R b/R/bootvalidate.R index 7e31fc6..b1f9027 100644 --- a/R/bootvalidate.R +++ b/R/bootvalidate.R @@ -25,6 +25,12 @@ #' over all pairings. Higher-is-better metrics (`auc`, `accuracy`, `rsq`, ...) are #' handled on the corresponding loss scale, so "worse" means lower. #' +#' Calibration metrics `calslope` and `citl` (binary outcomes) have an +#' ideal value (1 and 0) rather than a direction: their optimism and +#' corrected values follow `rms::validate()`'s `Slope` (the apparent slope +#' of a logistic regression is 1, so corrected = 1 - optimism), while +#' `gamma` and `est632plus` are `NA`. +#' #' The OOB metric is computed per replicate and averaged, which for #' non-additive metrics (`auc`, `ece`) is the usual approximation of the #' per-observation leave-one-out bootstrap. @@ -37,6 +43,8 @@ #' @param metrics Character vector of metric names. Defaults to #' `c("rmse", "mse", "rsq")` for regression and #' `c("auc", "brier", "logloss", "accuracy")` for classification. +#' Binary outcomes also accept `"calslope"` (calibration slope) and +#' `"citl"` (calibration-in-the-large). #' @param seed Optional seed for the bootstrap draws. #' @param ncores Optional number of CPU cores for the replicates. #' @param ... Passed to [fit()]. @@ -153,7 +161,7 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) { # Score a fitted model on `newdata`; keeps the predictions for the # no-information rate. .bv_score <- function(fit_obj, newdata, truth, task, metrics) { - prob_metrics <- c("logloss", "brier", "auc", "auc_weighted", "ece", "mce") + prob_metrics <- .prob_metrics if (task == "regression") { pred <- as.numeric(predict(fit_obj, newdata = newdata, type = "response")) vals <- vapply(metrics, function(m) .loss(truth, pred, task, m), numeric(1)) @@ -187,6 +195,7 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) { # Over all n^2 pairings the scores are independent of the outcome, so the AUC is exactly 0.5; # the sampled pairs would only add noise. if (m %in% c("auc", "auc_weighted")) return(0.5) + if (m %in% names(.target_metrics)) return(NA_real_) suppressWarnings(.loss(truth[i], scored$pred[j], task, m, prob_matrix = prob)) }, numeric(1), USE.NAMES = FALSE) } @@ -194,6 +203,10 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) { .bv_estimates <- function(app, boot, orig, oob, gamma, direction) { optimism <- mean(boot - orig) oob_mean <- mean(oob) + if (identical(direction, "target")) { + return(c(apparent = app, optimism = optimism, corrected = app - optimism, oob = oob_mean, + est632 = 0.368 * app + 0.632 * oob_mean, est632plus = NA_real_, gamma = NA_real_)) + } # .632+ on the loss scale (larger = worse). s <- if (identical(direction, "min")) 1 else -1 a <- s * app diff --git a/R/compare.R b/R/compare.R index 8ce74cb..1816876 100644 --- a/R/compare.R +++ b/R/compare.R @@ -352,6 +352,7 @@ compare <- function(data, formula, models, specs = NULL, .comparison_score <- function(x, metric) { direction <- metric_direction(metric[1]) + if (direction == "target") return(abs(x - .target_metrics[[metric[1]]])) if (direction == "min") x else -x } diff --git a/R/evaluate.R b/R/evaluate.R index f61e6a5..df63556 100644 --- a/R/evaluate.R +++ b/R/evaluate.R @@ -290,7 +290,7 @@ plot.funcml_eval <- function(x, ...) { train_data <- data[fold$train, , drop = FALSE] test_data <- data[fold$test, , drop = FALSE] fit_fold <- fit(formula, train_data, model, spec = spec, ...) - prob_metrics <- c("logloss", "brier", "auc", "auc_weighted", "ece", "mce") + prob_metrics <- .prob_metrics type_use <- type %||% if (task == "regression") "response" else if (any(metrics %in% prob_metrics)) "prob" else "class" preds <- predict(fit_fold, newdata = test_data, type = type_use) truth <- y_all[fold$test] diff --git a/R/metrics.R b/R/metrics.R index fd39ae9..53aeeab 100644 --- a/R/metrics.R +++ b/R/metrics.R @@ -2,7 +2,8 @@ #' #' @param direction Optional character filter: `"maximize"` or `"minimize"`. #' @param columns Optional character vector of columns to return. -#' @return Data frame of metric metadata. +#' @return Data frame of metric metadata. `direction` is `"maximize"`, `"minimize"` or +#' `"target"` (calibration metrics with an ideal value: `calslope` 1, `citl` 0). #' @examples #' listmetrics() #' listmetrics(direction = "minimize") @@ -30,13 +31,13 @@ listmetrics <- function(direction = NULL, columns = NULL) { "rmse", "mae", "mse", "medae", "mape", "rsq", "accuracy", "precision", "recall", "specificity", "f1", "balanced_accuracy", "logloss", "brier", "auc", "auc_weighted", - "ece", "mce" + "ece", "mce", "calslope", "citl" ), direction = c( "minimize", "minimize", "minimize", "minimize", "minimize", "maximize", "maximize", "maximize", "maximize", "maximize", "maximize", "maximize", "minimize", "minimize", "maximize", "maximize", - "minimize", "minimize" + "minimize", "minimize", "target", "target" ), summary = c( "Root mean squared error for regression predictions.", @@ -56,13 +57,15 @@ listmetrics <- function(direction = NULL, columns = NULL) { "Area under the ROC curve.", "Weighted multiclass area under the ROC curve.", "Expected calibration error for binary classification.", - "Maximum calibration error for binary classification." + "Maximum calibration error for binary classification.", + "Calibration slope (logistic recalibration) for binary classification.", + "Calibration-in-the-large (intercept with the logit as offset) for binary classification." ), range = c( "[0, Inf)", "[0, Inf)", "[0, Inf)", "[0, Inf)", "[0, Inf)", "(-Inf, 1]", "[0, 1]", "[0, 1]", "[0, 1]", "[0, 1]", "[0, 1]", "[0, 1]", "[0, Inf)", "[0, 1] binary, [0, 2] multiclass", "[0, 1]", "[0, 1]", - "[0, 1]", "[0, 1]" + "[0, 1]", "[0, 1]", "(-Inf, Inf)", "(-Inf, Inf)" ), stringsAsFactors = FALSE ) @@ -101,6 +104,8 @@ listmetrics <- function(direction = NULL, columns = NULL) { #' returned, as close to equal size as the ties allow. #' @param positive Optional positive/event class for binary classification. #' Defaults to the second factor level. +#' @details `calslope()` and `citl()` are the `slope` and `intercept` rows of +#' [calibrationfit()]: ideal values 1 and 0. #' @return Numeric scalar metric. #' @name metrics #' @examples @@ -144,6 +149,8 @@ listmetrics <- function(direction = NULL, columns = NULL) { #' calibrationcurve(truth_cls, prob_cls[, "yes"]) #' ece(truth_cls, prob_cls[, "yes"]) #' mce(truth_cls, prob_cls[, "yes"]) +#' calslope(truth_cls, prob_cls[, "yes"]) +#' citl(truth_cls, prob_cls[, "yes"]) #' @export rmse <- function(truth, pred) { sqrt(mean((truth - pred)^2)) @@ -553,13 +560,30 @@ mce <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), pos max(curve$abs_gap[nonempty]) } +#' @rdname metrics +#' @export +calslope <- function(truth, prob, positive = NULL) { + calibrationfit(truth, prob, positive = positive)$estimate[2] +} + +#' @rdname metrics +#' @export +citl <- function(truth, prob, positive = NULL) { + calibrationfit(truth, prob, positive = positive)$estimate[1] +} + +# Metrics scored from predicted probabilities. +.prob_metrics <- c("logloss", "brier", "auc", "auc_weighted", "ece", "mce", "calslope", "citl") +# Calibration metrics: ideal value 1 (slope) or 0 (CITL), neither "higher" nor "lower" is better. +.target_metrics <- c(calslope = 1, citl = 0) + .loss <- function(truth, pred, task, metric, prob_matrix = NULL) { metric <- match.arg(metric, c( "rmse", "mae", "mse", "medae", "mape", "rsq", "logloss", "brier", "accuracy", "precision", "recall", - "specificity", "f1", "balanced_accuracy", "auc", "auc_weighted", "ece", "mce" + "specificity", "f1", "balanced_accuracy", "auc", "auc_weighted", "ece", "mce", "calslope", "citl" )) - if (task == "regression" && metric %in% c("logloss", "brier", "accuracy", "precision", "recall", "specificity", "f1", "balanced_accuracy", "auc", "auc_weighted", "ece", "mce")) { + if (task == "regression" && metric %in% c("accuracy", "precision", "recall", "specificity", "f1", "balanced_accuracy", .prob_metrics)) { stop(sprintf("Metric '%s' is classification-only.", metric), call. = FALSE) } if (task == "classification" && metric %in% c("rmse", "mae", "mse", "medae", "mape", "rsq")) { @@ -578,7 +602,7 @@ mce <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), pos truth <- factor(truth) pred_class <- if (is.null(pred)) NULL else factor(pred, levels = levels(truth)) - if (metric %in% c("logloss", "brier", "auc", "auc_weighted", "ece", "mce")) { + if (metric %in% .prob_metrics) { if (is.null(prob_matrix)) stop(sprintf("Metric '%s' requires probabilities.", metric), call. = FALSE) prob_matrix <- .normalize_prob_matrix(prob_matrix, levels(truth)) } @@ -595,6 +619,8 @@ mce <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), pos brier = brier(truth, prob_matrix), ece = ece(truth, prob_matrix), mce = mce(truth, prob_matrix), + calslope = calslope(truth, prob_matrix), + citl = citl(truth, prob_matrix), auc = auc(truth, prob_matrix), auc_weighted = aucweighted(truth, prob_matrix) ) diff --git a/R/tune.R b/R/tune.R index 04c772f..09f10bd 100644 --- a/R/tune.R +++ b/R/tune.R @@ -54,6 +54,7 @@ tune <- function(data, formula, model, grid = NULL, resampling = cv(5), set.seed(seed) } dirs <- metric_direction(metric) + if (dirs == "target") stop(sprintf("Metric '%s' is a calibration measure, not a tuning criterion.", metric), call. = FALSE) search_grid <- .select_tuning_configs(grid, search = search, n_evals = n_evals) nested <- NULL if (!is.null(outer_resampling)) { @@ -131,6 +132,7 @@ tune <- function(data, formula, model, grid = NULL, resampling = cv(5), } metric_direction <- function(metric) { + if (metric %in% names(.target_metrics)) return("target") if (metric %in% c("rmse", "mae", "mse", "medae", "mape", "logloss", "brier", "ece", "mce")) "min" else "max" } @@ -271,7 +273,7 @@ plot.funcml_tune <- function(x, ...) { type_use <- type %||% if (fit_obj$task == "regression") { "response" - } else if (metric %in% c("logloss", "brier", "auc", "auc_weighted", "ece", "mce")) { + } else if (metric %in% .prob_metrics) { "prob" } else { "class" diff --git a/man/bootvalidate.Rd b/man/bootvalidate.Rd index 9c21c0d..2d1c183 100644 --- a/man/bootvalidate.Rd +++ b/man/bootvalidate.Rd @@ -29,7 +29,9 @@ bootvalidate( \item{metrics}{Character vector of metric names. Defaults to \code{c("rmse", "mse", "rsq")} for regression and -\code{c("auc", "brier", "logloss", "accuracy")} for classification.} +\code{c("auc", "brier", "logloss", "accuracy")} for classification. +Binary outcomes also accept \code{"calslope"} (calibration slope) and +\code{"citl"} (calibration-in-the-large).} \item{seed}{Optional seed for the bootstrap draws.} @@ -73,6 +75,12 @@ over all pairings. Higher-is-better metrics (\code{auc}, \code{accuracy}, \code{ handled on the corresponding loss scale, so "worse" means lower. } +Calibration metrics \code{calslope} and \code{citl} (binary outcomes) have an +ideal value (1 and 0) rather than a direction: their optimism and +corrected values follow \code{rms::validate()}'s \code{Slope} (the apparent slope +of a logistic regression is 1, so corrected = 1 - optimism), while +\code{gamma} and \code{est632plus} are \code{NA}. + The OOB metric is computed per replicate and averaged, which for non-additive metrics (\code{auc}, \code{ece}) is the usual approximation of the per-observation leave-one-out bootstrap. diff --git a/man/listmetrics.Rd b/man/listmetrics.Rd index f7599f7..88324c4 100644 --- a/man/listmetrics.Rd +++ b/man/listmetrics.Rd @@ -12,7 +12,8 @@ listmetrics(direction = NULL, columns = NULL) \item{columns}{Optional character vector of columns to return.} } \value{ -Data frame of metric metadata. +Data frame of metric metadata. \code{direction} is \code{"maximize"}, \code{"minimize"} or +\code{"target"} (calibration metrics with an ideal value: \code{calslope} 1, \code{citl} 0). } \description{ List available metrics used in scoring and resampling summaries. diff --git a/man/metrics.Rd b/man/metrics.Rd index 948f0f1..67ce30b 100644 --- a/man/metrics.Rd +++ b/man/metrics.Rd @@ -21,6 +21,8 @@ \alias{calibrationcurve} \alias{ece} \alias{mce} +\alias{calslope} +\alias{citl} \title{Regression and classification metrics.} \usage{ rmse(truth, pred) @@ -78,6 +80,10 @@ mce( strategy = c("quantile", "uniform"), positive = NULL ) + +calslope(truth, prob, positive = NULL) + +citl(truth, prob, positive = NULL) } \arguments{ \item{truth}{Observed outcomes.} @@ -124,6 +130,10 @@ Numeric scalar metric. \description{ Base R implementations used across evaluation and interpretation utilities. } +\details{ +\code{calslope()} and \code{citl()} are the \code{slope} and \code{intercept} rows of +\code{\link[=calibrationfit]{calibrationfit()}}: ideal values 1 and 0. +} \examples{ truth_reg <- c(3, 5, 2.5, 7) pred_reg <- c(2.8, 4.9, 2.7, 6.8) @@ -165,4 +175,6 @@ aucweighted(truth_multi, prob_multi) calibrationcurve(truth_cls, prob_cls[, "yes"]) ece(truth_cls, prob_cls[, "yes"]) mce(truth_cls, prob_cls[, "yes"]) +calslope(truth_cls, prob_cls[, "yes"]) +citl(truth_cls, prob_cls[, "yes"]) } diff --git a/tests/testthat/test-book-bugs-4.R b/tests/testthat/test-book-bugs-4.R new file mode 100644 index 0000000..fbbefd4 --- /dev/null +++ b/tests/testthat/test-book-bugs-4.R @@ -0,0 +1,35 @@ +test_that("calslope and citl match calibrationfit() and glm", { + set.seed(1); p <- runif(300); y <- rbinom(300, 1, plogis(0.8 * qlogis(p) - 0.3)) + lp <- qlogis(p) + expect_equal(calslope(y, p), unname(coef(glm(y ~ lp, family = binomial()))[2])) + expect_equal(citl(y, p), unname(coef(glm(y ~ 1, family = binomial(), offset = lp))[1])) + expect_true(all(c("calslope", "citl") %in% listmetrics()$metric)) +}) + +test_that("bootvalidate() calibration slope matches rms::validate() Slope", { + skip_if_not_installed("rms") + set.seed(4); n <- 300 + d <- data.frame(x1 = rnorm(n), x2 = rnorm(n), x3 = rnorm(n)) + d$y <- factor(rbinom(n, 1, plogis(0.7 * d$x1 - 0.4 * d$x2))) + f <- y ~ x1 + x2 + x3 + bv <- bootvalidate(d, f, "glm", B = 30, seed = 1, metrics = c("calslope", "citl")) + s <- bv$summary + set.seed(1); v <- rms::validate(rms::lrm(f, data = d, x = TRUE, y = TRUE), B = 30) + expect_equal(s$apparent, c(1, 0), tolerance = 1e-6) + expect_equal(s$corrected[1], v["Slope", "index.corrected"], tolerance = 1e-6) + expect_true(all(is.na(s$est632plus)) && all(is.na(s$gamma))) + # CITL optimism from scratch on the same bootstrap rows + o <- vapply(bv$indices, function(i) { + m <- glm(f, binomial, d[i, ]); lo <- function(dd) predict(m, dd) + ci <- function(dd) coef(glm(dd$y ~ 1, family = binomial(), offset = lo(dd)))[1] + ci(d[i, ]) - ci(d) + }, numeric(1)) + expect_equal(s$optimism[2], mean(o), tolerance = 1e-6) +}) + +test_that("calibration metrics work in evaluate() and are refused by tune()", { + set.seed(2); d <- data.frame(x = rnorm(200)); d$y <- factor(rbinom(200, 1, plogis(d$x))) + ev <- evaluate(d, y ~ x, model = "glm", resampling = cv(3, seed = 1), metrics = c("calslope", "citl")) + expect_true(all(is.finite(ev$summary$mean))) + expect_error(tune(d, y ~ x, model = "glmnet", grid = data.frame(alpha = c(0, 1)), metric = "calslope"), "calibration") +}) diff --git a/tests/testthat/test-core.R b/tests/testthat/test-core.R index c0e28b9..8918148 100644 --- a/tests/testthat/test-core.R +++ b/tests/testthat/test-core.R @@ -198,7 +198,7 @@ test_that("listmetrics returns metric metadata", { expect_s3_class(metrics_tbl, "data.frame") expect_named(metrics_tbl, c("metric", "direction", "summary", "range")) expect_true(all(c("rmse", "mae", "accuracy", "auc", "ece") %in% metrics_tbl$metric)) - expect_true(all(metrics_tbl$direction %in% c("maximize", "minimize"))) + expect_true(all(metrics_tbl$direction %in% c("maximize", "minimize", "target"))) expect_true(all(nzchar(metrics_tbl$summary))) expect_true(all(nzchar(metrics_tbl$range))) })