diff --git a/DESCRIPTION b/DESCRIPTION index 53de553..1032875 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: funcml Type: Package Title: Functional Machine Learning Framework -Version: 0.9.11 +Version: 0.9.12 Depends: R (>= 3.5.0) Authors@R: c( person(given = "Imad", family = "El Badisy", role = c("aut", "cre"), email = "elbadisyimad@gmail.com") diff --git a/NAMESPACE b/NAMESPACE index cb87ca4..8ff59f1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -65,6 +65,7 @@ export(bootvalidate) export(brier) export(calibration_curve) export(calibrationcurve) +export(calibrationfit) export(compare) export(cv) export(dca) diff --git a/NEWS.md b/NEWS.md index f137707..9a0ddeb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,25 @@ +# funcml 0.9.12 + +* `auc()` overflowed integer arithmetic in the pair count (positives times + negatives) and returned `NA` for large samples. This broke + `bootvalidate()` with `metrics = "auc"` (and the default classification + metrics) for n of about 400 and more, where the no-information value was + computed on 250,000 sampled pairs. Fixed; the no-information AUC in + `bootvalidate()` is now exactly 0.5 (bug fix: `gamma` and `est632plus` + for `auc` change slightly at large n). +* `coef()` on a `glmnet` fit returns the coefficients at the selected + lambda (`$state$lambda_sel`), as `coef(enginemodel(m), s = m$state$lambda_sel)`, + a matrix with one column per class for multinomial fits. +* `calibrationcurve()`, `ece()` and `mce()` with `strategy = "quantile"` + collapsed to few bins when tied scores made quantile break points + coincide. The bins are now cut between distinct scores, giving + `min(bins, number of distinct scores)` bins of near equal size (bug fix: + results change only for tied scores). +* New `calibrationfit()` returns the calibration intercept + (calibration-in-the-large), the calibration slope and the recalibration + intercept with Wald confidence intervals, matching `glm()` and + `rms::val.prob()`. + # funcml 0.9.11 * The `xgboost` learner accepts a regression `objective` in `spec` diff --git a/R/bootvalidate.R b/R/bootvalidate.R index 538b392..7e31fc6 100644 --- a/R/bootvalidate.R +++ b/R/bootvalidate.R @@ -21,7 +21,8 @@ #' capped at `gamma`. The no-information value `gamma` is the metric #' computed on every pairing of an observed outcome with an apparent #' prediction (a random sample of 250,000 pairings when `n^2` exceeds -#' that). Higher-is-better metrics (`auc`, `accuracy`, `rsq`, ...) are +#' that). For `auc` and `auc_weighted`, `gamma` is exactly 0.5, the AUC +#' over all pairings. Higher-is-better metrics (`auc`, `accuracy`, `rsq`, ...) are #' handled on the corresponding loss scale, so "worse" means lower. #' #' The OOB metric is computed per replicate and averaged, which for @@ -183,6 +184,9 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) { } prob <- if (is.null(scored$prob)) NULL else scored$prob[j, , drop = FALSE] vapply(metrics, function(m) { + # Over all n^2 pairings the scores are independent of the outcome, so the AUC is exactly 0.5; + # the sampled pairs would only add noise. + if (m %in% c("auc", "auc_weighted")) return(0.5) suppressWarnings(.loss(truth[i], scored$pred[j], task, m, prob_matrix = prob)) }, numeric(1), USE.NAMES = FALSE) } diff --git a/R/fit.R b/R/fit.R index 270b079..0fefc98 100644 --- a/R/fit.R +++ b/R/fit.R @@ -375,7 +375,10 @@ listtunablelearners <- function(...) { #' @param ... Additional arguments passed to the underlying method. #' @return `print()` and `summary()` return the input object invisibly. #' `predict()` returns predictions in the requested format. `coef()` -#' returns a named numeric coefficient vector when available. +#' returns a named numeric coefficient vector for `glm`/`lm` learners and, +#' for `glmnet`, the coefficients at the selected lambda +#' (`object$state$lambda_sel`), a matrix with one column per class for a +#' multinomial fit. #' #' @name fit-methods #' @aliases print.funcml_fit summary.funcml_fit predict.funcml_fit coef.funcml_fit @@ -433,7 +436,17 @@ enginemodel <- function(fit) { #' @rdname fit-methods #' @export coef.funcml_fit <- function(object, ...) { - if (is.null(object$state$state)) stop("Coefficients not available for this model.", call. = FALSE) - if (!inherits(object$state$state, c("glm", "lm"))) stop("Coefficients not available for this model.", call. = FALSE) - stats::coef(object$state$state) + st <- object$state + if (inherits(st$state, "glmnet")) return(.glmnet_coef(st)) + if (!inherits(st$state, c("glm", "lm"))) stop("Coefficients not available for this model.", call. = FALSE) + stats::coef(st$state) +} + +# glmnet coefficients at the selected lambda as a named vector (a matrix with one column per class for +# multinomial fits), without the constant column padded to single-predictor designs. +.glmnet_coef <- function(st) { + cf <- stats::coef(st$state, s = st$lambda_sel) + dense <- function(x) stats::setNames(as.numeric(x), rownames(x))[rownames(x) != ".pad"] + if (!is.list(cf)) return(dense(cf)) + do.call(cbind, lapply(cf, dense)) } diff --git a/R/metrics.R b/R/metrics.R index 1937f2a..fd39ae9 100644 --- a/R/metrics.R +++ b/R/metrics.R @@ -94,7 +94,11 @@ listmetrics <- function(direction = NULL, columns = NULL) { #' For multiclass `auc()`, aggregation mode: `"macro"` or `"weighted"` #' (class-frequency weighted one-vs-rest AUC). Ignored for binary AUC. #' @param bins Number of bins for calibration summaries. -#' @param strategy Binning strategy: `"quantile"` or `"uniform"`. +#' @param strategy Binning strategy: `"quantile"` (bins of about equal size) +#' or `"uniform"` (equal-width bins on 0-1). When tied scores make +#' quantile break points coincide, the quantile bins are cut only between +#' distinct scores, so `min(bins, number of distinct scores)` bins are +#' returned, as close to equal size as the ties allow. #' @param positive Optional positive/event class for binary classification. #' Defaults to the second factor level. #' @return Numeric scalar metric. @@ -286,6 +290,8 @@ balancedaccuracy <- function(truth, pred_class) { if (m == 0L || n == 0L) { return(NA_real_) } + # Counts as doubles: m * n overflows integer arithmetic once n exceeds ~46,000 pairs per class. + m <- as.numeric(m); n <- as.numeric(n) (sum(pos_ranks) - n * (n + 1) / 2) / (m * n) } @@ -409,6 +415,24 @@ aucweighted <- function(truth, prob) { ) } +# Quantile bins for tied scores: cut only between distinct values, choosing each cut so the next bin holds +# about 1/k of the remaining rows (k bins left), and keeping a distinct value for every bin still to come. +# This gives min(bins, number of distinct scores) bins, as close to equal size as the ties allow. +.tied_quantile_breaks <- function(prob, bins) { + tb <- table(prob[!is.na(prob)]) + u <- as.numeric(names(tb)); cnt <- as.numeric(tb); nu <- length(u) + k <- min(bins, nu) + cuts <- integer(); start <- 1L + for (b in seq_len(k - 1L)) { + left <- k - b + last <- nu - left + cs <- cumsum(cnt[start:last]) + end <- start - 1L + which.min(abs(cs - sum(cnt[start:nu]) / (left + 1))) + cuts <- c(cuts, end); start <- end + 1L + } + c(u[1], (u[cuts] + u[cuts + 1L]) / 2, u[nu]) +} + .calibration_breaks <- function(prob, bins = 10, strategy = c("quantile", "uniform")) { strategy <- match.arg(strategy) if (!is.numeric(bins) || length(bins) != 1L || bins < 1L) { @@ -417,6 +441,7 @@ aucweighted <- function(truth, prob) { bins <- as.integer(bins) if (strategy == "quantile") { breaks <- unique(stats::quantile(prob, probs = seq(0, 1, length.out = bins + 1L), na.rm = TRUE, names = FALSE)) + if (length(breaks) < bins + 1L) breaks <- .tied_quantile_breaks(prob, bins) } else { breaks <- seq(0, 1, length.out = bins + 1L) } @@ -464,6 +489,48 @@ calibrationcurve <- function(truth, prob, bins = 10, strategy = c("quantile", "u .rbind_dt(out) } +#' Calibration intercept and slope. +#' +#' Logistic recalibration of predicted probabilities for a binary outcome. +#' The calibration intercept (calibration-in-the-large) is `a` in +#' `logit P(y = 1) = a + logit(p)`, a logistic model with `logit(p)` as an +#' offset; it is 0 when the mean prediction matches the event rate and is +#' negative when risks are overestimated. The calibration slope is `b` in +#' `logit P(y = 1) = a' + b * logit(p)`; it is 1 for a well calibrated model +#' and below 1 when predictions are too extreme (overfitting). Both are +#' fitted by [stats::glm()] with Wald confidence intervals. The slope matches +#' `rms::val.prob()`'s `"Slope"`; `rms::val.prob()`'s `"Intercept"` is the +#' recalibration intercept `a'`, returned here as `recalintercept`. +#' +#' @param truth Binary outcome (factor or vector with two values). +#' @param prob Predicted probability of the positive class. +#' @param positive Positive class; defaults to the second level. +#' @param level Confidence level. +#' @return A data frame with one row per term (`intercept`, `slope`, +#' `recalintercept`) and columns `term`, `estimate`, `se`, `lower`, +#' `upper`. +#' @references +#' Van Calster B, McLernon DJ, van Smeden M, Wynants L, Steyerberg EW (2019). +#' Calibration: the Achilles heel of predictive analytics. BMC Medicine 17, 230. +#' @examples +#' set.seed(1) +#' p <- runif(300) +#' y <- rbinom(300, 1, plogis(0.8 * qlogis(p) - 0.3)) +#' calibrationfit(y, p) +#' @export +calibrationfit <- function(truth, prob, positive = NULL, level = 0.95) { + d <- .binary_truth_prob(truth, prob, positive = positive) + y <- d$truth + lp <- stats::qlogis(pmin(pmax(d$prob, 1e-15), 1 - 1e-15)) + ci <- function(m, j, term) { + est <- stats::coef(m)[j]; se <- sqrt(diag(stats::vcov(m)))[j]; z <- stats::qnorm(1 - (1 - level) / 2) + data.frame(term = term, estimate = est, se = se, lower = est - z * se, upper = est + z * se, row.names = NULL) + } + m0 <- stats::glm(y ~ 1, family = stats::binomial(), offset = lp) + m1 <- stats::glm(y ~ lp, family = stats::binomial()) + rbind(ci(m0, 1, "intercept"), ci(m1, 2, "slope"), ci(m1, 1, "recalintercept")) +} + #' @rdname metrics #' @export ece <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), positive = NULL) { diff --git a/man/bootvalidate.Rd b/man/bootvalidate.Rd index 30fcd9b..9c21c0d 100644 --- a/man/bootvalidate.Rd +++ b/man/bootvalidate.Rd @@ -68,7 +68,8 @@ over replicates. capped at \code{gamma}. The no-information value \code{gamma} is the metric computed on every pairing of an observed outcome with an apparent prediction (a random sample of 250,000 pairings when \code{n^2} exceeds -that). Higher-is-better metrics (\code{auc}, \code{accuracy}, \code{rsq}, ...) are +that). For \code{auc} and \code{auc_weighted}, \code{gamma} is exactly 0.5, the AUC +over all pairings. Higher-is-better metrics (\code{auc}, \code{accuracy}, \code{rsq}, ...) are handled on the corresponding loss scale, so "worse" means lower. } diff --git a/man/calibrationfit.Rd b/man/calibrationfit.Rd new file mode 100644 index 0000000..5d5af53 --- /dev/null +++ b/man/calibrationfit.Rd @@ -0,0 +1,44 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/metrics.R +\name{calibrationfit} +\alias{calibrationfit} +\title{Calibration intercept and slope.} +\usage{ +calibrationfit(truth, prob, positive = NULL, level = 0.95) +} +\arguments{ +\item{truth}{Binary outcome (factor or vector with two values).} + +\item{prob}{Predicted probability of the positive class.} + +\item{positive}{Positive class; defaults to the second level.} + +\item{level}{Confidence level.} +} +\value{ +A data frame with one row per term (\code{intercept}, \code{slope}, +\code{recalintercept}) and columns \code{term}, \code{estimate}, \code{se}, \code{lower}, +\code{upper}. +} +\description{ +Logistic recalibration of predicted probabilities for a binary outcome. +The calibration intercept (calibration-in-the-large) is \code{a} in +\verb{logit P(y = 1) = a + logit(p)}, a logistic model with \code{logit(p)} as an +offset; it is 0 when the mean prediction matches the event rate and is +negative when risks are overestimated. The calibration slope is \code{b} in +\verb{logit P(y = 1) = a' + b * logit(p)}; it is 1 for a well calibrated model +and below 1 when predictions are too extreme (overfitting). Both are +fitted by \code{\link[stats:glm]{stats::glm()}} with Wald confidence intervals. The slope matches +\code{rms::val.prob()}'s \code{"Slope"}; \code{rms::val.prob()}'s \code{"Intercept"} is the +recalibration intercept \verb{a'}, returned here as \code{recalintercept}. +} +\examples{ +set.seed(1) +p <- runif(300) +y <- rbinom(300, 1, plogis(0.8 * qlogis(p) - 0.3)) +calibrationfit(y, p) +} +\references{ +Van Calster B, McLernon DJ, van Smeden M, Wynants L, Steyerberg EW (2019). +Calibration: the Achilles heel of predictive analytics. BMC Medicine 17, 230. +} diff --git a/man/fit-methods.Rd b/man/fit-methods.Rd index 38471a7..3f8ca88 100644 --- a/man/fit-methods.Rd +++ b/man/fit-methods.Rd @@ -44,7 +44,10 @@ \value{ \code{print()} and \code{summary()} return the input object invisibly. \code{predict()} returns predictions in the requested format. \code{coef()} -returns a named numeric coefficient vector when available. +returns a named numeric coefficient vector for \code{glm}/\code{lm} learners and, +for \code{glmnet}, the coefficients at the selected lambda +(\code{object$state$lambda_sel}), a matrix with one column per class for a +multinomial fit. } \description{ These methods provide the standard \code{print()}, \code{summary()}, \code{predict()}, and diff --git a/man/metrics.Rd b/man/metrics.Rd index 6c87ce7..948f0f1 100644 --- a/man/metrics.Rd +++ b/man/metrics.Rd @@ -112,7 +112,11 @@ per class (multiclass).} \item{bins}{Number of bins for calibration summaries.} -\item{strategy}{Binning strategy: \code{"quantile"} or \code{"uniform"}.} +\item{strategy}{Binning strategy: \code{"quantile"} (bins of about equal size) +or \code{"uniform"} (equal-width bins on 0-1). When tied scores make +quantile break points coincide, the quantile bins are cut only between +distinct scores, so \verb{min(bins, number of distinct scores)} bins are +returned, as close to equal size as the ties allow.} } \value{ Numeric scalar metric. diff --git a/tests/testthat/test-book-bugs-3.R b/tests/testthat/test-book-bugs-3.R new file mode 100644 index 0000000..31a59eb --- /dev/null +++ b/tests/testthat/test-book-bugs-3.R @@ -0,0 +1,76 @@ +test_that("auc and bootvalidate work for large n (no integer overflow)", { + set.seed(1); d <- data.frame(x = rnorm(400)); d$y <- factor(rbinom(400, 1, plogis(d$x))) + bv <- bootvalidate(d, y ~ x, "glm", B = 3, seed = 1, metrics = "auc") + expect_true(all(is.finite(unlist(bv$summary[, -1])))) + expect_identical(bv$summary$gamma, 0.5) + y <- rep(0:1, 60000); p <- runif(120000) + expect_equal(auc(factor(y), p), mean(outer(p[y == 1][1:300], p[y == 0][1:300], ">")), tolerance = 0.05) + expect_true(is.finite(auc(factor(y), p))) +}) + +test_that("bootvalidate() gives finite estimates for every metric at n = 5000", { + set.seed(2); d <- data.frame(x = rnorm(5000), z = rnorm(5000)); d$y <- factor(rbinom(5000, 1, plogis(d$x))) + ms <- c("auc", "auc_weighted", "brier", "logloss", "accuracy", "precision", "recall", "specificity", + "f1", "balanced_accuracy", "ece", "mce") + s <- bootvalidate(d, y ~ x + z, "glm", B = 4, seed = 1, metrics = ms)$summary + expect_true(all(is.finite(as.matrix(s[, -1])))) + expect_equal(s$gamma[1:2], c(0.5, 0.5)) + expect_true(all(s$est632plus >= pmin(s$apparent, s$gamma) - 1e-12 & + s$est632plus <= pmax(s$apparent, s$oob, s$gamma) + 1e-12)) + # all-pairs Brier: mean over i, j of (y_i - p_j)^2 (estimated from sampled pairs) + yy <- d$y == "1"; pp <- fitted(glm(y ~ x + z, binomial, d)) + expect_equal(s$gamma[s$metric == "brier"], mean(yy) - 2 * mean(yy) * mean(pp) + mean(pp^2), tolerance = 0.01) +}) + +test_that("bootvalidate() optimism-corrected C matches rms::validate() at n = 600", { + skip_if_not_installed("rms") + set.seed(4); n <- 600 + d <- data.frame(x1 = rnorm(n), x2 = rnorm(n), x3 = rnorm(n)) + d$y <- factor(rbinom(n, 1, plogis(0.7 * d$x1 - 0.4 * d$x2))) + f <- y ~ x1 + x2 + x3 + s <- bootvalidate(d, f, "glm", B = 40, seed = 1, metrics = "auc")$summary + set.seed(1); v <- rms::validate(rms::lrm(f, data = d, x = TRUE, y = TRUE), B = 40) + expect_equal(s$corrected, 0.5 + v["Dxy", "index.corrected"] / 2, tolerance = 0.01) +}) + +test_that("coef() on a glmnet fit returns the coefficients at the selected lambda", { + set.seed(3); d <- data.frame(x = rnorm(200), z = rnorm(200)); d$y <- factor(rbinom(200, 1, plogis(d$x))) + m <- fit(y ~ x + z, d, "glmnet") + ref <- as.matrix(coef(enginemodel(m), s = m$state$lambda_sel)) + expect_equal(coef(m), setNames(ref[, 1], rownames(ref))) + m1 <- fit(y ~ x, d, "glmnet", spec = list(lambda = 0.01)) + expect_named(coef(m1), c("(Intercept)", "x")) + mm <- fit(Species ~ ., iris, "glmnet") + expect_equal(dim(coef(mm)), c(5L, 3L)) + expect_equal(colnames(coef(mm)), levels(iris$Species)) +}) + +test_that("quantile calibration bins honour `bins` on tied scores", { + p <- rep(c(0.02, 0.1, 0.3, 0.6, 0.9, 0.97), c(380, 50, 50, 40, 30, 150)) + set.seed(1); y <- rbinom(length(p), 1, p) + cc <- calibrationcurve(y, p, bins = 5) + expect_equal(nrow(cc), 5L) + expect_equal(sum(cc$n), length(p)) + expect_true(all(cc$n > 0)) + expect_equal(nrow(calibrationcurve(y, p, bins = 10)), 6L) + expect_equal(nrow(calibrationcurve(y, rep(0.4, length(y)), bins = 5)), 1L) + # untied scores keep the plain quantile bins + q <- runif(200); z <- rbinom(200, 1, q) + expect_equal(calibrationcurve(z, q, bins = 4)$n, rep(50L, 4)) + expect_equal(ece(y, p, bins = 5), sum(cc$n / sum(cc$n) * cc$abs_gap)) +}) + +test_that("calibrationfit() matches glm and rms::val.prob()", { + set.seed(1); p <- runif(500); y <- rbinom(500, 1, plogis(0.7 * qlogis(p) - 0.3)) + cf <- calibrationfit(y, p) + expect_equal(cf$term, c("intercept", "slope", "recalintercept")) + m0 <- glm(y ~ 1, binomial, offset = qlogis(p)); m1 <- glm(y ~ qlogis(p), binomial) + expect_equal(cf$estimate, unname(c(coef(m0), coef(m1)[2:1])), tolerance = 1e-8) + expect_equal(cf$lower[2], unname(confint.default(m1)[2, 1]), tolerance = 1e-8) + if (requireNamespace("rms", quietly = TRUE)) { + vp <- rms::val.prob(p, y, pl = FALSE) + expect_equal(cf$estimate[2:3], unname(vp[c("Slope", "Intercept")]), tolerance = 1e-6) + } + f <- factor(ifelse(y == 1, "yes", "no")) + expect_equal(calibrationfit(f, p, positive = "yes")$estimate, cf$estimate) +})