Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Package: funcml
Type: Package
Title: Functional Machine Learning Framework
Version: 0.9.11
Version: 0.9.12
Depends: R (>= 3.5.0)
Authors@R: c(
person(given = "Imad", family = "El Badisy", role = c("aut", "cre"), email = "elbadisyimad@gmail.com")
Expand Down
1 change: 1 addition & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ export(bootvalidate)
export(brier)
export(calibration_curve)
export(calibrationcurve)
export(calibrationfit)
export(compare)
export(cv)
export(dca)
Expand Down
22 changes: 22 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,25 @@
# funcml 0.9.12

* `auc()` overflowed integer arithmetic in the pair count (positives times
negatives) and returned `NA` for large samples. This broke
`bootvalidate()` with `metrics = "auc"` (and the default classification
metrics) for n of about 400 and more, where the no-information value was
computed on 250,000 sampled pairs. Fixed; the no-information AUC in
`bootvalidate()` is now exactly 0.5 (bug fix: `gamma` and `est632plus`
for `auc` change slightly at large n).
* `coef()` on a `glmnet` fit returns the coefficients at the selected
lambda (`$state$lambda_sel`), as `coef(enginemodel(m), s = m$state$lambda_sel)`,
a matrix with one column per class for multinomial fits.
* `calibrationcurve()`, `ece()` and `mce()` with `strategy = "quantile"`
collapsed to few bins when tied scores made quantile break points
coincide. The bins are now cut between distinct scores, giving
`min(bins, number of distinct scores)` bins of near equal size (bug fix:
results change only for tied scores).
* New `calibrationfit()` returns the calibration intercept
(calibration-in-the-large), the calibration slope and the recalibration
intercept with Wald confidence intervals, matching `glm()` and
`rms::val.prob()`.

# funcml 0.9.11

* The `xgboost` learner accepts a regression `objective` in `spec`
Expand Down
6 changes: 5 additions & 1 deletion R/bootvalidate.R
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
#' capped at `gamma`. The no-information value `gamma` is the metric
#' computed on every pairing of an observed outcome with an apparent
#' prediction (a random sample of 250,000 pairings when `n^2` exceeds
#' that). Higher-is-better metrics (`auc`, `accuracy`, `rsq`, ...) are
#' that). For `auc` and `auc_weighted`, `gamma` is exactly 0.5, the AUC
#' over all pairings. Higher-is-better metrics (`auc`, `accuracy`, `rsq`, ...) are
#' handled on the corresponding loss scale, so "worse" means lower.
#'
#' The OOB metric is computed per replicate and averaged, which for
Expand Down Expand Up @@ -183,6 +184,9 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) {
}
prob <- if (is.null(scored$prob)) NULL else scored$prob[j, , drop = FALSE]
vapply(metrics, function(m) {
# Over all n^2 pairings the scores are independent of the outcome, so the AUC is exactly 0.5;
# the sampled pairs would only add noise.
if (m %in% c("auc", "auc_weighted")) return(0.5)
suppressWarnings(.loss(truth[i], scored$pred[j], task, m, prob_matrix = prob))
}, numeric(1), USE.NAMES = FALSE)
}
Expand Down
21 changes: 17 additions & 4 deletions R/fit.R
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,10 @@ listtunablelearners <- function(...) {
#' @param ... Additional arguments passed to the underlying method.
#' @return `print()` and `summary()` return the input object invisibly.
#' `predict()` returns predictions in the requested format. `coef()`
#' returns a named numeric coefficient vector when available.
#' returns a named numeric coefficient vector for `glm`/`lm` learners and,
#' for `glmnet`, the coefficients at the selected lambda
#' (`object$state$lambda_sel`), a matrix with one column per class for a
#' multinomial fit.
#'
#' @name fit-methods
#' @aliases print.funcml_fit summary.funcml_fit predict.funcml_fit coef.funcml_fit
Expand Down Expand Up @@ -433,7 +436,17 @@ enginemodel <- function(fit) {
#' @rdname fit-methods
#' @export
coef.funcml_fit <- function(object, ...) {
if (is.null(object$state$state)) stop("Coefficients not available for this model.", call. = FALSE)
if (!inherits(object$state$state, c("glm", "lm"))) stop("Coefficients not available for this model.", call. = FALSE)
stats::coef(object$state$state)
st <- object$state
if (inherits(st$state, "glmnet")) return(.glmnet_coef(st))
if (!inherits(st$state, c("glm", "lm"))) stop("Coefficients not available for this model.", call. = FALSE)
stats::coef(st$state)
}

# glmnet coefficients at the selected lambda as a named vector (a matrix with one column per class for
# multinomial fits), without the constant column padded to single-predictor designs.
.glmnet_coef <- function(st) {
cf <- stats::coef(st$state, s = st$lambda_sel)
dense <- function(x) stats::setNames(as.numeric(x), rownames(x))[rownames(x) != ".pad"]
if (!is.list(cf)) return(dense(cf))
do.call(cbind, lapply(cf, dense))
}
69 changes: 68 additions & 1 deletion R/metrics.R
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,11 @@ listmetrics <- function(direction = NULL, columns = NULL) {
#' For multiclass `auc()`, aggregation mode: `"macro"` or `"weighted"`
#' (class-frequency weighted one-vs-rest AUC). Ignored for binary AUC.
#' @param bins Number of bins for calibration summaries.
#' @param strategy Binning strategy: `"quantile"` or `"uniform"`.
#' @param strategy Binning strategy: `"quantile"` (bins of about equal size)
#' or `"uniform"` (equal-width bins on 0-1). When tied scores make
#' quantile break points coincide, the quantile bins are cut only between
#' distinct scores, so `min(bins, number of distinct scores)` bins are
#' returned, as close to equal size as the ties allow.
#' @param positive Optional positive/event class for binary classification.
#' Defaults to the second factor level.
#' @return Numeric scalar metric.
Expand Down Expand Up @@ -286,6 +290,8 @@ balancedaccuracy <- function(truth, pred_class) {
if (m == 0L || n == 0L) {
return(NA_real_)
}
# Counts as doubles: m * n overflows integer arithmetic once n exceeds ~46,000 pairs per class.
m <- as.numeric(m); n <- as.numeric(n)
(sum(pos_ranks) - n * (n + 1) / 2) / (m * n)
}

Expand Down Expand Up @@ -409,6 +415,24 @@ aucweighted <- function(truth, prob) {
)
}

# Quantile bins for tied scores: cut only between distinct values, choosing each cut so the next bin holds
# about 1/k of the remaining rows (k bins left), and keeping a distinct value for every bin still to come.
# This gives min(bins, number of distinct scores) bins, as close to equal size as the ties allow.
.tied_quantile_breaks <- function(prob, bins) {
tb <- table(prob[!is.na(prob)])
u <- as.numeric(names(tb)); cnt <- as.numeric(tb); nu <- length(u)
k <- min(bins, nu)
cuts <- integer(); start <- 1L
for (b in seq_len(k - 1L)) {
left <- k - b
last <- nu - left
cs <- cumsum(cnt[start:last])
end <- start - 1L + which.min(abs(cs - sum(cnt[start:nu]) / (left + 1)))
cuts <- c(cuts, end); start <- end + 1L
}
c(u[1], (u[cuts] + u[cuts + 1L]) / 2, u[nu])
}

.calibration_breaks <- function(prob, bins = 10, strategy = c("quantile", "uniform")) {
strategy <- match.arg(strategy)
if (!is.numeric(bins) || length(bins) != 1L || bins < 1L) {
Expand All @@ -417,6 +441,7 @@ aucweighted <- function(truth, prob) {
bins <- as.integer(bins)
if (strategy == "quantile") {
breaks <- unique(stats::quantile(prob, probs = seq(0, 1, length.out = bins + 1L), na.rm = TRUE, names = FALSE))
if (length(breaks) < bins + 1L) breaks <- .tied_quantile_breaks(prob, bins)
} else {
breaks <- seq(0, 1, length.out = bins + 1L)
}
Expand Down Expand Up @@ -464,6 +489,48 @@ calibrationcurve <- function(truth, prob, bins = 10, strategy = c("quantile", "u
.rbind_dt(out)
}

#' Calibration intercept and slope.
#'
#' Logistic recalibration of predicted probabilities for a binary outcome.
#' The calibration intercept (calibration-in-the-large) is `a` in
#' `logit P(y = 1) = a + logit(p)`, a logistic model with `logit(p)` as an
#' offset; it is 0 when the mean prediction matches the event rate and is
#' negative when risks are overestimated. The calibration slope is `b` in
#' `logit P(y = 1) = a' + b * logit(p)`; it is 1 for a well calibrated model
#' and below 1 when predictions are too extreme (overfitting). Both are
#' fitted by [stats::glm()] with Wald confidence intervals. The slope matches
#' `rms::val.prob()`'s `"Slope"`; `rms::val.prob()`'s `"Intercept"` is the
#' recalibration intercept `a'`, returned here as `recalintercept`.
#'
#' @param truth Binary outcome (factor or vector with two values).
#' @param prob Predicted probability of the positive class.
#' @param positive Positive class; defaults to the second level.
#' @param level Confidence level.
#' @return A data frame with one row per term (`intercept`, `slope`,
#' `recalintercept`) and columns `term`, `estimate`, `se`, `lower`,
#' `upper`.
#' @references
#' Van Calster B, McLernon DJ, van Smeden M, Wynants L, Steyerberg EW (2019).
#' Calibration: the Achilles heel of predictive analytics. BMC Medicine 17, 230.
#' @examples
#' set.seed(1)
#' p <- runif(300)
#' y <- rbinom(300, 1, plogis(0.8 * qlogis(p) - 0.3))
#' calibrationfit(y, p)
#' @export
calibrationfit <- function(truth, prob, positive = NULL, level = 0.95) {
d <- .binary_truth_prob(truth, prob, positive = positive)
y <- d$truth
lp <- stats::qlogis(pmin(pmax(d$prob, 1e-15), 1 - 1e-15))
ci <- function(m, j, term) {
est <- stats::coef(m)[j]; se <- sqrt(diag(stats::vcov(m)))[j]; z <- stats::qnorm(1 - (1 - level) / 2)
data.frame(term = term, estimate = est, se = se, lower = est - z * se, upper = est + z * se, row.names = NULL)
}
m0 <- stats::glm(y ~ 1, family = stats::binomial(), offset = lp)
m1 <- stats::glm(y ~ lp, family = stats::binomial())
rbind(ci(m0, 1, "intercept"), ci(m1, 2, "slope"), ci(m1, 1, "recalintercept"))
}

#' @rdname metrics
#' @export
ece <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), positive = NULL) {
Expand Down
3 changes: 2 additions & 1 deletion man/bootvalidate.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

44 changes: 44 additions & 0 deletions man/calibrationfit.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion man/fit-methods.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion man/metrics.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

76 changes: 76 additions & 0 deletions tests/testthat/test-book-bugs-3.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
test_that("auc and bootvalidate work for large n (no integer overflow)", {
set.seed(1); d <- data.frame(x = rnorm(400)); d$y <- factor(rbinom(400, 1, plogis(d$x)))
bv <- bootvalidate(d, y ~ x, "glm", B = 3, seed = 1, metrics = "auc")
expect_true(all(is.finite(unlist(bv$summary[, -1]))))
expect_identical(bv$summary$gamma, 0.5)
y <- rep(0:1, 60000); p <- runif(120000)
expect_equal(auc(factor(y), p), mean(outer(p[y == 1][1:300], p[y == 0][1:300], ">")), tolerance = 0.05)
expect_true(is.finite(auc(factor(y), p)))
})

test_that("bootvalidate() gives finite estimates for every metric at n = 5000", {
set.seed(2); d <- data.frame(x = rnorm(5000), z = rnorm(5000)); d$y <- factor(rbinom(5000, 1, plogis(d$x)))
ms <- c("auc", "auc_weighted", "brier", "logloss", "accuracy", "precision", "recall", "specificity",
"f1", "balanced_accuracy", "ece", "mce")
s <- bootvalidate(d, y ~ x + z, "glm", B = 4, seed = 1, metrics = ms)$summary
expect_true(all(is.finite(as.matrix(s[, -1]))))
expect_equal(s$gamma[1:2], c(0.5, 0.5))
expect_true(all(s$est632plus >= pmin(s$apparent, s$gamma) - 1e-12 &
s$est632plus <= pmax(s$apparent, s$oob, s$gamma) + 1e-12))
# all-pairs Brier: mean over i, j of (y_i - p_j)^2 (estimated from sampled pairs)
yy <- d$y == "1"; pp <- fitted(glm(y ~ x + z, binomial, d))
expect_equal(s$gamma[s$metric == "brier"], mean(yy) - 2 * mean(yy) * mean(pp) + mean(pp^2), tolerance = 0.01)
})

test_that("bootvalidate() optimism-corrected C matches rms::validate() at n = 600", {
skip_if_not_installed("rms")
set.seed(4); n <- 600
d <- data.frame(x1 = rnorm(n), x2 = rnorm(n), x3 = rnorm(n))
d$y <- factor(rbinom(n, 1, plogis(0.7 * d$x1 - 0.4 * d$x2)))
f <- y ~ x1 + x2 + x3
s <- bootvalidate(d, f, "glm", B = 40, seed = 1, metrics = "auc")$summary
set.seed(1); v <- rms::validate(rms::lrm(f, data = d, x = TRUE, y = TRUE), B = 40)
expect_equal(s$corrected, 0.5 + v["Dxy", "index.corrected"] / 2, tolerance = 0.01)
})

test_that("coef() on a glmnet fit returns the coefficients at the selected lambda", {
set.seed(3); d <- data.frame(x = rnorm(200), z = rnorm(200)); d$y <- factor(rbinom(200, 1, plogis(d$x)))
m <- fit(y ~ x + z, d, "glmnet")
ref <- as.matrix(coef(enginemodel(m), s = m$state$lambda_sel))
expect_equal(coef(m), setNames(ref[, 1], rownames(ref)))
m1 <- fit(y ~ x, d, "glmnet", spec = list(lambda = 0.01))
expect_named(coef(m1), c("(Intercept)", "x"))
mm <- fit(Species ~ ., iris, "glmnet")
expect_equal(dim(coef(mm)), c(5L, 3L))
expect_equal(colnames(coef(mm)), levels(iris$Species))
})

test_that("quantile calibration bins honour `bins` on tied scores", {
p <- rep(c(0.02, 0.1, 0.3, 0.6, 0.9, 0.97), c(380, 50, 50, 40, 30, 150))
set.seed(1); y <- rbinom(length(p), 1, p)
cc <- calibrationcurve(y, p, bins = 5)
expect_equal(nrow(cc), 5L)
expect_equal(sum(cc$n), length(p))
expect_true(all(cc$n > 0))
expect_equal(nrow(calibrationcurve(y, p, bins = 10)), 6L)
expect_equal(nrow(calibrationcurve(y, rep(0.4, length(y)), bins = 5)), 1L)
# untied scores keep the plain quantile bins
q <- runif(200); z <- rbinom(200, 1, q)
expect_equal(calibrationcurve(z, q, bins = 4)$n, rep(50L, 4))
expect_equal(ece(y, p, bins = 5), sum(cc$n / sum(cc$n) * cc$abs_gap))
})

test_that("calibrationfit() matches glm and rms::val.prob()", {
set.seed(1); p <- runif(500); y <- rbinom(500, 1, plogis(0.7 * qlogis(p) - 0.3))
cf <- calibrationfit(y, p)
expect_equal(cf$term, c("intercept", "slope", "recalintercept"))
m0 <- glm(y ~ 1, binomial, offset = qlogis(p)); m1 <- glm(y ~ qlogis(p), binomial)
expect_equal(cf$estimate, unname(c(coef(m0), coef(m1)[2:1])), tolerance = 1e-8)
expect_equal(cf$lower[2], unname(confint.default(m1)[2, 1]), tolerance = 1e-8)
if (requireNamespace("rms", quietly = TRUE)) {
vp <- rms::val.prob(p, y, pl = FALSE)
expect_equal(cf$estimate[2:3], unname(vp[c("Slope", "Intercept")]), tolerance = 1e-6)
}
f <- factor(ifelse(y == 1, "yes", "no"))
expect_equal(calibrationfit(f, p, positive = "yes")$estimate, cf$estimate)
})
Loading