Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Package: funcml
Type: Package
Title: Functional Machine Learning Framework
Version: 0.9.12
Version: 0.9.13
Depends: R (>= 3.5.0)
Authors@R: c(
person(given = "Imad", family = "El Badisy", role = c("aut", "cre"), email = "elbadisyimad@gmail.com")
Expand Down
2 changes: 2 additions & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ export(brier)
export(calibration_curve)
export(calibrationcurve)
export(calibrationfit)
export(calslope)
export(citl)
export(compare)
export(cv)
export(dca)
Expand Down
4 changes: 4 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# funcml 0.9.13

* New calibration metrics `calslope()` (calibration slope) and `citl()` (calibration-in-the-large), the `slope` and `intercept` rows of `calibrationfit()`. As metric names `"calslope"` and `"citl"` they work in `bootvalidate()` and `evaluate()` and are listed by `listmetrics()` (direction `"target"`: ideal values 1 and 0). In `bootvalidate()` the optimism-corrected slope equals `rms::validate()`'s corrected `Slope` when both are seeded identically; `gamma` and `est632plus` are `NA` for these metrics. `tune()` refuses them as tuning criteria; `compare()` ranks them by distance to the ideal value.

# funcml 0.9.12

* `auc()` overflowed integer arithmetic in the pair count (positives times
Expand Down
15 changes: 14 additions & 1 deletion R/bootvalidate.R
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@
#' over all pairings. Higher-is-better metrics (`auc`, `accuracy`, `rsq`, ...) are
#' handled on the corresponding loss scale, so "worse" means lower.
#'
#' Calibration metrics `calslope` and `citl` (binary outcomes) have an
#' ideal value (1 and 0) rather than a direction: their optimism and
#' corrected values follow `rms::validate()`'s `Slope` (the apparent slope
#' of a logistic regression is 1, so corrected = 1 - optimism), while
#' `gamma` and `est632plus` are `NA`.
#'
#' The OOB metric is computed per replicate and averaged, which for
#' non-additive metrics (`auc`, `ece`) is the usual approximation of the
#' per-observation leave-one-out bootstrap.
Expand All @@ -37,6 +43,8 @@
#' @param metrics Character vector of metric names. Defaults to
#' `c("rmse", "mse", "rsq")` for regression and
#' `c("auc", "brier", "logloss", "accuracy")` for classification.
#' Binary outcomes also accept `"calslope"` (calibration slope) and
#' `"citl"` (calibration-in-the-large).
#' @param seed Optional seed for the bootstrap draws.
#' @param ncores Optional number of CPU cores for the replicates.
#' @param ... Passed to [fit()].
Expand Down Expand Up @@ -153,7 +161,7 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) {
# Score a fitted model on `newdata`; keeps the predictions for the
# no-information rate.
.bv_score <- function(fit_obj, newdata, truth, task, metrics) {
prob_metrics <- c("logloss", "brier", "auc", "auc_weighted", "ece", "mce")
prob_metrics <- .prob_metrics
if (task == "regression") {
pred <- as.numeric(predict(fit_obj, newdata = newdata, type = "response"))
vals <- vapply(metrics, function(m) .loss(truth, pred, task, m), numeric(1))
Expand Down Expand Up @@ -187,13 +195,18 @@ print.funcml_bootvalidate <- function(x, digits = 4L, ...) {
# Over all n^2 pairings the scores are independent of the outcome, so the AUC is exactly 0.5;
# the sampled pairs would only add noise.
if (m %in% c("auc", "auc_weighted")) return(0.5)
if (m %in% names(.target_metrics)) return(NA_real_)
suppressWarnings(.loss(truth[i], scored$pred[j], task, m, prob_matrix = prob))
}, numeric(1), USE.NAMES = FALSE)
}

.bv_estimates <- function(app, boot, orig, oob, gamma, direction) {
optimism <- mean(boot - orig)
oob_mean <- mean(oob)
if (identical(direction, "target")) {
return(c(apparent = app, optimism = optimism, corrected = app - optimism, oob = oob_mean,
est632 = 0.368 * app + 0.632 * oob_mean, est632plus = NA_real_, gamma = NA_real_))
}
# .632+ on the loss scale (larger = worse).
s <- if (identical(direction, "min")) 1 else -1
a <- s * app
Expand Down
1 change: 1 addition & 0 deletions R/compare.R
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,7 @@ compare <- function(data, formula, models, specs = NULL,

.comparison_score <- function(x, metric) {
direction <- metric_direction(metric[1])
if (direction == "target") return(abs(x - .target_metrics[[metric[1]]]))
if (direction == "min") x else -x
}

Expand Down
2 changes: 1 addition & 1 deletion R/evaluate.R
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ plot.funcml_eval <- function(x, ...) {
train_data <- data[fold$train, , drop = FALSE]
test_data <- data[fold$test, , drop = FALSE]
fit_fold <- fit(formula, train_data, model, spec = spec, ...)
prob_metrics <- c("logloss", "brier", "auc", "auc_weighted", "ece", "mce")
prob_metrics <- .prob_metrics
type_use <- type %||% if (task == "regression") "response" else if (any(metrics %in% prob_metrics)) "prob" else "class"
preds <- predict(fit_fold, newdata = test_data, type = type_use)
truth <- y_all[fold$test]
Expand Down
42 changes: 34 additions & 8 deletions R/metrics.R
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
#'
#' @param direction Optional character filter: `"maximize"` or `"minimize"`.
#' @param columns Optional character vector of columns to return.
#' @return Data frame of metric metadata.
#' @return Data frame of metric metadata. `direction` is `"maximize"`, `"minimize"` or
#' `"target"` (calibration metrics with an ideal value: `calslope` 1, `citl` 0).
#' @examples
#' listmetrics()
#' listmetrics(direction = "minimize")
Expand Down Expand Up @@ -30,13 +31,13 @@ listmetrics <- function(direction = NULL, columns = NULL) {
"rmse", "mae", "mse", "medae", "mape", "rsq",
"accuracy", "precision", "recall", "specificity", "f1",
"balanced_accuracy", "logloss", "brier", "auc", "auc_weighted",
"ece", "mce"
"ece", "mce", "calslope", "citl"
),
direction = c(
"minimize", "minimize", "minimize", "minimize", "minimize", "maximize",
"maximize", "maximize", "maximize", "maximize", "maximize",
"maximize", "minimize", "minimize", "maximize", "maximize",
"minimize", "minimize"
"minimize", "minimize", "target", "target"
),
summary = c(
"Root mean squared error for regression predictions.",
Expand All @@ -56,13 +57,15 @@ listmetrics <- function(direction = NULL, columns = NULL) {
"Area under the ROC curve.",
"Weighted multiclass area under the ROC curve.",
"Expected calibration error for binary classification.",
"Maximum calibration error for binary classification."
"Maximum calibration error for binary classification.",
"Calibration slope (logistic recalibration) for binary classification.",
"Calibration-in-the-large (intercept with the logit as offset) for binary classification."
),
range = c(
"[0, Inf)", "[0, Inf)", "[0, Inf)", "[0, Inf)", "[0, Inf)", "(-Inf, 1]",
"[0, 1]", "[0, 1]", "[0, 1]", "[0, 1]", "[0, 1]",
"[0, 1]", "[0, Inf)", "[0, 1] binary, [0, 2] multiclass", "[0, 1]", "[0, 1]",
"[0, 1]", "[0, 1]"
"[0, 1]", "[0, 1]", "(-Inf, Inf)", "(-Inf, Inf)"
),
stringsAsFactors = FALSE
)
Expand Down Expand Up @@ -101,6 +104,8 @@ listmetrics <- function(direction = NULL, columns = NULL) {
#' returned, as close to equal size as the ties allow.
#' @param positive Optional positive/event class for binary classification.
#' Defaults to the second factor level.
#' @details `calslope()` and `citl()` are the `slope` and `intercept` rows of
#' [calibrationfit()]: ideal values 1 and 0.
#' @return Numeric scalar metric.
#' @name metrics
#' @examples
Expand Down Expand Up @@ -144,6 +149,8 @@ listmetrics <- function(direction = NULL, columns = NULL) {
#' calibrationcurve(truth_cls, prob_cls[, "yes"])
#' ece(truth_cls, prob_cls[, "yes"])
#' mce(truth_cls, prob_cls[, "yes"])
#' calslope(truth_cls, prob_cls[, "yes"])
#' citl(truth_cls, prob_cls[, "yes"])
#' @export
rmse <- function(truth, pred) {
sqrt(mean((truth - pred)^2))
Expand Down Expand Up @@ -553,13 +560,30 @@ mce <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), pos
max(curve$abs_gap[nonempty])
}

#' @rdname metrics
#' @export
calslope <- function(truth, prob, positive = NULL) {
calibrationfit(truth, prob, positive = positive)$estimate[2]
}

#' @rdname metrics
#' @export
citl <- function(truth, prob, positive = NULL) {
calibrationfit(truth, prob, positive = positive)$estimate[1]
}

# Metrics scored from predicted probabilities.
.prob_metrics <- c("logloss", "brier", "auc", "auc_weighted", "ece", "mce", "calslope", "citl")
# Calibration metrics: ideal value 1 (slope) or 0 (CITL), neither "higher" nor "lower" is better.
.target_metrics <- c(calslope = 1, citl = 0)

.loss <- function(truth, pred, task, metric, prob_matrix = NULL) {
metric <- match.arg(metric, c(
"rmse", "mae", "mse", "medae", "mape", "rsq",
"logloss", "brier", "accuracy", "precision", "recall",
"specificity", "f1", "balanced_accuracy", "auc", "auc_weighted", "ece", "mce"
"specificity", "f1", "balanced_accuracy", "auc", "auc_weighted", "ece", "mce", "calslope", "citl"
))
if (task == "regression" && metric %in% c("logloss", "brier", "accuracy", "precision", "recall", "specificity", "f1", "balanced_accuracy", "auc", "auc_weighted", "ece", "mce")) {
if (task == "regression" && metric %in% c("accuracy", "precision", "recall", "specificity", "f1", "balanced_accuracy", .prob_metrics)) {
stop(sprintf("Metric '%s' is classification-only.", metric), call. = FALSE)
}
if (task == "classification" && metric %in% c("rmse", "mae", "mse", "medae", "mape", "rsq")) {
Expand All @@ -578,7 +602,7 @@ mce <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), pos

truth <- factor(truth)
pred_class <- if (is.null(pred)) NULL else factor(pred, levels = levels(truth))
if (metric %in% c("logloss", "brier", "auc", "auc_weighted", "ece", "mce")) {
if (metric %in% .prob_metrics) {
if (is.null(prob_matrix)) stop(sprintf("Metric '%s' requires probabilities.", metric), call. = FALSE)
prob_matrix <- .normalize_prob_matrix(prob_matrix, levels(truth))
}
Expand All @@ -595,6 +619,8 @@ mce <- function(truth, prob, bins = 10, strategy = c("quantile", "uniform"), pos
brier = brier(truth, prob_matrix),
ece = ece(truth, prob_matrix),
mce = mce(truth, prob_matrix),
calslope = calslope(truth, prob_matrix),
citl = citl(truth, prob_matrix),
auc = auc(truth, prob_matrix),
auc_weighted = aucweighted(truth, prob_matrix)
)
Expand Down
4 changes: 3 additions & 1 deletion R/tune.R
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ tune <- function(data, formula, model, grid = NULL, resampling = cv(5),
set.seed(seed)
}
dirs <- metric_direction(metric)
if (dirs == "target") stop(sprintf("Metric '%s' is a calibration measure, not a tuning criterion.", metric), call. = FALSE)
search_grid <- .select_tuning_configs(grid, search = search, n_evals = n_evals)
nested <- NULL
if (!is.null(outer_resampling)) {
Expand Down Expand Up @@ -131,6 +132,7 @@ tune <- function(data, formula, model, grid = NULL, resampling = cv(5),
}

metric_direction <- function(metric) {
if (metric %in% names(.target_metrics)) return("target")
if (metric %in% c("rmse", "mae", "mse", "medae", "mape", "logloss", "brier", "ece", "mce")) "min" else "max"
}

Expand Down Expand Up @@ -271,7 +273,7 @@ plot.funcml_tune <- function(x, ...) {

type_use <- type %||% if (fit_obj$task == "regression") {
"response"
} else if (metric %in% c("logloss", "brier", "auc", "auc_weighted", "ece", "mce")) {
} else if (metric %in% .prob_metrics) {
"prob"
} else {
"class"
Expand Down
10 changes: 9 additions & 1 deletion man/bootvalidate.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion man/listmetrics.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions man/metrics.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 35 additions & 0 deletions tests/testthat/test-book-bugs-4.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
test_that("calslope and citl match calibrationfit() and glm", {
set.seed(1); p <- runif(300); y <- rbinom(300, 1, plogis(0.8 * qlogis(p) - 0.3))
lp <- qlogis(p)
expect_equal(calslope(y, p), unname(coef(glm(y ~ lp, family = binomial()))[2]))
expect_equal(citl(y, p), unname(coef(glm(y ~ 1, family = binomial(), offset = lp))[1]))
expect_true(all(c("calslope", "citl") %in% listmetrics()$metric))
})

test_that("bootvalidate() calibration slope matches rms::validate() Slope", {
skip_if_not_installed("rms")
set.seed(4); n <- 300
d <- data.frame(x1 = rnorm(n), x2 = rnorm(n), x3 = rnorm(n))
d$y <- factor(rbinom(n, 1, plogis(0.7 * d$x1 - 0.4 * d$x2)))
f <- y ~ x1 + x2 + x3
bv <- bootvalidate(d, f, "glm", B = 30, seed = 1, metrics = c("calslope", "citl"))
s <- bv$summary
set.seed(1); v <- rms::validate(rms::lrm(f, data = d, x = TRUE, y = TRUE), B = 30)
expect_equal(s$apparent, c(1, 0), tolerance = 1e-6)
expect_equal(s$corrected[1], v["Slope", "index.corrected"], tolerance = 1e-6)
expect_true(all(is.na(s$est632plus)) && all(is.na(s$gamma)))
# CITL optimism from scratch on the same bootstrap rows
o <- vapply(bv$indices, function(i) {
m <- glm(f, binomial, d[i, ]); lo <- function(dd) predict(m, dd)
ci <- function(dd) coef(glm(dd$y ~ 1, family = binomial(), offset = lo(dd)))[1]
ci(d[i, ]) - ci(d)
}, numeric(1))
expect_equal(s$optimism[2], mean(o), tolerance = 1e-6)
})

test_that("calibration metrics work in evaluate() and are refused by tune()", {
set.seed(2); d <- data.frame(x = rnorm(200)); d$y <- factor(rbinom(200, 1, plogis(d$x)))
ev <- evaluate(d, y ~ x, model = "glm", resampling = cv(3, seed = 1), metrics = c("calslope", "citl"))
expect_true(all(is.finite(ev$summary$mean)))
expect_error(tune(d, y ~ x, model = "glmnet", grid = data.frame(alpha = c(0, 1)), metric = "calslope"), "calibration")
})
2 changes: 1 addition & 1 deletion tests/testthat/test-core.R
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ test_that("listmetrics returns metric metadata", {
expect_s3_class(metrics_tbl, "data.frame")
expect_named(metrics_tbl, c("metric", "direction", "summary", "range"))
expect_true(all(c("rmse", "mae", "accuracy", "auc", "ece") %in% metrics_tbl$metric))
expect_true(all(metrics_tbl$direction %in% c("maximize", "minimize")))
expect_true(all(metrics_tbl$direction %in% c("maximize", "minimize", "target")))
expect_true(all(nzchar(metrics_tbl$summary)))
expect_true(all(nzchar(metrics_tbl$range)))
})
Expand Down
Loading