From bb2492f0a08c5ab06fd744b2a8e0893c7feed57c Mon Sep 17 00:00:00 2001 From: ielbadisy Date: Sat, 14 Nov 2026 16:19:54 +0000 Subject: [PATCH] Compute binary precision, recall, specificity and F1 for the positive class --- R/metrics.R | 77 +++++++++++++++++++++++------- man/metrics.Rd | 26 ++++++---- tests/testthat/test-core.R | 14 ++++-- tests/testthat/test-metric-fixes.R | 32 +++++++++++++ 4 files changed, 117 insertions(+), 32 deletions(-) diff --git a/R/metrics.R b/R/metrics.R index e927abc..1937f2a 100644 --- a/R/metrics.R +++ b/R/metrics.R @@ -46,11 +46,11 @@ listmetrics <- function(direction = NULL, columns = NULL) { "Mean absolute percentage error for regression predictions.", "Coefficient of determination for regression predictions.", "Classification accuracy.", - "Macro-averaged classification precision.", - "Macro-averaged classification recall.", - "Macro-averaged classification specificity.", - "Macro-averaged F1 score.", - "Macro-averaged balanced accuracy.", + "Precision of the positive class (binary), macro-averaged (multiclass).", + "Recall (sensitivity) of the positive class (binary), macro-averaged (multiclass).", + "Specificity of the positive class (binary), macro-averaged (multiclass).", + "F1 score of the positive class (binary), macro-averaged (multiclass).", + "Balanced accuracy (macro-averaged recall).", "Negative log-likelihood for classification probabilities.", "Brier score for classification probabilities.", "Area under the ROC curve.", @@ -85,12 +85,18 @@ listmetrics <- function(direction = NULL, columns = NULL) { #' @param pred_class Predicted class labels (classification). #' @param prob Probability vector (binary) or probability matrix with one column #' per class (multiclass). -#' @param average For multiclass AUC, aggregation mode: `"macro"` or -#' `"weighted"` (class-frequency weighted one-vs-rest AUC). Ignored for -#' binary AUC. +#' @param average For `precision()`, `recall()`, `specificity()` and `f1()`: +#' `"binary"` (default) computes the metric for the positive class of a +#' binary outcome (the second factor level, or `positive`), so `recall()` is +#' sensitivity; `"macro"` averages the one-vs-rest metric over all classes +#' with equal weight. Outcomes with more than two classes are always +#' macro-averaged. `balancedaccuracy()` is the macro-averaged recall. +#' For multiclass `auc()`, aggregation mode: `"macro"` or `"weighted"` +#' (class-frequency weighted one-vs-rest AUC). Ignored for binary AUC. #' @param bins Number of bins for calibration summaries. #' @param strategy Binning strategy: `"quantile"` or `"uniform"`. #' @param positive Optional positive/event class for binary classification. +#' Defaults to the second factor level. #' @return Numeric scalar metric. #' @name metrics #' @examples @@ -232,27 +238,33 @@ accuracy <- function(truth, pred_class) { #' @rdname metrics #' @export -precision <- function(truth, pred_class) { - .macro_class_metric(truth, pred_class, numerator = function(tp, fp, fn, tn) tp, denominator = function(tp, fp, fn, tn) tp + fp) +precision <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) { + .class_metric(truth, pred_class, average = average, positive = positive, + numerator = function(tp, fp, fn, tn) tp, + denominator = function(tp, fp, fn, tn) tp + fp) } #' @rdname metrics #' @export -recall <- function(truth, pred_class) { - .macro_class_metric(truth, pred_class, numerator = function(tp, fp, fn, tn) tp, denominator = function(tp, fp, fn, tn) tp + fn) +recall <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) { + .class_metric(truth, pred_class, average = average, positive = positive, + numerator = function(tp, fp, fn, tn) tp, + denominator = function(tp, fp, fn, tn) tp + fn) } #' @rdname metrics #' @export -specificity <- function(truth, pred_class) { - .macro_class_metric(truth, pred_class, numerator = function(tp, fp, fn, tn) tn, denominator = function(tp, fp, fn, tn) tn + fp) +specificity <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) { + .class_metric(truth, pred_class, average = average, positive = positive, + numerator = function(tp, fp, fn, tn) tn, + denominator = function(tp, fp, fn, tn) tn + fp) } #' @rdname metrics #' @export -f1 <- function(truth, pred_class) { - prec <- precision(truth, pred_class) - rec <- recall(truth, pred_class) +f1 <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) { + prec <- precision(truth, pred_class, average = average, positive = positive) + rec <- recall(truth, pred_class, average = average, positive = positive) if (!is.finite(prec) || !is.finite(rec) || (prec + rec) == 0) { return(NA_real_) } @@ -262,7 +274,7 @@ f1 <- function(truth, pred_class) { #' @rdname metrics #' @export balancedaccuracy <- function(truth, pred_class) { - recall(truth, pred_class) + recall(truth, pred_class, average = "macro") } .binary_auc <- function(truth_binary, prob) { @@ -318,6 +330,35 @@ aucweighted <- function(truth, prob) { auc(truth, prob, average = "weighted") } +# Binary outcomes: the metric for the positive class (second level unless +# `positive` is given). Multiclass outcomes, or average = "macro": the +# unweighted mean of the one-vs-rest metric over classes. +.class_metric <- function(truth, pred_class, average = c("binary", "macro"), + positive = NULL, numerator, denominator) { + average <- match.arg(average) + if (!is.factor(truth)) { + truth <- factor(truth) + } + lev <- levels(truth) + if (average == "macro" || length(lev) != 2L) { + return(.macro_class_metric(truth, pred_class, numerator, denominator)) + } + positive <- positive %||% lev[2L] + if (length(positive) != 1L || !(positive %in% lev)) { + stop("`positive` must be one of the outcome levels.", call. = FALSE) + } + pred_class <- factor(pred_class, levels = lev) + tp <- sum(truth == positive & pred_class == positive, na.rm = TRUE) + fp <- sum(truth != positive & pred_class == positive, na.rm = TRUE) + fn <- sum(truth == positive & pred_class != positive, na.rm = TRUE) + tn <- sum(truth != positive & pred_class != positive, na.rm = TRUE) + denom <- denominator(tp, fp, fn, tn) + if (denom == 0) { + return(NA_real_) + } + numerator(tp, fp, fn, tn) / denom +} + .macro_class_metric <- function(truth, pred_class, numerator, denominator) { if (!is.factor(truth)) { truth <- factor(truth) diff --git a/man/metrics.Rd b/man/metrics.Rd index 02ee0c3..6c87ce7 100644 --- a/man/metrics.Rd +++ b/man/metrics.Rd @@ -41,13 +41,13 @@ brier(truth, prob_matrix) accuracy(truth, pred_class) -precision(truth, pred_class) +precision(truth, pred_class, average = c("binary", "macro"), positive = NULL) -recall(truth, pred_class) +recall(truth, pred_class, average = c("binary", "macro"), positive = NULL) -specificity(truth, pred_class) +specificity(truth, pred_class, average = c("binary", "macro"), positive = NULL) -f1(truth, pred_class) +f1(truth, pred_class, average = c("binary", "macro"), positive = NULL) balancedaccuracy(truth, pred_class) @@ -95,18 +95,24 @@ classes it returns the multiclass Brier score \item{pred_class}{Predicted class labels (classification).} +\item{average}{For \code{precision()}, \code{recall()}, \code{specificity()} and \code{f1()}: +\code{"binary"} (default) computes the metric for the positive class of a +binary outcome (the second factor level, or \code{positive}), so \code{recall()} is +sensitivity; \code{"macro"} averages the one-vs-rest metric over all classes +with equal weight. Outcomes with more than two classes are always +macro-averaged. \code{balancedaccuracy()} is the macro-averaged recall. +For multiclass \code{auc()}, aggregation mode: \code{"macro"} or \code{"weighted"} +(class-frequency weighted one-vs-rest AUC). Ignored for binary AUC.} + +\item{positive}{Optional positive/event class for binary classification. +Defaults to the second factor level.} + \item{prob}{Probability vector (binary) or probability matrix with one column per class (multiclass).} -\item{average}{For multiclass AUC, aggregation mode: \code{"macro"} or -\code{"weighted"} (class-frequency weighted one-vs-rest AUC). Ignored for -binary AUC.} - \item{bins}{Number of bins for calibration summaries.} \item{strategy}{Binning strategy: \code{"quantile"} or \code{"uniform"}.} - -\item{positive}{Optional positive/event class for binary classification.} } \value{ Numeric scalar metric. diff --git a/tests/testthat/test-core.R b/tests/testthat/test-core.R index 9a4dd98..c0e28b9 100644 --- a/tests/testthat/test-core.R +++ b/tests/testthat/test-core.R @@ -48,10 +48,16 @@ test_that("metrics are sane", { expect_equal(auc(truth_cls, prob[,2]), 1) expect_equal(aucweighted(truth_cls, prob[,2]), 1) expect_equal(accuracy(truth_cls, pred_cls), 0.75) - expect_equal(precision(truth_cls, pred_cls), 5 / 6) - expect_equal(recall(truth_cls, pred_cls), 0.75) - expect_equal(specificity(truth_cls, pred_cls), 0.75) - expect_equal(f1(truth_cls, pred_cls), 0.7894737, tolerance = 1e-7) + # binary default: positive class "b" + expect_equal(precision(truth_cls, pred_cls), 2 / 3) + expect_equal(recall(truth_cls, pred_cls), 1) + expect_equal(specificity(truth_cls, pred_cls), 0.5) + expect_equal(f1(truth_cls, pred_cls), 0.8) + # macro averaging over both classes + expect_equal(precision(truth_cls, pred_cls, average = "macro"), 5 / 6) + expect_equal(recall(truth_cls, pred_cls, average = "macro"), 0.75) + expect_equal(specificity(truth_cls, pred_cls, average = "macro"), 0.75) + expect_equal(f1(truth_cls, pred_cls, average = "macro"), 0.7894737, tolerance = 1e-7) expect_equal(balancedaccuracy(truth_cls, pred_cls), 0.75) curve <- calibrationcurve(truth_cls, prob[, 2], bins = 2, strategy = "uniform") expect_equal(nrow(curve), 2) diff --git a/tests/testthat/test-metric-fixes.R b/tests/testthat/test-metric-fixes.R index c7460c6..08266fa 100644 --- a/tests/testthat/test-metric-fixes.R +++ b/tests/testthat/test-metric-fixes.R @@ -10,3 +10,35 @@ test_that("brier() and logloss() accept a positive-class probability vector", { truth3 <- factor(c("a", "b", "c")) expect_error(brier(truth3, c(0.2, 0.3, 0.4)), "binary") }) + +test_that("binary recall, precision, specificity and f1 use the positive class", { + t <- factor(c(0, 0, 0, 0, 1, 1, 1, 1, 1, 1)) + p <- factor(c(0, 0, 1, 1, 1, 1, 1, 1, 0, 0)) + expect_equal(recall(t, p), 4 / 6) + expect_equal(precision(t, p), 4 / 6) + expect_equal(specificity(t, p), 2 / 4) + expect_equal(f1(t, p), 4 / 6) + expect_equal(balancedaccuracy(t, p), (4 / 6 + 2 / 4) / 2) + expect_equal(recall(t, p, average = "macro"), balancedaccuracy(t, p)) + expect_equal(recall(t, p, positive = "0"), 2 / 4) + expect_false(isTRUE(all.equal(recall(t, p), balancedaccuracy(t, p)))) +}) + +test_that("multiclass class metrics stay macro-averaged", { + t <- factor(c("a", "a", "b", "b", "c", "c")) + p <- factor(c("a", "b", "b", "b", "c", "a"), levels = levels(t)) + expect_equal(recall(t, p), mean(c(1 / 2, 1, 1 / 2))) + expect_equal(recall(t, p), recall(t, p, average = "macro")) +}) + +test_that("compare() reports distinct recall, specificity and balanced accuracy", { + set.seed(5) + d <- data.frame(x1 = rnorm(200), x2 = rnorm(200)) + d$y <- factor(ifelse(runif(200) < plogis(-1 + d$x1), "yes", "no"), levels = c("no", "yes")) + ev <- evaluate(d, y ~ x1 + x2, model = "glm", resampling = cv(3, seed = 1), + metrics = c("recall", "specificity", "balanced_accuracy")) + s <- ev$summary + vals <- s$mean[match(c("recall", "specificity", "balanced_accuracy"), s$metric)] + expect_false(isTRUE(all.equal(vals[1], vals[3]))) + expect_equal(vals[3], (vals[1] + vals[2]) / 2, tolerance = 0.05) +})