Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 59 additions & 18 deletions R/metrics.R
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ listmetrics <- function(direction = NULL, columns = NULL) {
"Mean absolute percentage error for regression predictions.",
"Coefficient of determination for regression predictions.",
"Classification accuracy.",
"Macro-averaged classification precision.",
"Macro-averaged classification recall.",
"Macro-averaged classification specificity.",
"Macro-averaged F1 score.",
"Macro-averaged balanced accuracy.",
"Precision of the positive class (binary), macro-averaged (multiclass).",
"Recall (sensitivity) of the positive class (binary), macro-averaged (multiclass).",
"Specificity of the positive class (binary), macro-averaged (multiclass).",
"F1 score of the positive class (binary), macro-averaged (multiclass).",
"Balanced accuracy (macro-averaged recall).",
"Negative log-likelihood for classification probabilities.",
"Brier score for classification probabilities.",
"Area under the ROC curve.",
Expand Down Expand Up @@ -85,12 +85,18 @@ listmetrics <- function(direction = NULL, columns = NULL) {
#' @param pred_class Predicted class labels (classification).
#' @param prob Probability vector (binary) or probability matrix with one column
#' per class (multiclass).
#' @param average For multiclass AUC, aggregation mode: `"macro"` or
#' `"weighted"` (class-frequency weighted one-vs-rest AUC). Ignored for
#' binary AUC.
#' @param average For `precision()`, `recall()`, `specificity()` and `f1()`:
#' `"binary"` (default) computes the metric for the positive class of a
#' binary outcome (the second factor level, or `positive`), so `recall()` is
#' sensitivity; `"macro"` averages the one-vs-rest metric over all classes
#' with equal weight. Outcomes with more than two classes are always
#' macro-averaged. `balancedaccuracy()` is the macro-averaged recall.
#' For multiclass `auc()`, aggregation mode: `"macro"` or `"weighted"`
#' (class-frequency weighted one-vs-rest AUC). Ignored for binary AUC.
#' @param bins Number of bins for calibration summaries.
#' @param strategy Binning strategy: `"quantile"` or `"uniform"`.
#' @param positive Optional positive/event class for binary classification.
#' Defaults to the second factor level.
#' @return Numeric scalar metric.
#' @name metrics
#' @examples
Expand Down Expand Up @@ -232,27 +238,33 @@ accuracy <- function(truth, pred_class) {

#' @rdname metrics
#' @export
precision <- function(truth, pred_class) {
.macro_class_metric(truth, pred_class, numerator = function(tp, fp, fn, tn) tp, denominator = function(tp, fp, fn, tn) tp + fp)
precision <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) {
.class_metric(truth, pred_class, average = average, positive = positive,
numerator = function(tp, fp, fn, tn) tp,
denominator = function(tp, fp, fn, tn) tp + fp)
}

#' @rdname metrics
#' @export
recall <- function(truth, pred_class) {
.macro_class_metric(truth, pred_class, numerator = function(tp, fp, fn, tn) tp, denominator = function(tp, fp, fn, tn) tp + fn)
recall <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) {
.class_metric(truth, pred_class, average = average, positive = positive,
numerator = function(tp, fp, fn, tn) tp,
denominator = function(tp, fp, fn, tn) tp + fn)
}

#' @rdname metrics
#' @export
specificity <- function(truth, pred_class) {
.macro_class_metric(truth, pred_class, numerator = function(tp, fp, fn, tn) tn, denominator = function(tp, fp, fn, tn) tn + fp)
specificity <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) {
.class_metric(truth, pred_class, average = average, positive = positive,
numerator = function(tp, fp, fn, tn) tn,
denominator = function(tp, fp, fn, tn) tn + fp)
}

#' @rdname metrics
#' @export
f1 <- function(truth, pred_class) {
prec <- precision(truth, pred_class)
rec <- recall(truth, pred_class)
f1 <- function(truth, pred_class, average = c("binary", "macro"), positive = NULL) {
prec <- precision(truth, pred_class, average = average, positive = positive)
rec <- recall(truth, pred_class, average = average, positive = positive)
if (!is.finite(prec) || !is.finite(rec) || (prec + rec) == 0) {
return(NA_real_)
}
Expand All @@ -262,7 +274,7 @@ f1 <- function(truth, pred_class) {
#' @rdname metrics
#' @export
balancedaccuracy <- function(truth, pred_class) {
recall(truth, pred_class)
recall(truth, pred_class, average = "macro")
}

.binary_auc <- function(truth_binary, prob) {
Expand Down Expand Up @@ -318,6 +330,35 @@ aucweighted <- function(truth, prob) {
auc(truth, prob, average = "weighted")
}

# Binary outcomes: the metric for the positive class (second level unless
# `positive` is given). Multiclass outcomes, or average = "macro": the
# unweighted mean of the one-vs-rest metric over classes.
.class_metric <- function(truth, pred_class, average = c("binary", "macro"),
positive = NULL, numerator, denominator) {
average <- match.arg(average)
if (!is.factor(truth)) {
truth <- factor(truth)
}
lev <- levels(truth)
if (average == "macro" || length(lev) != 2L) {
return(.macro_class_metric(truth, pred_class, numerator, denominator))
}
positive <- positive %||% lev[2L]
if (length(positive) != 1L || !(positive %in% lev)) {
stop("`positive` must be one of the outcome levels.", call. = FALSE)
}
pred_class <- factor(pred_class, levels = lev)
tp <- sum(truth == positive & pred_class == positive, na.rm = TRUE)
fp <- sum(truth != positive & pred_class == positive, na.rm = TRUE)
fn <- sum(truth == positive & pred_class != positive, na.rm = TRUE)
tn <- sum(truth != positive & pred_class != positive, na.rm = TRUE)
denom <- denominator(tp, fp, fn, tn)
if (denom == 0) {
return(NA_real_)
}
numerator(tp, fp, fn, tn) / denom
}

.macro_class_metric <- function(truth, pred_class, numerator, denominator) {
if (!is.factor(truth)) {
truth <- factor(truth)
Expand Down
26 changes: 16 additions & 10 deletions man/metrics.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 10 additions & 4 deletions tests/testthat/test-core.R
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,16 @@ test_that("metrics are sane", {
expect_equal(auc(truth_cls, prob[,2]), 1)
expect_equal(aucweighted(truth_cls, prob[,2]), 1)
expect_equal(accuracy(truth_cls, pred_cls), 0.75)
expect_equal(precision(truth_cls, pred_cls), 5 / 6)
expect_equal(recall(truth_cls, pred_cls), 0.75)
expect_equal(specificity(truth_cls, pred_cls), 0.75)
expect_equal(f1(truth_cls, pred_cls), 0.7894737, tolerance = 1e-7)
# binary default: positive class "b"
expect_equal(precision(truth_cls, pred_cls), 2 / 3)
expect_equal(recall(truth_cls, pred_cls), 1)
expect_equal(specificity(truth_cls, pred_cls), 0.5)
expect_equal(f1(truth_cls, pred_cls), 0.8)
# macro averaging over both classes
expect_equal(precision(truth_cls, pred_cls, average = "macro"), 5 / 6)
expect_equal(recall(truth_cls, pred_cls, average = "macro"), 0.75)
expect_equal(specificity(truth_cls, pred_cls, average = "macro"), 0.75)
expect_equal(f1(truth_cls, pred_cls, average = "macro"), 0.7894737, tolerance = 1e-7)
expect_equal(balancedaccuracy(truth_cls, pred_cls), 0.75)
curve <- calibrationcurve(truth_cls, prob[, 2], bins = 2, strategy = "uniform")
expect_equal(nrow(curve), 2)
Expand Down
32 changes: 32 additions & 0 deletions tests/testthat/test-metric-fixes.R
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,35 @@ test_that("brier() and logloss() accept a positive-class probability vector", {
truth3 <- factor(c("a", "b", "c"))
expect_error(brier(truth3, c(0.2, 0.3, 0.4)), "binary")
})

test_that("binary recall, precision, specificity and f1 use the positive class", {
t <- factor(c(0, 0, 0, 0, 1, 1, 1, 1, 1, 1))
p <- factor(c(0, 0, 1, 1, 1, 1, 1, 1, 0, 0))
expect_equal(recall(t, p), 4 / 6)
expect_equal(precision(t, p), 4 / 6)
expect_equal(specificity(t, p), 2 / 4)
expect_equal(f1(t, p), 4 / 6)
expect_equal(balancedaccuracy(t, p), (4 / 6 + 2 / 4) / 2)
expect_equal(recall(t, p, average = "macro"), balancedaccuracy(t, p))
expect_equal(recall(t, p, positive = "0"), 2 / 4)
expect_false(isTRUE(all.equal(recall(t, p), balancedaccuracy(t, p))))
})

test_that("multiclass class metrics stay macro-averaged", {
t <- factor(c("a", "a", "b", "b", "c", "c"))
p <- factor(c("a", "b", "b", "b", "c", "a"), levels = levels(t))
expect_equal(recall(t, p), mean(c(1 / 2, 1, 1 / 2)))
expect_equal(recall(t, p), recall(t, p, average = "macro"))
})

test_that("compare() reports distinct recall, specificity and balanced accuracy", {
set.seed(5)
d <- data.frame(x1 = rnorm(200), x2 = rnorm(200))
d$y <- factor(ifelse(runif(200) < plogis(-1 + d$x1), "yes", "no"), levels = c("no", "yes"))
ev <- evaluate(d, y ~ x1 + x2, model = "glm", resampling = cv(3, seed = 1),
metrics = c("recall", "specificity", "balanced_accuracy"))
s <- ev$summary
vals <- s$mean[match(c("recall", "specificity", "balanced_accuracy"), s$metric)]
expect_false(isTRUE(all.equal(vals[1], vals[3])))
expect_equal(vals[3], (vals[1] + vals[2]) / 2, tolerance = 0.05)
})
Loading