diff --git a/R/metrics.R b/R/metrics.R index e059d8b..e927abc 100644 --- a/R/metrics.R +++ b/R/metrics.R @@ -75,6 +75,8 @@ listmetrics <- function(direction = NULL, columns = NULL) { #' @param truth Observed outcomes. #' @param pred Predicted numeric values or class labels. #' @param prob_matrix Matrix or vector of predicted probabilities (classification). +#' For `logloss()` and `brier()` with a binary outcome, a plain numeric +#' vector is read as the probability of the second (positive) factor level. #' For `brier()`, a binary task returns the standard binary Brier score #' `mean((p - y)^2)`, with `p` the probability of the second (positive) #' class and `y` its 0/1 indicator (range 0 to 1). With more than two @@ -172,10 +174,36 @@ mape <- function(truth, pred) { mean(abs((truth[keep] - pred[keep]) / truth[keep])) } +# Coerce probabilities to a matrix with one column per class level, named by +# level. A plain vector is read as the probability of the second (positive) +# level and is only accepted for a binary outcome. +.metric_prob_matrix <- function(truth, prob_matrix) { + lev <- if (is.factor(truth)) levels(truth) else levels(factor(truth)) + if (is.null(dim(prob_matrix))) { + if (length(lev) != 2L) { + stop("A probability vector is only accepted for a binary outcome; ", + "supply a matrix with one column per class level.", call. = FALSE) + } + prob_matrix <- as.numeric(prob_matrix) + prob_matrix <- cbind(1 - prob_matrix, prob_matrix) + colnames(prob_matrix) <- lev + return(prob_matrix) + } + prob_matrix <- as.matrix(prob_matrix) + if (is.null(colnames(prob_matrix))) { + if (ncol(prob_matrix) != length(lev)) { + stop("Probability matrix has no column names and its column count does not ", + "match the number of class levels.", call. = FALSE) + } + colnames(prob_matrix) <- lev + } + prob_matrix +} + #' @rdname metrics #' @export logloss <- function(truth, prob_matrix) { - if (!is.matrix(prob_matrix)) prob_matrix <- cbind(1 - prob_matrix, prob_matrix) + prob_matrix <- .metric_prob_matrix(truth, prob_matrix) truth_idx <- match(truth, colnames(prob_matrix)) p <- prob_matrix[cbind(seq_along(truth_idx), truth_idx)] -mean(log(p + 1e-15)) @@ -184,7 +212,7 @@ logloss <- function(truth, prob_matrix) { #' @rdname metrics #' @export brier <- function(truth, prob_matrix) { - if (!is.matrix(prob_matrix)) prob_matrix <- cbind(1 - prob_matrix, prob_matrix) + prob_matrix <- .metric_prob_matrix(truth, prob_matrix) truth_onehot <- matrix(0, nrow = length(truth), ncol = ncol(prob_matrix)) colnames(truth_onehot) <- colnames(prob_matrix) truth_onehot[cbind(seq_along(truth), match(truth, colnames(prob_matrix)))] <- 1 diff --git a/man/metrics.Rd b/man/metrics.Rd index 7105737..02ee0c3 100644 --- a/man/metrics.Rd +++ b/man/metrics.Rd @@ -85,6 +85,8 @@ mce( \item{pred}{Predicted numeric values or class labels.} \item{prob_matrix}{Matrix or vector of predicted probabilities (classification). +For \code{logloss()} and \code{brier()} with a binary outcome, a plain numeric +vector is read as the probability of the second (positive) factor level. For \code{brier()}, a binary task returns the standard binary Brier score \code{mean((p - y)^2)}, with \code{p} the probability of the second (positive) class and \code{y} its 0/1 indicator (range 0 to 1). With more than two diff --git a/tests/testthat/test-metric-fixes.R b/tests/testthat/test-metric-fixes.R new file mode 100644 index 0000000..c7460c6 --- /dev/null +++ b/tests/testthat/test-metric-fixes.R @@ -0,0 +1,12 @@ +library(funcml) + +test_that("brier() and logloss() accept a positive-class probability vector", { + truth <- factor(c("a", "b", "b", "a", "b"), levels = c("a", "b")) + p <- c(0.2, 0.7, 0.4, 0.5, 0.9) + expect_equal(brier(truth, p), mean((p - (truth == "b"))^2)) + expect_equal(brier(truth, p), brier(truth, cbind(a = 1 - p, b = p))) + expect_equal(logloss(truth, p), -mean(log(ifelse(truth == "b", p, 1 - p))), tolerance = 1e-10) + expect_equal(logloss(truth, p), logloss(truth, cbind(a = 1 - p, b = p))) + truth3 <- factor(c("a", "b", "c")) + expect_error(brier(truth3, c(0.2, 0.3, 0.4)), "binary") +})