diff --git a/DESCRIPTION b/DESCRIPTION index d5b6bc0..a38a75f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: funcml Type: Package Title: Functional Machine Learning Framework -Version: 0.9.9 +Version: 0.9.10 Depends: R (>= 3.5.0) Authors@R: c( person(given = "Imad", family = "El Badisy", role = c("aut", "cre"), email = "elbadisyimad@gmail.com") diff --git a/NEWS.md b/NEWS.md index 2941078..c39aa71 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,32 @@ +# funcml 0.9.10 + +* `brier()` and `logloss()` given a plain probability vector for a binary + outcome returned `mean(p^2)` and `NA` respectively. The vector is now read + as the probability of the second (positive) level, so `brier()` returns + `mean((p - y)^2)`; a vector with a multiclass outcome is an error. +* `precision()`, `recall()`, `specificity()` and `f1()` macro-averaged over + both classes of a binary outcome, so recall and specificity always equalled + balanced accuracy (and `compare()` showed the same value for all three). + For binary outcomes they are now computed for the positive class (second + factor level, or `positive`). New `average = c("binary", "macro")` and + `positive` arguments; multiclass outcomes stay macro-averaged and + `balancedaccuracy()` is unchanged. +* `estimate()` reported the spread of the unit-level effects as the standard + error, which ignores uncertainty in the outcome model (SE 0 and a + zero-width CI for an additive `glm`). With `model = "glm"` the normal + interval now uses a delta-method SE of the g-computation contrast; other + learners and `interval = "bootstrap"` use the SD of `n_boot` bootstrap + refits. `$bootstrap` now keeps the replicate estimates and their SD. +* The `stacking` learner fitted its meta-learner on in-sample base-learner + predictions. It now uses K-fold out-of-fold predictions (new spec entry + `folds`, default 5, reproducible with `seed`) and refits the base learners + on all rows for prediction. The meta-features are stored at + `$state$meta_x`. +* `compare()` checks for missing values before drawing folds and stops with + an error that names `compare()`, the affected columns and the number of + incomplete rows (before, the error came from `evaluate()` or from fold + generation). The behaviour is documented in `?compare` and `?evaluate`. + # funcml 0.9.9 * `compare()` now draws the resampling folds once (from `seed` or the diff --git a/R/compare.R b/R/compare.R index e6a617e..8ce74cb 100644 --- a/R/compare.R +++ b/R/compare.R @@ -44,6 +44,11 @@ #' @param inner_resampling Optional resampling object for the inner tuning #' loop when `nested = TRUE`. Defaults to the specification of `resampling`. #' @param ... Additional arguments passed to `evaluate()` or `tune()` / `fit()`. +#' @details Missing values: if any variable used by `formula` contains `NA`, +#' `compare()` stops before resampling with an error naming the affected +#' columns and the number of incomplete rows. Impute or drop incomplete rows +#' first. Predictor `NA` is accepted only when every learner in `models` +#' handles it natively (currently `fastgbm`); the outcome must be complete. #' @return A `funcml_compare` object: a list with `results` (one row per #' learner and metric), `details` (per-learner evaluation and tuning #' objects), `predictions` (out-of-fold predictions of every learner, with a @@ -83,6 +88,16 @@ compare <- function(data, formula, models, specs = NULL, set.seed(seed) } + # Check missing values once, before folds are drawn, so the error names + # compare() and the affected columns. Predictor NA is allowed only when + # every learner handles it natively. + native_na <- all(vapply(models, function(m) isTRUE(funcml_registry(m)$supports$na), logical(1))) + .check_no_missing_model_vars( + formula, data, + outcome_only = native_na && is.null(list(...)$na_action), + caller = "compare" + ) + y <- model.response(model.frame(formula, data, na.action = stats::na.pass)) task <- infer_task(y) if (task == "classification") { diff --git a/R/evaluate.R b/R/evaluate.R index 380f301..f61e6a5 100644 --- a/R/evaluate.R +++ b/R/evaluate.R @@ -217,13 +217,13 @@ plot.funcml_eval <- function(x, ...) { themefuncml() } -.check_no_missing_model_vars <- function(formula, data, outcome_only = FALSE) { +.check_no_missing_model_vars <- function(formula, data, outcome_only = FALSE, caller = "evaluate") { mf <- stats::model.frame(formula, data, na.action = stats::na.pass) if (isTRUE(outcome_only)) { if (anyNA(stats::model.response(mf))) { stop(sprintf( - "`evaluate()` needs a complete outcome: %d row(s) have NA in '%s'. Remove them first.", - sum(is.na(stats::model.response(mf))), names(mf)[1L] + "`%s()` needs a complete outcome: %d row(s) have NA in '%s'. Remove them first.", + caller, sum(is.na(stats::model.response(mf))), names(mf)[1L] ), call. = FALSE) } return(invisible(TRUE)) @@ -233,10 +233,10 @@ plot.funcml_eval <- function(x, ...) { n_rows <- sum(!stats::complete.cases(mf)) stop(sprintf( paste0( - "`evaluate()` does not handle missing values: %d row(s) have NA in ", + "`%s()` does not handle missing values: %d row(s) have NA in ", "model variable(s) %s. Impute or remove incomplete rows first." ), - n_rows, + caller, n_rows, paste(sprintf("'%s'", names(mf)[has_na]), collapse = ", ") ), call. = FALSE) } diff --git a/README.md b/README.md index c5cc211..be55cfb 100644 --- a/README.md +++ b/README.md @@ -158,25 +158,25 @@ listmetrics() #> 16 auc_weighted maximize #> 17 ece minimize #> 18 mce minimize -#> summary -#> 1 Root mean squared error for regression predictions. -#> 2 Mean absolute error for regression predictions. -#> 3 Mean squared error for regression predictions. -#> 4 Median absolute error for regression predictions. -#> 5 Mean absolute percentage error for regression predictions. -#> 6 Coefficient of determination for regression predictions. -#> 7 Classification accuracy. -#> 8 Macro-averaged classification precision. -#> 9 Macro-averaged classification recall. -#> 10 Macro-averaged classification specificity. -#> 11 Macro-averaged F1 score. -#> 12 Macro-averaged balanced accuracy. -#> 13 Negative log-likelihood for classification probabilities. -#> 14 Brier score for classification probabilities. -#> 15 Area under the ROC curve. -#> 16 Weighted multiclass area under the ROC curve. -#> 17 Expected calibration error for binary classification. -#> 18 Maximum calibration error for binary classification. +#> summary +#> 1 Root mean squared error for regression predictions. +#> 2 Mean absolute error for regression predictions. +#> 3 Mean squared error for regression predictions. +#> 4 Median absolute error for regression predictions. +#> 5 Mean absolute percentage error for regression predictions. +#> 6 Coefficient of determination for regression predictions. +#> 7 Classification accuracy. +#> 8 Precision of the positive class (binary), macro-averaged (multiclass). +#> 9 Recall (sensitivity) of the positive class (binary), macro-averaged (multiclass). +#> 10 Specificity of the positive class (binary), macro-averaged (multiclass). +#> 11 F1 score of the positive class (binary), macro-averaged (multiclass). +#> 12 Balanced accuracy (macro-averaged recall). +#> 13 Negative log-likelihood for classification probabilities. +#> 14 Brier score for classification probabilities. +#> 15 Area under the ROC curve. +#> 16 Weighted multiclass area under the ROC curve. +#> 17 Expected calibration error for binary classification. +#> 18 Maximum calibration error for binary classification. #> range #> 1 [0, Inf) #> 2 [0, Inf) @@ -303,10 +303,10 @@ eval_obj #> model: xgboost | task: classification #> metric mean sd n std_error conf_level conf_low conf_high #> 1 accuracy 0.7473 0.0146 4 0.0073 0.95 0.7241 0.7706 -#> 2 precision 0.6932 0.0234 4 0.0117 0.95 0.6559 0.7304 -#> 3 recall 0.6405 0.0178 4 0.0089 0.95 0.6121 0.6689 -#> 4 specificity 0.6405 0.0178 4 0.0089 0.95 0.6121 0.6689 -#> 5 f1 0.6657 0.0195 4 0.0098 0.95 0.6347 0.6968 +#> 2 precision 0.6066 0.0405 4 0.0203 0.95 0.5421 0.6711 +#> 3 recall 0.3842 0.0360 4 0.0180 0.95 0.3269 0.4415 +#> 4 specificity 0.8968 0.0192 4 0.0096 0.95 0.8663 0.9273 +#> 5 f1 0.4695 0.0331 4 0.0166 0.95 0.4168 0.5222 #> 6 balanced_accuracy 0.6405 0.0178 4 0.0089 0.95 0.6121 0.6689 #> 7 logloss 0.4925 0.0088 4 0.0044 0.95 0.4785 0.5065 #> 8 brier 0.1639 0.0038 4 0.0019 0.95 0.1579 0.1699 @@ -664,7 +664,7 @@ est_obj <- estimate( est_obj #> ATE via g-computation #> Treatment: smoke (Yes vs No) -#> Estimate: 0.0897 | SE: 0.0006 | 95% normal CI [0.0886, 0.0908] +#> Estimate: 0.0897 | SE: 0.0118 | 95% normal CI [0.0665, 0.1129] ``` By default, `plot()` shows the model-implied potential outcome @@ -744,14 +744,15 @@ citation("funcml") #> To cite package 'funcml' in publications use: #> #> El Badisy I (????). _funcml: Functional Machine Learning Framework_. -#> R package version 0.9.8, . +#> R package version 0.9.10, +#> . #> #> A BibTeX entry for LaTeX users is #> #> @Manual{, #> title = {funcml: Functional Machine Learning Framework}, #> author = {Imad {El Badisy}}, -#> note = {R package version 0.9.8}, +#> note = {R package version 0.9.10}, #> url = {https://CRAN.R-project.org/package=funcml}, #> } ``` diff --git a/man/compare.Rd b/man/compare.Rd index 53199b7..6266b8e 100644 --- a/man/compare.Rd +++ b/man/compare.Rd @@ -94,6 +94,12 @@ which is the configuration you would deploy. \code{nested = FALSE} restores the older, faster behaviour of tuning and scoring on the same folds; that estimate is optimistic (the best of many configurations on the very folds used to pick it) and should not be reported as a performance estimate. + +Missing values: if any variable used by \code{formula} contains \code{NA}, +\code{compare()} stops before resampling with an error naming the affected +columns and the number of incomplete rows. Impute or drop incomplete rows +first. Predictor \code{NA} is accepted only when every learner in \code{models} +handles it natively (currently \code{fastgbm}); the outcome must be complete. } \examples{ cmp <- compare( diff --git a/man/figures/README-evaluate-plot-1.png b/man/figures/README-evaluate-plot-1.png index d5dfef3..a154074 100644 Binary files a/man/figures/README-evaluate-plot-1.png and b/man/figures/README-evaluate-plot-1.png differ diff --git a/man/figures/README-interpret-ale-1.png b/man/figures/README-interpret-ale-1.png index e05f931..f3b5caa 100644 Binary files a/man/figures/README-interpret-ale-1.png and b/man/figures/README-interpret-ale-1.png differ diff --git a/man/figures/README-interpret-ceteris-paribus-1.png b/man/figures/README-interpret-ceteris-paribus-1.png index 382873a..4f936f0 100644 Binary files a/man/figures/README-interpret-ceteris-paribus-1.png and b/man/figures/README-interpret-ceteris-paribus-1.png differ diff --git a/man/figures/README-interpret-pdp-1.png b/man/figures/README-interpret-pdp-1.png index 08adaa3..9d28be4 100644 Binary files a/man/figures/README-interpret-pdp-1.png and b/man/figures/README-interpret-pdp-1.png differ diff --git a/tests/testthat/test-metric-fixes.R b/tests/testthat/test-metric-fixes.R index 08266fa..bb87a48 100644 --- a/tests/testthat/test-metric-fixes.R +++ b/tests/testthat/test-metric-fixes.R @@ -42,3 +42,29 @@ test_that("compare() reports distinct recall, specificity and balanced accuracy" expect_false(isTRUE(all.equal(vals[1], vals[3]))) expect_equal(vals[3], (vals[1] + vals[2]) / 2, tolerance = 0.05) }) + +test_that("compare() and evaluate() stop on NA with an error naming the columns", { + d <- mtcars + d$wt[c(2, 5)] <- NA + d$hp[7] <- NA + expect_error( + evaluate(d, mpg ~ wt + hp, model = "glm", resampling = cv(3, seed = 1)), + "`evaluate\\(\\)`.*3 row\\(s\\).*'wt', 'hp'" + ) + expect_error( + compare(d, mpg ~ wt + hp, models = c("glm", "rpart"), resampling = cv(3, seed = 1)), + "`compare\\(\\)`.*3 row\\(s\\).*'wt', 'hp'" + ) + di <- iris + di$Species[c(3, 60)] <- NA + expect_error( + compare(di, Species ~ ., models = "rpart", resampling = cv(3, seed = 1)), + "`compare\\(\\)`.*'Species'" + ) + # complete rows still work after dropping the incomplete ones + cc <- d[stats::complete.cases(d[, c("mpg", "wt", "hp")]), ] + expect_s3_class( + compare(cc, mpg ~ wt + hp, models = "glm", resampling = cv(3, seed = 1)), + "funcml_compare" + ) +})