diff --git a/CHANGELOG.md b/CHANGELOG.md index 889b503d..a9d46df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,44 @@ All notable changes to mlquantify will be documented in this file. ## [Unreleased] +### New Features + +- **`PWKAC`: the PWK quantifier as Barranquero et al. (2013) propose it, + with leave-one-out rates.** `mlquantify.neighbors` had only `PWK`, which is + `class PWK(CC)` -- the weighted k-NN rule under plain classify-and-count. + That is the classifier of the paper inside a different quantifier. The + method the paper proposes is an adjusted count, and its whole argument for + nearest neighbours is the way the rates are estimated: the correction is + only as good as the `TPR` and `FPR` it divides by, and k-NN is the one + learner that can afford leave-one-out, because the neighbour query is made + once and every fold reads from it. `PWKAC` is that method. + + `PWKAC` replaces the cross-validation of `ACC` with one neighbour query: + it asks for `n_neighbors + 1` neighbours of each training point and drops + the first, which is the point itself. Cross-validation would estimate the + rates of a classifier trained on a fraction of the data and cost k + neighbour structures; resubstitution -- labelling the training set with the + fitted rule -- would let each point vote for itself at distance zero, and + under the PWK weights that self-vote is the heaviest of all when the point + belongs to the minority class. + + Measured on 12 binary datasets, against the truth (the rates of the fitted + rule on a held-out half), leave-one-out is closest on 6 of them with a mean + deviation of 0.0769, resubstitution on 5 with 0.0903, and 5-fold + cross-validation on 1 with 0.0877. The spread matters more than the + average: resubstitution is accurate where the problem is easy and + optimistic where it is not. On `haberman` the true `TPR` is 0.415 and + resubstitution reports 0.675; on `sonar`, 0.729 against 0.857 -- precisely + the datasets where the adjustment is doing work. + + The correction itself is what `PWK` lacks. On an imbalanced problem + (400 examples, 25% positives) `PWK` returns 0.385 against a true 0.250, + while `PWKAC` returns 0.254. + + `PWK` is unchanged and stays in the API: classify-and-count over the + weighted rule is a legitimate baseline, and several comparative studies + catalogue PWK that way. + ### Breaking Changes - **`MLPE` no longer takes an estimator, and is no longer aggregative.** @@ -68,6 +106,13 @@ All notable changes to mlquantify will be documented in this file. ### Documentation +- **`PWK`'s reference pointed at the wrong paper.** It cited Barranquero, + Diez & del Coz (2013), *Quantification-Oriented Learning Based on Reliable + Classifiers*, *Pattern Recognition* 48(2), 591-604 -- which is the Q-measure + paper, from 2015. PWK is from Barranquero, Gonzalez, Diez & del Coz (2013), + *On the study of nearest neighbor algorithms for prevalence estimation in + binary problems*, *Pattern Recognition* 46(2), 472-482. + - `CDE` documents when the iteration helps and when it turns against you. The cost update has an amplifying fixed point, which is what corrects `CC`'s shrinkage towards the training prevalence and also what turns a weak scorer diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 84cd9180..07c4666c 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -2163,6 +2163,25 @@ reference on core concepts, see the :ref:`Foundations >> from mlquantify.neighbors import PWKAC + >>> from sklearn.datasets import make_classification + >>> X, y = make_classification(n_samples=200, random_state=42) + >>> q = PWKAC(n_neighbors=10).fit(X, y) + >>> q.predict(X) + {0: ..., 1: ...} + + References + ---------- + .. dropdown:: References + + .. [1] Barranquero, J., Gonzalez, P., Diez, J., & del Coz, J. J. (2013). + On the study of nearest neighbor algorithms for prevalence + estimation in binary problems. + *Pattern Recognition*, 46(2), 472-482. + .. [2] Forman, G. (2008). Quantifying Counts and Costs via + Classification. *Data Mining and Knowledge Discovery*, + 17(2), 164-206. + """ + + _parameter_constraints = { + "alpha": [Interval(1, None, inclusive_right=False)], + "n_neighbors": [Interval(1, None, inclusive_right=False)], + "algorithm": [Options(["auto", "ball_tree", "kd_tree", "brute"])], + "metric": [str], + "leaf_size": [Interval(1, None, inclusive_right=False)], + "p": [Interval(1, None, inclusive_right=False)], + "metric_params": [dict, type(None)], + "n_jobs": [Interval(-1, None, inclusive_right=False), type(None)], + "strategy": [Options(["ovr", "ovo"])], + } + + def __init__(self, + alpha=1, + n_neighbors=10, + algorithm="auto", + metric="euclidean", + leaf_size=30, + p=2, + metric_params=None, + n_jobs=None, + strategy="ovr"): + self.alpha = alpha + self.n_neighbors = n_neighbors + self.algorithm = algorithm + self.metric = metric + self.leaf_size = leaf_size + self.p = p + self.metric_params = metric_params + self.n_jobs = n_jobs + # Like PWK, the estimator is fixed to the quantification k-NN instead + # of being supplied by the user. The cross-validation parameters of + # ACC stay at their defaults and are never used: the rates come from + # leave-one-out. + super().__init__( + estimator=PWKCLF(alpha=alpha, + n_neighbors=n_neighbors, + algorithm=algorithm, + metric=metric, + leaf_size=leaf_size, + p=p, + metric_params=metric_params, + n_jobs=n_jobs), + strategy=strategy, + n_jobs=n_jobs, + ) + + def _fit_estimator_predictions(self, X, y, estimator_fitted=False, + cv=None, stratified=None, shuffle=None, + random_state=None, cv_prediction="refit"): + """Fit the weighted k-NN and label the training set leave-one-out. + + Replaces the cross-validation of + :class:`~mlquantify.counting._base.BaseAdjustCount`; the + cross-validation arguments are accepted and ignored. + """ + X = np.asarray(X) + y = np.asarray(y) + if estimator_fitted: + self.estimator_ = self.estimator + return self.estimator_.predict(X), y + self.estimator_ = self.estimator.fit(X, y) + return self._leave_one_out(X), y + + def _leave_one_out(self, X): + """Label each training point with the neighbours that are not itself.""" + estimator = self.estimator_ + n_samples = X.shape[0] + classes = estimator.classes_ + # One query for the whole training set: ask for one neighbour more + # than the rule uses and drop the first column, which is the point + # itself at distance zero. + n_query = min(estimator.nbrs.n_neighbors + 1, n_samples) + indices = estimator.nbrs.kneighbors( + X, n_neighbors=n_query, return_distance=False)[:, 1:] + if indices.shape[1] == 0: + # A single training point has no other neighbour to vote. + return np.repeat(classes[0], n_samples) + + labels = np.asarray(estimator.y_train)[indices] + codes = np.searchsorted(classes, labels) + counts = np.zeros((n_samples, len(classes)), dtype=float) + np.add.at(counts, (np.arange(n_samples)[:, None], codes), 1.0) + counts *= estimator.class_weights + return classes[np.argmax(counts, axis=1)] diff --git a/tests/test_neighbors.py b/tests/test_neighbors.py index 9c7d8324..fe089726 100644 --- a/tests/test_neighbors.py +++ b/tests/test_neighbors.py @@ -1,7 +1,8 @@ import pytest import numpy as np -from mlquantify.neighbors import PWK +from mlquantify.neighbors import PWK, PWKAC +from mlquantify.neighbors._classification import PWKCLF def _assert_valid_prevalence(prevalence, n_classes): @@ -39,3 +40,86 @@ def test_pwk_params(binary_dataset): # Test high neighbors count q = PWK(n_neighbors=len(X) - 1).fit(X, y) + + +# --------------------------------------------------------------------------- +# PWKAC: the adjusted count over the weighted k-NN, with leave-one-out rates +# --------------------------------------------------------------------------- + +def test_pwkac_fit_predict(binary_dataset): + X, y = binary_dataset + preds = PWKAC(n_neighbors=10).fit(X, y).predict(X) + _assert_valid_prevalence(preds, n_classes=2) + + +def test_pwkac_multiclass(multiclass_dataset): + X, y = multiclass_dataset + preds = PWKAC(n_neighbors=10).fit(X, y).predict(X) + _assert_valid_prevalence(preds, n_classes=3) + + +def test_pwkac_train_predictions_are_leave_one_out(binary_dataset): + """The stored training labels must not be the resubstitution ones. + + Resubstitution lets each point vote for itself, which is what makes the + rates optimistic; leave-one-out drops that vote. On data where the rule + makes mistakes the two labelings have to differ. + """ + X, y = binary_dataset + q = PWKAC(n_neighbors=10).fit(X, y) + reinsercao = q.estimator_.predict(X) + assert q.train_predictions.shape == y.shape + assert not np.array_equal(q.train_predictions, reinsercao) + acerto_loo = np.mean(q.train_predictions == y) + acerto_reinsercao = np.mean(reinsercao == y) + assert acerto_loo <= acerto_reinsercao + + +def test_pwkac_leave_one_out_matches_an_explicit_loop(binary_dataset): + """The single-query trick must agree with refitting without each point.""" + X, y = binary_dataset + X, y = X[:60], y[:60] + q = PWKAC(n_neighbors=5).fit(X, y) + + esperado = np.empty_like(y) + for i in range(len(X)): + resto = np.ones(len(X), dtype=bool) + resto[i] = False + clf = PWKCLF(alpha=1, n_neighbors=5).fit(X[resto], y[resto]) + esperado[i] = clf.predict(X[i:i + 1])[0] + + # Leaving a point out also changes the class weights of the rule, by one + # example out of sixty, so a handful of borderline points may flip. The + # bulk has to agree. + assert np.mean(q.train_predictions == esperado) > 0.95 + + +def test_pwkac_corrects_the_bias_of_pwk(): + """On an imbalanced problem the adjustment must beat the plain count.""" + from sklearn.datasets import make_classification + + X, y = make_classification(n_samples=400, n_features=10, n_informative=5, + weights=[0.25, 0.75], random_state=42) + verdade = np.bincount(y)[1] / len(y) + contagem = np.asarray(PWK(n_neighbors=10).fit(X, y).predict(X))[1] + ajustada = np.asarray(PWKAC(n_neighbors=10).fit(X, y).predict(X))[1] + assert abs(ajustada - verdade) < abs(contagem - verdade) + + +def test_pwkac_params(binary_dataset): + X, y = binary_dataset + q = PWKAC(alpha=2, n_neighbors=7, metric="manhattan") + assert q.get_params()["alpha"] == 2 + assert q.get_params()["n_neighbors"] == 7 + q.fit(X, y) + assert q.estimator_.n_neighbors == 7 + _assert_valid_prevalence(q.predict(X), n_classes=2) + + +def test_pwkac_more_neighbors_than_samples(): + """``n_neighbors`` above the training size must not break the query.""" + rng = np.random.default_rng(0) + X = rng.normal(size=(8, 3)) + y = np.array([0, 0, 0, 0, 1, 1, 1, 1]) + preds = PWKAC(n_neighbors=20).fit(X, y).predict(X) + _assert_valid_prevalence(preds, n_classes=2)