diff --git a/mlquantify/matching/_histogram.py b/mlquantify/matching/_histogram.py index 8c199750..d0c0f69f 100644 --- a/mlquantify/matching/_histogram.py +++ b/mlquantify/matching/_histogram.py @@ -77,6 +77,11 @@ class MatchingHistogramQuantifier(BaseMatchingQuantifier): Number of score columns used to build histograms. bin_strategy : str or None, default=None Aggregation strategy across bin sizes (``'median'`` or ``'mean'``). + bin_edges : {'fixed', 'auto'}, default='fixed' + Where the bin edges are placed. ``'fixed'`` spans ``range``, + ``[0, 1]`` by default, which is right for histograms of posteriors. + ``'auto'`` fits per-feature edges to the training data and leaves + the outer bins unbounded, which is what raw features need. Attributes ---------- @@ -110,6 +115,7 @@ def __init__( histogram_features=None, bin_strategy=None, laplace_smoothing=False, + bin_edges="fixed", ): if bins_size is None: bins_size = np.append(np.linspace(2, 20, 10), 30).astype(int) @@ -122,6 +128,7 @@ def __init__( self.histogram_features = histogram_features self.bin_strategy = bin_strategy self.laplace_smoothing = laplace_smoothing + self.bin_edges = bin_edges super().__init__( representation=HistogramRepresentation( bins=bins_size, @@ -129,6 +136,7 @@ def __init__( features=histogram_features, partition_blocks=bin_strategy in {"median", "mean"}, laplace_smoothing=laplace_smoothing, + bin_edges=bin_edges, ), normalize=True, ) @@ -636,6 +644,17 @@ def __init__( distance="hellinger", solver="ternary", strategy=strategy, + # HDx bins raw features, which are on the data's own scale, so + # the bin edges come from the training features -- as GHDx and + # GHDy, the generalized siblings, already did. The score-space + # siblings (HDy, DyS) keep the fixed [0, 1] edges, which is + # right for posteriors. With fixed edges here every feature + # value outside [0, 1] fell out of the histogram, and the + # estimate stopped depending on the test sample. Fitted edges + # keep their outer bins open, at -inf and +inf, so test values + # beyond the training range land in the end bins instead of + # being dropped. + bin_edges="auto", ) def fit(self, X, y, sample_weight=None): diff --git a/tests/test_matching.py b/tests/test_matching.py index 9a636043..19d3ad46 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -57,6 +57,69 @@ def test_histogram_feature_matching_binary(binary_dataset): assert q.best_distance_ is not None +def _shifted_split(prevalence, seed=0, n_train=800, n_test=600): + """Balanced training split and a test sample at the given prevalence.""" + from sklearn.datasets import make_classification + + X, y = make_classification( + n_samples=4000, n_features=6, n_informative=4, n_redundant=0, + n_classes=2, random_state=seed, + ) + rng = np.random.default_rng(seed) + order = rng.permutation(len(y)) + X, y = X[order], y[order] + + train = np.concatenate([ + np.where(y == 0)[0][:n_train // 2], + np.where(y == 1)[0][:n_train // 2], + ]) + rest = np.setdiff1d(np.arange(len(y)), train) + n_pos = int(round(n_test * prevalence)) + test = np.concatenate([ + rest[y[rest] == 1][:n_pos], + rest[y[rest] == 0][:n_test - n_pos], + ]) + return X[train], y[train], X[test], y[test] + + +@pytest.mark.parametrize("prevalence", [0.1, 0.5, 0.9]) +def test_hdx_tracks_the_test_prevalence_on_unscaled_features(prevalence): + """HDx must bin raw features on their own scale. + + The bin edges used to be fixed to [0, 1], which is right for the + score-space siblings but discards every feature value outside that range. + The estimate then stopped depending on the test sample: it came out near + 0.64 whether the true prevalence was 0.1 or 0.9. + """ + X_train, y_train, X_test, y_test = _shifted_split(prevalence) + true_prevalence = float(np.mean(y_test == 1)) + + q = HDx(bins_size=[8]).fit(X_train, y_train) + with config_context(prevalence_return_type="array"): + estimate = np.asarray(q.predict(X_test), dtype=float) + + _assert_valid_prevalence(estimate, n_classes=2) + assert abs(estimate[1] - true_prevalence) < 0.1 + + +def test_hdx_is_invariant_to_feature_scaling(): + """Binning on the data's own scale makes a monotone rescaling irrelevant.""" + from sklearn.preprocessing import MinMaxScaler + + X_train, y_train, X_test, _ = _shifted_split(0.3) + scaler = MinMaxScaler().fit(X_train) + + def estimate(A, B): + q = HDx(bins_size=[8]).fit(A, y_train) + with config_context(prevalence_return_type="array"): + return np.asarray(q.predict(B), dtype=float) + + raw = estimate(X_train, X_test) + scaled = estimate(scaler.transform(X_train), scaler.transform(X_test)) + + np.testing.assert_allclose(raw, scaled, atol=0.05) + + def test_hdy_matches_median_grid_recipe(): train_scores = np.asarray( [