diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c166db..6f2ce68 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,9 +5,8 @@ name: tests on: push: - branches: [ "main", "dev" ] + branches: [ "**" ] pull_request: - branches: [ "main", "dev" ] jobs: test: @@ -16,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 @@ -33,7 +32,7 @@ jobs: - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest diff --git a/.gitignore b/.gitignore index 8455419..c4898a8 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,5 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +# Pane worktrees (local only) +worktrees/ diff --git a/PyNomaly/__init__.py b/PyNomaly/__init__.py index 0cd3ebf..0e94304 100644 --- a/PyNomaly/__init__.py +++ b/PyNomaly/__init__.py @@ -3,6 +3,10 @@ from PyNomaly.loop import ( LocalOutlierProbability, + LoOP, + __version__, +) +from PyNomaly.exceptions import ( PyNomalyError, ValidationError, ClusterSizeError, @@ -11,8 +15,10 @@ __all__ = [ "LocalOutlierProbability", + "LoOP", "PyNomalyError", "ValidationError", "ClusterSizeError", "MissingValuesError", + "__version__", ] diff --git a/PyNomaly/_compat.py b/PyNomaly/_compat.py new file mode 100644 index 0000000..d6c11a5 --- /dev/null +++ b/PyNomaly/_compat.py @@ -0,0 +1,198 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +""" +Optional scikit-learn integration. + +When scikit-learn is installed, LocalOutlierProbability inherits from +``BaseEstimator`` and ``OutlierMixin`` for full ecosystem compatibility +(pipelines, grid search, ``check_estimator``). + +When it is not installed, lightweight fallbacks provide the same public +surface (``get_params``, ``set_params``, ``fit_predict``, ``check_is_fitted``) +so PyNomaly works as a standalone package. + +Install the optional dependency with:: + + pip install PyNomaly[sklearn] +""" + +from __future__ import annotations + +import inspect + +try: + from sklearn.base import BaseEstimator, OutlierMixin + from sklearn.exceptions import NotFittedError + from sklearn.utils.validation import check_array, check_is_fitted + + _SKLEARN_AVAILABLE = True +except ImportError: # pragma: no cover - exercised only without sklearn + _SKLEARN_AVAILABLE = False + + class NotFittedError(ValueError, AttributeError): + """Raised when an estimator method is used before fit.""" + + class BaseEstimator: + """Minimal BaseEstimator providing get_params / set_params.""" + + def get_params(self, deep=True): + params = {} + for name, param in inspect.signature(self.__init__).parameters.items(): + if name == "self": + continue + params[name] = getattr(self, name, param.default) + return params + + def set_params(self, **params): + if not params: + return self + valid = self.get_params(deep=True) + for key, value in params.items(): + if key not in valid: + raise ValueError( + f"Invalid parameter {key!r} for estimator " + f"{type(self).__name__}. Valid parameters are: " + f"{sorted(valid)!r}." + ) + setattr(self, key, value) + return self + + def __repr__(self): + params = self.get_params(deep=False) + filtered = { + k: v + for k, v in params.items() + if v is not inspect.Parameter.empty + } + inner = ", ".join(f"{k}={v!r}" for k, v in filtered.items()) + return f"{type(self).__name__}({inner})" + + class OutlierMixin: + """Minimal OutlierMixin providing fit_predict.""" + + def fit_predict(self, X, y=None, **fit_params): + return self.fit(X, y, **fit_params).predict(X) + + def check_is_fitted(estimator, attributes=None, *, msg=None): + if attributes is None: + fitted = [ + a + for a in vars(estimator) + if a.endswith("_") and not a.startswith("__") + ] + if not fitted: + raise NotFittedError( + msg + or ( + f"This {type(estimator).__name__} instance is not " + f"fitted yet. Call 'fit' with appropriate arguments " + f"before using this estimator." + ) + ) + return + + if isinstance(attributes, str): + attributes = [attributes] + missing = [a for a in attributes if not hasattr(estimator, a)] + if missing: + raise NotFittedError( + msg + or ( + f"This {type(estimator).__name__} instance is not fitted " + f"yet. Call 'fit' with appropriate arguments before using " + f"this estimator." + ) + ) + + def check_array( + array, + *, + accept_sparse=False, + dtype="numeric", + ensure_2d=True, + ensure_min_samples=1, + ensure_min_features=1, + copy=False, + **kwargs, + ): + """Fallback input validator approximating sklearn.utils.check_array.""" + import numpy as np + + try: + from scipy.sparse import issparse + except ImportError: + + def issparse(x): + return False + + if issparse(array): + if accept_sparse is False or accept_sparse == []: + raise TypeError( + "A sparse matrix was passed, but dense data is required. " + "Use X.toarray() to convert to a dense numpy array." + ) + array = array.toarray() + + if hasattr(array, "values") and hasattr(array, "columns"): + array = array.values + + arr = np.asarray(array) + if copy: + arr = np.array(arr, copy=True) + + if ensure_2d and arr.ndim == 1: + raise ValueError( + "Expected 2D array, got 1D array instead:\n" + f"array={arr}.\n" + "Reshape your data either using array.reshape(-1, 1) if " + "your data has a single feature or array.reshape(1, -1) if " + "it contains a single sample." + ) + if ensure_2d and arr.ndim != 2: + raise ValueError( + f"Expected 2D array, got array with ndim={arr.ndim} instead." + ) + + if arr.size == 0: + n_samples = arr.shape[0] if arr.ndim >= 1 else 0 + n_features = arr.shape[1] if arr.ndim >= 2 else 0 + if n_samples < ensure_min_samples: + raise ValueError( + f"Found array with {n_samples} sample(s) " + f"(shape={arr.shape}) while a minimum of " + f"{ensure_min_samples} is required." + ) + if n_features < ensure_min_features: + raise ValueError( + f"Found array with {n_features} feature(s) " + f"(shape={arr.shape}) while a minimum of " + f"{ensure_min_features} is required." + ) + + if np.iscomplexobj(arr): + raise ValueError("Complex data not supported") + + if dtype == "numeric": + try: + arr = arr.astype(float, copy=False) + except (ValueError, TypeError) as exc: + raise ValueError( + "Unable to convert array of strings to floating numbers." + ) from exc + + if ensure_2d: + if arr.shape[0] < ensure_min_samples: + raise ValueError( + f"Found array with {arr.shape[0]} sample(s) " + f"(shape={arr.shape}) while a minimum of " + f"{ensure_min_samples} is required." + ) + if arr.shape[1] < ensure_min_features: + raise ValueError( + f"Found array with {arr.shape[1]} feature(s) " + f"(shape={arr.shape}) while a minimum of " + f"{ensure_min_features} is required." + ) + + return arr diff --git a/PyNomaly/_distance.py b/PyNomaly/_distance.py new file mode 100644 index 0000000..98a1fcb --- /dev/null +++ b/PyNomaly/_distance.py @@ -0,0 +1,262 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +import os +import sys +import warnings +from typing import Tuple + +import numpy as np + +from PyNomaly._utils import Utils + +try: + from scipy.spatial.distance import cdist as _scipy_cdist +except ImportError: + _scipy_cdist = None + +try: + import numba + + def _numba_distance_kernel_parallel(clust_points_vector, n_neighbors): + n = clust_points_vector.shape[0] + d_features = clust_points_vector.shape[1] + local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) + local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) + for i in numba.prange(n): + for j in range(n): + if i == j: + continue + d = 0.0 + for f in range(d_features): + diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] + d += diff_val * diff_val + d = d**0.5 + idx_max = 0 + for k in range(1, n_neighbors): + if local_distances[i, k] > local_distances[i, idx_max]: + idx_max = k + if d < local_distances[i, idx_max]: + local_distances[i, idx_max] = d + local_indexes[i, idx_max] = j + return local_distances, local_indexes + + _numba_kernel_parallel = numba.jit( + _numba_distance_kernel_parallel, nopython=True, parallel=True, cache=True + ) + + def _numba_distance_kernel_sequential(clust_points_vector, n_neighbors): + n = clust_points_vector.shape[0] + d_features = clust_points_vector.shape[1] + local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) + local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) + for i in range(n): + for j in range(n): + if i == j: + continue + d = 0.0 + for f in range(d_features): + diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] + d += diff_val * diff_val + d = d**0.5 + idx_max = 0 + for k in range(1, n_neighbors): + if local_distances[i, k] > local_distances[i, idx_max]: + idx_max = k + if d < local_distances[i, idx_max]: + local_distances[i, idx_max] = d + local_indexes[i, idx_max] = j + return local_distances, local_indexes + + _numba_kernel_sequential = numba.jit( + _numba_distance_kernel_sequential, nopython=True, cache=True + ) + +except ImportError: + pass + + +class DistanceMixin: + """Mixin providing distance computation methods for LocalOutlierProbability.""" + + @staticmethod + def _euclidean(vector1: np.ndarray, vector2: np.ndarray) -> np.ndarray: + """ + Calculates the euclidean distance between two observations in the + input data. + :param vector1: a numpy array corresponding to observation 1. + :param vector2: a numpy array corresponding to observation 2. + :return: the euclidean distance between the two observations. + """ + diff = vector1 - vector2 + return np.dot(diff, diff) ** 0.5 + + def _assign_distances(self, data_store: np.ndarray) -> np.ndarray: + _dist = getattr( + self, "distance_matrix_", getattr(self, "distance_matrix", None) + ) + _neigh = getattr( + self, "neighbor_matrix_", getattr(self, "neighbor_matrix", None) + ) + + if _dist is None or _neigh is None: + raise ValueError( + "Distance or neighbor matrix is missing. Ensure inputs are valid." + ) + + for vec, cluster_id in zip(range(_dist.shape[0]), self._cluster_labels()): + data_store[vec][0] = cluster_id + data_store[vec][1] = _dist[vec] + data_store[vec][2] = _neigh[vec] + return data_store + + @staticmethod + def _compute_distance_and_neighbor_matrix( + clust_points_vector: np.ndarray, + indices: np.ndarray, + distances: np.ndarray, + indexes: np.ndarray, + ) -> Tuple[np.ndarray, np.ndarray, int]: + """ + This helper method provides the heavy lifting for the _distances + method and is only intended for use therein. The code has been + written so that it can make full use of Numba's jit capabilities if + desired. + """ + for i in range(clust_points_vector.shape[0]): + for j in range(i + 1, clust_points_vector.shape[0]): + global_i = indices[0][i] + global_j = indices[0][j] + + diff = clust_points_vector[i] - clust_points_vector[j] + d = np.dot(diff, diff) ** 0.5 + + idx_max = distances[global_i].argmax() + if d < distances[global_i][idx_max]: + distances[global_i][idx_max] = d + indexes[global_i][idx_max] = global_j + + idx_max = distances[global_j].argmax() + if d < distances[global_j][idx_max]: + distances[global_j][idx_max] = d + indexes[global_j][idx_max] = global_i + + yield distances, indexes, i + + def _distances_vectorized(self, clusters, distances, indexes, progress_bar) -> None: + """Vectorized kNN distance computation with chunked progress.""" + progress = "=" + total_points = sum(cv.shape[0] for cv, _ in clusters) + completed = 0 + + for clust_points_vector, global_indices in clusters: + n = clust_points_vector.shape[0] + + if clust_points_vector.ndim == 1: + clust_points_vector = clust_points_vector.reshape(-1, 1) + + chunk_size = min(256, n) + for chunk_start in range(0, n, chunk_size): + chunk_end = min(chunk_start + chunk_size, n) + chunk = clust_points_vector[chunk_start:chunk_end] + + if _scipy_cdist is not None: + dist = _scipy_cdist(chunk, clust_points_vector, metric="euclidean") + else: + diff = ( + chunk[:, np.newaxis, :] - clust_points_vector[np.newaxis, :, :] + ) + dist = np.sqrt((diff**2).sum(axis=2)) + + row_idx = np.arange(chunk_end - chunk_start) + dist[row_idx, row_idx + chunk_start] = np.inf + + n_neighbors = self._effective_n_neighbors() + knn_idx = np.argpartition(dist, n_neighbors, axis=1)[ + :, :n_neighbors + ] + knn_dists = np.take_along_axis(dist, knn_idx, axis=1) + + chunk_global = global_indices[chunk_start:chunk_end] + distances[chunk_global] = knn_dists + indexes[chunk_global] = global_indices[knn_idx] + + completed += chunk_end - chunk_start + if progress_bar: + progress = Utils.emit_progress_bar( + progress, completed, total_points + ) + + def _distances_numba( + self, clusters, distances, indexes, progress_bar, parallel=False + ) -> None: + """Numba-accelerated distance computation.""" + progress = "=" + kernel = _numba_kernel_parallel if parallel else _numba_kernel_sequential + + for idx, (clust_points_vector, global_indices) in enumerate(clusters): + if clust_points_vector.ndim == 1: + clust_points_vector = clust_points_vector.reshape(-1, 1) + + local_dists, local_idxs = kernel( + clust_points_vector.astype(np.float64), + self._effective_n_neighbors(), + ) + + distances[global_indices] = local_dists + indexes[global_indices] = global_indices[local_idxs] + + if progress_bar: + progress = Utils.emit_progress_bar(progress, idx + 1, len(clusters)) + + def _distances(self, progress_bar: bool = False) -> None: + """ + Provides the distances between each observation and it's closest + neighbors. When input data is provided, calculates the euclidean + distance between every observation. Otherwise, the user-provided + distance matrix is used. + :return: the updated storage matrix that collects information on + each observation. + """ + n_neighbors = self._effective_n_neighbors() + distances = np.full( + [self._n_observations(), n_neighbors], 9e10, dtype=float + ) + indexes = np.full([self._n_observations(), n_neighbors], 9e10, dtype=float) + + _data = getattr(self, "data_", getattr(self, "data", None)) + self.points_vector_ = np.asarray(_data, dtype=float) + + cluster_labels = self._cluster_labels() + cluster_ids = sorted(set(cluster_labels)) + + clusters = [] + for cluster_id in cluster_ids: + indices = np.where(cluster_labels == cluster_id)[0] + clust_points_vector = np.array( + self.points_vector_[indices], dtype=np.float64 + ) + clusters.append((clust_points_vector, indices)) + + n_jobs = getattr(self, "_fit_n_jobs", self.n_jobs) + if n_jobs == -1: + n_jobs = os.cpu_count() or 1 + + use_numba = getattr(self, "_fit_use_numba", self.use_numba) + if use_numba: + self._distances_numba( + clusters, distances, indexes, progress_bar, parallel=(n_jobs > 1) + ) + else: + if n_jobs > 1: + warnings.warn( + "n_jobs > 1 requires use_numba=True for parallel " + "processing. Install Numba and set use_numba=True " + "to enable parallelism. Falling back to sequential.", + UserWarning, + ) + self._distances_vectorized(clusters, distances, indexes, progress_bar) + + # Assure that computed data is strictly stored in private fitted attributes + self.distance_matrix_ = distances + self.neighbor_matrix_ = indexes diff --git a/PyNomaly/_pipeline.py b/PyNomaly/_pipeline.py new file mode 100644 index 0000000..b38312c --- /dev/null +++ b/PyNomaly/_pipeline.py @@ -0,0 +1,321 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +from math import erf, sqrt + +import numpy as np + +try: + from scipy.special import erf as _scipy_erf +except ImportError: + _scipy_erf = None + + +class PipelineMixin: + """Mixin providing the LoOP scoring pipeline for LocalOutlierProbability.""" + + @staticmethod + def _standard_distance(cardinality: float, sum_squared_distance: float) -> float: + """ + Calculates the standard distance of an observation. + :param cardinality: the cardinality of the input observation. + :param sum_squared_distance: the sum squared distance between all + neighbors of the input observation. + :return: the standard distance. + """ + division_result = sum_squared_distance / cardinality + st_dist = sqrt(division_result) + return st_dist + + @staticmethod + def _prob_distance(extent: int, standard_distance: float) -> float: + """ + Calculates the probabilistic distance of an observation. + :param extent: the extent value specified during initialization. + :param standard_distance: the standard distance of the input + observation. + :return: the probabilistic distance. + """ + return extent * standard_distance + + @staticmethod + def _prob_outlier_factor( + probabilistic_distance: np.ndarray, ev_prob_dist: np.ndarray + ) -> np.ndarray: + """ + Calculates the probabilistic outlier factor of an observation. + :param probabilistic_distance: the probabilistic distance of the + input observation. + :param ev_prob_dist: + :return: the probabilistic outlier factor. + """ + if np.all(probabilistic_distance == ev_prob_dist): + return np.zeros(probabilistic_distance.shape) + else: + ev_prob_dist[ev_prob_dist == 0.0] = 1.0e-8 + result = np.divide(probabilistic_distance, ev_prob_dist) - 1.0 + return result + + @staticmethod + def _norm_prob_outlier_factor( + extent: float, ev_probabilistic_outlier_factor: list + ) -> list: + """ + Calculates the normalized probabilistic outlier factor of an + observation. + :param extent: the extent value specified during initialization. + :param ev_probabilistic_outlier_factor: the expected probabilistic + outlier factor of the input observation. + :return: the normalized probabilistic outlier factor. + """ + ev_arr = np.array(ev_probabilistic_outlier_factor, dtype=float) + return (extent * np.sqrt(ev_arr)).tolist() + + @staticmethod + def _local_outlier_probability( + plof_val: np.ndarray, nplof_val: np.ndarray + ) -> np.ndarray: + """ + Calculates the local outlier probability of an observation. + :param plof_val: the probabilistic outlier factor of the input + observation. + :param nplof_val: the normalized probabilistic outlier factor of the + input observation. + :return: the local outlier probability. + """ + if np.all(plof_val == nplof_val): + return np.zeros(plof_val.shape) + plof_f = np.asarray(plof_val, dtype=float) + nplof_f = np.asarray(nplof_val, dtype=float) + if _scipy_erf is not None: + return np.maximum(0, _scipy_erf(plof_f / (nplof_f * np.sqrt(2.0)))) + erf_vec = np.vectorize(erf) + return np.maximum(0, erf_vec(plof_f / (nplof_f * np.sqrt(2.0)))) + + def _n_observations(self) -> int: + """ + Calculates the number of observations in the data. + :return: the number of observations in the input data if data. + """ + _data = getattr(self, "data_", getattr(self, "data", None)) + _dist = getattr( + self, "distance_matrix_", getattr(self, "distance_matrix", None) + ) + + if _data is not None: + return np.asarray(_data).shape[0] + if _dist is not None: + return np.asarray(_dist).shape[0] + return 0 + + def _store(self) -> np.ndarray: + """ + Initializes the storage matrix that includes the input value, + cluster labels, local outlier probability, etc. for the input data. + :return: an empty numpy array of shape [n_observations, 3]. + """ + return np.empty([self._n_observations(), 3], dtype=object) + + def _cluster_labels(self) -> np.ndarray: + """ + Returns a numpy array of cluster labels that corresponds to the + input labels or that is an array of all 0 values to indicate all + points belong to the same cluster. + :return: a numpy array of cluster labels. + """ + _labels = getattr( + self, "cluster_labels_", getattr(self, "cluster_labels", None) + ) + _data = getattr(self, "data_", getattr(self, "data", None)) + _dist = getattr( + self, "distance_matrix_", getattr(self, "distance_matrix", None) + ) + + if _labels is None: + if _data is not None: + return np.array([0] * np.asarray(_data).shape[0]) + if _dist is not None: + return np.array([0] * np.asarray(_dist).shape[0]) + return np.array([0]) + return np.array(_labels) + + def _ssd(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the sum squared distance between neighbors for each + observation in the input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + self._cluster_labels_u = np.unique(data_store[:, 0]) + ssd_array = np.empty([self._n_observations(), 1]) + for cluster_id in self._cluster_labels_u: + indices = np.where(data_store[:, 0] == cluster_id) + cluster_distances = np.take(data_store[:, 1], indices).tolist() + ssd = np.power(cluster_distances[0], 2).sum(axis=1) + for i, j in zip(indices[0], ssd): + ssd_array[i] = j + data_store = np.hstack((data_store, ssd_array)) + return data_store + + def _standard_distances(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculated the standard distance for each observation in the input + data. First calculates the cardinality and then calculates the standard + distance with respect to each observation. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + ssd_vals = data_store[:, 3].astype(float) + std_distances = np.sqrt(ssd_vals / self._effective_n_neighbors()) + return np.hstack((data_store, std_distances.reshape(-1, 1))) + + def _prob_distances(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the probabilistic distance for each observation in the + input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + prob_distances = self.extent * data_store[:, 4].astype(float) + return np.hstack((data_store, prob_distances.reshape(-1, 1))) + + def _prob_distances_ev(self, data_store) -> np.ndarray: + """ + Calculates the expected value of the probabilistic distance for + each observation in the input data with respect to the cluster the + observation belongs to. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + prob_set_distance_ev = np.empty([self._n_observations(), 1]) + for cluster_id in self._cluster_labels_u: + indices = np.where(data_store[:, 0] == cluster_id)[0] + for index in indices: + nbrhood = data_store[index][2].astype(int) + nbrhood_prob_distances = np.take(data_store[:, 5], nbrhood).astype( + float + ) + nbrhood_prob_distances_nonan = nbrhood_prob_distances[ + np.logical_not(np.isnan(nbrhood_prob_distances)) + ] + prob_set_distance_ev[index] = nbrhood_prob_distances_nonan.mean() + + self.prob_distances_ev_ = prob_set_distance_ev + return np.hstack((data_store, prob_set_distance_ev)) + + def _prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the probabilistic local outlier factor for each + observation in the input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + return np.hstack( + ( + data_store, + np.array( + [ + np.apply_along_axis( + self._prob_outlier_factor, + 0, + data_store[:, 5], + data_store[:, 6], + ) + ] + ).T, + ) + ) + + def _prob_local_outlier_factors_ev(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the expected value of the probabilistic local outlier factor + for each observation in the input data with respect to the cluster the + observation belongs to. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + prob_local_outlier_factor_ev_dict = {} + for cluster_id in self._cluster_labels_u: + indices = np.where(data_store[:, 0] == cluster_id) + prob_local_outlier_factors = np.take(data_store[:, 7], indices).astype( + float + ) + prob_local_outlier_factors_nonan = prob_local_outlier_factors[ + np.logical_not(np.isnan(prob_local_outlier_factors)) + ] + prob_local_outlier_factor_ev_dict[cluster_id] = np.power( + prob_local_outlier_factors_nonan, 2 + ).sum() / float(prob_local_outlier_factors_nonan.size) + data_store = np.hstack( + ( + data_store, + np.array( + [ + [ + prob_local_outlier_factor_ev_dict[x] + for x in data_store[:, 0].tolist() + ] + ] + ).T, + ) + ) + return data_store + + def _norm_prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the normalized probabilistic local outlier factor for each + observation in the input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + return np.hstack( + ( + data_store, + np.array( + [ + self._norm_prob_outlier_factor( + self.extent, data_store[:, 8].tolist() + ) + ] + ).T, + ) + ) + + def _local_outlier_probabilities(self, data_store: np.ndarray) -> np.ndarray: + """ + Calculates the local outlier probability for each observation in the + input data. + :param data_store: the storage matrix that collects information on + each observation. + :return: the updated storage matrix that collects information on + each observation. + """ + return np.hstack( + ( + data_store, + np.array( + [ + np.apply_along_axis( + self._local_outlier_probability, + 0, + data_store[:, 7], + data_store[:, 9], + ) + ] + ).T, + ) + ) diff --git a/PyNomaly/_utils.py b/PyNomaly/_utils.py new file mode 100644 index 0000000..a8d9a72 --- /dev/null +++ b/PyNomaly/_utils.py @@ -0,0 +1,33 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +import sys +from python_utils.terminal import get_terminal_size + + +class Utils: + @staticmethod + def emit_progress_bar(progress: str, index: int, total: int) -> str: + """ + A progress bar that is continuously updated in Python's standard + out. + :param progress: a string printed to stdout that is updated and later + returned. + :param index: the current index of the iteration within the tracked + process. + :param total: the total length of the tracked process. + :return: progress string. + """ + + w, h = get_terminal_size() + sys.stdout.write("\r") + if total < w: + block_size = int(w / total) + else: + block_size = int(total / w) + if index % block_size == 0: + progress += "=" + percent = index / total + sys.stdout.write("[ %s ] %.2f%%" % (progress, percent * 100)) + sys.stdout.flush() + return progress diff --git a/PyNomaly/_validation.py b/PyNomaly/_validation.py new file mode 100644 index 0000000..5d209a9 --- /dev/null +++ b/PyNomaly/_validation.py @@ -0,0 +1,361 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + +"""Input validation helpers for LocalOutlierProbability.""" + +from __future__ import annotations + +import warnings +from functools import wraps +from typing import TYPE_CHECKING, Union + +import numpy as np + +from PyNomaly._compat import check_array +from PyNomaly.exceptions import ClusterSizeError, MissingValuesError + +if TYPE_CHECKING: # pragma: no cover + import pandas as pd # noqa: F401 (type hints only; pandas is optional) + +try: + from scipy.sparse import issparse as _issparse + + _SCIPY_AVAILABLE = True +except ImportError: # pragma: no cover + + _SCIPY_AVAILABLE = False + + def _issparse(x): + return False + + +def _is_sparse_like(obj) -> bool: + """Return True for scipy sparse matrices (with or without scipy installed).""" + if _issparse(obj): + return True + obj_type = type(obj) + module = getattr(obj_type, "__module__", "") or "" + return module.startswith("scipy.sparse") + + +def _require_scipy_for_sparse(obj) -> None: + """Raise a clear error when sparse input is passed without scipy.""" + if _is_sparse_like(obj) and not _SCIPY_AVAILABLE: + raise ImportError( + "Sparse matrix input requires scipy. " + "Install it with: pip install scipy" + ) + + +class ValidationMixin: + """Mixin providing input validation methods for LocalOutlierProbability.""" + + def _validate_data_matrix( + self, + X, + *, + ensure_2d: bool = True, + reset: bool = False, + ) -> np.ndarray: + """ + Validate and densify feature matrices for fit / predict / stream. + + Sparse inputs are accepted and converted to dense arrays. This keeps + the public API sparse-capable while the LoOP kernels operate on dense + float64 data. + """ + if X is None: + raise ValueError("Input data must be provided.") + + # Preserve DataFrame column names before densifying / converting. + feature_names = None + if hasattr(X, "columns"): + try: + feature_names = np.asarray(X.columns, dtype=object) + except Exception: + feature_names = None + + if _is_sparse_like(X): + _require_scipy_for_sparse(X) + X = X.toarray() + + # Historical LoOP API accepts 1-d series and reshapes to a single feature. + arr_probe = np.asarray(X) + if getattr(arr_probe, "ndim", None) == 1: + if arr_probe.__class__.__name__ != "ndarray" or not isinstance(X, np.ndarray): + warnings.warn( + "Provided data or distance matrix must be in ndarray or DataFrame.", + UserWarning, + ) + X = np.asarray(arr_probe, dtype=float).reshape(-1, 1) + + # Prefer PyNomaly's MissingValuesError over sklearn's generic ValueError. + try: + probe = np.asarray(X, dtype=float) + except (ValueError, TypeError): + probe = None + if probe is not None and (np.any(np.isnan(probe)) or np.any(np.isinf(probe))): + raise MissingValuesError( + "Input contains NaN, infinity or a value too large for " + "dtype('float64')." + ) + + arr = check_array( + X, + accept_sparse=False, + dtype="numeric", + ensure_2d=ensure_2d, + ensure_min_samples=1, + ensure_min_features=1, + ) + arr = np.asarray(arr, dtype=float) + + if np.any(np.isnan(arr)) or np.any(np.isinf(arr)): + raise MissingValuesError( + "Input contains NaN, infinity or a value too large for " + "dtype('float64')." + ) + + if reset: + self.n_features_in_ = arr.shape[1] if arr.ndim == 2 else 1 + if feature_names is not None and len(feature_names) == self.n_features_in_: + self.feature_names_in_ = feature_names + + return arr + + @staticmethod + def _convert_to_array(obj: Union["pd.DataFrame", np.ndarray]) -> np.ndarray: + """ + Converts the input data to a numpy array if it is a Pandas DataFrame + or validates it is already a numpy array. + """ + if _is_sparse_like(obj): + _require_scipy_for_sparse(obj) + obj = obj.toarray() + + if obj.__class__.__name__ in ("DataFrame", "Series"): + arr = np.asarray(obj.values) + elif obj.__class__.__name__ == "ndarray": + arr = obj + else: + warnings.warn( + "Provided data or distance matrix must be in ndarray or DataFrame.", + UserWarning, + ) + arr = np.asarray(obj, dtype=float) + if arr.ndim == 0: + arr = np.array([obj], dtype=float) + + if arr.size == 0: + if arr.ndim >= 2 and arr.shape[1] == 0: + raise ValueError( + f"Found array with 0 feature(s) (shape={arr.shape}) while a " + f"minimum of 1 is required." + ) + raise ValueError( + f"Found array with 0 sample(s) (shape={arr.shape}) while a " + f"minimum of 1 is required." + ) + + if np.iscomplexobj(arr): + raise ValueError("Complex data not supported.") + + return arr.astype(float, copy=False) + + @staticmethod + def _convert_observation(obj: Union["pd.DataFrame", np.ndarray]) -> np.ndarray: + """ + Convert a single observation to a 1-D float vector. + + Accepts dense arrays, pandas objects, lists, and scipy sparse rows. + Sparse matrices are densified (requires scipy). + """ + arr = ValidationMixin._convert_to_array(obj) + if arr.ndim == 2 and arr.shape[0] == 1: + return arr.ravel() + return arr + + def _validate_inputs(self): + """ + Validates that either data or a distance/neighbor matrix pair is set. + """ + _data = getattr(self, "data_", getattr(self, "data", None)) + _dist = getattr( + self, "distance_matrix_", getattr(self, "distance_matrix", None) + ) + _neigh = getattr( + self, "neighbor_matrix_", getattr(self, "neighbor_matrix", None) + ) + + if all(v is None for v in [_data, _dist]): + warnings.warn("Data or a distance matrix must be provided.", UserWarning) + return False + if all(v is not None for v in [_data, _dist]): + warnings.warn( + "Only one of the following may be provided: data or a " + "distance matrix (not both).", + UserWarning, + ) + return False + + if _data is not None: + points_vector = self._validate_data_matrix(_data, ensure_2d=True, reset=True) + self.data_ = points_vector + return points_vector, _dist, _neigh + + if all(matrix is not None for matrix in [_neigh, _dist]): + dist_vector = self._convert_to_array(_dist) + neigh_vector = self._convert_to_array(_neigh) + else: + warnings.warn( + "A neighbor index matrix and distance matrix must both be " + "provided when not using raw input data.", + UserWarning, + ) + return False + + if _dist.shape != _neigh.shape: + warnings.warn( + "The shape of the distance and neighbor index matrices must match.", + UserWarning, + ) + return False + if (_dist.shape[1] != self.n_neighbors) or ( + _neigh.shape[1] != self.n_neighbors + ): + warnings.warn( + "The shape of the distance or neighbor index matrix does not " + "match the number of neighbors specified.", + UserWarning, + ) + return False + + self.distance_matrix_ = dist_vector + self.neighbor_matrix_ = neigh_vector + return _data, dist_vector, neigh_vector + + def _check_cluster_size(self) -> None: + """Raises ClusterSizeError if any cluster is too small for n_neighbors.""" + n_neighbors = getattr(self, "n_neighbors_", self.n_neighbors) + c_labels = self._cluster_labels() + for cluster_id in set(c_labels): + c_size = np.where(c_labels == cluster_id)[0].shape[0] + if c_size <= n_neighbors: + raise ClusterSizeError( + "Number of neighbors specified larger than smallest " + "cluster. Specify a number of neighbors smaller than " + "the smallest cluster size (observations in smallest " + "cluster minus one)." + ) + + def _check_n_neighbors(self) -> bool: + """ + Validate n_neighbors without mutating the public parameter. + + The effective value used during fit is stored on ``n_neighbors_``. + """ + n_neighbors = self.n_neighbors + n_obs = self._n_observations() + + if not n_neighbors > 0: + warnings.warn( + "n_neighbors must be greater than 0. Fit with 10 instead.", + UserWarning, + ) + n_neighbors = 10 + ok = False + else: + ok = True + + if n_neighbors >= n_obs: + adjusted = max(n_obs - 1, 1) + warnings.warn( + "n_neighbors must be less than the number of observations." + f" Fit with {adjusted} instead.", + UserWarning, + ) + n_neighbors = adjusted + ok = False + + self.n_neighbors_ = n_neighbors + return ok + + def _check_extent(self) -> bool: + """Validate that extent is 1, 2, or 3.""" + if self.extent not in [1, 2, 3]: + warnings.warn("extent parameter (lambda) must be 1, 2, or 3.", UserWarning) + return False + return True + + def _check_missing_values(self) -> None: + """Raise MissingValuesError if fitted data contains NaN/Inf.""" + _data = getattr(self, "data_", getattr(self, "data", None)) + if _data is not None: + arr = self._convert_to_array(_data) + if np.any(np.isnan(arr)) or np.any(np.isinf(arr)): + raise MissingValuesError( + "Input contains NaN, infinity or a value too large for " + "dtype('float64')." + ) + + def _check_is_fit(self) -> bool: + """Return whether the estimator has been successfully fit.""" + if getattr(self, "is_fit_", False) is False: + warnings.warn( + "Must fit on historical data by calling fit() prior to " + "calling stream(x).", + UserWarning, + ) + return False + return True + + def _check_no_cluster_labels(self) -> bool: + """Return False when multi-cluster labels are present (stream unsupported).""" + if len(set(self._cluster_labels())) > 1: + warnings.warn( + "Stream approach does not support clustered data. " + "Automatically refit using single cluster of points.", + UserWarning, + ) + return False + return True + + def _effective_n_neighbors(self) -> int: + """Return the neighbor count used by the current fit.""" + return getattr(self, "n_neighbors_", self.n_neighbors) + + +def validate_init_types(*types): + """ + Soft type checker used at fit-time for legacy constructor parameters. + """ + + def decorator(f): + @wraps(f) + def new_f(self, *args, **kwds): + attr_names = [ + "extent", + "n_neighbors", + "use_numba", + "n_jobs", + "progress_bar", + "data", + "distance_matrix", + "neighbor_matrix", + "cluster_labels", + ] + for attr, expected_type in zip(attr_names, types[1:]): + val = getattr(self, attr, None) + if val is not None: + if type(val).__name__ == "DataFrame": + val = np.array(val) + if not isinstance(val, expected_type): + warnings.warn( + "Argument %r is not of type %s." % (attr, expected_type), + UserWarning, + ) + return f(self, *args, **kwds) + + return new_f + + return decorator diff --git a/PyNomaly/exceptions.py b/PyNomaly/exceptions.py new file mode 100644 index 0000000..7e04eff --- /dev/null +++ b/PyNomaly/exceptions.py @@ -0,0 +1,26 @@ +# Authors: Valentino Constantinou +# License: Apache 2.0 + + +class PyNomalyError(Exception): + """Base exception for PyNomaly.""" + + pass + + +class ValidationError(PyNomalyError): + """Raised when input validation fails.""" + + pass + + +class ClusterSizeError(ValidationError): + """Raised when cluster size is smaller than n_neighbors.""" + + pass + + +class MissingValuesError(ValidationError, ValueError): + """Raised when data contains missing values.""" + + pass diff --git a/PyNomaly/loop.py b/PyNomaly/loop.py index 31a32b1..cabd820 100644 --- a/PyNomaly/loop.py +++ b/PyNomaly/loop.py @@ -1,1046 +1,476 @@ -from math import erf, sqrt -import numpy as np -import os -from python_utils.terminal import get_terminal_size +# Authors: Valentino Constantinou +# License: Apache 2.0 + +"""Local Outlier Probability (LoOP) estimator.""" + +from __future__ import annotations + import sys -from typing import Tuple, Union import warnings -try: - from scipy.spatial.distance import cdist as _scipy_cdist -except ImportError: - _scipy_cdist = None - -try: - from scipy.special import erf as _scipy_erf -except ImportError: - _scipy_erf = None - -try: - import numba - - def _numba_distance_kernel_parallel(clust_points_vector, n_neighbors): - n = clust_points_vector.shape[0] - d_features = clust_points_vector.shape[1] - local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) - local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) - for i in numba.prange(n): - for j in range(n): - if i == j: - continue - d = 0.0 - for f in range(d_features): - diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] - d += diff_val * diff_val - d = d ** 0.5 - idx_max = 0 - for k in range(1, n_neighbors): - if local_distances[i, k] > local_distances[i, idx_max]: - idx_max = k - if d < local_distances[i, idx_max]: - local_distances[i, idx_max] = d - local_indexes[i, idx_max] = j - return local_distances, local_indexes - - _numba_kernel_parallel = numba.jit( - _numba_distance_kernel_parallel, nopython=True, parallel=True, cache=True - ) - - def _numba_distance_kernel_sequential(clust_points_vector, n_neighbors): - n = clust_points_vector.shape[0] - d_features = clust_points_vector.shape[1] - local_distances = np.full((n, n_neighbors), 9e10, dtype=np.float64) - local_indexes = np.full((n, n_neighbors), 0, dtype=np.int64) - for i in range(n): - for j in range(n): - if i == j: - continue - d = 0.0 - for f in range(d_features): - diff_val = clust_points_vector[i, f] - clust_points_vector[j, f] - d += diff_val * diff_val - d = d ** 0.5 - idx_max = 0 - for k in range(1, n_neighbors): - if local_distances[i, k] > local_distances[i, idx_max]: - idx_max = k - if d < local_distances[i, idx_max]: - local_distances[i, idx_max] = d - local_indexes[i, idx_max] = j - return local_distances, local_indexes - - _numba_kernel_sequential = numba.jit( - _numba_distance_kernel_sequential, nopython=True, cache=True - ) +import numpy as np -except ImportError: - pass +from PyNomaly._compat import ( + BaseEstimator, + OutlierMixin, + check_array, + check_is_fitted, +) +from PyNomaly._distance import DistanceMixin +from PyNomaly._pipeline import PipelineMixin +from PyNomaly._validation import ValidationMixin, validate_init_types +from PyNomaly.exceptions import ( # noqa: F401 (re-exported for backward compatibility) + ClusterSizeError, + MissingValuesError, + PyNomalyError, + ValidationError, +) __author__ = "Valentino Constantinou" -__version__ = "0.4.0" +__version__ = "1.0.0" __license__ = "Apache License, Version 2.0" -# Custom Exceptions -class PyNomalyError(Exception): - """Base exception for PyNomaly.""" - pass - - -class ValidationError(PyNomalyError): - """Raised when input validation fails.""" - pass - - -class ClusterSizeError(ValidationError): - """Raised when cluster size is smaller than n_neighbors.""" - pass - - -class MissingValuesError(ValidationError): - """Raised when data contains missing values.""" - pass - - -class Utils: - @staticmethod - def emit_progress_bar(progress: str, index: int, total: int) -> str: - """ - A progress bar that is continuously updated in Python's standard - out. - :param progress: a string printed to stdout that is updated and later - returned. - :param index: the current index of the iteration within the tracked - process. - :param total: the total length of the tracked process. - :return: progress string. - """ +class LocalOutlierProbability( + OutlierMixin, ValidationMixin, DistanceMixin, PipelineMixin, BaseEstimator +): + """Local Outlier Probability (LoOP) estimator. - w, h = get_terminal_size() - sys.stdout.write("\r") - if total < w: - block_size = int(w / total) - else: - block_size = int(total / w) - if index % block_size == 0: - progress += "=" - percent = index / total - sys.stdout.write("[ %s ] %.2f%%" % (progress, percent * 100)) - sys.stdout.flush() - return progress - - -class LocalOutlierProbability(object): - """ - :param data: a Pandas DataFrame or Numpy array of float data - :param extent: an integer value [1, 2, 3] that controls the statistical - extent, e.g. lambda times the standard deviation from the mean (optional, - default 3) - :param n_neighbors: the total number of neighbors to consider w.r.t. each - sample (optional, default 10) - :param cluster_labels: a numpy array of cluster assignments w.r.t. each - sample (optional, default None) - :param n_jobs: controls Numba thread-level parallelism via prange. - Use -1 to use all available CPU cores, or 1 for sequential processing. - Only effective when use_numba=True (optional, default 1) - :return: - """ """ - - Based on the work of Kriegel, Kröger, Schubert, and Zimek (2009) in LoOP: - Local Outlier Probabilities. + Parameters ---------- + extent : int, default=3 + Statistical extent (lambda); must be 1, 2, or 3. + n_neighbors : int, default=10 + Number of neighbors considered for each sample. + use_numba : bool, default=False + Whether to use Numba JIT acceleration for distance computation. + n_jobs : int, default=1 + Numba parallelism. ``-1`` uses all cores. Only effective with + ``use_numba=True``. + progress_bar : bool, default=False + Whether to display a progress bar during distance computation. + + Notes + ----- + Fitted attributes end with ``_`` (scikit-learn convention). Legacy + read aliases without the trailing underscore remain available for + *output* attributes such as ``local_outlier_probabilities``. Init / + configuration attributes (e.g. ``distance_matrix``) are never aliased + to fitted state — that would break ``get_params`` / ``check_estimator``. + + Sparse matrices (``scipy.sparse``) are accepted for ``fit``, ``predict``, + ``decision_function``, and ``stream``. They are converted to dense arrays + internally before distance computation. **scipy** is a soft dependency for + sparse input — install it with ``pip install scipy`` (included in the + ``PyNomaly[all]`` extra). References ---------- - .. [1] Breunig M., Kriegel H.-P., Ng R., Sander, J. LOF: Identifying - Density-based Local Outliers. ACM SIGMOD - International Conference on Management of Data (2000). - .. [2] Kriegel H.-P., Kröger P., Schubert E., Zimek A. LoOP: Local Outlier - Probabilities. 18th ACM conference on - Information and knowledge management, CIKM (2009). - .. [3] Goldstein M., Uchida S. A Comparative Evaluation of Unsupervised - Anomaly Detection Algorithms for Multivariate Data. PLoS ONE 11(4): - e0152173 (2016). - .. [4] Hamlet C., Straub J., Russell M., Kerlin S. An incremental and - approximate local outlier probability algorithm for intrusion - detection and its evaluation. Journal of Cyber Security Technology - (2016). + .. [1] Breunig M., Kriegel H.-P., Ng R., Sander, J. LOF: Identifying + Density-based Local Outliers. ACM SIGMOD (2000). + .. [2] Kriegel H.-P., Kröger P., Schubert E., Zimek A. LoOP: Local Outlier + Probabilities. CIKM (2009). """ - """ - Validation methods. - These methods validate inputs and raise exceptions or warnings as appropriate. - """ + _DATA_PARAMS = ("data", "distance_matrix", "neighbor_matrix", "cluster_labels") - @staticmethod - def _convert_to_array(obj: Union["pd.DataFrame", np.ndarray]) -> np.ndarray: - """ - Converts the input data to a numpy array if it is a Pandas DataFrame - or validates it is already a numpy array. - :param obj: user-provided input data. - :return: a vector of values to be used in calculating the local - outlier probability. - """ - if obj.__class__.__name__ == "DataFrame": - points_vector = obj.values - return points_vector - elif obj.__class__.__name__ == "ndarray": - points_vector = obj - return points_vector - else: - warnings.warn( - "Provided data or distance matrix must be in ndarray " - "or DataFrame.", - UserWarning, - ) - if isinstance(obj, list): - points_vector = np.array(obj) - return points_vector - points_vector = np.array([obj]) - return points_vector + # ------------------------------------------------------------------ + # Backward-compatible read aliases for *fitted output* attributes. + # These are NOT constructor parameters, so they do not affect + # get_params() / set_params() / check_estimator param comparisons. + # ------------------------------------------------------------------ + @property + def points_vector(self): + return getattr(self, "points_vector_", None) - def _validate_inputs(self): - """ - Validates the inputs provided during initialization to ensure - that the needed objects are provided. - :return: a tuple of (data, distance_matrix, neighbor_matrix) or - raises a warning for invalid inputs. - """ - if all(v is None for v in [self.data, self.distance_matrix]): - warnings.warn( - "Data or a distance matrix must be provided.", UserWarning - ) - return False - elif all(v is not None for v in [self.data, self.distance_matrix]): - warnings.warn( - "Only one of the following may be provided: data or a " - "distance matrix (not both).", - UserWarning, - ) - return False - if self.data is not None: - points_vector = self._convert_to_array(self.data) - return points_vector, self.distance_matrix, self.neighbor_matrix - if all( - matrix is not None - for matrix in [self.neighbor_matrix, self.distance_matrix] - ): - dist_vector = self._convert_to_array(self.distance_matrix) - neigh_vector = self._convert_to_array(self.neighbor_matrix) - else: - warnings.warn( - "A neighbor index matrix and distance matrix must both be " - "provided when not using raw input data.", - UserWarning, - ) - return False - if self.distance_matrix.shape != self.neighbor_matrix.shape: - warnings.warn( - "The shape of the distance and neighbor " - "index matrices must match.", - UserWarning, - ) - return False - elif (self.distance_matrix.shape[1] != self.n_neighbors) or ( - self.neighbor_matrix.shape[1] != self.n_neighbors - ): - warnings.warn( - "The shape of the distance or " - "neighbor index matrix does not " - "match the number of neighbors " - "specified.", - UserWarning, - ) - return False - return self.data, dist_vector, neigh_vector + @property + def prob_distances(self): + return getattr(self, "prob_distances_", None) - def _check_cluster_size(self) -> None: - """ - Validates the cluster labels to ensure that the smallest cluster - size (number of observations in the cluster) is larger than the - specified number of neighbors. - :raises ClusterSizeError: if any cluster is too small. - """ - c_labels = self._cluster_labels() - for cluster_id in set(c_labels): - c_size = np.where(c_labels == cluster_id)[0].shape[0] - if c_size <= self.n_neighbors: - raise ClusterSizeError( - "Number of neighbors specified larger than smallest " - "cluster. Specify a number of neighbors smaller than " - "the smallest cluster size (observations in smallest " - "cluster minus one)." - ) + @property + def prob_distances_ev(self): + return getattr(self, "prob_distances_ev_", None) - def _check_n_neighbors(self) -> bool: - """ - Validates the specified number of neighbors to ensure that it is - greater than 0 and that the specified value is less than the total - number of observations. - :return: a boolean indicating whether validation has passed without - adjustment. - """ - if not self.n_neighbors > 0: - self.n_neighbors = 10 - warnings.warn( - "n_neighbors must be greater than 0." - " Fit with " + str(self.n_neighbors) + " instead.", - UserWarning, - ) - return False - elif self.n_neighbors >= self._n_observations(): - self.n_neighbors = self._n_observations() - 1 - warnings.warn( - "n_neighbors must be less than the number of observations." - " Fit with " + str(self.n_neighbors) + " instead.", - UserWarning, - ) - return True + @property + def norm_prob_local_outlier_factor(self): + return getattr(self, "norm_prob_local_outlier_factor_", None) - def _check_extent(self) -> bool: - """ - Validates the specified extent parameter to ensure it is either 1, - 2, or 3. - :return: a boolean indicating whether validation has passed. - """ - if self.extent not in [1, 2, 3]: - warnings.warn( - "extent parameter (lambda) must be 1, 2, or 3.", UserWarning - ) - return False - return True + @property + def local_outlier_probabilities(self): + return getattr(self, "local_outlier_probabilities_", None) - def _check_missing_values(self) -> None: - """ - Validates the provided data to ensure that it contains no - missing values. - :raises MissingValuesError: if data contains NaN values. - """ - if np.any(np.isnan(self.data)): - raise MissingValuesError( - "Method does not support missing values in input data." - ) + @property + def is_fit(self): + return getattr(self, "is_fit_", False) - def _check_is_fit(self) -> bool: - """ - Checks that the model was fit prior to calling the stream() method. - :return: a boolean indicating whether the model has been fit. - """ - if self.is_fit is False: - warnings.warn( - "Must fit on historical data by calling fit() prior to " - "calling stream(x).", - UserWarning, - ) - return False - return True - - def _check_no_cluster_labels(self) -> bool: - """ - Checks to see if cluster labels are attempting to be used in - stream() and, if so, returns False. As PyNomaly does not accept - clustering algorithms as input, the stream approach does not - support clustering. - :return: a boolean indicating whether single cluster (no labels). - """ - if len(set(self._cluster_labels())) > 1: - warnings.warn( - "Stream approach does not support clustered data. " - "Automatically refit using single cluster of points.", - UserWarning, - ) - return False - return True - - """ - Decorators. - """ - - def accepts(*types): - """ - A decorator that facilitates a form of type checking for the inputs - which can be used in Python 3.4-3.7 in lieu of Python 3.5+'s type - hints. - :param types: the input types of the objects being passed as arguments - in __init__. - :return: a decorator. - """ - - def decorator(f): - assert len(types) == f.__code__.co_argcount - - def new_f(*args, **kwds): - for a, t in zip(args, types): - if type(a).__name__ == "DataFrame": - a = np.array(a) - if isinstance(a, t) is False: - warnings.warn( - "Argument %r is not of type %s" % (a, t), UserWarning - ) - opt_types = { - "distance_matrix": {"type": types[2]}, - "neighbor_matrix": {"type": types[3]}, - "extent": {"type": types[4]}, - "n_neighbors": {"type": types[5]}, - "cluster_labels": {"type": types[6]}, - "use_numba": {"type": types[7]}, - "n_jobs": {"type": types[8]}, - "progress_bar": {"type": types[9]}, - } - for x in kwds: - opt_types[x]["value"] = kwds[x] - for k in opt_types: - try: - if ( - isinstance(opt_types[k]["value"], opt_types[k]["type"]) - is False - ): - warnings.warn( - "Argument %r is not of type %s." - % (k, opt_types[k]["type"]), - UserWarning, - ) - except KeyError: - pass - return f(*args, **kwds) - - new_f.__name__ = f.__name__ - return new_f - - return decorator - - @accepts( - object, - np.ndarray, - np.ndarray, - np.ndarray, - (int, np.integer), - (int, np.integer), - list, - bool, - (int, np.integer), - bool, - ) def __init__( self, - data=None, - distance_matrix=None, - neighbor_matrix=None, extent=3, n_neighbors=10, - cluster_labels=None, use_numba=False, n_jobs=1, progress_bar=False, + data=None, + distance_matrix=None, + neighbor_matrix=None, + cluster_labels=None, ) -> None: - self.data = data - self.distance_matrix = distance_matrix - self.neighbor_matrix = neighbor_matrix + # Configuration parameters only — never mutate these in fit(). self.extent = extent self.n_neighbors = n_neighbors - self.cluster_labels = cluster_labels self.use_numba = use_numba self.n_jobs = n_jobs - self.points_vector = None - self.prob_distances = None - self.prob_distances_ev = None - self.norm_prob_local_outlier_factor = None - self.local_outlier_probabilities = None - self._objects = {} self.progress_bar = progress_bar - self.is_fit = False - if self.use_numba is True and "numba" not in sys.modules: - self.use_numba = False + # Deprecated data-in-constructor path (backward compatible). + _locals = { + "data": data, + "distance_matrix": distance_matrix, + "neighbor_matrix": neighbor_matrix, + "cluster_labels": cluster_labels, + } + for param in self._DATA_PARAMS: + if _locals[param] is not None: + warnings.warn( + f"Passing '{param}' to __init__ is deprecated. " + "Pass it to fit() instead. This will raise an error " + "in a future version.", + FutureWarning, + stacklevel=2, + ) + + self.data = data + self.distance_matrix = distance_matrix + self.neighbor_matrix = neighbor_matrix + self.cluster_labels = cluster_labels + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + # Sparse inputs are densified in validation; tag as supported. + tags.input_tags.sparse = True + return tags + + def _reset_state(self) -> None: + """Clear fitted state so the estimator can be re-fit.""" + self.points_vector_ = None + self.prob_distances_ = None + self.prob_distances_ev_ = None + self.norm_prob_local_outlier_factor_ = None + self.local_outlier_probabilities_ = None + self.data_ = None + self.distance_matrix_ = None + self.neighbor_matrix_ = None + self.cluster_labels_ = None + self.n_neighbors_ = self.n_neighbors + self._objects_ = {} + self.is_fit_ = False + + def _resolve_runtime_flags(self): + """Return (use_numba, n_jobs) without mutating public params.""" + use_numba = self.use_numba + if use_numba is True and "numba" not in sys.modules: warnings.warn( - "Numba is not available, falling back to pure python mode.", UserWarning + "Numba is not available, falling back to pure python mode.", + UserWarning, ) + use_numba = False - if self.n_jobs < -1 or self.n_jobs == 0: + n_jobs = self.n_jobs + if n_jobs < -1 or n_jobs == 0: warnings.warn( "n_jobs must be -1 or a positive integer. Defaulting to 1.", UserWarning, ) - self.n_jobs = 1 - - self._validate_inputs() - self._check_extent() - - """ - Private methods. - """ - - @staticmethod - def _standard_distance(cardinality: float, sum_squared_distance: float) -> float: - """ - Calculates the standard distance of an observation. - :param cardinality: the cardinality of the input observation. - :param sum_squared_distance: the sum squared distance between all - neighbors of the input observation. - :return: the standard distance. - #""" - division_result = sum_squared_distance / cardinality - st_dist = sqrt(division_result) - return st_dist - - @staticmethod - def _prob_distance(extent: int, standard_distance: float) -> float: - """ - Calculates the probabilistic distance of an observation. - :param extent: the extent value specified during initialization. - :param standard_distance: the standard distance of the input - observation. - :return: the probabilistic distance. - """ - return extent * standard_distance - - @staticmethod - def _prob_outlier_factor( - probabilistic_distance: np.ndarray, ev_prob_dist: np.ndarray - ) -> np.ndarray: - """ - Calculates the probabilistic outlier factor of an observation. - :param probabilistic_distance: the probabilistic distance of the - input observation. - :param ev_prob_dist: - :return: the probabilistic outlier factor. - """ - if np.all(probabilistic_distance == ev_prob_dist): - return np.zeros(probabilistic_distance.shape) - else: - ev_prob_dist[ev_prob_dist == 0.0] = 1.0e-8 - result = np.divide(probabilistic_distance, ev_prob_dist) - 1.0 - return result - - @staticmethod - def _norm_prob_outlier_factor( - extent: float, ev_probabilistic_outlier_factor: list - ) -> list: - """ - Calculates the normalized probabilistic outlier factor of an - observation. - :param extent: the extent value specified during initialization. - :param ev_probabilistic_outlier_factor: the expected probabilistic - outlier factor of the input observation. - :return: the normalized probabilistic outlier factor. - """ - ev_arr = np.array(ev_probabilistic_outlier_factor, dtype=float) - return (extent * np.sqrt(ev_arr)).tolist() - - @staticmethod - def _local_outlier_probability( - plof_val: np.ndarray, nplof_val: np.ndarray - ) -> np.ndarray: - """ - Calculates the local outlier probability of an observation. - :param plof_val: the probabilistic outlier factor of the input - observation. - :param nplof_val: the normalized probabilistic outlier factor of the - input observation. - :return: the local outlier probability. - """ - if np.all(plof_val == nplof_val): - return np.zeros(plof_val.shape) - plof_f = np.asarray(plof_val, dtype=float) - nplof_f = np.asarray(nplof_val, dtype=float) - if _scipy_erf is not None: - return np.maximum(0, _scipy_erf(plof_f / (nplof_f * np.sqrt(2.0)))) - erf_vec = np.vectorize(erf) - return np.maximum(0, erf_vec(plof_f / (nplof_f * np.sqrt(2.0)))) - - def _n_observations(self) -> int: - """ - Calculates the number of observations in the data. - :return: the number of observations in the input data. - """ - if self.data is not None: - return len(self.data) - return len(self.distance_matrix) - - def _store(self) -> np.ndarray: - """ - Initializes the storage matrix that includes the input value, - cluster labels, local outlier probability, etc. for the input data. - :return: an empty numpy array of shape [n_observations, 3]. - """ - return np.empty([self._n_observations(), 3], dtype=object) - - def _cluster_labels(self) -> np.ndarray: - """ - Returns a numpy array of cluster labels that corresponds to the - input labels or that is an array of all 0 values to indicate all - points belong to the same cluster. - :return: a numpy array of cluster labels. - """ - if self.cluster_labels is None: - if self.data is not None: - return np.array([0] * len(self.data)) - return np.array([0] * len(self.distance_matrix)) - return np.array(self.cluster_labels) - - @staticmethod - def _euclidean(vector1: np.ndarray, vector2: np.ndarray) -> np.ndarray: - """ - Calculates the euclidean distance between two observations in the - input data. - :param vector1: a numpy array corresponding to observation 1. - :param vector2: a numpy array corresponding to observation 2. - :return: the euclidean distance between the two observations. - """ - diff = vector1 - vector2 - return np.dot(diff, diff) ** 0.5 - - def _assign_distances(self, data_store: np.ndarray) -> np.ndarray: - """ - Takes a distance matrix, produced by _distances or provided through - user input, and assigns distances for each observation to the storage - matrix, data_store. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - for vec, cluster_id in zip( - range(self.distance_matrix.shape[0]), self._cluster_labels() - ): - data_store[vec][0] = cluster_id - data_store[vec][1] = self.distance_matrix[vec] - data_store[vec][2] = self.neighbor_matrix[vec] - return data_store - - @staticmethod - def _compute_distance_and_neighbor_matrix( - clust_points_vector: np.ndarray, - indices: np.ndarray, - distances: np.ndarray, - indexes: np.ndarray, - ) -> Tuple[np.ndarray, np.ndarray, int]: - """ - This helper method provides the heavy lifting for the _distances - method and is only intended for use therein. The code has been - written so that it can make full use of Numba's jit capabilities if - desired. - """ - for i in range(clust_points_vector.shape[0]): - for j in range(i + 1, clust_points_vector.shape[0]): - # Global index of the points - global_i = indices[0][i] - global_j = indices[0][j] - - # Compute Euclidean distance - diff = clust_points_vector[i] - clust_points_vector[j] - d = np.dot(diff, diff) ** 0.5 - - # Update distance and neighbor index for global_i - idx_max = distances[global_i].argmax() - if d < distances[global_i][idx_max]: - distances[global_i][idx_max] = d - indexes[global_i][idx_max] = global_j - - # Update distance and neighbor index for global_j - idx_max = distances[global_j].argmax() - if d < distances[global_j][idx_max]: - distances[global_j][idx_max] = d - indexes[global_j][idx_max] = global_i - - yield distances, indexes, i - - def _distances_vectorized( - self, clusters, distances, indexes, progress_bar - ) -> None: - """Vectorized kNN distance computation with chunked progress.""" - progress = "=" - total_points = sum(cv.shape[0] for cv, _ in clusters) - completed = 0 - - for clust_points_vector, global_indices in clusters: - n = clust_points_vector.shape[0] - - if clust_points_vector.ndim == 1: - clust_points_vector = clust_points_vector.reshape(-1, 1) - - chunk_size = min(256, n) - for chunk_start in range(0, n, chunk_size): - chunk_end = min(chunk_start + chunk_size, n) - chunk = clust_points_vector[chunk_start:chunk_end] - - if _scipy_cdist is not None: - dist = _scipy_cdist( - chunk, clust_points_vector, metric="euclidean" - ) - else: - diff = ( - chunk[:, np.newaxis, :] - - clust_points_vector[np.newaxis, :, :] - ) - dist = np.sqrt((diff ** 2).sum(axis=2)) - - row_idx = np.arange(chunk_end - chunk_start) - dist[row_idx, row_idx + chunk_start] = np.inf - - knn_idx = np.argpartition(dist, self.n_neighbors, axis=1)[ - :, : self.n_neighbors - ] - knn_dists = np.take_along_axis(dist, knn_idx, axis=1) - - chunk_global = global_indices[chunk_start:chunk_end] - distances[chunk_global] = knn_dists - indexes[chunk_global] = global_indices[knn_idx] - - completed += chunk_end - chunk_start - if progress_bar: - progress = Utils.emit_progress_bar( - progress, completed, total_points - ) - - def _distances_numba( - self, clusters, distances, indexes, progress_bar, parallel=False - ) -> None: - """Numba-accelerated distance computation.""" - progress = "=" - kernel = _numba_kernel_parallel if parallel else _numba_kernel_sequential - - for idx, (clust_points_vector, global_indices) in enumerate(clusters): - if clust_points_vector.ndim == 1: - clust_points_vector = clust_points_vector.reshape(-1, 1) - - local_dists, local_idxs = kernel( - clust_points_vector.astype(np.float64), self.n_neighbors - ) + n_jobs = 1 + return use_numba, n_jobs - distances[global_indices] = local_dists - indexes[global_indices] = global_indices[local_idxs] - - if progress_bar: - progress = Utils.emit_progress_bar( - progress, idx + 1, len(clusters) - ) - - def _distances(self, progress_bar: bool = False) -> None: - """ - Provides the distances between each observation and it's closest - neighbors. When input data is provided, calculates the euclidean - distance between every observation. Otherwise, the user-provided - distance matrix is used. - :return: the updated storage matrix that collects information on - each observation. - """ - distances = np.full( - [self._n_observations(), self.n_neighbors], 9e10, dtype=float - ) - indexes = np.full( - [self._n_observations(), self.n_neighbors], 9e10, dtype=float - ) - self.points_vector = self._convert_to_array(self.data) - - cluster_labels = self._cluster_labels() - cluster_ids = sorted(set(cluster_labels)) - - clusters = [] - for cluster_id in cluster_ids: - indices = np.where(cluster_labels == cluster_id) - clust_points_vector = np.array( - self.points_vector.take(indices, axis=0)[0], dtype=np.float64 - ) - clusters.append((clust_points_vector, indices[0])) - - n_jobs = self.n_jobs - if n_jobs == -1: - n_jobs = os.cpu_count() or 1 - - if self.use_numba: - self._distances_numba( - clusters, distances, indexes, progress_bar, - parallel=(n_jobs > 1) - ) - else: - if n_jobs > 1: + @validate_init_types( + object, + (int, np.integer), + (int, np.integer), + bool, + (int, np.integer), + bool, + np.ndarray, + np.ndarray, + np.ndarray, + list, + ) + def fit( + self, + X=None, + y=None, + data=None, + distance_matrix=None, + neighbor_matrix=None, + cluster_labels=None, + ) -> "LocalOutlierProbability": + """ + Fit the LoOP model. + + Parameters + ---------- + X : array-like of shape (n_samples, n_features), optional + Training data (scikit-learn style). Sparse matrices are densified. + y : ignored + Present for API consistency. For backward compatibility only, + a 2-d ``y`` with more than one column is treated as a deprecated + positional ``distance_matrix`` when no raw data (``X`` / ``data``) + is given. + data, distance_matrix, neighbor_matrix, cluster_labels + Explicit keyword alternatives to ``X`` / precomputed neighbors. + """ + if X is not None: + data = X + + # Legacy positional fit(data, distance_matrix) support. Only applies + # when no raw data is given and y cannot be a label vector, so that + # sklearn-style fit(X, y) with (n, 1) labels is not hijacked. + if y is not None and distance_matrix is None and data is None: + y_array = np.asarray(y) + if y_array.ndim == 2 and y_array.shape[1] > 1: + distance_matrix = y + y = None warnings.warn( - "n_jobs > 1 requires use_numba=True for parallel " - "processing. Install Numba and set use_numba=True " - "to enable parallelism. Falling back to sequential.", - UserWarning, + "Passing 'distance_matrix' as the second positional " + "argument is deprecated. Use keyword arguments instead.", + FutureWarning, ) - self._distances_vectorized( - clusters, distances, indexes, progress_bar - ) - - self.distance_matrix = distances - self.neighbor_matrix = indexes - - def _ssd(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the sum squared distance between neighbors for each - observation in the input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - self.cluster_labels_u = np.unique(data_store[:, 0]) - ssd_array = np.empty([self._n_observations(), 1]) - for cluster_id in self.cluster_labels_u: - indices = np.where(data_store[:, 0] == cluster_id) - cluster_distances = np.take(data_store[:, 1], indices).tolist() - ssd = np.power(cluster_distances[0], 2).sum(axis=1) - for i, j in zip(indices[0], ssd): - ssd_array[i] = j - data_store = np.hstack((data_store, ssd_array)) - return data_store - - def _standard_distances(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculated the standard distance for each observation in the input - data. First calculates the cardinality and then calculates the standard - distance with respect to each observation. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - ssd_vals = data_store[:, 3].astype(float) - std_distances = np.sqrt(ssd_vals / self.n_neighbors) - return np.hstack((data_store, std_distances.reshape(-1, 1))) - def _prob_distances(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the probabilistic distance for each observation in the - input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - prob_distances = self.extent * data_store[:, 4].astype(float) - return np.hstack((data_store, prob_distances.reshape(-1, 1))) + use_numba, n_jobs = self._resolve_runtime_flags() + self._fit_use_numba = use_numba + self._fit_n_jobs = n_jobs - def _prob_distances_ev(self, data_store) -> np.ndarray: - """ - Calculates the expected value of the probabilistic distance for - each observation in the input data with respect to the cluster the - observation belongs to. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - prob_set_distance_ev = np.empty([self._n_observations(), 1]) - for cluster_id in self.cluster_labels_u: - indices = np.where(data_store[:, 0] == cluster_id)[0] - for index in indices: - # Global neighbor indices for the current point - nbrhood = data_store[index][2].astype(int) # Ensure global indices - nbrhood_prob_distances = np.take(data_store[:, 5], nbrhood).astype( - float - ) - nbrhood_prob_distances_nonan = nbrhood_prob_distances[ - np.logical_not(np.isnan(nbrhood_prob_distances)) - ] - prob_set_distance_ev[index] = nbrhood_prob_distances_nonan.mean() - - self.prob_distances_ev = prob_set_distance_ev - return np.hstack((data_store, prob_set_distance_ev)) + self._check_extent() + self._reset_state() - def _prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the probabilistic local outlier factor for each - observation in the input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - return np.hstack( - ( - data_store, - np.array( - [ - np.apply_along_axis( - self._prob_outlier_factor, - 0, - data_store[:, 5], - data_store[:, 6], - ) - ] - ).T, - ) + self.data_ = data if data is not None else self.data + self.distance_matrix_ = ( + distance_matrix if distance_matrix is not None else self.distance_matrix ) - - def _prob_local_outlier_factors_ev(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the expected value of the probabilistic local outlier factor - for each observation in the input data with respect to the cluster the - observation belongs to. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - prob_local_outlier_factor_ev_dict = {} - for cluster_id in self.cluster_labels_u: - indices = np.where(data_store[:, 0] == cluster_id) - prob_local_outlier_factors = np.take(data_store[:, 7], indices).astype( - float - ) - prob_local_outlier_factors_nonan = prob_local_outlier_factors[ - np.logical_not(np.isnan(prob_local_outlier_factors)) - ] - prob_local_outlier_factor_ev_dict[cluster_id] = np.power( - prob_local_outlier_factors_nonan, 2 - ).sum() / float(prob_local_outlier_factors_nonan.size) - data_store = np.hstack( - ( - data_store, - np.array( - [ - [ - prob_local_outlier_factor_ev_dict[x] - for x in data_store[:, 0].tolist() - ] - ] - ).T, - ) + self.neighbor_matrix_ = ( + neighbor_matrix if neighbor_matrix is not None else self.neighbor_matrix ) - return data_store - - def _norm_prob_local_outlier_factors(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the normalized probabilistic local outlier factor for each - observation in the input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - return np.hstack( - ( - data_store, - np.array( - [ - self._norm_prob_outlier_factor( - self.extent, data_store[:, 8].tolist() - ) - ] - ).T, - ) + self.cluster_labels_ = ( + cluster_labels if cluster_labels is not None else self.cluster_labels ) - def _local_outlier_probabilities(self, data_store: np.ndarray) -> np.ndarray: - """ - Calculates the local outlier probability for each observation in the - input data. - :param data_store: the storage matrix that collects information on - each observation. - :return: the updated storage matrix that collects information on - each observation. - """ - return np.hstack( - ( - data_store, - np.array( - [ - np.apply_along_axis( - self._local_outlier_probability, - 0, - data_store[:, 7], - data_store[:, 9], - ) - ] - ).T, - ) - ) - - """ - Public methods - """ + if self._validate_inputs() is False: + return self - def fit(self) -> "LocalOutlierProbability": - """ - Calculates the local outlier probability for each observation in the - input data according to the input parameters extent, n_neighbors, and - cluster_labels. - :return: self, which contains the local outlier probabilities as - self.local_outlier_probabilities. - :raises ClusterSizeError: if any cluster is smaller than n_neighbors. - :raises MissingValuesError: if data contains missing values. - """ + n_obs = self._n_observations() + if n_obs < 2: + raise ValueError( + f"Found array with {n_obs} sample(s) while a minimum of 2 " + f"is required by LocalOutlierProbability." + ) self._check_n_neighbors() self._check_cluster_size() - if self.data is not None: + + _data = getattr(self, "data_", None) + if _data is not None: self._check_missing_values() store = self._store() - if self.data is not None: + if _data is not None: self._distances(progress_bar=self.progress_bar) + store = self._assign_distances(store) store = self._ssd(store) store = self._standard_distances(store) store = self._prob_distances(store) - self.prob_distances = store[:, 5] + self.prob_distances_ = np.asarray(store[:, 5], dtype=float) store = self._prob_distances_ev(store) store = self._prob_local_outlier_factors(store) store = self._prob_local_outlier_factors_ev(store) store = self._norm_prob_local_outlier_factors(store) - self.norm_prob_local_outlier_factor = store[:, 9].max() + self.norm_prob_local_outlier_factor_ = float( + np.asarray(store[:, 9], dtype=float).max() + ) store = self._local_outlier_probabilities(store) - self.local_outlier_probabilities = store[:, 10] - - self.is_fit = True + self.local_outlier_probabilities_ = np.asarray(store[:, 10], dtype=float) + if _data is not None and self.points_vector_ is not None: + self.n_features_in_ = ( + self.points_vector_.shape[1] + if self.points_vector_.ndim == 2 + else 1 + ) + elif self.distance_matrix_ is not None: + self.n_features_in_ = 1 + + # sklearn outlier API: decision_function = score_samples - offset_ + # score_samples = -LoOP probability (higher is more normal) + # offset_ = -0.5 so decision >= 0 iff probability <= 0.5 + self.offset_ = -0.5 + self.is_fit_ = True return self - def stream(self, x: np.ndarray) -> np.ndarray: - """ - Calculates the local outlier probability for an individual sample - according to the input parameters extent, n_neighbors, and - cluster_labels after first calling fit(). Observations are assigned - a local outlier probability against the mean of expected values of - probabilistic distance and the normalized probabilistic outlier - factor from the earlier model, provided when calling fit(). - distance - :param x: an observation to score for its local outlier probability. - :return: the local outlier probability of the input observation. - """ + def stream(self, x: np.ndarray) -> float: + """Score a single observation against the fitted model.""" + if self._check_is_fit() is False: + self.fit() + + _data = getattr(self, "data_", getattr(self, "data", None)) + _dist = getattr( + self, "distance_matrix_", getattr(self, "distance_matrix", None) + ) + _clust = getattr(self, "cluster_labels_", getattr(self, "cluster_labels", None)) orig_cluster_labels = None if self._check_no_cluster_labels() is False: - orig_cluster_labels = self.cluster_labels - self.cluster_labels = np.array([0] * len(self.data)) + orig_cluster_labels = _clust + if getattr(self, "points_vector_", None) is not None: + self.cluster_labels_ = np.array([0] * self.points_vector_.shape[0]) - if self._check_is_fit() is False: - self.fit() + point_vector = self._convert_observation(x) + n_neighbors = self._effective_n_neighbors() + distances = np.full([1, n_neighbors], 9e10, dtype=float) - point_vector = self._convert_to_array(x) - distances = np.full([1, self.n_neighbors], 9e10, dtype=float) - if self.data is not None: - matrix = self.points_vector + if _data is not None: + matrix = getattr(self, "points_vector_", None) else: - matrix = self.distance_matrix - # When using distance matrix mode, x is a scalar distance value. - # Extract scalar from array to avoid NumPy assignment errors. + matrix = _dist if point_vector.size == 1: point_vector = float(point_vector.flat[0]) + + if matrix is None: + from PyNomaly._compat import NotFittedError + + raise NotFittedError( + "This LocalOutlierProbability instance is not fitted yet. " + "Call 'fit' with appropriate arguments before using this " + "estimator." + ) + for p in range(0, matrix.shape[0]): - if self.data is not None: - d = self._euclidean(matrix[p, :], point_vector) + if _data is not None: + ref_point = matrix[p] if matrix.ndim == 1 else matrix[p, :] + d = float(self._euclidean(ref_point, point_vector)) else: - d = point_vector + d = float(point_vector) + idx_max = distances[0].argmax() if d < distances[0][idx_max]: distances[0][idx_max] = d ssd = np.power(distances, 2).sum() - std_dist = np.sqrt(np.divide(ssd, self.n_neighbors)) + std_dist = np.sqrt(np.divide(ssd, n_neighbors)) prob_dist = self._prob_distance(self.extent, std_dist) plof = self._prob_outlier_factor( - np.array(prob_dist), np.array(self.prob_distances_ev.mean()) + np.array(prob_dist), np.array(self.prob_distances_ev_.mean()) ) - loop = self._local_outlier_probability( - plof, self.norm_prob_local_outlier_factor + loop_score = self._local_outlier_probability( + plof, self.norm_prob_local_outlier_factor_ ) if orig_cluster_labels is not None: - self.cluster_labels = orig_cluster_labels + self.cluster_labels_ = orig_cluster_labels + + return float(np.asarray(loop_score, dtype=float).ravel()[0]) + + def _score_matrix(self, X) -> np.ndarray: + """ + Return LoOP probabilities for ``X``. + + Training rows are matched to fitted scores so predict / decision + function stay order- and subset-invariant. Unseen rows use ``stream``. + """ + check_is_fitted(self, ["is_fit_"]) + + if _issparse_safe(X): + from PyNomaly._validation import _require_scipy_for_sparse + + _require_scipy_for_sparse(X) + X = X.toarray() + + X_probe = np.asarray(X) + if X_probe.ndim == 1 and getattr(self, "n_features_in_", None) == 1: + X = X_probe.reshape(-1, 1) + + X_array = check_array( + X, + accept_sparse=False, + dtype="numeric", + ensure_2d=True, + ) + X_array = np.asarray(X_array, dtype=float) + + if np.any(np.isnan(X_array)) or np.any(np.isinf(X_array)): + raise ValueError( + "Input contains NaN, infinity or a value too large for " + "dtype('float64')." + ) + + if hasattr(self, "n_features_in_"): + n_features = X_array.shape[1] + if n_features != self.n_features_in_: + raise ValueError( + f"X has {n_features} features, but {type(self).__name__} " + f"is expecting {self.n_features_in_} features as input." + ) + + train = getattr(self, "points_vector_", None) + fitted = getattr(self, "local_outlier_probabilities_", None) + scores = np.empty(X_array.shape[0], dtype=float) + + if train is not None and fitted is not None and train.ndim == 2: + for i, row in enumerate(X_array): + matches = np.where(np.all(np.isclose(train, row), axis=1))[0] + if matches.size: + scores[i] = fitted[matches[0]] + else: + scores[i] = float(self.stream(row)) + return scores + + for i, row in enumerate(X_array): + scores[i] = float(self.stream(row)) + return scores + + def decision_function(self, X): + """ + Signed outlier score (higher is more normal). + + Equivalent to ``score_samples(X) - offset_``. Inliers satisfy + ``decision_function(X) >= 0``. + """ + return self.score_samples(X) - self.offset_ + + def score_samples(self, X): + """ + Opposite of the LoOP probability (higher is more normal). + """ + return -1.0 * self._score_matrix(X) + + def predict(self, X): + """ + Predict inliers (1) and outliers (-1). + + Labels are derived from ``decision_function(X) >= 0``. + """ + decision = self.decision_function(X) + return np.where(decision >= 0, 1, -1) + + +def _issparse_safe(x) -> bool: + try: + from scipy.sparse import issparse + + return issparse(x) + except ImportError: + return False + + +LoOP = LocalOutlierProbability - return loop +__all__ = [ + "LocalOutlierProbability", + "LoOP", + # Exceptions live in PyNomaly.exceptions; re-exported here so that + # ``from PyNomaly.loop import ClusterSizeError`` keeps working. + "PyNomalyError", + "ValidationError", + "ClusterSizeError", + "MissingValuesError", +] diff --git a/README.md b/README.md index 2a1ac49..5f4245d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ scores in the range of [0,1] that are directly interpretable as the probability PyNomaly is a core library of [deepchecks](https://github.com/deepchecks/deepchecks), [OmniDocBench](https://github.com/opendatalab/OmniDocBench) and [pysad](https://github.com/selimfirat/pysad). [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![PyPi](https://img.shields.io/badge/pypi-0.4.0-blue.svg)](https://pypi.python.org/pypi/PyNomaly/0.4.0) +[![PyPi](https://img.shields.io/badge/pypi-1.0.0-blue.svg)](https://pypi.python.org/pypi/PyNomaly/1.0.0) [![Total Downloads](https://static.pepy.tech/badge/pynomaly)](https://pepy.tech/projects/pynomaly) [![Monthly Downloads](https://static.pepy.tech/badge/pynomaly/month)](https://pepy.tech/projects/pynomaly) ![Tests](https://github.com/vc1492a/PyNomaly/actions/workflows/tests.yml/badge.svg) @@ -60,7 +60,7 @@ LoOP scores are computed through a pipeline of steps that transform raw distance For full mathematical detail, see the [original paper](http://www.dbs.ifi.lmu.de/Publikationen/Papers/LoOP1649.pdf) by Kriegel et al. (2009). For the full API and additional documentation, see the [PyNomaly documentation](https://vc1492a.github.io/PyNomaly/). ## Dependencies -- Python 3.8 - 3.14 +- Python 3.9 - 3.14 - numpy >= 1.16.3 - python-utils >= 2.3.0 - (optional) numba >= 0.45.1 @@ -72,6 +72,9 @@ distance between observations, providing a reduction in computation time requirement and PyNomaly may still be used solely with numpy if desired (details below). When scipy is available, PyNomaly uses its optimized distance computation and error function implementations for additional performance gains. +Sparse matrices (`scipy.sparse`) are also accepted as input and converted to +dense arrays internally; **scipy is required for sparse input** (`pip install scipy` +or `pip install PyNomaly[sparse]`). ## Quick Start @@ -89,19 +92,19 @@ conda install conda-forge::pynomaly Then you can do something like this: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data).fit() +from PyNomaly import LoOP +m = LoOP(n_neighbors=10).fit(data) scores = m.local_outlier_probabilities print(scores) ``` where *data* is a NxM (N rows, M columns; 2-dimensional) set of data as either a Pandas DataFrame or Numpy array. -LocalOutlierProbability sets the *extent* (in integer in value of 1, 2, or 3) and *n_neighbors* (must be greater than 0) parameters with the default +`LocalOutlierProbability` (also available as `LoOP`) sets the *extent* (an integer value of 1, 2, or 3) and *n_neighbors* (must be greater than 0) parameters with the default values of 3 and 10, respectively. You're free to set these parameters on your own as below: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -111,10 +114,10 @@ of varying density occur within the same set of data. When using *cluster_labels sample is calculated with respect to its cluster assignment. ```python -from PyNomaly import loop +from PyNomaly import LoOP from sklearn.cluster import DBSCAN db = DBSCAN(eps=0.6, min_samples=50).fit(data) -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20, cluster_labels=list(db.labels_)).fit() +m = LoOP(extent=2, n_neighbors=20).fit(data, cluster_labels=list(db.labels_)) scores = m.local_outlier_probabilities print(scores) ``` @@ -130,8 +133,8 @@ For large datasets, Numba's just-in-time (JIT) compilation can significantly speed up distance computation. Enable it with `use_numba=True`: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20, use_numba=True).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20, use_numba=True).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -140,11 +143,8 @@ To go further, set `n_jobs=-1` to enable Numba's thread-level parallelism (`prange`), which distributes work across all available CPU cores: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability( - data, extent=2, n_neighbors=20, - use_numba=True, n_jobs=-1 -).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20, use_numba=True, n_jobs=-1).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -172,8 +172,8 @@ You may choose to print progress bars _with or without_ the use of Numba by passing `progress_bar=True` to `LocalOutlierProbability()`: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, use_numba=True, n_jobs=-1, progress_bar=True).fit() +from PyNomaly import LoOP +m = LoOP(use_numba=True, n_jobs=-1, progress_bar=True).fit(data) ``` Progress bars are supported in both sequential and Numba execution modes. @@ -212,11 +212,10 @@ PyNomaly provides custom exceptions that can be caught and handled in your appli These exceptions are exported from the package and can be imported directly: ```python -from PyNomaly import loop -from PyNomaly.loop import ClusterSizeError, MissingValuesError +from PyNomaly import LoOP, ClusterSizeError, MissingValuesError try: - m = loop.LocalOutlierProbability(data, n_neighbors=50, cluster_labels=labels).fit() + m = LoOP(n_neighbors=50).fit(data, cluster_labels=labels) except ClusterSizeError: print("Reduce n_neighbors or use larger clusters.") except MissingValuesError: @@ -261,9 +260,9 @@ values for both *extent* (0.997) and *n_neighbors* (10). ```python db = DBSCAN(eps=0.9, min_samples=10).fit(iris) -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities -m_clust = loop.LocalOutlierProbability(iris, cluster_labels=list(db.labels_)).fit() +m_clust = loop.LocalOutlierProbability().fit(iris, cluster_labels=list(db.labels_)) scores_clust = m_clust.local_outlier_probabilities ``` @@ -348,7 +347,7 @@ data = np.array([ [421.5, 90.3, 50.0] ]) -scores = loop.LocalOutlierProbability(data, n_neighbors=3).fit().local_outlier_probabilities +scores = loop.LocalOutlierProbability(n_neighbors=3).fit(data).local_outlier_probabilities print(scores) ``` @@ -364,7 +363,7 @@ Similar to the above: ```python data = np.random.rand(100, 5) -scores = loop.LocalOutlierProbability(data).fit().local_outlier_probabilities +scores = loop.LocalOutlierProbability().fit(data).local_outlier_probabilities print(scores) ``` @@ -398,11 +397,13 @@ neigh.fit(data) d, idx = neigh.kneighbors(data, return_distance=True) # Remove self-distances - you MUST do this to preserve the same results as intended by the definition of LoOP -indices = np.delete(indices, 0, 1) -distances = np.delete(distances, 0, 1) +idx = np.delete(idx, 0, 1) +d = np.delete(d, 0, 1) # Fit and return scores -m = loop.LocalOutlierProbability(distance_matrix=d, neighbor_matrix=idx, n_neighbors=n_neighbors+1).fit() +m = loop.LocalOutlierProbability(n_neighbors=n_neighbors).fit( + distance_matrix=d, neighbor_matrix=idx +) scores = m.local_outlier_probabilities ``` @@ -436,12 +437,12 @@ iris_test = iris.iloc[:, 0:4].tail(30) Fit to each set. ```python -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities iris['scores'] = scores_noclust -m_train = loop.LocalOutlierProbability(iris_train, n_neighbors=10) -m_train.fit() +m_train = loop.LocalOutlierProbability(n_neighbors=10) +m_train.fit(iris_train) iris_train_scores = m_train.local_outlier_probabilities ``` diff --git a/changelog.md b/changelog.md index 9b41565..ada9ae2 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,37 @@ All notable changes to PyNomaly will be documented in this Changelog. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 1.0.0 +### Changed +- **Breaking (with backward compatibility):** Adopted scikit-learn API conventions. + Data parameters (`data`, `distance_matrix`, `neighbor_matrix`, `cluster_labels`) + are now passed to `fit()` instead of `__init__()`. Configuration parameters + (`extent`, `n_neighbors`, `use_numba`, `n_jobs`, `progress_bar`) remain in + the constructor + ([Issue #7](https://github.com/vc1492a/PyNomaly/issues/7)). +- Passing data parameters to `__init__()` still works but emits a + `FutureWarning` and will be removed in a future version. +- Calling `fit()` multiple times with different data is now supported + (re-fitting resets internal state). +- `stream()` now returns a Python `float` instead of a NumPy array. +- Exception classes (`PyNomalyError`, `ValidationError`, `ClusterSizeError`, + `MissingValuesError`) now live in the new `PyNomaly.exceptions` module. + They remain importable from `PyNomaly.loop` and from `PyNomaly` for + backward compatibility. +- Passing data as a positional argument to the constructor + (e.g. `LocalOutlierProbability(data)`) no longer works: the first + positional parameter is now `extent`. Pass data to `fit()` instead + (e.g. `LocalOutlierProbability().fit(data)`). Keyword form + `LocalOutlierProbability(data=data)` still works with a `FutureWarning`. +### Added +- `LoOP` class alias for `LocalOutlierProbability`, enabling + `from PyNomaly import LoOP` + ([Issue #5](https://github.com/vc1492a/PyNomaly/issues/5)). +- `__version__` is now exported from the package (`from PyNomaly import __version__`). +### Removed +- Support for Python 3.8. PyNomaly 1.0.0 requires Python 3.9 or newer + (`python_requires='>=3.9'`). + ## 0.4.0 ### Added - Parallel distance computation via Numba `prange` and the new `n_jobs` diff --git a/docs/api.md b/docs/api.md index a37d3a9..79b5c50 100644 --- a/docs/api.md +++ b/docs/api.md @@ -22,7 +22,7 @@ clf = loop.LocalOutlierProbability( | Parameter | Type | Default | Description | |---|---|---|---| -| `data` | `np.ndarray` or `pd.DataFrame` | `None` | Input data as a 2D array with shape (n_observations, n_features). Mutually exclusive with `distance_matrix`. | +| `data` | `np.ndarray`, `pd.DataFrame`, or `scipy.sparse` matrix | `None` | Input data as a 2D array with shape (n_observations, n_features). Sparse matrices are accepted and densified internally (requires scipy). Mutually exclusive with `distance_matrix`. | | `distance_matrix` | `np.ndarray` or `pd.DataFrame` | `None` | Pre-computed distance matrix with shape (n_observations, n_neighbors). Must be provided together with `neighbor_matrix`. Mutually exclusive with `data`. | | `neighbor_matrix` | `np.ndarray` or `pd.DataFrame` | `None` | Pre-computed neighbor index matrix with shape (n_observations, n_neighbors). Required when `distance_matrix` is provided. | | `extent` | `int` | `3` | Controls scoring sensitivity. Must be 1, 2, or 3. Corresponds to lambda times the standard deviation from the mean (1 = ~68%, 2 = ~95%, 3 = ~99.7%). | @@ -47,7 +47,7 @@ clf.fit() -> LocalOutlierProbability Calculates the Local Outlier Probability for each observation in the input data. -**Returns**: `self` -- the fitted model instance. Access scores via `clf.local_outlier_probabilities`. +**Returns**: `self` -- the fitted model instance. Access scores via `clf.local_outlier_probabilities_`. **Raises**: @@ -59,7 +59,7 @@ Calculates the Local Outlier Probability for each observation in the input data. #### `stream(x)` ```python -clf.stream(x) -> np.ndarray +clf.stream(x) -> float ``` Calculates the Local Outlier Probability for an individual observation against the fitted model. Must be called after `fit()`. @@ -70,7 +70,7 @@ Calculates the Local Outlier Probability for an individual observation against t |---|---|---| | `x` | `np.ndarray` | A single observation to score. When using raw data mode, this should be a 1D array with the same number of features as the training data. When using distance matrix mode, this should be a scalar distance value. | -**Returns**: `np.ndarray` -- the Local Outlier Probability of the input observation (a value in [0, 1]). +**Returns**: `float` -- the Local Outlier Probability of the input observation (a value in [0, 1]). !!! warning The stream approach does **not** support clustered data. If `cluster_labels` were provided during `fit()`, PyNomaly will automatically refit using a single cluster and issue a `UserWarning`. @@ -83,22 +83,25 @@ Attributes available after calling `fit()`: | Attribute | Type | Description | |---|---|---| -| `local_outlier_probabilities` | `np.ndarray` | Array of LoOP scores for each observation, with values in [0, 1]. | -| `prob_distances` | `np.ndarray` | Probabilistic distances for each observation. | -| `prob_distances_ev` | `np.ndarray` | Expected values of probabilistic distances for each observation's neighborhood. | -| `norm_prob_local_outlier_factor` | `float` | Maximum normalized probabilistic local outlier factor across all observations. Used internally by `stream()`. | -| `is_fit` | `bool` | Whether the model has been fit. | +| `local_outlier_probabilities_` | `np.ndarray` | Array of LoOP scores for each observation, with values in [0, 1]. | +| `prob_distances_` | `np.ndarray` | Probabilistic distances for each observation. | +| `prob_distances_ev_` | `np.ndarray` | Expected values of probabilistic distances for each observation's neighborhood. | +| `norm_prob_local_outlier_factor_` | `float` | Maximum normalized probabilistic local outlier factor across all observations. Used internally by `stream()`. | +| `is_fit_` | `bool` | Whether the model has been fit. | | `n_neighbors` | `int` | The number of neighbors used (may differ from the value passed to the constructor if it was adjusted). | --- ## Exceptions -All exceptions are importable from `PyNomaly.loop` or directly from `PyNomaly`: +All exceptions are defined in `PyNomaly.exceptions`. They are also importable +directly from `PyNomaly`, and from `PyNomaly.loop` for backward compatibility: ```python -from PyNomaly import ClusterSizeError, MissingValuesError +from PyNomaly.exceptions import PyNomalyError, ValidationError, ClusterSizeError, MissingValuesError # or +from PyNomaly import ClusterSizeError, MissingValuesError +# or (backward compatible) from PyNomaly.loop import PyNomalyError, ValidationError, ClusterSizeError, MissingValuesError ``` diff --git a/docs/changelog.md b/docs/changelog.md index 7214f7b..c217d79 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,37 @@ All notable changes to PyNomaly will be documented in this Changelog. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 1.0.0 +### Changed +- **Breaking (with backward compatibility):** Adopted scikit-learn API conventions. + Data parameters (`data`, `distance_matrix`, `neighbor_matrix`, `cluster_labels`) + are now passed to `fit()` instead of `__init__()`. Configuration parameters + (`extent`, `n_neighbors`, `use_numba`, `n_jobs`, `progress_bar`) remain in + the constructor + ([Issue #7](https://github.com/vc1492a/PyNomaly/issues/7)). +- Passing data parameters to `__init__()` still works but emits a + `FutureWarning` and will be removed in a future version. +- Calling `fit()` multiple times with different data is now supported + (re-fitting resets internal state). +- `stream()` now returns a Python `float` instead of a NumPy array. +- Exception classes (`PyNomalyError`, `ValidationError`, `ClusterSizeError`, + `MissingValuesError`) now live in the new `PyNomaly.exceptions` module. + They remain importable from `PyNomaly.loop` and from `PyNomaly` for + backward compatibility. +- Passing data as a positional argument to the constructor + (e.g. `LocalOutlierProbability(data)`) no longer works: the first + positional parameter is now `extent`. Pass data to `fit()` instead + (e.g. `LocalOutlierProbability().fit(data)`). Keyword form + `LocalOutlierProbability(data=data)` still works with a `FutureWarning`. +### Added +- `LoOP` class alias for `LocalOutlierProbability`, enabling + `from PyNomaly import LoOP` + ([Issue #5](https://github.com/vc1492a/PyNomaly/issues/5)). +- `__version__` is now exported from the package (`from PyNomaly import __version__`). +### Removed +- Support for Python 3.8. PyNomaly 1.0.0 requires Python 3.9 or newer + (`python_requires='>=3.9'`). + ## 0.4.0 ### Added - Parallel distance computation via Numba `prange` and the new `n_jobs` diff --git a/docs/examples.md b/docs/examples.md index 827d6ff..672f989 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -30,9 +30,9 @@ Cluster the data using DBSCAN and generate two sets of scores. In both cases, we ```python db = DBSCAN(eps=0.9, min_samples=10).fit(iris) -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities -m_clust = loop.LocalOutlierProbability(iris, cluster_labels=list(db.labels_)).fit() +m_clust = loop.LocalOutlierProbability().fit(iris, cluster_labels=list(db.labels_)) scores_clust = m_clust.local_outlier_probabilities ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index ca72734..f98adf6 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,11 +2,14 @@ ## Dependencies -- Python 3.8 - 3.14 +- Python 3.9 - 3.14 - numpy >= 1.16.3 - python-utils >= 2.3.0 - (optional) numba >= 0.45.1 -- (optional) scipy >= 1.3.0 +- (optional) scipy >= 1.3.0 — performance optimizations and **sparse matrix input** + (`scipy.sparse` CSR/CSC matrices are accepted and densified internally) +- (optional) scikit-learn >= 1.0 — full estimator API / pipeline integration + (`pip install PyNomaly[sklearn]`) Numba just-in-time (JIT) compiles the function which calculates the Euclidean distance between observations, providing a reduction in computation time @@ -17,6 +20,17 @@ When scipy is available, PyNomaly uses its optimized distance computation (`scipy.spatial.distance.cdist`) and error function (`scipy.special.erf`) implementations for additional performance gains. +Sparse matrices from `scipy.sparse` (e.g. CSR, CSC) can be passed to `fit()`, +`predict()`, `decision_function()`, and `stream()`. They are converted to dense +NumPy arrays before distance computation. **scipy must be installed** to use +sparse input: + +```shell +pip install scipy +# or +pip install PyNomaly[sparse] +``` + ## Installation Install from the Python Package Index: @@ -34,19 +48,19 @@ conda install conda-forge::pynomaly ## Quick Start ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data).fit() +from PyNomaly import LoOP +m = LoOP().fit(data) scores = m.local_outlier_probabilities print(scores) ``` where `data` is a NxM (N rows, M columns; 2-dimensional) set of data as either a Pandas DataFrame or Numpy array. -`LocalOutlierProbability` sets the `extent` (an integer value of 1, 2, or 3) and `n_neighbors` (must be greater than 0) parameters with the default values of 3 and 10, respectively: +`LocalOutlierProbability` (also available as `LoOP`) sets the `extent` (an integer value of 1, 2, or 3) and `n_neighbors` (must be greater than 0) parameters with the default values of 3 and 10, respectively: ```python -from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20).fit() +from PyNomaly import LoOP +m = LoOP(extent=2, n_neighbors=20).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -56,12 +70,10 @@ print(scores) This implementation of LoOP includes an optional `cluster_labels` parameter. This is useful in cases where regions of varying density occur within the same set of data. When using `cluster_labels`, the Local Outlier Probability of a sample is calculated with respect to its cluster assignment. ```python -from PyNomaly import loop +from PyNomaly import LoOP from sklearn.cluster import DBSCAN db = DBSCAN(eps=0.6, min_samples=50).fit(data) -m = loop.LocalOutlierProbability( - data, extent=2, n_neighbors=20, cluster_labels=list(db.labels_) -).fit() +m = LoOP(extent=2, n_neighbors=20).fit(data, cluster_labels=list(db.labels_)) scores = m.local_outlier_probabilities print(scores) ``` diff --git a/docs/user-guide.md b/docs/user-guide.md index 2933e99..7da1dc4 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -29,7 +29,7 @@ For large datasets, Numba's just-in-time (JIT) compilation can significantly spe ```python from PyNomaly import loop -m = loop.LocalOutlierProbability(data, extent=2, n_neighbors=20, use_numba=True).fit() +m = loop.LocalOutlierProbability(extent=2, n_neighbors=20, use_numba=True).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -39,9 +39,8 @@ To go further, set `n_jobs=-1` to enable Numba's thread-level parallelism (`pran ```python from PyNomaly import loop m = loop.LocalOutlierProbability( - data, extent=2, n_neighbors=20, - use_numba=True, n_jobs=-1 -).fit() + extent=2, n_neighbors=20, use_numba=True, n_jobs=-1 +).fit(data) scores = m.local_outlier_probabilities print(scores) ``` @@ -55,13 +54,46 @@ This provides **2-3x speedups** on multi-core machines (benchmarked on 8 cores). Numba must be installed to use JIT compilation. PyNomaly has been tested with Numba versions 0.45.1 through 0.65.1. +## Sparse Matrix Input + +LoOP accepts `scipy.sparse` matrices (CSR, CSC, etc.) wherever raw feature data +is expected — `fit(X)`, `predict(X)`, `decision_function(X)`, and `stream(x)`. +Sparse inputs are **densified internally** before distance computation; there is +no sparse-native distance kernel yet. + +**scipy is a soft dependency for sparse input.** If you pass a sparse matrix +without scipy installed, PyNomaly raises a clear `ImportError` with install +instructions. Install scipy with: + +```shell +pip install scipy +# or +pip install PyNomaly[sparse] +``` + +Example: + +```python +from scipy.sparse import csr_matrix +from PyNomaly import LoOP + +X = csr_matrix([[1, 0, 2], [0, 3, 0], [4, 0, 5]], dtype=float) +clf = LoOP(n_neighbors=2).fit(X) +scores = clf.local_outlier_probabilities_ +``` + +!!! note + Densifying very large sparse matrices can use significant memory. For + extremely high-dimensional sparse data, consider dimensionality reduction + or sampling before fitting LoOP. + ## Progress Bars You may choose to print progress bars _with or without_ the use of Numba by passing `progress_bar=True` to `LocalOutlierProbability()`: ```python from PyNomaly import loop -m = loop.LocalOutlierProbability(data, use_numba=True, n_jobs=-1, progress_bar=True).fit() +m = loop.LocalOutlierProbability(use_numba=True, n_jobs=-1, progress_bar=True).fit(data) ``` Progress bars are supported in both sequential and Numba execution modes. @@ -83,7 +115,7 @@ data = np.array([ [421.5, 90.3, 50.0] ]) -scores = loop.LocalOutlierProbability(data, n_neighbors=3).fit().local_outlier_probabilities +scores = loop.LocalOutlierProbability(n_neighbors=3).fit(data).local_outlier_probabilities print(scores) ``` @@ -120,12 +152,12 @@ neigh.fit(data) d, idx = neigh.kneighbors(data, return_distance=True) # Remove self-distances -indices = np.delete(indices, 0, 1) -distances = np.delete(distances, 0, 1) +idx = np.delete(idx, 0, 1) +d = np.delete(d, 0, 1) -m = loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, n_neighbors=n_neighbors+1 -).fit() +m = loop.LocalOutlierProbability(n_neighbors=n_neighbors).fit( + distance_matrix=d, neighbor_matrix=idx +) scores = m.local_outlier_probabilities ``` @@ -150,8 +182,8 @@ iris_test = iris.iloc[:, 0:4].tail(30) Fit the model on training data: ```python -m_train = loop.LocalOutlierProbability(iris_train, n_neighbors=10) -m_train.fit() +m_train = loop.LocalOutlierProbability(n_neighbors=10) +m_train.fit(iris_train) iris_train_scores = m_train.local_outlier_probabilities ``` @@ -189,13 +221,10 @@ PyNomaly provides custom exceptions that can be caught and handled in your appli These exceptions are exported from the package and can be imported directly: ```python -from PyNomaly import loop -from PyNomaly.loop import ClusterSizeError, MissingValuesError +from PyNomaly import LoOP, ClusterSizeError, MissingValuesError try: - m = loop.LocalOutlierProbability( - data, n_neighbors=50, cluster_labels=labels - ).fit() + m = LoOP(n_neighbors=50).fit(data, cluster_labels=labels) except ClusterSizeError: print("Reduce n_neighbors or use larger clusters.") except MissingValuesError: diff --git a/examples/iris.py b/examples/iris.py index 24b71d6..d095778 100644 --- a/examples/iris.py +++ b/examples/iris.py @@ -11,9 +11,9 @@ db = DBSCAN(eps=0.9, min_samples=10).fit(iris) -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities -m_clust = loop.LocalOutlierProbability(iris, cluster_labels=list(db.labels_)).fit() +m_clust = loop.LocalOutlierProbability().fit(iris, cluster_labels=list(db.labels_)) scores_clust = m_clust.local_outlier_probabilities diff --git a/examples/iris_dist_grid.py b/examples/iris_dist_grid.py index 78092d2..1257aba 100644 --- a/examples/iris_dist_grid.py +++ b/examples/iris_dist_grid.py @@ -29,8 +29,9 @@ neigh.fit(iris) d, idx = neigh.kneighbors(iris, return_distance=True) - m = loop.LocalOutlierProbability(distance_matrix=d, - neighbor_matrix=idx).fit() + m = loop.LocalOutlierProbability().fit( + distance_matrix=d, neighbor_matrix=idx + ) iris['scores'] = m.local_outlier_probabilities ax = fig.add_subplot(3, 3, i, projection='3d') diff --git a/examples/multiple_gaussian_2d.py b/examples/multiple_gaussian_2d.py index 56fc7a0..e50ba28 100644 --- a/examples/multiple_gaussian_2d.py +++ b/examples/multiple_gaussian_2d.py @@ -8,7 +8,7 @@ print(df) # fit LoOP according to the original settings outlined in the paper # -m = loop.LocalOutlierProbability(df[['x', 'y']], n_neighbors=20, extent=3).fit() +m = loop.LocalOutlierProbability(n_neighbors=20, extent=3).fit(df[['x', 'y']]) scores = m.local_outlier_probabilities print(scores) @@ -25,4 +25,3 @@ plt.text(df['x'].loc[i] * (1 + 0.01), df['y'].loc[i] * (1 + 0.01), round(scores[i], 2), fontsize=8) plt.show() - diff --git a/examples/numba_speed_diff.py b/examples/numba_speed_diff.py index 25260ee..0384a9e 100644 --- a/examples/numba_speed_diff.py +++ b/examples/numba_speed_diff.py @@ -10,11 +10,10 @@ def main(): # Vectorized NumPy (default, no Numba) t1 = time.time() scores_numpy = loop.LocalOutlierProbability( - data, n_neighbors=3, use_numba=False, progress_bar=True - ).fit().local_outlier_probabilities + ).fit(data).local_outlier_probabilities t2 = time.time() seconds_no_numba = t2 - t1 print("\nComputation took " + str(seconds_no_numba) + " seconds without Numba JIT.") @@ -22,11 +21,10 @@ def main(): # Numba JIT (sequential) t3 = time.time() scores_numba = loop.LocalOutlierProbability( - data, n_neighbors=3, use_numba=True, progress_bar=True - ).fit().local_outlier_probabilities + ).fit(data).local_outlier_probabilities t4 = time.time() seconds_numba = t4 - t3 print("\nComputation took " + str(seconds_numba) + " seconds with Numba JIT (sequential).") @@ -34,12 +32,11 @@ def main(): # Numba JIT (parallel, n_jobs=-1) t5 = time.time() scores_numba_par = loop.LocalOutlierProbability( - data, n_neighbors=3, use_numba=True, n_jobs=-1, progress_bar=True - ).fit().local_outlier_probabilities + ).fit(data).local_outlier_probabilities t6 = time.time() seconds_numba_par = t6 - t5 print("\nComputation took " + str(seconds_numba_par) + " seconds with Numba JIT (parallel, n_jobs=-1).") diff --git a/examples/parallel_benchmark.py b/examples/parallel_benchmark.py index 432a165..638631f 100644 --- a/examples/parallel_benchmark.py +++ b/examples/parallel_benchmark.py @@ -42,13 +42,12 @@ def generate_clustered_data(n_per_cluster, n_clusters, n_features=N_FEATURES, se def time_fit(data, cluster_labels, n_neighbors, use_numba, n_jobs): clf = loop.LocalOutlierProbability( - data, n_neighbors=n_neighbors, - cluster_labels=cluster_labels, + n_neighbors=n_neighbors, use_numba=use_numba, - n_jobs=n_jobs + n_jobs=n_jobs, ) t0 = time.perf_counter() - clf.fit() + clf.fit(data, cluster_labels=cluster_labels) return time.perf_counter() - t0 diff --git a/examples/stream.py b/examples/stream.py index c00d90f..3ecfb3e 100644 --- a/examples/stream.py +++ b/examples/stream.py @@ -12,12 +12,12 @@ iris_train = iris.iloc[:, 0:4].head(120) iris_test = iris.iloc[:, 0:4].tail(30) -m = loop.LocalOutlierProbability(iris).fit() +m = loop.LocalOutlierProbability().fit(iris) scores_noclust = m.local_outlier_probabilities iris['scores'] = scores_noclust -m_train = loop.LocalOutlierProbability(iris_train, n_neighbors=10) -m_train.fit() +m_train = loop.LocalOutlierProbability(n_neighbors=10) +m_train.fit(iris_train) iris_train_scores = m_train.local_outlier_probabilities iris_test_scores = [] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..61e1c76 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[tool:pytest] +norecursedirs = .* build dist venv worktrees *.egg-info + +[flake8] +exclude = + .git, + __pycache__, + build, + dist, + venv, + worktrees, + *.egg-info diff --git a/setup.py b/setup.py index a73829b..8895c72 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='PyNomaly', packages=['PyNomaly'], - version='0.4.0', + version='1.0.0', description='A Python 3 implementation of LoOP: Local Outlier ' 'Probabilities, a local density based outlier detection ' 'method providing an outlier score in the range of [0,1].', @@ -16,10 +16,29 @@ long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/vc1492a/PyNomaly', - download_url='https://github.com/vc1492a/PyNomaly/archive/0.4.0.tar.gz', + download_url='https://github.com/vc1492a/PyNomaly/archive/1.0.0.tar.gz', keywords=['outlier', 'anomaly', 'detection', 'machine', 'learning', 'probability'], - classifiers=[], + classifiers=[ + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Topic :: Scientific/Engineering', + ], + python_requires='>=3.9', license='Apache License, Version 2.0', - install_requires=['numpy', 'python-utils'] + install_requires=['numpy', 'python-utils'], + extras_require={ + 'sklearn': ['scikit-learn>=1.0', 'scipy>=1.3.0'], + 'sparse': ['scipy>=1.3.0'], + 'numba': ['numba'], + 'all': ['scikit-learn>=1.0', 'numba', 'scipy>=1.3.0'], + }, ) diff --git a/tests/test_loop.py b/tests/test_loop.py index 939a686..48522d2 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,20 +1,23 @@ # Authors: Valentino Constantinou # License: Apache 2.0 -from PyNomaly import loop -from PyNomaly.loop import ClusterSizeError, MissingValuesError - import logging +import re +import sys +import warnings from typing import Tuple + import numpy as np -from numpy.testing import assert_array_equal, assert_array_almost_equal import pandas as pd import pytest +from numpy.testing import assert_array_almost_equal, assert_array_equal from sklearn.datasets import load_iris from sklearn.metrics import roc_auc_score from sklearn.neighbors import NearestNeighbors from sklearn.utils import check_random_state -import sys +from sklearn.utils.estimator_checks import check_estimator + +from PyNomaly import ClusterSizeError, MissingValuesError, loop logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) @@ -155,6 +158,29 @@ def X_n1000() -> np.ndarray: return X +def test_sklearn_api_compliance(): + """ + Check scikit-learn compatibility as referenced in the developer guide: + https://scikit-learn.org/stable/developers/develop.html#rolling-your-own-estimator + + Requires the optional sklearn extra: ``pip install PyNomaly[sklearn]``. + + ``check_fit1d`` is an expected failure: LoOP historically accepts 1-d + inputs (e.g. univariate series) by reshaping to ``(-1, 1)`` rather than + raising. + """ + pytest.importorskip("sklearn") + check_estimator( + loop.LoOP(), + expected_failed_checks={ + "check_fit1d": ( + "LoOP accepts 1-d inputs and reshapes them to (-1, 1) for " + "univariate / time-series use cases." + ), + }, + ) + + def test_loop(X_n8) -> None: """ Tests the basic functionality and asserts that the anomalous observations @@ -164,8 +190,8 @@ def test_loop(X_n8) -> None: :return: None """ # Test LocalOutlierProbability: - clf = loop.LocalOutlierProbability(X_n8, n_neighbors=5, use_numba=NUMBA) - score = clf.fit().local_outlier_probabilities + clf = loop.LocalOutlierProbability(n_neighbors=5, use_numba=NUMBA) + score = clf.fit(X_n8).local_outlier_probabilities_ share_outlier = 2.0 / 8.0 predictions = [-1 if s > share_outlier else 1 for s in score] assert_array_equal(predictions, 6 * [1] + 2 * [-1]) @@ -177,8 +203,8 @@ def test_loop(X_n8) -> None: X_df = pd.DataFrame(X_n8) # Test LocalOutlierProbability: - clf = loop.LocalOutlierProbability(X_df, n_neighbors=5, use_numba=NUMBA) - score = clf.fit().local_outlier_probabilities + clf = loop.LocalOutlierProbability(n_neighbors=5, use_numba=NUMBA) + score = clf.fit(X_df).local_outlier_probabilities_ share_outlier = 2.0 / 8.0 predictions = [-1 if s > share_outlier else 1 for s in score] assert_array_equal(predictions, 6 * [1] + 2 * [-1]) @@ -194,8 +220,8 @@ def test_regression(X_n20_scores) -> None: the same result given the same dataset """ input_data, expected_scores = X_n20_scores - clf = loop.LocalOutlierProbability(input_data).fit() - scores = clf.local_outlier_probabilities + clf = loop.LocalOutlierProbability().fit(input_data) + scores = clf.local_outlier_probabilities_ assert_array_almost_equal(scores, expected_scores, 6) @@ -214,15 +240,13 @@ def test_loop_performance(X_n120) -> None: # fit the model clf = loop.LocalOutlierProbability( - X_test, n_neighbors=X_test.shape[0] - 1, - # test the progress bar progress_bar=True, use_numba=NUMBA, ) # predict scores (the lower, the more normal) - score = clf.fit().local_outlier_probabilities + score = clf.fit(X_test).local_outlier_probabilities_ share_outlier = X_outliers.shape[0] / X_test.shape[0] X_pred = [-1 if s > share_outlier else 1 for s in score] @@ -237,16 +261,12 @@ def test_input_nodata(X_n140_outliers) -> None: :param X_n140_outliers: A pytest Fixture that generates 140 observations. :return: None """ - with pytest.warns(UserWarning) as record: + with pytest.warns(UserWarning, match="Data or a distance matrix must be provided.") as record: # attempt to fit loop without data or a distance matrix - loop.LocalOutlierProbability( + clf = loop.LocalOutlierProbability( n_neighbors=X_n140_outliers.shape[0] - 1, use_numba=NUMBA ) - - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert record[0].message.args[0] == "Data or a distance matrix must be provided." + clf.fit() def test_input_incorrect_type(X_n140_outliers) -> None: @@ -256,22 +276,15 @@ def test_input_incorrect_type(X_n140_outliers) -> None: :param X_n140_outliers: A pytest Fixture that generates 140 observations. :return: None """ - with pytest.warns(UserWarning) as record: + + match_str = re.escape("Argument 'n_neighbors' is not of type (, ).") + with pytest.warns(UserWarning, match=match_str): # attempt to fit loop with a string input for n_neighbors - loop.LocalOutlierProbability( - X_n140_outliers, + clf = loop.LocalOutlierProbability( n_neighbors=str(X_n140_outliers.shape[0] - 1), use_numba=NUMBA, ) - - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] - == "Argument 'n_neighbors' is not of type (, " - ")." - ) + clf.fit() def test_input_neighbor_zero(X_n120) -> None: @@ -281,19 +294,12 @@ def test_input_neighbor_zero(X_n120) -> None: :param X_n120: A pytest Fixture that generates 120 observations. :return: None """ - clf = loop.LocalOutlierProbability(X_n120, n_neighbors=0, use_numba=NUMBA) - with pytest.warns(UserWarning) as record: + clf = loop.LocalOutlierProbability(n_neighbors=0, use_numba=NUMBA) + with pytest.warns(UserWarning, match="n_neighbors must be greater than 0. Fit with 10 instead."): # attempt to fit loop with a 0 neighbor count - clf.fit() + clf.fit(X_n120) - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] - == "n_neighbors must be greater than 0. Fit with 10 instead." - ) def test_input_distonly(X_n120) -> None: @@ -308,18 +314,12 @@ def test_input_distonly(X_n120) -> None: neigh.fit(X_n120) d, idx = neigh.kneighbors(X_n120, n_neighbors=10, return_distance=True) - with pytest.warns(UserWarning) as record: + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + match_str = re.escape("A neighbor index matrix and distance matrix must both be provided when not using raw input data.") + with pytest.warns(UserWarning, match=match_str): # attempt to fit loop with a distance matrix and no neighbor matrix - loop.LocalOutlierProbability(distance_matrix=d, use_numba=NUMBA) + clf.fit(distance_matrix=d) - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] - == "A neighbor index matrix and distance matrix must both " - "be provided when not using raw input data." - ) def test_input_neighboronly(X_n120) -> None: @@ -334,14 +334,11 @@ def test_input_neighboronly(X_n120) -> None: neigh.fit(X_n120) d, idx = neigh.kneighbors(X_n120, n_neighbors=10, return_distance=True) - with pytest.warns(UserWarning) as record: + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + + with pytest.warns(UserWarning, match="Data or a distance matrix must be provided.") as record: # attempt to fit loop with a neighbor matrix and no distance matrix - loop.LocalOutlierProbability(neighbor_matrix=idx, use_numba=NUMBA) - - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert record[0].message.args[0] == "Data or a distance matrix must be provided." + clf.fit(neighbor_matrix=idx) def test_input_too_many(X_n120) -> None: @@ -355,21 +352,12 @@ def test_input_too_many(X_n120) -> None: neigh = NearestNeighbors(metric="euclidean") neigh.fit(X_n120) d, idx = neigh.kneighbors(X_n120, n_neighbors=10, return_distance=True) - - with pytest.warns(UserWarning) as record: + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + match_str = re.escape("Only one of the following may be provided: data or a distance matrix (not both).") + with pytest.warns(UserWarning, match=match_str): # attempt to fit loop with data and a distance matrix - loop.LocalOutlierProbability( - X_n120, distance_matrix=d, neighbor_matrix=idx, use_numba=NUMBA - ) + clf.fit(X_n120, distance_matrix=d, neighbor_matrix=idx) - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] - == "Only one of the following may be provided: data or a " - "distance matrix (not both)." - ) def test_distance_neighbor_shape_mismatch(X_n120) -> None: @@ -388,20 +376,12 @@ def test_distance_neighbor_shape_mismatch(X_n120) -> None: neigh_2 = NearestNeighbors(metric="euclidean") neigh_2.fit(X_n120) d_2, idx_2 = neigh.kneighbors(X_n120, n_neighbors=5, return_distance=True) + clf = loop.LocalOutlierProbability(n_neighbors=5, use_numba=NUMBA) - with pytest.warns(UserWarning) as record: + with pytest.warns(UserWarning, match="The shape of the distance and neighbor " + "index matrices must match.") as record: # attempt to fit loop with a mismatch in shapes - loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx_2, n_neighbors=5, use_numba=NUMBA - ) - - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] == "The shape of the distance and neighbor " - "index matrices must match." - ) + clf.fit(distance_matrix=d, neighbor_matrix=idx_2) def test_input_neighbor_mismatch(X_n120) -> None: @@ -415,22 +395,15 @@ def test_input_neighbor_mismatch(X_n120) -> None: neigh = NearestNeighbors(metric="euclidean") neigh.fit(X_n120) d, idx = neigh.kneighbors(X_n120, n_neighbors=5, return_distance=True) + clf = loop.LocalOutlierProbability(n_neighbors=10, use_numba=NUMBA) - with pytest.warns(UserWarning) as record: - # attempt to fit loop with a neighbor size mismatch - loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, n_neighbors=10, use_numba=NUMBA - ) - - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] == "The shape of the distance or " + with pytest.warns(UserWarning, match=("The shape of the distance or " "neighbor index matrix does not " "match the number of neighbors " - "specified." - ) + "specified.")) as record: + # attempt to fit loop with a neighbor size mismatch + clf.fit(distance_matrix=d, neighbor_matrix=idx) + def test_loop_dist_matrix(X_n120) -> None: @@ -446,12 +419,10 @@ def test_loop_dist_matrix(X_n120) -> None: d, idx = neigh.kneighbors(X_n120, n_neighbors=10, return_distance=True) # fit loop using data and distance matrix - clf1 = loop.LocalOutlierProbability(X_n120, use_numba=NUMBA) - clf2 = loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, use_numba=NUMBA - ) - scores1 = clf1.fit().local_outlier_probabilities - scores2 = clf2.fit().local_outlier_probabilities + clf1 = loop.LocalOutlierProbability(use_numba=NUMBA) + clf2 = loop.LocalOutlierProbability(use_numba=NUMBA) + scores1 = clf1.fit(X_n120).local_outlier_probabilities_ + scores2 = clf2.fit(distance_matrix=d, neighbor_matrix=idx).local_outlier_probabilities_ # compare the agreement between the results assert np.abs(scores2 - scores1).all() <= 0.1 @@ -466,14 +437,14 @@ def test_lambda_values(X_n140_outliers) -> None: :return: None """ # Fit the model with different extent (lambda) values - clf1 = loop.LocalOutlierProbability(X_n140_outliers, extent=1, use_numba=NUMBA) - clf2 = loop.LocalOutlierProbability(X_n140_outliers, extent=2, use_numba=NUMBA) - clf3 = loop.LocalOutlierProbability(X_n140_outliers, extent=3, use_numba=NUMBA) + clf1 = loop.LocalOutlierProbability(extent=1, use_numba=NUMBA) + clf2 = loop.LocalOutlierProbability(extent=2, use_numba=NUMBA) + clf3 = loop.LocalOutlierProbability(extent=3, use_numba=NUMBA) # predict scores (the lower, the more normal) - score1 = clf1.fit().local_outlier_probabilities - score2 = clf2.fit().local_outlier_probabilities - score3 = clf3.fit().local_outlier_probabilities + score1 = clf1.fit(X_n140_outliers).local_outlier_probabilities_ + score2 = clf2.fit(X_n140_outliers).local_outlier_probabilities_ + score3 = clf3.fit(X_n140_outliers).local_outlier_probabilities_ # Get the mean of all the scores score_mean1 = np.mean(score1) @@ -494,22 +465,185 @@ def test_parameters(X_n120) -> None: :return: None """ # fit the model - clf = loop.LocalOutlierProbability(X_n120, use_numba=NUMBA).fit() + clf = loop.LocalOutlierProbability(use_numba=NUMBA).fit(X_n120) # check that the model has attributes post fit assert hasattr(clf, "n_neighbors") and clf.n_neighbors is not None assert hasattr(clf, "extent") and clf.extent is not None assert hasattr(clf, "cluster_labels") and clf._cluster_labels() is not None - assert hasattr(clf, "prob_distances") and clf.prob_distances is not None - assert hasattr(clf, "prob_distances_ev") and clf.prob_distances_ev is not None + assert hasattr(clf, "prob_distances") and clf.prob_distances_ is not None + assert hasattr(clf, "prob_distances_ev") and clf.prob_distances_ev_ is not None assert ( hasattr(clf, "norm_prob_local_outlier_factor") - and clf.norm_prob_local_outlier_factor is not None + and clf.norm_prob_local_outlier_factor_ is not None ) assert ( hasattr(clf, "local_outlier_probabilities") - and clf.local_outlier_probabilities is not None + and clf.local_outlier_probabilities_ is not None + ) + + +def test_public_api_contract(X_n120) -> None: + """ + Guards against unintentional breaking changes to the public API of + LocalOutlierProbability. Exercises every non-underscored constructor + keyword argument, every fit() keyword argument, public method, and + public attribute so that a rename or removal of any of them causes this + test to fail. Additions to the API are unaffected. + + Written against the 1.0.0 scikit-learn-style API: the constructor takes + configuration only, data goes to ``fit(X, y=None, ...)``, and fitted + state lives on ``_``-suffixed attributes (with read-only legacy aliases). + :param X_n120: A pytest Fixture that generates 120 observations. + :return: None + """ + # exercise every public constructor keyword argument by name; a rename + # or removal raises a TypeError here. Data parameters are no longer + # constructor arguments in 1.0.0 -- they are exercised via fit() below. + clf = loop.LocalOutlierProbability( + extent=3, + n_neighbors=10, + use_numba=NUMBA, + n_jobs=1, + progress_bar=False, + ) + + # sklearn contract: get_params() exposes exactly the constructor's + # parameters -- the configuration parameters plus the deprecated data + # keywords, which stay constructor arguments (with a FutureWarning) + # until they are removed in a future version + assert set(clf.get_params()) == { + "extent", + "n_neighbors", + "use_numba", + "n_jobs", + "progress_bar", + "data", + "distance_matrix", + "neighbor_matrix", + "cluster_labels", + } + + # exercise every public method by name, and every fit() keyword + # argument by name (a rename or removal raises a TypeError here) + assert hasattr(clf, "fit") and callable(clf.fit) + fitted = clf.fit( + X_n120, + y=None, + distance_matrix=None, + neighbor_matrix=None, + cluster_labels=None, ) + assert fitted is clf + for method_name in ( + "fit", + "fit_predict", + "predict", + "decision_function", + "score_samples", + "stream", + "get_params", + "set_params", + ): + assert hasattr(fitted, method_name) and callable( + getattr(fitted, method_name) + ), f"missing public method: {method_name}" + fitted.stream(X_n120[0]) + fitted.predict(X_n120) + fitted.decision_function(X_n120) + fitted.score_samples(X_n120) + + # every public (non-underscored) attribute expected to exist post-fit: + # configuration parameters, the deprecated constructor data inputs, + # the `_`-suffixed fitted state, and the read-only legacy aliases + expected_public_attributes = [ + # configuration parameters (constructor) + "extent", + "n_neighbors", + "use_numba", + "n_jobs", + "progress_bar", + # deprecated constructor data inputs (always present, None here) + "data", + "distance_matrix", + "neighbor_matrix", + "cluster_labels", + # fitted state (scikit-learn convention) + "data_", + "distance_matrix_", + "neighbor_matrix_", + "cluster_labels_", + "n_neighbors_", + "n_features_in_", + "offset_", + "points_vector_", + "prob_distances_", + "prob_distances_ev_", + "norm_prob_local_outlier_factor_", + "local_outlier_probabilities_", + "is_fit_", + # read-only legacy aliases for fitted output attributes + "points_vector", + "prob_distances", + "prob_distances_ev", + "norm_prob_local_outlier_factor", + "local_outlier_probabilities", + "is_fit", + ] + for attr_name in expected_public_attributes: + assert hasattr(fitted, attr_name), f"missing public attribute: {attr_name}" + + # the legacy aliases must read the canonical fitted attributes + assert fitted.local_outlier_probabilities is fitted.local_outlier_probabilities_ + assert fitted.is_fit is fitted.is_fit_ + + # exercise every public constructor argument positionally, in the + # documented 1.0.0 order, using a distinct value per slot. Swapping two + # parameters of the same type (e.g. extent/n_neighbors/n_jobs, or + # use_numba/progress_bar) doesn't raise an error -- it silently + # assigns the wrong value -- so each assigned attribute is checked + # against the value passed at its expected position. + positional_clf = loop.LocalOutlierProbability(2, 15, False, 1, True) + assert positional_clf.extent == 2 + assert positional_clf.n_neighbors == 15 + assert positional_clf.use_numba is False + assert positional_clf.n_jobs == 1 + assert positional_clf.progress_bar is True + + # data is positional in fit(), sklearn-style: fit(X, y) + positional_fit = loop.LocalOutlierProbability(n_neighbors=10).fit( + X_n120, None + ) + assert positional_fit.data_ is X_n120 + assert positional_fit.is_fit_ is True + + # the `LoOP` alias is part of the public API + from PyNomaly import LoOP + + assert LoOP is loop.LocalOutlierProbability + assert loop.LoOP is loop.LocalOutlierProbability + + # the exception hierarchy is part of the public API: customers import + # these names directly (as this test file does at the top) and may + # catch on the base classes, so both the import paths and the + # hierarchy itself must remain stable. The canonical home is + # PyNomaly.exceptions; PyNomaly.loop and PyNomaly re-export them. + from PyNomaly import exceptions + from PyNomaly.loop import ( + PyNomalyError, + ValidationError, + ClusterSizeError as ImportedClusterSizeError, + MissingValuesError as ImportedMissingValuesError, + ) + + assert PyNomalyError is exceptions.PyNomalyError + assert ValidationError is exceptions.ValidationError + assert ImportedClusterSizeError is exceptions.ClusterSizeError + assert ImportedMissingValuesError is exceptions.MissingValuesError + assert issubclass(ImportedClusterSizeError, ValidationError) + assert issubclass(ImportedMissingValuesError, ValidationError) + assert issubclass(ValidationError, PyNomalyError) + assert issubclass(PyNomalyError, Exception) def test_n_neighbors() -> None: @@ -517,21 +651,26 @@ def test_n_neighbors() -> None: Tests the functionality of providing a large number of neighbors that is greater than the number of observations (software defaults to the data input size and provides a UserWarning). + + The public ``n_neighbors`` parameter is left unchanged (sklearn + immutability); the effective value used during fit is ``n_neighbors_``. :return: None """ X = iris.data - clf = loop.LocalOutlierProbability(X, n_neighbors=500, use_numba=NUMBA).fit() - assert clf.n_neighbors == X.shape[0] - 1 + clf = loop.LocalOutlierProbability(n_neighbors=500, use_numba=NUMBA).fit(X) + assert clf.n_neighbors == 500 + assert clf.n_neighbors_ == X.shape[0] - 1 - clf = loop.LocalOutlierProbability(X, n_neighbors=500, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=500, use_numba=NUMBA) with pytest.warns(UserWarning) as record: - clf.fit() + clf.fit(X) # check that only one warning was raised assert len(record) == 1 - assert clf.n_neighbors == X.shape[0] - 1 + assert clf.n_neighbors == 500 + assert clf.n_neighbors_ == X.shape[0] - 1 def test_extent() -> None: @@ -541,13 +680,10 @@ def test_extent() -> None: :return: None """ X = np.array([[1, 1], [1, 0]]) - clf = loop.LocalOutlierProbability(X, n_neighbors=2, extent=4, use_numba=NUMBA) - - with pytest.warns(UserWarning) as record: - clf.fit() - - # check that only one warning was raised - assert len(record) == 1 + clf = loop.LocalOutlierProbability(n_neighbors=2, extent=4, use_numba=NUMBA) + match_str = re.escape("extent parameter (lambda) must be 1, 2, or 3.") + with pytest.warns(UserWarning, match=match_str): + clf.fit(X) def test_data_format() -> None: @@ -558,13 +694,10 @@ def test_data_format() -> None: :return: None """ X = [1.3, 1.1, 0.9, 1.4, 1.5, 3.2] - clf = loop.LocalOutlierProbability(X, n_neighbors=3, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=3, use_numba=NUMBA) - with pytest.warns(UserWarning) as record: - clf.fit() - - # check that only one warning was raised - assert len(record) == 1 + with pytest.warns(UserWarning, match="Provided data or distance matrix must be in ndarray") as record: + clf.fit(X) def test_missing_values() -> None: @@ -574,15 +707,15 @@ def test_missing_values() -> None: :return: None """ X = np.array([1.3, 1.1, 0.9, 1.4, 1.5, np.nan, 3.2]) - clf = loop.LocalOutlierProbability(X, n_neighbors=3, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(n_neighbors=3, use_numba=NUMBA) with pytest.raises(MissingValuesError) as record: - clf.fit() + clf.fit(X) # check that the message matches assert ( str(record.value) - == "Method does not support missing values in input data." + == "Input contains NaN, infinity or a value too large for dtype('float64')." ) @@ -599,11 +732,11 @@ def test_small_cluster_size(X_n140_outliers) -> None: cluster_labels = a + b clf = loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=50, cluster_labels=cluster_labels, use_numba=NUMBA + n_neighbors=50, use_numba=NUMBA ) with pytest.raises(ClusterSizeError) as record: - clf.fit() + clf.fit(X_n140_outliers, cluster_labels=cluster_labels) # check that the message matches assert ( @@ -625,18 +758,13 @@ def test_stream_fit(X_n140_outliers) -> None: # Fit the model X_train = X_n140_outliers[0:138] X_test = X_n140_outliers[139] - clf = loop.LocalOutlierProbability(X_train, use_numba=NUMBA) + clf = loop.LocalOutlierProbability(use_numba=NUMBA) + clf.data = X_train + match_str = re.escape("Must fit on historical data by calling fit() prior to calling stream(x).") - with pytest.warns(UserWarning) as record: + with pytest.warns(UserWarning, match=match_str): clf.stream(X_test) - # check that the message matches - messages = [i.message.args[0] for i in record] - assert ( - "Must fit on historical data by calling fit() prior to " - "calling stream(x)." in messages - ) - def test_stream_distance(X_n140_outliers) -> None: """ @@ -655,10 +783,10 @@ def test_stream_distance(X_n140_outliers) -> None: d, idx = neigh.kneighbors(X_train, n_neighbors=10, return_distance=True) # Fit the models in standard and distance matrix form - m = loop.LocalOutlierProbability(X_train, use_numba=NUMBA).fit() + m = loop.LocalOutlierProbability(use_numba=NUMBA).fit(X_train) m_dist = loop.LocalOutlierProbability( - distance_matrix=d, neighbor_matrix=idx, use_numba=NUMBA - ).fit() + use_numba=NUMBA + ).fit(distance_matrix=d, neighbor_matrix=idx) # Collect the scores X_test_scores = [] @@ -694,19 +822,13 @@ def test_stream_cluster(X_n140_outliers) -> None: X_train = X_n140_outliers[0:138] X_test = X_n140_outliers[139] clf = loop.LocalOutlierProbability( - X_train, cluster_labels=cluster_labels, use_numba=NUMBA - ).fit() + use_numba=NUMBA + ).fit(X_train, cluster_labels=cluster_labels) - with pytest.warns(UserWarning) as record: + with pytest.warns(UserWarning, match=("Stream approach does not support clustered data. " + "Automatically refit using single cluster of points.")) as record: clf.stream(X_test) - # check that only one warning was raised - assert len(record) == 1 - # check that the message matches - assert ( - record[0].message.args[0] == "Stream approach does not support clustered data. " - "Automatically refit using single cluster of points." - ) def test_stream_performance(X_n140_outliers) -> None: @@ -722,12 +844,12 @@ def test_stream_performance(X_n140_outliers) -> None: X_test = X_n140_outliers[100:140] # Fit the models in standard and stream form - m = loop.LocalOutlierProbability(X_n140_outliers, use_numba=NUMBA).fit() - scores_noclust = m.local_outlier_probabilities + m = loop.LocalOutlierProbability(use_numba=NUMBA).fit(X_n140_outliers) + scores_noclust = m.local_outlier_probabilities_ - m_train = loop.LocalOutlierProbability(X_train, use_numba=NUMBA) - m_train.fit() - X_train_scores = m_train.local_outlier_probabilities + m_train = loop.LocalOutlierProbability(use_numba=NUMBA) + m_train.fit(X_train) + X_train_scores = m_train.local_outlier_probabilities_ X_test_scores = [] for idx in range(X_test.shape[0]): @@ -751,7 +873,7 @@ def test_progress_bar(X_n8) -> None: """ # attempt to use the progress bar on a small number of observations - loop.LocalOutlierProbability(X_n8, use_numba=NUMBA, progress_bar=True).fit() + loop.LocalOutlierProbability(use_numba=NUMBA, progress_bar=True).fit(X_n8) def test_data_flipping() -> None: @@ -765,22 +887,20 @@ def test_data_flipping() -> None: np.random.normal(2, 1, [n, 2]), np.random.normal(8, 1, [n, 2]), axis=0 ) clus = np.append(np.ones(n), 2 * np.ones(n)).tolist() - model = loop.LocalOutlierProbability(data, n_neighbors=5, cluster_labels=clus) - fit = model.fit() - res = fit.local_outlier_probabilities + model = loop.LocalOutlierProbability(n_neighbors=5) + fit = model.fit(data, cluster_labels=clus) + res = fit.local_outlier_probabilities_ data_flipped = np.flipud(data) clus_flipped = np.flipud(clus).tolist() - model2 = loop.LocalOutlierProbability( - data_flipped, n_neighbors=5, cluster_labels=clus_flipped - ) - fit2 = model2.fit() - res2 = np.flipud(fit2.local_outlier_probabilities) + model2 = loop.LocalOutlierProbability(n_neighbors=5) + fit2 = model2.fit(data_flipped, cluster_labels=clus_flipped) + res2 = np.flipud(fit2.local_outlier_probabilities_) assert_array_almost_equal(res, res2, decimal=6) assert_array_almost_equal( - fit.norm_prob_local_outlier_factor, - fit2.norm_prob_local_outlier_factor, + fit.norm_prob_local_outlier_factor_, + fit2.norm_prob_local_outlier_factor_, decimal=6, ) @@ -801,12 +921,12 @@ def test_distance_matrix_consistency(X_n120) -> None: distances = np.delete(distances, 0, 1) # Fit LoOP with and without distance matrix - clf_data = loop.LocalOutlierProbability(X_n120, n_neighbors=10) - clf_dist = loop.LocalOutlierProbability(distance_matrix=distances, neighbor_matrix=indices, n_neighbors=11) + clf_data = loop.LocalOutlierProbability(n_neighbors=10) + clf_dist = loop.LocalOutlierProbability(n_neighbors=10) # Attempt to retrieve scores and check types - scores_data = clf_data.fit().local_outlier_probabilities - scores_dist = clf_dist.fit().local_outlier_probabilities + scores_data = clf_data.fit(X_n120).local_outlier_probabilities_ + scores_dist = clf_dist.fit(distance_matrix=distances, neighbor_matrix=indices).local_outlier_probabilities_ # Debugging prints to investigate types and contents print("Type of scores_data:", type(scores_data)) @@ -835,8 +955,8 @@ def test_vectorized_1d_data() -> None: the ndim == 1 reshape branch in _distances_vectorized. """ X = np.array([1.0, 2.0, 3.0, 10.0, 11.0, 12.0, 50.0]) - clf = loop.LocalOutlierProbability(X, n_neighbors=3) - scores = clf.fit().local_outlier_probabilities + clf = loop.LocalOutlierProbability(n_neighbors=3) + scores = clf.fit(X).local_outlier_probabilities_ assert scores is not None assert len(scores) == len(X) assert scores[-1] > 0 @@ -849,9 +969,9 @@ def test_n_jobs_without_numba_warns(X_n120) -> None: """ with pytest.warns(UserWarning, match="n_jobs > 1 requires use_numba=True"): clf = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, n_jobs=2 + n_neighbors=10, n_jobs=2 ) - scores = clf.fit().local_outlier_probabilities + scores = clf.fit(X_n120).local_outlier_probabilities_ assert scores is not None assert len(scores) == len(X_n120) @@ -859,16 +979,15 @@ def test_n_jobs_without_numba_warns(X_n120) -> None: def test_n_jobs_negative_two() -> None: """ - Tests that n_jobs=-2 (invalid) produces a warning and defaults to 1. + Tests that n_jobs=-2 (invalid) produces a warning and uses an effective + value of 1 without mutating the public parameter (sklearn immutability). """ - X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) - - with pytest.warns(UserWarning) as record: - clf = loop.LocalOutlierProbability(X, n_neighbors=2, n_jobs=-2) + clf = loop.LocalOutlierProbability(n_neighbors=2, n_jobs=-2) + with pytest.warns(UserWarning, match="n_jobs must be -1 or a positive integer") as record: + clf.fit() - messages = [r.message.args[0] for r in record] - assert any("n_jobs must be -1 or a positive integer" in m for m in messages) - assert clf.n_jobs == 1 + assert clf.n_jobs == -2 + assert clf._fit_n_jobs == 1 def test_vectorized_progress_bar_single_cluster(X_n120) -> None: @@ -877,25 +996,23 @@ def test_vectorized_progress_bar_single_cluster(X_n120) -> None: on single-cluster data. """ clf = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, n_jobs=1, progress_bar=True + n_neighbors=10, n_jobs=1, progress_bar=True ) - scores = clf.fit().local_outlier_probabilities + scores = clf.fit(X_n120).local_outlier_probabilities_ assert scores is not None assert len(scores) == len(X_n120) def test_n_jobs_invalid() -> None: """ - Tests that invalid n_jobs values produce a warning and default to 1. + Tests that invalid n_jobs values produce a warning and use an effective + value of 1 without mutating the public parameter (sklearn immutability). """ - X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) - - with pytest.warns(UserWarning) as record: - clf = loop.LocalOutlierProbability(X, n_neighbors=2, n_jobs=0) - - messages = [r.message.args[0] for r in record] - assert any("n_jobs must be -1 or a positive integer" in m for m in messages) - assert clf.n_jobs == 1 + clf = loop.LocalOutlierProbability(n_neighbors=2, n_jobs=0) + with pytest.warns(UserWarning, match="n_jobs must be -1 or a positive integer") as record: + clf.fit() + assert clf.n_jobs == 0 + assert clf._fit_n_jobs == 1 # --- Numba-specific tests --- @@ -915,11 +1032,11 @@ def test_numba_sequential_equivalence(X_n8) -> None: Tests that use_numba=True with n_jobs=1 produces equivalent results to the default vectorized path. """ - clf_vec = loop.LocalOutlierProbability(X_n8, n_neighbors=5, use_numba=False) - scores_vec = clf_vec.fit().local_outlier_probabilities + clf_vec = loop.LocalOutlierProbability(n_neighbors=5, use_numba=False) + scores_vec = clf_vec.fit(X_n8).local_outlier_probabilities_ - clf_numba = loop.LocalOutlierProbability(X_n8, n_neighbors=5, use_numba=True) - scores_numba = clf_numba.fit().local_outlier_probabilities + clf_numba = loop.LocalOutlierProbability(n_neighbors=5, use_numba=True) + scores_numba = clf_numba.fit(X_n8).local_outlier_probabilities_ assert_array_almost_equal(scores_vec, scores_numba, decimal=6) @@ -935,16 +1052,18 @@ def test_numba_parallel_equivalence(X_n140_outliers) -> None: cluster_labels = a + b clf_seq = loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=10, cluster_labels=cluster_labels, - use_numba=True, n_jobs=1 + n_neighbors=10, use_numba=True, n_jobs=1 ) - scores_seq = clf_seq.fit().local_outlier_probabilities + scores_seq = clf_seq.fit( + X_n140_outliers, cluster_labels=cluster_labels + ).local_outlier_probabilities_ clf_par = loop.LocalOutlierProbability( - X_n140_outliers, n_neighbors=10, cluster_labels=cluster_labels, - use_numba=True, n_jobs=2 + n_neighbors=10, use_numba=True, n_jobs=2 ) - scores_par = clf_par.fit().local_outlier_probabilities + scores_par = clf_par.fit( + X_n140_outliers, cluster_labels=cluster_labels + ).local_outlier_probabilities_ assert_array_almost_equal(scores_seq, scores_par, decimal=10) @@ -955,9 +1074,9 @@ def test_numba_with_progress_bar(X_n120) -> None: Tests that the Numba path works with the progress bar enabled. """ clf = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, use_numba=True, progress_bar=True + n_neighbors=10, use_numba=True, progress_bar=True ) - scores = clf.fit().local_outlier_probabilities + scores = clf.fit(X_n120).local_outlier_probabilities_ assert scores is not None assert len(scores) == len(X_n120) @@ -969,13 +1088,273 @@ def test_numba_prange_single_cluster(X_n120) -> None: and produces equivalent results to the sequential path. """ clf_seq = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, use_numba=True, n_jobs=1 + n_neighbors=10, use_numba=True, n_jobs=1 ) - scores_seq = clf_seq.fit().local_outlier_probabilities + scores_seq = clf_seq.fit(X_n120).local_outlier_probabilities_ clf_par = loop.LocalOutlierProbability( - X_n120, n_neighbors=10, use_numba=True, n_jobs=-1 + n_neighbors=10, use_numba=True, n_jobs=-1 ) - scores_par = clf_par.fit().local_outlier_probabilities + scores_par = clf_par.fit(X_n120).local_outlier_probabilities_ assert_array_almost_equal(scores_seq, scores_par, decimal=6) + + +# --- scikit-learn API convention tests (1.0.0) --- + + +def test_exceptions_importable_from_loop() -> None: + """ + Tests that the exception classes, which moved to ``PyNomaly.exceptions`` + in 1.0.0, remain importable from ``PyNomaly.loop`` for backward + compatibility and are the same objects as the canonical ones. + """ + from PyNomaly import exceptions + from PyNomaly.loop import ( + ClusterSizeError as LoopClusterSizeError, + MissingValuesError as LoopMissingValuesError, + PyNomalyError as LoopPyNomalyError, + ValidationError as LoopValidationError, + ) + + assert LoopPyNomalyError is exceptions.PyNomalyError + assert LoopValidationError is exceptions.ValidationError + assert LoopClusterSizeError is exceptions.ClusterSizeError + assert LoopMissingValuesError is exceptions.MissingValuesError + for name in ( + "PyNomalyError", + "ValidationError", + "ClusterSizeError", + "MissingValuesError", + ): + assert name in loop.__all__ + + +def test_numerical_equivalence_old_style(X_n8) -> None: + """ + Tests that old-style (deprecated) and new-style API produce identical + results. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + clf_old = loop.LocalOutlierProbability(data=X_n8, n_neighbors=5) + scores_old = clf_old.fit().local_outlier_probabilities_ + + clf_new = loop.LocalOutlierProbability(n_neighbors=5) + scores_new = clf_new.fit(X_n8).local_outlier_probabilities_ + + assert_array_equal(scores_old, scores_new) + + +def test_old_style_future_warning(X_n8) -> None: + """ + Tests that passing data to __init__ emits a FutureWarning. + """ + with pytest.warns(FutureWarning, match="Passing 'data' to __init__"): + loop.LocalOutlierProbability(data=X_n8, n_neighbors=5) + + +def test_fit_overrides_init(X_n8, X_n120) -> None: + """ + Tests that data passed to fit() takes precedence over data passed + to __init__. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + clf = loop.LocalOutlierProbability(data=X_n8, n_neighbors=5) + + clf.fit(X_n120) + scores = clf.local_outlier_probabilities_ + assert len(scores) == 120 + + +def test_fit_with_2d_label_y_is_ignored(X_n120) -> None: + """ + Tests that a (n, 1) label array passed as y (sklearn-style fit(X, y)) + is ignored rather than being hijacked as a legacy positional + distance_matrix, which previously left the model unfitted. + """ + y = np.ones((X_n120.shape[0], 1)) + clf = loop.LocalOutlierProbability(n_neighbors=10).fit(X_n120, y) + assert clf.is_fit_ is True + assert clf.local_outlier_probabilities_ is not None + assert len(clf.local_outlier_probabilities_) == len(X_n120) + + +@pytest.mark.parametrize( + "y_shape", [(120,), (120, 1), (120, 2)], ids=["1d", "n_by_1", "n_by_2"] +) +def test_fit_ignores_y_when_X_is_given(X_n120, y_shape) -> None: + """ + Regression test for the fit(X, y) review finding on PR #92: when raw + data X is supplied, any label array y (1-d, a single column, or a + multi-column 2-d array) must be ignored without triggering the legacy + positional distance_matrix fallback, and the result must equal fit(X). + """ + expected = ( + loop.LocalOutlierProbability(n_neighbors=10) + .fit(X_n120) + .local_outlier_probabilities_ + ) + + y = np.ones(y_shape) + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + clf = loop.LocalOutlierProbability(n_neighbors=10).fit(X_n120, y) + + assert clf.is_fit_ is True + # raw-data path was used: y did not become the distance matrix + assert clf.data_ is X_n120 + assert clf.distance_matrix is None + assert_array_equal(clf.local_outlier_probabilities_, expected) + + +def test_refit(X_n8, X_n120) -> None: + """ + Tests that calling fit() twice with different data produces correct + results each time. + """ + clf = loop.LocalOutlierProbability(n_neighbors=5) + + clf.fit(X_n8) + scores_first = clf.local_outlier_probabilities_.copy() + assert len(scores_first) == 8 + + clf.fit(X_n120) + scores_second = clf.local_outlier_probabilities_ + assert len(scores_second) == 120 + assert clf.is_fit_ is True + + +def test_loop_alias_import() -> None: + """ + Tests that LoOP alias can be imported and is the same class. + """ + from PyNomaly import LocalOutlierProbability, LoOP + assert LoOP is LocalOutlierProbability + +def test_legacy_positional_arguments(X_n120) -> None: + """ + Tests that passing a distance matrix as the second positional argument (y) + is correctly routed to distance_matrix and raises a FutureWarning. + """ + neigh = NearestNeighbors(metric="euclidean") + neigh.fit(X_n120) + d, idx = neigh.kneighbors(X_n120, n_neighbors=10, return_distance=True) + + clf = loop.LocalOutlierProbability(n_neighbors=10) + + match_str = re.escape("Passing 'distance_matrix' as the second positional argument is deprecated") + with pytest.warns(FutureWarning, match=match_str): + clf.fit(None, d, neighbor_matrix=idx) + + assert clf.distance_matrix_ is not None + assert clf.local_outlier_probabilities_ is not None + + +def test_standard_y_argument_ignored(X_n120) -> None: + """ + Tests that passing standard 1D target labels to y does not trigger + the distance_matrix routing logic. + """ + y_labels = np.zeros(120) + + clf = loop.LocalOutlierProbability(n_neighbors=10) + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + clf.fit(X_n120, y_labels) + + assert clf.local_outlier_probabilities_ is not None + + +def test_decision_function(X_n20_scores) -> None: + """ + Tests that decision_function follows the sklearn outlier convention: + ``score_samples(X) - offset_`` with ``offset_ = -0.5``, so inliers satisfy + ``decision_function(X) >= 0`` iff LoOP probability <= 0.5. + """ + input_data, expected_scores = X_n20_scores + clf = loop.LocalOutlierProbability().fit(input_data) + + expected_decision_scores = -1.0 * expected_scores - clf.offset_ + + decision_scores = clf.decision_function(input_data) + + assert_array_almost_equal(decision_scores, expected_decision_scores, decimal=6) + + +def test_predict(X_n20_scores) -> None: + """ + Tests that predict returns exact 1 and -1 labels based on the 0.5 + probability threshold applied to the known historical fixture results. + """ + input_data, expected_scores = X_n20_scores + clf = loop.LocalOutlierProbability().fit(input_data) + + # predict returns -1 for outliers (prob >= 0.5), 1 for inliers (prob < 0.5) + expected_predictions = np.where(expected_scores >= 0.5, -1, 1) + + predictions = clf.predict(input_data) + + assert_array_equal(predictions, expected_predictions) + + +# --- Sparse matrix tests --- +# Require scipy for sparse input. Skipped when scipy is not installed. + +_scipy = pytest.importorskip("scipy", reason="scipy required for sparse matrix tests") +from scipy.sparse import csr_matrix, csc_matrix + + +def test_sparse_fit_csr(X_n8) -> None: + """CSR sparse matrices can be passed to fit().""" + X_sparse = csr_matrix(X_n8, dtype=float) + clf = loop.LocalOutlierProbability(n_neighbors=5) + scores = clf.fit(X_sparse).local_outlier_probabilities_ + assert scores is not None + assert len(scores) == X_n8.shape[0] + + +def test_sparse_fit_csc_predict(X_n8) -> None: + """CSC sparse matrices work for fit() and predict().""" + X_sparse = csc_matrix(X_n8, dtype=float) + clf = loop.LocalOutlierProbability(n_neighbors=5).fit(X_sparse) + labels = clf.predict(X_sparse) + assert labels.shape == (X_n8.shape[0],) + assert set(labels).issubset({-1, 1}) + + +def test_sparse_equivalent_to_dense(X_n8) -> None: + """Sparse and dense inputs produce identical LoOP scores.""" + clf_dense = loop.LocalOutlierProbability(n_neighbors=5).fit(X_n8) + clf_sparse = loop.LocalOutlierProbability(n_neighbors=5).fit( + csr_matrix(X_n8, dtype=float) + ) + assert_array_almost_equal( + clf_dense.local_outlier_probabilities_, + clf_sparse.local_outlier_probabilities_, + decimal=6, + ) + + +def test_sparse_stream_row(X_n8) -> None: + """stream() accepts a single sparse observation row.""" + clf = loop.LocalOutlierProbability(n_neighbors=5).fit( + csr_matrix(X_n8, dtype=float) + ) + row_sparse = csr_matrix(X_n8[0:1], dtype=float) + score = clf.stream(row_sparse) + assert isinstance(score, (float, np.floating)) + assert 0.0 <= score <= 1.0 + + +def test_sparse_requires_scipy(X_n8) -> None: + """Sparse input without scipy installed raises a clear ImportError.""" + from unittest.mock import patch + + X_sparse = csr_matrix(X_n8, dtype=float) + clf = loop.LocalOutlierProbability(n_neighbors=5) + with patch("PyNomaly._validation._SCIPY_AVAILABLE", False): + with pytest.raises(ImportError, match="Sparse matrix input requires scipy"): + clf.fit(X_sparse)