From 2cca660c776ed9dd9b9b4513f4272b84bc786fc7 Mon Sep 17 00:00:00 2001 From: gudoshnikovn Date: Tue, 13 Oct 2026 11:52:23 +1000 Subject: [PATCH] test: sanitizers, fuzzing and memory bounds tools/sanitizers.py builds libphash, its decoders and the extension with Clang's AddressSanitizer + UndefinedBehaviorSanitizer (and libFuzzer's coverage hooks, so one build serves the tests and the fuzzers) or with ThreadSanitizer, runs a command with the runtime preloaded into the uninstrumented interpreter, and checks the logs. Any report fails; a leak fails when the extension itself allocated the block. CPython leaves thousands of unfreed blocks, and an LSan suppression cannot single out the extension: it matches any frame, and every stack passes the interpreter. The leak check runs when the tests are done (conftest), not at exit: the interpreter's teardown buries it there, and silencing the exit check through `exitcode` would also turn ASan aborts into success, as ASan, LSan and UBSan share that flag. Clang, not GCC: GCC's object-size check reports stb_image_resize2's scatter tables at -O3, which Clang does not and libphash's own sanitizer job (Clang) never sees. libphash is built in Release, as wheels are: its Debug install names -lpng16 in libphash.pc while the archive is libpng16d.a. The address build leaves out the function type check (-fno-sanitize=function): the fuzzers run under atheris' copy of the runtime, built by an older LLVM that has no handler for it, and the extension would not load. ci.yml runs the suite under both sanitizers on every push and PR, then each fuzz target for a minute; fuzz.yml gives them a quarter of an hour each. The fuzz targets cover what the binding adds over libphash, which fuzzes its decoders in C: encoded bytes, array layouts against their C-ordered copy (where ASan sees any read past a zero-copy view) and digests. The load target found that libphash 2.0.1 hashes a truncated TGA differently from load to load: stb_image leaves the missing rows uninitialized (16 loads of the same 74 bytes gave 11 hashes on a plain build). The target compares the two loads' outcome and dimensions only, and the input is in the corpus. tests/test_memory.py bounds the growth of Python objects and of resident memory over hundreds of repetitions of every path (success, load and hash errors, GC of an unclosed context, batch, a raising callback); a deliberate leak of one object or one context per step fails it. It also holds a batch's peak to libphash's estimate, four bytes per pixel and the file per worker (measured: exactly that for 1, 2 and 4 workers), plus the files the binding reads itself on Windows. Under a sanitizer it skips: CPython allocates through malloc there and some of its caches fill slowly enough to look like growth. mutmut leaves it out: it would multiply the run's time and catches no mutant the other tests miss. On a free-threaded build the two scenarios whose callbacks run on libphash's workers are a strict xfail: cffi 2.1.1 keeps a few objects of every C thread that enters an extern "Python" callback (an empty one included; a minimal cffi module without libphash shows it, a ctypes callback from the same threads does not, and GIL builds stay flat), and each batch starts its own workers. Valgrind is not run: ASan and LSan cover its findings here, and CPython under Valgrind is slow and noisy. --- .github/workflows/ci.yml | 59 ++++- .github/workflows/fuzz.yml | 72 ++++++ CLAUDE.md | 11 +- pyproject.toml | 30 ++- tests/conftest.py | 25 ++ tests/fuzz/digest/bits-pair | 1 + tests/fuzz/digest/coefficients-pair | Bin 0 -> 85 bytes tests/fuzz/digest/flat-coefficients | Bin 0 -> 85 bytes tests/fuzz/digest/from-hex-bad-digit | Bin 0 -> 10 bytes tests/fuzz/digest/from-hex-bits | Bin 0 -> 70 bytes tests/fuzz/digest/hash-from-hex | 1 + tests/fuzz/digest/histogram-pair | Bin 0 -> 221 bytes tests/fuzz/digest/unspecified-and-bits | Bin 0 -> 21 bytes tests/fuzz/digest/vector16-pair | Bin 0 -> 41 bytes tests/fuzz/load/truncated-tga | Bin 0 -> 74 bytes tests/fuzz/pixels/channel-gaps-bgr | Bin 0 -> 265 bytes tests/fuzz/pixels/column-step-bgra | Bin 0 -> 265 bytes tests/fuzz/pixels/empty | Bin 0 -> 265 bytes tests/fuzz/pixels/reversed-rows-gray | Bin 0 -> 265 bytes tests/fuzz/pixels/row-gaps-rgb | Bin 0 -> 265 bytes tests/fuzz/pixels/swapped-axes-one-channel | Bin 0 -> 265 bytes tests/test_fuzz_corpus.py | 48 ++++ tests/test_memory.py | 267 +++++++++++++++++++++ tools/fuzz.py | 53 ++++ tools/fuzz_targets.py | 259 ++++++++++++++++++++ tools/sanitizers.py | 216 +++++++++++++++++ typings/atheris.pyi | 12 + uv.lock | 59 +++++ 28 files changed, 1105 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/fuzz.yml create mode 100644 tests/fuzz/digest/bits-pair create mode 100644 tests/fuzz/digest/coefficients-pair create mode 100644 tests/fuzz/digest/flat-coefficients create mode 100644 tests/fuzz/digest/from-hex-bad-digit create mode 100644 tests/fuzz/digest/from-hex-bits create mode 100644 tests/fuzz/digest/hash-from-hex create mode 100644 tests/fuzz/digest/histogram-pair create mode 100644 tests/fuzz/digest/unspecified-and-bits create mode 100644 tests/fuzz/digest/vector16-pair create mode 100644 tests/fuzz/load/truncated-tga create mode 100644 tests/fuzz/pixels/channel-gaps-bgr create mode 100644 tests/fuzz/pixels/column-step-bgra create mode 100644 tests/fuzz/pixels/empty create mode 100644 tests/fuzz/pixels/reversed-rows-gray create mode 100644 tests/fuzz/pixels/row-gaps-rgb create mode 100644 tests/fuzz/pixels/swapped-axes-one-channel create mode 100644 tests/test_fuzz_corpus.py create mode 100644 tests/test_memory.py create mode 100644 tools/fuzz.py create mode 100644 tools/fuzz_targets.py create mode 100644 tools/sanitizers.py create mode 100644 typings/atheris.pyi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55a3e64..9131376 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,7 +188,7 @@ jobs: - name: Install the wheel into a fresh environment run: | uv venv "$RUNNER_TEMP/venv" - VIRTUAL_ENV="$RUNNER_TEMP/venv" uv pip install dist/*.whl pytest pytest-timeout hypothesis + VIRTUAL_ENV="$RUNNER_TEMP/venv" uv pip install dist/*.whl pytest pytest-timeout hypothesis psutil - name: pytest run: '"$RUNNER_TEMP/venv/bin/python" -m pytest -rs' @@ -198,3 +198,60 @@ jobs: name: sdist path: dist/*.tar.gz if-no-files-found: error + + # The test suite with libphash, its decoders and the extension built under Clang's + # sanitizers by tools/sanitizers.py; the interpreter is not instrumented, so a report + # is about code of the extension. AddressSanitizer runs with UndefinedBehaviorSanitizer + # and LeakSanitizer, then fuzzes each target of tools/fuzz_targets.py for a minute + # from its corpus (fuzz.yml runs them longer); ThreadSanitizer watches libphash's + # worker threads and the callbacks they make into Python. + sanitizers: + strategy: + fail-fast: false + matrix: + sanitizer: [ address, thread ] + runs-on: ubuntu-24.04 + timeout-minutes: 30 + name: sanitizers (${{ matrix.sanitizer }}) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/setup + + - name: Install Clang and its sanitizer runtimes + run: | + sudo apt-get update + sudo apt-get install -y clang libclang-rt-18-dev + + - name: Build under the sanitizer + run: uv run --no-project python tools/sanitizers.py build ${{ matrix.sanitizer }} + + - name: pytest + run: > + .venv/bin/python tools/sanitizers.py run ${{ matrix.sanitizer }} sanitizer-logs -- + .venv/bin/python -m pytest -rs + + - name: Fuzz each target for a minute + if: matrix.sanitizer == 'address' + run: | + mkdir -p fuzz-crashes + for target in load pixels digest; do + .venv/bin/python tools/sanitizers.py run fuzz sanitizer-logs -- \ + .venv/bin/python tools/fuzz.py "$target" -max_total_time=60 -max_len=65536 \ + -artifact_prefix=fuzz-crashes/ + done + + - name: Sanitizer reports + if: ${{ !cancelled() }} + run: .venv/bin/python tools/sanitizers.py check sanitizer-logs + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: sanitizers-${{ matrix.sanitizer }} + path: | + sanitizer-logs/ + fuzz-crashes/ + if-no-files-found: ignore diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..a66bae6 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,72 @@ +name: Fuzz + +# Each fuzz target of tools/fuzz_targets.py for a quarter of an hour, in the +# AddressSanitizer build of tools/sanitizers.py; the sanitizer job of ci.yml gives them a +# minute each. Every run starts from the corpus in tests/fuzz/: an input that fails a +# target is uploaded, and goes into the corpus with the fix. It runs weekly, by hand, +# and when a target, the corpus or this workflow changes. +on: + push: + branches: [ main, 'release/**' ] + paths: &fuzz-paths + - tools/fuzz.py + - tools/fuzz_targets.py + - tests/fuzz/** + - .github/workflows/fuzz.yml + pull_request: + branches: [ main, 'release/**' ] + paths: *fuzz-paths + schedule: + - cron: '53 4 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + fuzz: + strategy: + fail-fast: false + matrix: + target: [ load, pixels, digest ] + runs-on: ubuntu-24.04 + timeout-minutes: 45 + name: fuzz (${{ matrix.target }}) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/setup + + - name: Install Clang and its sanitizer runtimes + run: | + sudo apt-get update + sudo apt-get install -y clang libclang-rt-18-dev + + - name: Build under AddressSanitizer + run: uv run --no-project python tools/sanitizers.py build address + + - name: Fuzz for 15 minutes + run: | + mkdir -p fuzz-crashes + .venv/bin/python tools/sanitizers.py run fuzz sanitizer-logs -- \ + .venv/bin/python tools/fuzz.py ${{ matrix.target }} -max_total_time=900 \ + -max_len=65536 -artifact_prefix=fuzz-crashes/ + + - name: Sanitizer reports + if: ${{ !cancelled() }} + run: .venv/bin/python tools/sanitizers.py check sanitizer-logs + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: fuzz-${{ matrix.target }} + path: | + sanitizer-logs/ + fuzz-crashes/ + if-no-files-found: ignore diff --git a/CLAUDE.md b/CLAUDE.md index 3744113..7648520 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,11 @@ uv run mypy && uv run basedpyright # both strict, over s uv run python tools/stub.py --check # _native.pyi matches the built module uv run typos uv build # sdist + wheel into dist/ +# Linux with Clang and libclang-rt-dev; the build replaces the editable install in .venv +uv run --no-project python tools/sanitizers.py build address # or: thread +.venv/bin/python tools/sanitizers.py run address logs -- .venv/bin/python -m pytest +.venv/bin/python tools/sanitizers.py run fuzz logs -- .venv/bin/python tools/fuzz.py pixels -max_total_time=60 +.venv/bin/python tools/sanitizers.py check logs ``` `uv sync` without `--inexact` removes packages that are not in the lock file from `.venv`. @@ -39,6 +44,8 @@ How the extension is built: - **`tools/cdef.py`** — generates the CFFI `cdef` from `native/libphash/include/libphash.h`: every function and constant of the header, nothing written by hand. - **`tools/ffi.py`** — the CFFI builder (`cffi_modules` in `setup.py`). - **`tools/stub.py`** — generates `src/libphash/_native.pyi` from the built module through cffi's introspection: a class per pointer and array type (`uint64_t_ptr`, `ph_digest_t_array`), struct fields, every `lib` function with the header's parameter names, `ffi.new`/`cast`/`from_buffer` overloaded on literal cdecls. Parameter types follow cffi's rules (a `T *` takes exactly `T *`/`T[]` or `ffi.NULL`; character pointers also take `bytes`). The classes exist only for type checkers: import them under `TYPE_CHECKING`. A test fails when the committed stub differs from the generated one. +- **`tools/sanitizers.py`** — the sanitizer builds (Clang; `address` is ASan + UBSan + libFuzzer's coverage hooks, `thread` is TSan) in `build/sanitizer-/`, runs with the runtime preloaded into the uninstrumented interpreter, and `check`, which fails on any report and on a leak whose block the extension itself allocated (the interpreter's own unfreed blocks are ignored; `tests/conftest.py` asks LeakSanitizer for the check when the tests are done). +- **`tools/fuzz_targets.py`** — fuzz targets of what the binding adds over libphash (which fuzzes its decoders in C): `load` (encoded bytes), `pixels` (array layouts against their C-ordered copy), `digest` (text form, round trips, comparison ranges). **`tools/fuzz.py`** runs one under atheris (Linux x86-64, `fuzz` dependency group); `typings/atheris.pyi` types the part of atheris the tools use. The corpus is `tests/fuzz//`, replayed on every platform by `tests/test_fuzz_corpus.py`. - **`setup.py`** — `build_ext` runs the CMake build before compiling the extension, so an sdist is made without a compiler. - **`MANIFEST.in`** — the sdist carries only what libphash's CMake build reads (`CMakeLists.txt`, `cmake/`, `include/`, `src/`, `vendor/`, licenses), enough to build offline. @@ -61,8 +68,8 @@ The public API is what `src/libphash/__init__.py` exports; every other module is Typing: the package passes mypy and basedpyright in strict mode, and every type a caller needs to annotate their own code (`Source`, `StrPath`, `PixelArray`, `AlgorithmLike`, the `Config` literals, the protocols) is exported from `libphash`. Public signatures use protocols, not `Any`. A runtime check of an argument for untyped callers takes it as `object`, so neither checker calls the check unnecessary. Lint: ruff with a broad rule set and pep257 docstrings (`pyproject.toml`); every exception to a rule is a per-file ignore with a comment or a `# noqa: CODE -- reason`, never a bare `noqa`. - **`tests/`** — pytest suite (`test_cdef.py` fails when a header function is neither wrapped nor listed as deliberately unwrapped). - `test_c_parity.py` builds `tests/c/reference.c` (through `c_reference.py`) against the libphash install of `tools/native.py` and requires the binding's hashes and error codes to equal C's for every fixture, setting and pixel layout; without the install or CMake it skips, and CI sets `LIBPHASH_REQUIRE_C_REFERENCE=1` so that it fails instead. `test_corpus.py` does the same for libphash's 200-photo site corpus and is collected only when `LIBPHASH_CORPUS` names the list of fetched photos. -- **`.github/workflows/`** — `ci.yml` (lint, types, tests on every platform, their line and branch coverage combined and held at 100 %, sdist) on every push and PR; `wheels.yml` only when a build file changes: every wheel by cibuildwheel (configuration in `pyproject.toml`), each wheel's hash table (`tools/hash_table.py`) compared across all platforms byte for byte, and a build with JPEG decoded by stb_image; `corpus.yml` (the photo corpus, network) weekly, by hand and when the submodule changes; `mutation.yml` (mutmut over `src/libphash/`) weekly, by hand and when the package changes: every mutant is caught or listed in `tests/equivalent_mutants.json` with its reason, and `tools/mutants.py` fails on any other survivor and on a listed mutant that no longer survives. `libphash-release-check.yml` opens a tracking issue weekly when `native/libphash` is behind libphash's newest release tag. + `test_c_parity.py` builds `tests/c/reference.c` (through `c_reference.py`) against the libphash install of `tools/native.py` and requires the binding's hashes and error codes to equal C's for every fixture, setting and pixel layout; without the install or CMake it skips, and CI sets `LIBPHASH_REQUIRE_C_REFERENCE=1` so that it fails instead. `test_corpus.py` does the same for libphash's 200-photo site corpus and is collected only when `LIBPHASH_CORPUS` names the list of fetched photos. `test_memory.py` repeats every path (success, errors, GC, batch, callbacks) and bounds the growth of Python objects (tracemalloc) and resident memory (psutil), and holds the peak of a batch to libphash's estimate (4 bytes per pixel and the file, per worker); under a sanitizer it skips, and LeakSanitizer checks the C side. +- **`.github/workflows/`** — `ci.yml` (lint, types, tests on every platform, their line and branch coverage combined and held at 100 %, sdist, the test suite under ASan/UBSan/LSan and under TSan, a minute of each fuzz target) on every push and PR; `fuzz.yml` (a quarter of an hour per fuzz target) weekly, by hand and when a target or the corpus changes; `wheels.yml` only when a build file changes: every wheel by cibuildwheel (configuration in `pyproject.toml`), each wheel's hash table (`tools/hash_table.py`) compared across all platforms byte for byte, and a build with JPEG decoded by stb_image; `corpus.yml` (the photo corpus, network) weekly, by hand and when the submodule changes; `mutation.yml` (mutmut over `src/libphash/`) weekly, by hand and when the package changes: every mutant is caught or listed in `tests/equivalent_mutants.json` with its reason, and `tools/mutants.py` fails on any other survivor and on a listed mutant that no longer survives. `libphash-release-check.yml` opens a tracking issue weekly when `native/libphash` is behind libphash's newest release tag. Code, comments, docstrings and tracked documentation are in English. diff --git a/pyproject.toml b/pyproject.toml index 6a910f3..fb89986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,14 @@ dependencies = ["cffi>=1.15.0"] "Homepage" = "https://github.com/gudoshnikovn/python-libphash" [project.optional-dependencies] -dev = ["pytest", "pytest-randomly", "pytest-timeout", "hypothesis", "coverage", "mypy", "basedpyright", "ruff", "typos", "setuptools", "types-setuptools", "types-cffi", "typing_extensions", "numpy", "Pillow", "mutmut; sys_platform != 'win32'"] +dev = ["pytest", "pytest-randomly", "pytest-timeout", "hypothesis", "psutil", "coverage", "mypy", "basedpyright", "ruff", "typos", "setuptools", "types-setuptools", "types-cffi", "types-psutil", "typing_extensions", "numpy", "Pillow", "mutmut; sys_platform != 'win32'"] + +# Not part of "dev": atheris has wheels for Linux x86-64 and CPython 3.12+ only, and the +# sanitizer job of CI, which runs the fuzzers, installs it with --group fuzz. +[dependency-groups] +fuzz = [ + "atheris>=3.1; sys_platform == 'linux' and platform_machine == 'x86_64' and python_version >= '3.12'", +] [tool.setuptools] zip-safe = false @@ -49,7 +56,7 @@ build-verbosity = 1 skip = ["*-musllinux_*"] # numpy and Pillow are optional at runtime; with them the array and Pillow tests run # against the wheel too. The sdist job of CI runs the suite without them. -test-requires = ["pytest", "pytest-timeout", "hypothesis", "numpy", "Pillow"] +test-requires = ["pytest", "pytest-timeout", "hypothesis", "psutil", "numpy", "Pillow"] test-command = "pytest {project}/tests -rs" # CMake and Ninja come from build-system.requires. nasm gives libjpeg-turbo its x86 @@ -106,14 +113,21 @@ exclude_also = ["if TYPE_CHECKING:"] [tool.mutmut] source_paths = ["src/libphash/"] -pytest_add_cli_args_test_selection = ["tests/"] -also_copy = ["native/libphash/include", "native/libphash/tests/data", "tools"] +# The memory tests repeat every operation hundreds of times and catch no mutant the other +# tests miss; run for each mutant, they would multiply the run's time. +pytest_add_cli_args_test_selection = ["tests/", "--ignore=tests/test_memory.py"] +also_copy = [ + "native/libphash/include", + "native/libphash/tests/data", + "tools", + ".github/workflows", +] [tool.mypy] strict = true python_version = "3.10" files = ["src", "tests", "tools", "setup.py"] -mypy_path = ["src", "tools"] +mypy_path = ["src", "tools", "typings"] [tool.basedpyright] typeCheckingMode = "strict" @@ -156,6 +170,12 @@ convention = "pep257" "tests/c_reference.py" = ["S603", "PERF203"] # The build runs CMake from PATH or the build environment, with arguments it builds. "tools/native.py" = ["S603"] +# The sanitizer builds run uv and clang from PATH, and a run starts the command it is +# given. +"tools/sanitizers.py" = ["S603", "S607"] +# A fuzz target fails by raising, so its checks are asserts; an error is an outcome to +# record for each algorithm, not a reason to stop. +"tools/fuzz_targets.py" = ["S101", "PERF203"] # The stub's classes and parameters carry the names of the C types and of the header. "src/libphash/_native.pyi" = ["N801", "A002"] diff --git a/tests/conftest.py b/tests/conftest.py index 2cc055b..628bcbc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import ctypes import functools import os import re @@ -20,6 +21,30 @@ DATA = c_reference.DATA +def _sanitizer_runtime() -> ctypes.CDLL | None: + """Return the process's own symbols if a sanitizer runtime is loaded into it.""" + if sys.platform == "win32": + return None + symbols = ctypes.CDLL(None) + return symbols if hasattr(symbols, "__sanitizer_print_stack_trace") else None + + +#: The sanitizer job runs the tests with AddressSanitizer or ThreadSanitizer preloaded. +SANITIZER = _sanitizer_runtime() + + +def pytest_unconfigure() -> None: + """Under LeakSanitizer, report the blocks nothing points to once the tests are done. + + The sanitizer job turns the check at exit off: by then the interpreter has dropped + its own references to memory it never frees, which would bury the report. Here the + interpreter still holds them, and tools/sanitizer_logs.py fails the job on a block + the extension allocated. + """ + if SANITIZER is not None and hasattr(SANITIZER, "__lsan_do_recoverable_leak_check"): + SANITIZER.__lsan_do_recoverable_leak_check() + + # libjpeg-turbo decodes JPEG unless the build was configured without it; stb_image then # does, with its own IDCT, no arithmetic coding and no decode scaling. The question is # asked inside tests, not at import: collection never calls into the package, so a diff --git a/tests/fuzz/digest/bits-pair b/tests/fuzz/digest/bits-pair new file mode 100644 index 0000000..6886a2e --- /dev/null +++ b/tests/fuzz/digest/bits-pair @@ -0,0 +1 @@ +4`¾1 iþÚ îè¹™\|)™ý¯å“%<ÖT¯Mú×' ®³þé#/Šò!žä‘ű ìµV;üo“B~ËÈþ) \ No newline at end of file diff --git a/tests/fuzz/digest/coefficients-pair b/tests/fuzz/digest/coefficients-pair new file mode 100644 index 0000000000000000000000000000000000000000..ba05b6437bd872fa618e6d67fa4c495b0662ffd0 GIT binary patch literal 85 zcmV-b0IL52Cj(XG&5lOgj?}lpc1FIxl0Y0vDYXe literal 0 HcmV?d00001 diff --git a/tests/fuzz/digest/flat-coefficients b/tests/fuzz/digest/flat-coefficients new file mode 100644 index 0000000000000000000000000000000000000000..c271989c15aab037d88611417aa117a67ef6baf9 GIT binary patch literal 85 zcmZQlXJ#N0s59%e?hf5nw4x}v>TF5cpKW}6{U2MBH$2(2OS8bF9000o?126yp literal 0 HcmV?d00001 diff --git a/tests/fuzz/digest/from-hex-bits b/tests/fuzz/digest/from-hex-bits new file mode 100644 index 0000000000000000000000000000000000000000..8aa4ac5ba936d78483d982c37a774bc56897d15c GIT binary patch literal 70 zcmV~$!4ZHk2n0Y&@_>NYP2iFl-`fAUd(`v2_Z-HaxMb*A1v;V9Ty)VWSxhY5A=q8W N0KzPdZmFFd{R6{x6KDVc literal 0 HcmV?d00001 diff --git a/tests/fuzz/digest/hash-from-hex b/tests/fuzz/digest/hash-from-hex new file mode 100644 index 0000000..12c0cb5 --- /dev/null +++ b/tests/fuzz/digest/hash-from-hex @@ -0,0 +1 @@ +00ff00ff12345678 \ No newline at end of file diff --git a/tests/fuzz/digest/histogram-pair b/tests/fuzz/digest/histogram-pair new file mode 100644 index 0000000000000000000000000000000000000000..a0bce4d8490ae2800ce28b42b93cdb77d2ae1573 GIT binary patch literal 221 zcmWkoNe%!p2$D@7+yCE(NgAsN2w!yEU~fTkc)au<>x+C> zVvXt(Xd#24c`8D2H>>0YsrF`LK!81EZ7)>Uep}#fT<5Gg0*lK{a&*QsqtpF~Ev4QW NR@9Iv;)bMeoP5dK1G4}C literal 0 HcmV?d00001 diff --git a/tests/fuzz/digest/unspecified-and-bits b/tests/fuzz/digest/unspecified-and-bits new file mode 100644 index 0000000000000000000000000000000000000000..06c029e7bc93dcb5025636e79ff07801a691ec0a GIT binary patch literal 21 RcmZQ#XJmi^cEY4P~1=(~x&xoGno;kZ*+yD5ozdjSa!~j3=5HJ7$ literal 0 HcmV?d00001 diff --git a/tests/fuzz/load/truncated-tga b/tests/fuzz/load/truncated-tga new file mode 100644 index 0000000000000000000000000000000000000000..ccac19beb674d4dbc1e547ccac5e1245eaf8a2e0 GIT binary patch literal 74 zcmZQzU}As)76uj(2?bULE@crn5yz%@k$uVmj$Cd5@c~V>0s9<#xZ-=}2 zSXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%jczJqze0_d@fPsR8goTEOh>41e zjE#}2 zSXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%jczJqze0_d@fPsR8goTEOh>41e zjE#}2 zSXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%jczJqze0_d@fPsR8goTEOh>41e zjE#}2 zSXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%jczJqze0_d@fPsR8goTEOh>41e zjE#}2 zSXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%jczJqze0_d@fPsR8goTEOh>41e zjE#}2 zSXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%jczJqze0_d@fPsR8goTEOh>41e zjE# list[Path]: + dirs = fuzz_targets.seed_dirs(target) + return sorted(p for d in dirs if d.is_dir() for p in d.iterdir() if p.is_file()) + + +CASES = [ + pytest.param(target, path, id=f"{target}/{path.name}") + for target in fuzz_targets.TARGETS + for path in corpus(target) +] + + +@pytest.mark.parametrize(("target", "path"), CASES) +def test_corpus_input(target: str, path: Path) -> None: + if target == "pixels": + pytest.importorskip("numpy") + fuzz_targets.TARGETS[target](path.read_bytes()) + + +def test_every_target_has_a_corpus() -> None: + assert all(corpus(target) for target in fuzz_targets.TARGETS) + + +def test_ci_fuzzes_every_target() -> None: + workflows = ROOT / ".github" / "workflows" + loop = re.search(r"for target in (.+); do", (workflows / "ci.yml").read_text()) + matrix = re.search(r"target: \[ (.+) \]", (workflows / "fuzz.yml").read_text()) + assert loop is not None + assert matrix is not None + assert loop.group(1).split() == list(fuzz_targets.TARGETS) + assert [t.strip() for t in matrix.group(1).split(",")] == list(fuzz_targets.TARGETS) diff --git a/tests/test_memory.py b/tests/test_memory.py new file mode 100644 index 0000000..ea82357 --- /dev/null +++ b/tests/test_memory.py @@ -0,0 +1,267 @@ +"""Memory: repeating an operation does not grow it, and a batch peaks where C says. + +Each scenario of the growth test runs once per step; after a warm-up (caches, free +lists, lazy imports) the step is repeated, and neither the Python objects it leaves +behind (tracemalloc) nor the process's resident memory, where a block the C side never +frees shows up, may grow past a bound far below what one leaked object or one leaked +image per step would add. + +Under a sanitizer nothing is measured: its allocator keeps freed memory in quarantine +and its shadow memory adds to every block, the interpreter allocates through malloc() +there, which some of its caches fill slowly enough to look like growth, and +LeakSanitizer checks the C side instead. +""" + +from __future__ import annotations + +import gc +import subprocess +import sys +import sysconfig +import tracemalloc +from typing import TYPE_CHECKING + +import psutil +import pytest +from conftest import SANITIZER, data + +import libphash as ph +from libphash._sources import READ_FILES_IN_PYTHON + +if TYPE_CHECKING: + from collections.abc import Callable + +pytestmark = pytest.mark.skipif( + SANITIZER is not None, reason="a sanitizer's allocator holds freed memory" +) + +STEPS = 400 +#: tracemalloc: about 10 bytes per step, less than the smallest Python object. +PYTHON_BOUND = 4 * 1024 +#: Resident memory: about 2.5 KiB per step, less than one leaked 32x32 RGB image. +RSS_BOUND = 1024 * 1024 + +ALGORITHMS = tuple(ph.Algorithm) + + +def load_file() -> None: + with ph.ImageContext() as ctx: + ctx.load_file(data("png/rgb8.png")) + for algo in ALGORITHMS: + ctx.hash(algo) + + +def load_bytes() -> None: + with ph.ImageContext() as ctx: + ctx.load_bytes(data("jpeg/s420.jpg").read_bytes()) + ctx.hashes(*ALGORITHMS) + + +def missing_file() -> None: + with ph.ImageContext() as ctx, pytest.raises(ph.ImageIOError): + ctx.load_file(data("png/rgb8.png").with_name("missing.png")) + + +def corrupt_bytes() -> None: + with ph.ImageContext() as ctx: + for name in ("corrupted.jpg", "png/broken_truncated.png", "png/broken_crc.png"): + with pytest.raises(ph.PhashError): + ctx.load_bytes(data(name).read_bytes()) + + +def refused_hashes() -> None: + gray = memoryview(bytes(range(256)) * 4).cast("B", (32, 32)) + flat = memoryview(bytes(32 * 32 * 3)).cast("B", (32, 32, 3)) + with ph.ImageContext() as ctx: + ctx.load_pixels(gray) + with pytest.raises(ph.RequiresColorError): + ctx.color_hash() + ctx.load_pixels(flat) + radial = ctx.radial() + with pytest.raises(ph.NoStructureError): + radial.similarity(radial) + + +def copied_pixels() -> None: + rgba = memoryview(bytes(range(256)) * 32).cast("B", (64, 32, 4)) + with ph.ImageContext() as ctx: + ctx.load_pixels(rgba[::2], channel_order="bgr") + for algo in ALGORITHMS: + ctx.hash(algo) + with pytest.raises(ValueError, match="shape"): + ctx.load_pixels(memoryview(bytes(10))) + + +def unclosed_context() -> None: + ctx = ph.ImageContext() + ctx.load_file(data("png/rgb8.png")) + ctx.phash() + del ctx + + +def closed_context() -> None: + ctx = ph.ImageContext() + ctx.close() + with pytest.raises(ValueError, match="closed"): + ctx.phash() + + +def invalid_config() -> None: + with pytest.raises(ph.InvalidArgumentError): + ph.Config(phash_dct_size=1) + + +def digests() -> None: + with ph.ImageContext() as ctx: + ctx.load_file(data("png/rgb8.png")) + a, b = ctx.bmh(), ctx.radial() + assert ph.Digest.from_hex(a.hex()) == a + a.similarity(a) + b.similarity(b) + with pytest.raises(ph.DigestKindError): + a.l2_distance(a) + with pytest.raises(ph.PhashError): + ph.Digest.from_hex("bits:zz") + ph.digest_info("radial") + + +def batch() -> None: + buffers = [data("png/rgb8.png").read_bytes(), b"not an image", b""] + results = ph.hash_buffers(buffers, ("phash", "dhash"), threads=2) + assert [r.ok for r in results] == [True, False, False] + + +def batch_callback_error() -> None: + def progress(done: int, total: int) -> bool: + raise KeyError(done, total) + + with pytest.raises(KeyError): + ph.hash_buffers( + [data("png/rgb8.png").read_bytes()] * 2, progress=progress, threads=2 + ) + + +SCENARIOS: dict[str, Callable[[], None]] = { + f.__name__: f + for f in ( + load_file, + load_bytes, + missing_file, + corrupt_bytes, + refused_hashes, + copied_pixels, + unclosed_context, + closed_context, + invalid_config, + digests, + batch, + batch_callback_error, + ) +} + + +def _repeat(step: Callable[[], None]) -> None: + for _ in range(STEPS): + step() + gc.collect() + + +def _growth(step: Callable[[], None], measure: Callable[[], int]) -> int: + """Return the smaller growth of ``measure`` over two runs of the steps. + + A leak grows memory in both. A table the interpreter enlarges once (the interned + strings, a free list), when the tests before this one have brought it to the edge, + grows it in one only. + """ + growths: list[int] = [] + for _ in range(2): + before = measure() + _repeat(step) + growths.append(measure() - before) + return min(growths) + + +def _traced() -> int: + return tracemalloc.get_traced_memory()[0] + + +#: The scenarios whose callbacks run on libphash's worker threads. +WORKER_CALLBACKS = {"batch", "batch_callback_error"} + +# On a free-threaded build, cffi 2.1.1 leaves a few Python objects behind for every +# thread of C that enters an extern "Python" callback, empty or not; a callback through +# ctypes from the same threads leaves none. Each batch starts its own workers, so these +# scenarios grow by a few hundred bytes per step there. Strict: the scenarios pass once +# cffi frees them, and the mark is then removed. +_CFFI_THREAD_LEAK = pytest.mark.xfail( + bool(sysconfig.get_config_var("Py_GIL_DISABLED")), + reason="cffi keeps objects of each C thread that calls into a free-threaded Python", + strict=True, +) + + +@pytest.mark.parametrize( + "step", + [ + pytest.param( + step, + id=name, + marks=[_CFFI_THREAD_LEAK] if name in WORKER_CALLBACKS else [], + ) + for name, step in SCENARIOS.items() + ], +) +def test_no_growth(step: Callable[[], None]) -> None: + _repeat(step) + rss_growth = _growth(step, lambda: psutil.Process().memory_info().rss) + tracemalloc.start() + try: + python_growth = _growth(step, _traced) + finally: + tracemalloc.stop() + + assert python_growth <= PYTHON_BOUND, f"Python objects grew by {python_growth} B" + assert rss_growth <= RSS_BOUND, f"resident memory grew by {rss_growth} B" + + +# Prints how far the peak resident memory of the process rose over a batch of copies +# of one image, two per worker: ru_maxrss (KiB on Linux, bytes on macOS) or, on +# Windows, the peak working set. +_PEAK_SCRIPT = """ +import sys +import libphash as ph + +def peak(): + if sys.platform == "win32": + import psutil + return psutil.Process().memory_info().peak_wset + import resource + scale = 1 if sys.platform == "darwin" else 1024 + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * scale + +path, threads = sys.argv[1], int(sys.argv[2]) +ph.hash(path) +before = peak() +assert all(r.ok for r in ph.hash_files([path] * 2 * threads, threads=threads)) +print(peak() - before) +""" + + +@pytest.mark.parametrize("threads", [1, 3]) +def test_batch_peak_is_an_image_per_worker(threads: int) -> None: + # libphash's batch guide: a worker holds about 4 bytes per pixel and the file. Where + # the binding reads the files itself (READ_FILES_IN_PYTHON), it also holds the + # encoded files of the chunk it hands to C, here every file of the batch. + path = data("photo_large.jpeg") + size = path.stat().st_size + pixels = 5472 * 3648 + files = 2 * threads if READ_FILES_IN_PYTHON else threads + estimate = threads * 4 * pixels + files * size + out = subprocess.run( # noqa: S603 -- this Python, with a fixed script + [sys.executable, "-c", _PEAK_SCRIPT, str(path), str(threads)], + capture_output=True, + text=True, + check=True, + ).stdout + growth = int(out) + assert growth <= estimate * 1.1 + 16 * 2**20, (growth, estimate) diff --git a/tools/fuzz.py b/tools/fuzz.py new file mode 100644 index 0000000..922cb27 --- /dev/null +++ b/tools/fuzz.py @@ -0,0 +1,53 @@ +"""Run one fuzz target of ``tools/fuzz_targets.py`` under atheris (libFuzzer). + + python tools/fuzz.py TARGET [libFuzzer options] [CORPUS_DIR ...] + +The arguments after the target go to libFuzzer as they are: ``-max_total_time=60``, +``-artifact_prefix=crashes/``, corpus directories, ``-merge=1 MIN_DIR CORPUS_DIR`` to +minimize a corpus. libFuzzer writes the inputs it finds into the first directory; the +target's seeds (``tests/fuzz//``, and libphash's seeds for ``load``) follow the +given directories, after a temporary one when none is given, and are only read. +atheris instruments the package's Python code for coverage; the C side of the binding +is checked by running the fuzzer in the AddressSanitizer build of the sanitizer job, +where a read past a buffer stops it as an assertion does. + +An input that fails a target goes into ``tests/fuzz//`` once the bug is fixed: +``tests/test_fuzz_corpus.py`` replays that corpus on every platform. + +atheris has wheels for Linux x86-64 only; it is in the ``fuzz`` dependency group. +""" + +from __future__ import annotations + +import shlex +import sys +import tempfile +from pathlib import Path + +# typings/atheris.pyi describes it where atheris itself is not installed. +import atheris # pyright: ignore[reportMissingModuleSource] + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + + +def main() -> None: + """Fuzz the target named by the first argument.""" + with atheris.instrument_imports(include=["libphash", "fuzz_targets"]): + import fuzz_targets # noqa: PLC0415 -- instrumented as it is imported + + if len(sys.argv) < 2 or sys.argv[1] not in fuzz_targets.TARGETS: + names = ", ".join(fuzz_targets.TARGETS) + sys.exit(f"usage: fuzz.py TARGET [libFuzzer options]; targets: {names}") + name, args = sys.argv[1], sys.argv[2:] + if not any(not a.startswith("-") for a in args): + args.append(tempfile.mkdtemp(prefix=f"fuzz-{name}-")) + args += [str(d) for d in fuzz_targets.seed_dirs(name) if d.is_dir()] + # libFuzzer's -merge and -fork modes start the fuzzer again through system(), as + # argv[0] followed by options: argv[0] is the command that leads back here. + command = shlex.join([sys.executable, str(Path(__file__).resolve()), name]) + atheris.Setup([command, *args], fuzz_targets.TARGETS[name]) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/tools/fuzz_targets.py b/tools/fuzz_targets.py new file mode 100644 index 0000000..041da8d --- /dev/null +++ b/tools/fuzz_targets.py @@ -0,0 +1,259 @@ +"""Fuzz targets of the binding: properties that hold for any input bytes. + +libphash fuzzes its decoders itself, in C. These targets fuzz what the binding adds on +top: the buffers and strides it hands to C, the error mapping, the digest text form and +the comparisons. Each target takes raw bytes and draws its arguments from them, so a +corpus file means the same on every platform and Python; ``tools/fuzz.py`` runs the +targets under atheris, ``tests/test_fuzz_corpus.py`` replays the corpus with pytest. + +A target fails by raising. A :class:`libphash.PhashError` is an answer, not a failure: +any input may be refused, but only with one of the documented errors. +""" + +from __future__ import annotations + +import importlib +import io +import math +from pathlib import Path +from typing import TYPE_CHECKING + +import libphash as ph + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +#: The corpus of each target: hand-made seeds and the inputs fuzzing has found. +CORPUS = PROJECT_ROOT / "tests" / "fuzz" +#: libphash's seeds for its own decoder fuzzer: a file per format and EXIF case. +LIBPHASH_SEEDS = PROJECT_ROOT / "native" / "libphash" / "tests" / "fuzz" / "seeds" + +#: A hash, or the name of the error that refused the input. +Outcome = int | ph.Digest | str + + +class Draw: + """Arguments drawn from the front of the input, the rest left as payload.""" + + def __init__(self, data: bytes) -> None: + self._data = data + self._pos = 0 + + def byte(self) -> int: + """Return the next byte, 0 past the end.""" + if self._pos >= len(self._data): + return 0 + self._pos += 1 + return self._data[self._pos - 1] + + def below(self, n: int) -> int: + """Return a number in ``range(n)`` (n <= 256).""" + return self.byte() % n + + def take(self, n: int) -> bytes: + """Return up to ``n`` next bytes.""" + chunk = self._data[self._pos : self._pos + n] + self._pos += len(chunk) + return chunk + + def rest(self) -> bytes: + """Return every byte not drawn yet.""" + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + + +#: The algorithms that compare loaded pixels. mHash costs more than all the others +#: together and reads the same image: it would only slow the target down. +PIXEL_ALGORITHMS = tuple(a for a in ph.Algorithm if a != ph.Algorithm.MHASH) + + +def _outcomes( + ctx: ph.ImageContext, algorithms: Iterable[ph.Algorithm] +) -> dict[ph.Algorithm, Outcome]: + result: dict[ph.Algorithm, Outcome] = {} + for algo in algorithms: + try: + result[algo] = ctx.hash(algo) + except ph.PhashError as e: + result[algo] = type(e).__name__ + return result + + +def _pixels_loader( + source: ph.PixelArray, order: ph.ChannelOrder = "rgb" +) -> Callable[[ph.ImageContext], None]: + return lambda ctx: ctx.load_pixels(source, order) + + +def _load_outcomes(load: Callable[[ph.ImageContext], None]) -> dict[str, Outcome]: + with ph.ImageContext() as ctx: + try: + load(ctx) + except ph.PhashError as e: + return {"load": type(e).__name__} + return {str(a): v for a, v in _outcomes(ctx, PIXEL_ALGORITHMS).items()} + + +#: libphash's own fuzzer caps images at 4 megapixels too: a file of a few bytes may +#: rightly ask for the default 256, more memory than a fuzzer is given. +LOAD_CONFIG = ph.Config(max_pixels=4 * 1024 * 1024) + + +def _decode(load: Callable[[ph.ImageContext], None]) -> str | tuple[int, int, int]: + with ph.ImageContext(LOAD_CONFIG) as ctx: + try: + load(ctx) + except ph.PhashError as e: + return type(e).__name__ + _outcomes(ctx, ph.Algorithm) + return ctx.width, ctx.height, ctx.channels + + +def load(data: bytes) -> None: + """Check that encoded bytes load alike from memory and from a file, and hash. + + Every hash of a loaded image runs, and only a :class:`libphash.PhashError` may + refuse it. The hashes of the two loads are not compared: libphash 2.0.1 decodes TGA + with stb_image, which leaves the rows past the end of a truncated file + uninitialized, so that such a file hashes differently from load to load. + """ + from_bytes = _decode(lambda ctx: ctx.load_bytes(data)) + from_file = _decode(lambda ctx: ctx.load_fileobj(io.BytesIO(data))) + assert from_bytes == from_file, (from_bytes, from_file) + + +def pixels(data: bytes) -> None: + """Check that a view of pixels hashes as its C-ordered copy, whatever its layout. + + The view is a crop of a larger array, with a row and a column step, gaps between + channels, reversed rows or swapped axes: the layouts the binding passes to C without + a copy (row gaps) and those it copies (anything else). It is loaded as an array and + as a memoryview, in RGB or BGR order, and each must hash like the contiguous array + in RGB order with the channels already swapped. + """ + # numpy is optional, as in the tests; only this target needs it. + np = importlib.import_module("numpy") + + draw = Draw(data) + height, width = draw.below(25), draw.below(25) + channels = (0, 1, 3, 4)[draw.below(4)] + row_step, col_step = 1 + draw.below(3), 1 + draw.below(3) + top, left = draw.below(4), draw.below(4) + flags = draw.byte() + bgr = channels >= 3 and bool(flags & 1) + + # Gaps between the channels of a pixel: every other channel of a wider array. + channel_step = 2 if channels and flags & 8 else 1 + rows, cols = top + height * row_step + draw.below(4), left + width * col_step + shape = (rows, cols) if channels == 0 else (rows, cols, channels * channel_step) + size = math.prod(shape) + payload = draw.rest() or b"\0" + filled = (payload * (size // len(payload) + 1))[:size] + view = np.frombuffer(filled, dtype=np.uint8).reshape(shape) + view = view[top : top + height * row_step : row_step, left::col_step][:, :width] + if channel_step > 1: + view = view[..., ::channel_step] + if flags & 2: + view = view[::-1] + if flags & 4: + view = view.swapaxes(0, 1) + + order: ph.ChannelOrder = "bgr" if bgr else "rgb" + reference = np.ascontiguousarray(view) + if bgr: + swap = [2, 1, 0, 3][:channels] + reference = np.ascontiguousarray(reference[..., swap]) + expected = _load_outcomes(_pixels_loader(reference)) + sources: tuple[ph.PixelArray, ...] = (view, memoryview(view)) + for source in sources: + got = _load_outcomes(_pixels_loader(source, order)) + assert got == expected, (type(source).__name__, view.strides, got, expected) + + +_COMPARISONS = ( + "hamming_distance", + "l2_distance", + "histogram_intersection", + "radial_similarity", + "similarity", +) +#: The range of each comparison's result, from the docstrings. +_RANGES: dict[str, tuple[float, float]] = { + "histogram_intersection": (0.0, 1.0), + "radial_similarity": (-1.0, 1.0), + "l2_distance": (0.0, math.inf), +} + + +def _digest(draw: Draw) -> ph.Digest | None: + size = 1 + draw.below(ph.PH_DIGEST_MAX_BYTES + 2) + kind = draw.below(8) - 1 + try: + return ph.Digest(draw.take(size), kind) + except ph.InvalidArgumentError: + return None + + +def _check_round_trip(d: ph.Digest) -> None: + assert ph.Digest.from_hex(d.hex()) == d, d + assert ph.Digest(bytes(d), d.kind) == d, d + + +def _check_comparisons(a: ph.Digest, b: ph.Digest) -> None: + for name in _COMPARISONS: + try: + value = getattr(a, name)(b) + except ph.PhashError: + continue + if name == "hamming_distance": + assert 0 <= value <= 8 * len(a), (name, value, a, b) + continue + if name == "similarity": + kind = a.kind if a.kind != ph.DigestKind.UNSPECIFIED else b.kind + low, high = (-1.0, 1.0) if kind == ph.DigestKind.COEFFICIENTS else (0, 1) + else: + low, high = _RANGES[name] + assert low <= value <= high, (name, value, a, b) + + +def digest(data: bytes) -> None: + """Check that digests from any bytes or text round-trip and compare within range.""" + draw = Draw(data) + mode = draw.below(3) + if mode == 0: + text = draw.rest().decode("latin-1") + try: + parsed = ph.Digest.from_hex(text) + except ph.PhashError: + return + _check_round_trip(parsed) + return + if mode == 1: + text = draw.rest().decode("latin-1") + try: + value = ph.hash_from_hex(text) + except ph.PhashError: + return + assert 0 <= value < 2**64, value + assert ph.hash_from_hex(ph.hash_to_hex(value)) == value, value + return + a, b = _digest(draw), _digest(draw) + for d in (a, b): + if d is not None: + _check_round_trip(d) + if a is not None and b is not None: + _check_comparisons(a, b) + + +TARGETS: dict[str, Callable[[bytes], None]] = { + "load": load, + "pixels": pixels, + "digest": digest, +} + + +def seed_dirs(target: str) -> list[Path]: + """Return the directories of a target's corpus.""" + return [CORPUS / target] + ([LIBPHASH_SEEDS] if target == "load" else []) diff --git a/tools/sanitizers.py b/tools/sanitizers.py new file mode 100644 index 0000000..54e44e6 --- /dev/null +++ b/tools/sanitizers.py @@ -0,0 +1,216 @@ +"""Build and run the binding under Clang's sanitizers (Linux). + +The sanitizer jobs of CI run these commands; on Linux with Clang and its runtimes +(``libclang-rt-dev``) they run the same way locally:: + + python tools/sanitizers.py build address # or: thread + python tools/sanitizers.py run address LOG_DIR -- .venv/bin/python -m pytest + python tools/sanitizers.py check LOG_DIR + +``build`` compiles libphash, its decoders and the extension with the sanitizer, into a +build directory of their own, and installs the package into ``.venv`` (editable) with +``uv sync``. ``address`` is AddressSanitizer with UndefinedBehaviorSanitizer and the +coverage hooks of libFuzzer, so that one build serves the tests and the fuzzers; +``thread`` is ThreadSanitizer. + +``run`` starts a command with the sanitizer's runtime preloaded, as the interpreter is +not built with it, and every report written to ``LOG_DIR``, one file per process. +``fuzz`` is the AddressSanitizer build under atheris' copy of the runtime, which carries +libFuzzer and so sees the coverage of the C code. + +``check`` fails on any report of a memory error, undefined behaviour or a data race, +and on a leak of the extension's. LeakSanitizer reports every block nothing points to, +and the interpreter holds blocks it never frees, so a leak counts only when the block +was allocated by the extension itself: the first frame of its stack past the +sanitizer's allocator is in ``libphash._native``, which holds libphash, its decoders +and the CFFI wrappers. A suppression file would not do: a suppression matches any frame +of a stack, and every stack of the extension passes through the interpreter's frames. +The tests ask for the leak check when they are done (``tests/conftest.py``); at exit +it is off, as the interpreter's teardown drops the references to its own blocks. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + +#: Compiler and linker flags of each build. The extension does not link the runtime +#: (no -shared-libasan), so it takes whichever copy ``run`` preloads. That copy is +#: atheris' for the fuzzers, built by an older LLVM than the runner's Clang; its +#: UndefinedBehaviorSanitizer has no handler for the function type check, which Clang +#: applies to C as part of ``undefined``, so the extension would not load with it. +FLAGS = { + "address": ( + "-fsanitize=address,undefined,fuzzer-no-link -fno-sanitize=function" + " -fno-sanitize-recover=all", + "-fsanitize=address,undefined", + ), + "thread": ("-fsanitize=thread", "-fsanitize=thread"), +} + +#: Each frame of a report ends with its module and offset, which ``check`` reads. +FRAME_FORMAT = 'stack_trace_format=" #%n %p %F %L %M"' + + +def build(sanitizer: str) -> None: + """Build the package with a sanitizer and install it into ``.venv``.""" + cflags, ldflags = FLAGS[sanitizer] + env = { + **os.environ, + "CC": "clang", + "CFLAGS": f"{cflags} -fno-omit-frame-pointer -g", + "LDFLAGS": ldflags, + "LIBPHASH_BUILD_DIR": str(PROJECT_ROOT / "build" / f"sanitizer-{sanitizer}"), + } + command = ["uv", "sync", "--locked", "--all-extras", "--inexact"] + command += ["--reinstall-package", "python-libphash"] + if sanitizer == "address": + command += ["--group", "fuzz"] + subprocess.run(command, cwd=PROJECT_ROOT, env=env, check=True) + + +def _clang_runtime(name: str) -> str: + machine = os.uname().machine + path = subprocess.run( + ["clang", f"-print-file-name=libclang_rt.{name}-{machine}.so"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + if not os.path.isabs(path): + sys.exit(f"Clang's {name} runtime is missing; install libclang-rt-dev") + return path + + +def run_environment(sanitizer: str, log_dir: Path) -> dict[str, str]: + """Return the environment that runs a command under a sanitizer.""" + log_dir.mkdir(parents=True, exist_ok=True) + log = f"log_path={log_dir.resolve()}/{sanitizer}" + env = dict(os.environ) + if sanitizer == "thread": + env["LD_PRELOAD"] = _clang_runtime("tsan") + env["TSAN_OPTIONS"] = f"halt_on_error=1:{log}" + return env + if sanitizer == "fuzz": + import atheris # noqa: PLC0415 -- only the fuzzers need it # pyright: ignore[reportMissingModuleSource] + + env["LD_PRELOAD"] = f"{atheris.path()}/asan_with_fuzzer.so" + # libFuzzer checks for leaks after every input, and the interpreter's caches + # would look like one; the tests' run checks for leaks instead. + leaks = "detect_leaks=0" + else: + env["LD_PRELOAD"] = _clang_runtime("asan") + leaks = "detect_leaks=1:leak_check_at_exit=0" + env["ASAN_OPTIONS"] = f"{leaks}:halt_on_error=1:{log}:{FRAME_FORMAT}" + env["UBSAN_OPTIONS"] = "halt_on_error=1:print_stacktrace=1" + # Python objects from the system allocator, where ASan sees each of them. + env["PYTHONMALLOC"] = "malloc" + return env + + +EXTENSION = re.compile(r"^_native\.") +RUNTIME = re.compile(r"^(lib(clang_rt\.|asan|ubsan)|asan_with_fuzzer)") +# The last "(module+0xoffset)" of a frame line. +MODULE = re.compile(r"\(([^()\s]+)\+0x[0-9a-f]+\)(?:\s*\(BuildId: [0-9a-f]+\))?\s*$") +LEAK = re.compile(r"^(Direct|Indirect) leak of (\d+) byte\(s\) in (\d+) object\(s\)") +ERROR = re.compile(r"ERROR: AddressSanitizer|runtime error:|WARNING: ThreadSanitizer") + + +@dataclass(frozen=True) +class Leak: + """One leak record of a LeakSanitizer report.""" + + size: int + frames: tuple[str, ...] + + @property + def origin(self) -> str: + """Return the module of the first frame past the sanitizer's allocator.""" + for frame in self.frames: + m = MODULE.search(frame) + module = m.group(1) if m else "?" + if not RUNTIME.match(module): + return module + return "?" + + +def leaks(text: str) -> list[Leak]: + """Return the leak records of a log.""" + found: list[Leak] = [] + lines = text.splitlines() + for i, line in enumerate(lines): + m = LEAK.match(line) + if not m: + continue + frames: list[str] = [] + for frame in lines[i + 1 :]: + if not frame.lstrip().startswith("#"): + break + frames.append(frame) + found.append(Leak(int(m.group(2)), tuple(frames))) + return found + + +def check(log_dir: Path) -> int: + """Print the findings of every log in a directory; return 1 if any fails.""" + failed = False + out: list[str] = [] + ignored: Counter[str] = Counter() + logs = ( + sorted(p for p in log_dir.iterdir() if p.is_file()) if log_dir.is_dir() else [] + ) + for log in logs: + text = log.read_text(encoding="utf-8", errors="replace") + if ERROR.search(text): + failed = True + out += [f"--- {log.name}: sanitizer report", text] + continue + for leak in leaks(text): + if EXTENSION.match(leak.origin): + failed = True + out += [f"--- {log.name}: leak in the extension", *leak.frames] + else: + ignored[leak.origin] += leak.size + out.append(f"{len(logs)} sanitizer log(s) in {log_dir}") + out += [ + f" not the extension's, ignored: {size} bytes allocated by {module}" + for module, size in ignored.most_common() + ] + sys.stdout.write("\n".join(out) + "\n") + return 1 if failed else 0 + + +def main(argv: list[str] | None = None) -> int: + """Run the subcommand the arguments name.""" + parser = argparse.ArgumentParser(description="Build and run under sanitizers.") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("build").add_argument("sanitizer", choices=FLAGS) + run = sub.add_parser("run") + run.add_argument("sanitizer", choices=[*FLAGS, "fuzz"]) + run.add_argument("log_dir", type=Path) + run.add_argument("argv", nargs=argparse.REMAINDER) + sub.add_parser("check").add_argument("log_dir", type=Path) + args = parser.parse_args(argv) + + if args.command == "build": + build(args.sanitizer) + return 0 + if args.command == "check": + return check(args.log_dir) + command: list[str] = args.argv[1:] if args.argv[:1] == ["--"] else args.argv + if not command: + parser.error("run needs a command after --") + env = run_environment(args.sanitizer, args.log_dir) + return subprocess.run(command, env=env, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/typings/atheris.pyi b/typings/atheris.pyi new file mode 100644 index 0000000..ef2f9eb --- /dev/null +++ b/typings/atheris.pyi @@ -0,0 +1,12 @@ +# The part of atheris the tools use: atheris ships no type information. +from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager + +def Setup( # noqa: N802 -- atheris's name + args: list[str], test_one_input: Callable[[bytes], None] +) -> list[str]: ... +def Fuzz() -> None: ... # noqa: N802 -- atheris's name +def instrument_imports( + include: Iterable[str] | None = None, exclude: Iterable[str] | None = None +) -> AbstractContextManager[None]: ... +def path() -> str: ... diff --git a/uv.lock b/uv.lock index 390f8f1..2051147 100644 --- a/uv.lock +++ b/uv.lock @@ -83,6 +83,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/f4/b54123680025c0b7253117418f023d1b2487f1102552acbdd9d8ee96b622/ast_serialize-0.12.1-cp39-abi3-win_arm64.whl", hash = "sha256:610a41351de68199de9a1434499083b4256c0df7658ec1cfc0a0a7b20b08d317", size = 1136560, upload-time = "2026-10-03T12:24:58.689Z" }, ] +[[package]] +name = "atheris" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/88/fd6ad595dafa9c7ce56dbfcaff0c7244988dac3af86c771166c6516ccf6b/atheris-3.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ec5e11f21a4c197fe91f7aea2b2de88e623c73a21fc07b105ac6329a1588457b", size = 36875908, upload-time = "2026-06-17T00:04:01.104Z" }, + { url = "https://files.pythonhosted.org/packages/4e/18/e19718c384fd7d801d0da7485407daef9af6194b6d8c8818175bec5efec6/atheris-3.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39", size = 36800563, upload-time = "2026-06-17T00:04:04.559Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ff/ae7a5bfe99033e510bea4ed09934e636d93777317a48147369bc0dc2b71f/atheris-3.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011", size = 36772569, upload-time = "2026-06-17T00:04:07.702Z" }, +] + [[package]] name = "basedpyright" version = "1.40.2" @@ -1196,6 +1206,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pycparser" version = "2.23" @@ -1274,17 +1312,24 @@ dev = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pillow" }, + { name = "psutil" }, { name = "pytest" }, { name = "pytest-randomly" }, { name = "pytest-timeout" }, { name = "ruff" }, { name = "setuptools" }, { name = "types-cffi" }, + { name = "types-psutil" }, { name = "types-setuptools" }, { name = "typing-extensions" }, { name = "typos" }, ] +[package.dev-dependencies] +fuzz = [ + { name = "atheris", marker = "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] + [package.metadata] requires-dist = [ { name = "basedpyright", marker = "extra == 'dev'" }, @@ -1295,18 +1340,23 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'" }, { name = "numpy", marker = "extra == 'dev'" }, { name = "pillow", marker = "extra == 'dev'" }, + { name = "psutil", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-randomly", marker = "extra == 'dev'" }, { name = "pytest-timeout", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'" }, { name = "setuptools", marker = "extra == 'dev'" }, { name = "types-cffi", marker = "extra == 'dev'" }, + { name = "types-psutil", marker = "extra == 'dev'" }, { name = "types-setuptools", marker = "extra == 'dev'" }, { name = "typing-extensions", marker = "extra == 'dev'" }, { name = "typos", marker = "extra == 'dev'" }, ] provides-extras = ["dev"] +[package.metadata.requires-dev] +fuzz = [{ name = "atheris", marker = "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = ">=3.1" }] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1614,6 +1664,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/fc/690c243ddac627901f1448f5cb29ca3b5463774a00ac932862c60d7ce5cf/types_cffi-2.1.0.20260827-py3-none-any.whl", hash = "sha256:851b6ffa5b962c577a2330b98ac0b60c56c904d086726276946388f58a0cffaf", size = 20525, upload-time = "2026-08-27T12:07:20.836Z" }, ] +[[package]] +name = "types-psutil" +version = "7.2.2.20260906" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/97/0a/f48b9b0ab5ba8599fd117309e345152bf82c756c72be2ff0f635780c2792/types_psutil-7.2.2.20260906.tar.gz", hash = "sha256:93abf22cf9a62b915f724e433bde702995ac274865425fd4a76d1d9b5828da1a", size = 27396, upload-time = "2026-09-06T06:35:24.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/81a86107486e1c23f5ca21bea1e9e5b18a50754242e7a0b82f41f9ae8384/types_psutil-7.2.2.20260906-py3-none-any.whl", hash = "sha256:db00baf7f96c3f63421c4d3d68d373923094a0dfddf13e922c5c5fbc42488159", size = 33383, upload-time = "2026-09-06T06:35:23.575Z" }, +] + [[package]] name = "types-setuptools" version = "84.0.0.20261006"