diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c7966cc..71fec582 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,6 +208,10 @@ jobs: if-no-files-found: error retention-days: 14 + coverage: + name: coverage + uses: ./.github/workflows/coverage.yml + package: name: native distributions # Qualify packages once, on the same vetted source as the GPU checks. @@ -221,7 +225,7 @@ jobs: ci-required: # PR checks do not satisfy the merge gate before package and GPU qualification. name: ${{ github.event_name == 'pull_request' && 'ci-pr-checks' || 'ci-required' }} - needs: [lint, test-cpu, test-gpu, package, conda] + needs: [lint, test-cpu, test-gpu, coverage, package, conda] if: ${{ always() }} runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -231,11 +235,12 @@ jobs: LINT_RESULT: ${{ needs.lint.result }} CPU_RESULT: ${{ needs.test-cpu.result }} GPU_RESULT: ${{ needs.test-gpu.result }} + COVERAGE_RESULT: ${{ needs.coverage.result }} REQUIRE_FULL_CI: ${{ github.event_name == 'push' && github.repository == 'NVIDIA/cuPhoton' }} PACKAGE_RESULT: ${{ needs.package.result }} CONDA_RESULT: ${{ needs.conda.result }} run: | - for result in "${LINT_RESULT}" "${CPU_RESULT}" "${CONDA_RESULT}"; do + for result in "${LINT_RESULT}" "${CPU_RESULT}" "${COVERAGE_RESULT}" "${CONDA_RESULT}"; do test "${result}" = success || exit 1 done if [ "${REQUIRE_FULL_CI}" = true ]; then diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..58cb13b9 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,278 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +name: coverage + +on: + workflow_call: + inputs: + gpu: + description: Include GPU tests and quickstarts + type: boolean + default: false + workflow_dispatch: + inputs: + gpu: + description: Include GPU tests and quickstarts + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: coverage-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +env: + CUPHOTON_XDR_BUILD_EXT: "0" + PIP_DISABLE_PIP_VERSION_CHECK: "1" + UV_HTTP_TIMEOUT: "300" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + +jobs: + test-cpu: + name: CPU coverage / py3.12 + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + CUPHOTON_XREP_TORCH_DEVICE: cpu + CUDA_VISIBLE_DEVICES: "" + PYTHON_VERSION: "3.12" + steps: + - name: Check GPU runner access + if: inputs.gpu && github.repository != 'NVIDIA/cuPhoton' + run: | + echo "GPU coverage requires the NVIDIA/cuPhoton GPU runner." >&2 + exit 1 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.12.1" + enable-cache: true + - run: uv python install "${PYTHON_VERSION}" + - run: >- + uv sync --locked --python "${PYTHON_VERSION}" + --extra dev --extra torch --extra viz --extra photometry + - run: uv run --no-sync coverage erase + - run: make ci-test-cpu PYTEST="coverage run -m pytest" + - name: Run synthetic CPU quickstarts + run: >- + uv run --locked --extra dev --extra torch --extra viz --extra photometry + coverage run examples/run_quickstarts.py --profile cpu + --output-dir "${RUNNER_TEMP}/cuphoton-quickstart" + - name: Upload CPU coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-data-cpu-py3.12-${{ github.sha }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + retention-days: 14 + + test-gpu: + name: GPU coverage / L40G / py3.12 + # Maintainers must select a trusted revision when enabling GPU coverage. + if: inputs.gpu && github.repository == 'NVIDIA/cuPhoton' + runs-on: linux-amd64-gpu-l40g-latest-1 + timeout-minutes: 20 + concurrency: + group: cuphoton-gpu + queue: max + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Record source and GPU + run: | + git rev-parse HEAD + nvidia-smi + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.12.1" + enable-cache: false + - run: uv python install 3.12 + - run: uv sync --locked --python 3.12 --extra dev --extra gpu --extra viz + - name: Require working CUDA 13 and GPU kernels + run: | + uv run --no-sync python - <<'PY' + import cupy as cp + import torch + + cp.show_config() + assert cp.cuda.runtime.getDeviceCount() >= 1 + assert 13000 <= cp.cuda.runtime.runtimeGetVersion() < 14000 + values = cp.arange(16, dtype=cp.float64) + assert float(cp.sum(values * values).item()) == 1240.0 + assert torch.cuda.is_available() + tensor = torch.from_dlpack(values) + assert tensor.is_cuda + assert float((tensor * tensor).sum().item()) == 1240.0 + print("CuPy and PyTorch CUDA kernels and DLPack passed") + PY + - run: uv run --no-sync coverage erase + - name: Run xFit GPU parity tests + run: >- + uv run --no-sync coverage run -m pytest -q -rs + --junitxml="${RUNNER_TEMP}/xfit-gpu.xml" + tests/xfit/test_models.py::test_cupy_fit_matches_numpy_when_cuda_is_available + tests/xfit/test_models.py::test_cupy_finite_difference_theta_fix_matches_numpy + tests/xfit/test_models.py::test_fit_dipoles_device_matches_portable_result_without_bulk_d2h + tests/xfit/test_solver.py::test_cupy_tall_jacobian_diagnostics_match_numpy + - name: Require all selected tests to execute + run: | + uv run --no-sync python - <<'PY' + import os + from pathlib import Path + import xml.etree.ElementTree as ET + + report = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "xfit-gpu.xml") + cases = report.findall(".//testcase") + assert len(cases) == 6, len(cases) + assert not report.findall(".//skipped"), "GPU tests must not skip" + assert not report.findall(".//failure") + assert not report.findall(".//error") + print("All 6 selected tests passed without skips") + PY + - name: Run synthetic GPU quickstarts + run: >- + uv run --no-sync coverage run examples/run_quickstarts.py + --require-gpu --output-dir "${RUNNER_TEMP}/cuphoton-quickstart" + - name: Upload GPU coverage data + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-data-gpu-l40g-py3.12-${{ github.sha }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + retention-days: 14 + - name: Upload test evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-l40g-${{ github.sha }} + path: | + ${{ runner.temp }}/xfit-gpu.xml + ${{ runner.temp }}/cuphoton-quickstart/**/summary.json + if-no-files-found: error + retention-days: 14 + + test-xdr: + name: Native xDR coverage / L40G / py3.12 + # Native coverage needs a GPU and runs only on vetted revisions. + if: >- + ${{ github.repository == 'NVIDIA/cuPhoton' && + (github.event_name == 'push' || + (github.event_name == 'workflow_dispatch' && inputs.gpu)) }} + runs-on: linux-amd64-gpu-l40g-latest-1 + timeout-minutes: 30 + concurrency: + group: cuphoton-gpu + queue: max + env: + CC: gcc + CXX: g++ + GCOV: gcov + KVIKIO_COMPAT_MODE: "ON" + CUDA_HOME: "" + CUDA_PATH: "" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Record source and GPU + run: | + git rev-parse HEAD + nvidia-smi + - name: Install native build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential pkg-config zlib1g-dev + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.12.1" + enable-cache: false + - run: uv python install 3.12 + - run: uv sync --locked --python 3.12 --extra dev --extra io + - name: Build reentrant CFITSIO + env: + CUPHOTON_XDR_CFITSIO_ROOT: ${{ runner.temp }}/cfitsio + run: bash scripts/wheels/prepare_cfitsio.sh + - name: Measure native xDR coverage + env: + CUPHOTON_XDR_CFITSIO_ROOT: ${{ runner.temp }}/cfitsio + run: make test-xdr-coverage + - name: Upload native xDR coverage + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: xdr-coverage-${{ github.sha }} + path: | + build/xdr-coverage/coverage-native.xml + build/xdr-coverage/coverage-native.json + build/xdr-coverage/junit.xml + build/xdr-coverage/coverage-revision.txt + if-no-files-found: error + retention-days: 14 + + coverage: + name: Coverage report + needs: [test-cpu, test-gpu, test-xdr] + if: >- + ${{ !cancelled() && needs.test-cpu.result == 'success' && + (needs.test-gpu.result == 'success' || + (needs.test-gpu.result == 'skipped' && !inputs.gpu)) && + (needs.test-xdr.result == 'success' || + (needs.test-xdr.result == 'skipped' && + !(github.repository == 'NVIDIA/cuPhoton' && + (github.event_name == 'push' || + (github.event_name == 'workflow_dispatch' && inputs.gpu))))) }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.12.1" + enable-cache: true + - run: uv python install 3.12 + - run: uv sync --locked --extra dev --python 3.12 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: coverage-data-*-${{ github.sha }} + path: coverage-data + merge-multiple: true + - name: Download native xDR coverage + if: needs.test-xdr.result == 'success' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: xdr-coverage-${{ github.sha }} + path: native-coverage + - name: Combine coverage and export Cobertura XML + run: | + uv run --no-sync coverage combine coverage-data + uv run --no-sync coverage xml + uv run --no-sync coverage report + git rev-parse HEAD > coverage-revision.txt + echo "Python GPU coverage requested: ${{ inputs.gpu }}" > coverage-profile.txt + echo "Native xDR coverage: ${{ needs.test-xdr.result }}" >> coverage-profile.txt + - name: Upload coverage report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-${{ github.sha }} + path: | + coverage.xml + coverage-revision.txt + coverage-profile.txt + .coverage + native-coverage/ + include-hidden-files: true + if-no-files-found: error + retention-days: 14 diff --git a/.gitignore b/.gitignore index 54a93fe3..2e82f97a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,12 @@ __pycache__/ .ruff_cache/ .mypy_cache/ .coverage +.coverage.* +/coverage.xml +/coverage-data/ +/native-coverage/ +/coverage-revision.txt +/coverage-profile.txt htmlcov/ build/ dist/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e597343..46e42a84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,6 +106,84 @@ Use `make format` for Python fixes and `make ci-lint` for the complete checks. Validation logs should be clean. If warnings are expected, describe them in the pull request. +### Coverage for static analysis + +Run the CPU suite with line and branch coverage from the repository root: + +```bash +make test-cpu-coverage +``` + +This clears previous coverage data and writes `coverage.xml` in Cobertura +format. Coverage includes Python under `src/cuphoton`, including unexecuted +modules, and excludes the generated `_version.py`. Child Python processes +and multiprocessing workers contribute to the report. Compiled C++/CUDA +code, scripts, and examples are outside this Python package measurement. +No minimum percentage is enforced while establishing a baseline. + +Each CI run collects coverage once in a separate Python 3.12/Linux CPU job, +running the CPU suite and synthetic quickstarts. The regular test matrix +runs without coverage instrumentation. Vetted CI pushes also run one native +xDR coverage job on an L40G runner. CI requires the applicable coverage jobs +to succeed and upload their reports. + +To collect an additional report on a chosen ref, run the `coverage` +workflow manually: + +```bash +gh workflow run coverage.yml --ref +``` + +Maintainers can add `--field gpu=true` to include GPU parity tests and quickstarts +and native xDR coverage on separate L40G jobs. Select a trusted revision before +enabling GPU coverage. These jobs share the regular CI GPU queue. + +The report job combines the data and uploads `coverage-`, +containing `coverage.xml`, the combined `.coverage` database, +`coverage-revision.txt`, and `coverage-profile.txt`. The profile records +whether Python GPU collection was requested and whether native xDR coverage +ran. When collected, `native-coverage/` contains the native Cobertura XML, +JSON, test results, and revision. Artifacts are retained for 14 days. + +Download the artifact from the desired CI run before scanning that revision: + +```bash +gh run download --name coverage- --dir . +test "$(git rev-parse HEAD)" = "$(cat coverage-revision.txt)" && \ + echo "Coverage matches this checkout" +``` + +Run this in a checkout of the recorded revision. Pull-request CI tests a +merge commit, so use the revision recorded in the artifact. +The XML report uses relative source paths so static-analysis tools can +import it from another machine's checkout. Configure the tool's Python +coverage input to use `coverage.xml` and its C++ Cobertura input to use +`native-coverage/coverage-native.xml` when present. + +#### Native xDR coverage + +On Linux with GCC, matching `gcov`, a CUDA 13-capable GPU, uv, and reentrant +CFITSIO available, run: + +```bash +CC=gcc CXX=g++ GCOV=gcov KVIKIO_COMPAT_MODE=ON \ + CUPHOTON_XDR_CFITSIO_ROOT=/path/to/cfitsio \ + CUDA_HOME= CUDA_PATH= make test-xdr-coverage +``` + +The target installs the pinned native build dependencies and uses their CUDA +SDK headers. It sets `CUPHOTON_XDR_BUILD_EXT=1` and +`CUPHOTON_XDR_COVERAGE=1` to compile xDR with GCC line/branch counters and +atomic counter updates. It builds into `build/xdr-coverage/`, runs native FITS +read, decompression, and buffer-ownership tests, and exports +`coverage-native.xml` and `coverage-native.json` there. The run fails if tests +skip or any xDR C++ source file has no measured execution. Normal builds use +the existing optimized compiler flags. + +This measures xDR's host C++ code. CUDA device kernels and third-party +libraries are outside the report. CI uses KvikIO compatibility-mode reads; +these tests do not establish GPUDirect Storage coverage. + ### Package and GPU CI Pull requests first run lint, CPU, and conda checks on GitHub-hosted runners. diff --git a/Makefile b/Makefile index e60d5f80..bd813385 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ GPU_EXTRAS = --extra dev --extra gpu --extra viz CORE_EXTRAS = --extra dev --extra photometry VIZ_EXTRAS = --extra dev --extra viz --extra photometry UV_RUN = uv run --locked +PYTEST = pytest + +.PHONY: test-cpu-coverage test-xdr-coverage coverage-report sync: uv sync --locked $(CPU_EXTRAS) @@ -40,7 +43,20 @@ test: $(UV_RUN) $(CPU_EXTRAS) pytest test-cpu: - CUDA_VISIBLE_DEVICES= CUPHOTON_XREP_TORCH_DEVICE=cpu $(UV_RUN) $(CPU_EXTRAS) pytest -rs + CUDA_VISIBLE_DEVICES= CUPHOTON_XREP_TORCH_DEVICE=cpu $(UV_RUN) $(CPU_EXTRAS) $(PYTEST) -rs + +test-cpu-coverage: + $(UV_RUN) --extra dev coverage erase + $(MAKE) test-cpu PYTEST="coverage run -m pytest" + $(MAKE) coverage-report + +coverage-report: + $(UV_RUN) --extra dev coverage combine + $(UV_RUN) --extra dev coverage xml + $(UV_RUN) --extra dev coverage report + +test-xdr-coverage: + $(UV_RUN) --extra dev --extra io bash scripts/coverage/xdr.sh test-core: $(UV_RUN) $(CORE_EXTRAS) pytest tests/core diff --git a/pyproject.toml b/pyproject.toml index f9f35cef..50fd484a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ dragon = [ "dragonhpc>=0.14.2,<0.15", ] dev = [ + "coverage>=7.10.6,<8", "mypy==1.19.1", "setuptools>=83.0.0", "pre-commit>=4.0", @@ -178,6 +179,18 @@ markers = [ pythonpath = ["src"] testpaths = ["tests"] +[tool.coverage.run] +branch = true +source = ["src/cuphoton"] +relative_files = true +parallel = true +concurrency = ["multiprocessing", "thread"] +patch = ["subprocess"] +omit = ["src/cuphoton/_version.py"] + +[tool.coverage.xml] +output = "coverage.xml" + [tool.cibuildwheel] build = "cp312-manylinux_* cp313-manylinux_* cp314-manylinux_*" build-frontend = { name = "build[uv]", args = ["--no-isolation"] } diff --git a/scripts/coverage/xdr.sh b/scripts/coverage/xdr.sh new file mode 100755 index 00000000..172fae39 --- /dev/null +++ b/scripts/coverage/xdr.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# Run through make test-xdr-coverage with reentrant CFITSIO available. +set -euo pipefail +cd "$(dirname "$0")/../.." + +bash scripts/wheels/install_build_dependencies.sh + +coverage_build=build/xdr-coverage +rm -rf "$coverage_build" +mkdir -p "$coverage_build" + +CUPHOTON_XDR_BUILD_EXT=1 CUPHOTON_XDR_COVERAGE=1 python setup.py \ + build_py --build-lib "$coverage_build/lib" \ + build_ext --force --build-temp "$coverage_build/obj" \ + --build-lib "$coverage_build/lib" + +export PYTHONPATH="$PWD/$coverage_build/lib" +python - <<'PY' +from pathlib import Path +from cuphoton.xdr import nvcomp_batch + +ext = nvcomp_batch._try_get_cpp_ext() +assert ext is not None, "Instrumented xDR extension did not load" +assert Path(ext.__file__).resolve().is_relative_to( + Path("build/xdr-coverage/lib").resolve() +), ext.__file__ +print(f"Testing instrumented extension: {ext.__file__}") +PY + +python -m pytest -q -rs -o "pythonpath=$PYTHONPATH" \ + --junitxml="$coverage_build/junit.xml" \ + tests/xdr/test_native.py tests/xdr/test_deflate_gpu.py + +python - <<'PY' +from xml.etree import ElementTree as ET + +report = ET.parse("build/xdr-coverage/junit.xml") +assert report.findall(".//testcase"), "No native tests executed" +assert not report.findall(".//skipped"), "Native tests must not skip" +assert not report.findall(".//failure") +assert not report.findall(".//error") +PY + +uv tool run --from gcovr==8.6 gcovr \ + --root . --filter 'src/cuphoton/xdr/src/' \ + --gcov-executable "${GCOV:-gcov}" \ + --xml "$coverage_build/coverage-native.xml" \ + --json "$coverage_build/coverage-native.json" \ + --print-summary "$coverage_build/obj" + +python - <<'PY' +from pathlib import Path +from xml.etree import ElementTree as ET + +report_path = Path("build/xdr-coverage/coverage-native.xml") +report = ET.parse(report_path) +# gcovr emits an absolute checkout root; make the report portable. +for source in report.findall("sources/source"): + source.text = "." +measured = { + item.attrib["filename"] + for item in report.findall(".//class") + if any(int(line.attrib["hits"]) > 0 for line in item.findall("lines/line")) +} +expected = {str(path) for path in Path("src/cuphoton/xdr/src").glob("*.cpp")} +assert expected and expected <= measured, (expected, measured) +report.write(report_path, encoding="utf-8", xml_declaration=True) +print("Measured execution in every xDR C++ translation unit") +PY + +git rev-parse HEAD > "$coverage_build/coverage-revision.txt" diff --git a/src/cuphoton/xdr/setup_package.py b/src/cuphoton/xdr/setup_package.py index 2e0ae5c5..726ab64c 100644 --- a/src/cuphoton/xdr/setup_package.py +++ b/src/cuphoton/xdr/setup_package.py @@ -313,6 +313,7 @@ def _find_cfitsio_paths(): def get_extensions(): + coverage = os.environ.get("CUPHOTON_XDR_COVERAGE", "0") == "1" cuda_include, cuda_lib = _find_cuda_toolkit() cufile_include = _find_cufile_include(cuda_include) package_paths = _find_gpu_package_paths() @@ -353,13 +354,18 @@ def get_extensions(): ("KVIKIO_CUFILE_VERSION_API_FOUND", "1"), ], extra_compile_args=[ - "-O3", + *( + ["-O0", "-g", "--coverage", "-fprofile-update=atomic"] + if coverage + else ["-O3"] + ), "-Wall", "-std=c++17", "-fvisibility=hidden", "-pthread", ], extra_link_args=[ + *(["--coverage"] if coverage else []), "-pthread", "-l:libnvcomp.so.5", "-l:libkvikio.so", diff --git a/tests/xdr/test_native.py b/tests/xdr/test_native.py new file mode 100644 index 00000000..7a1ad8ae --- /dev/null +++ b/tests/xdr/test_native.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Native FITS planning, device reads, and pooled allocation ownership.""" + +from __future__ import annotations + +import gc + +import numpy as np +import pytest +from astropy.io import fits + +from cuphoton.xdr import batch_to_device, nvcomp_batch + + +@pytest.fixture +def native(): + cp = pytest.importorskip("cupy") + try: + if cp.cuda.runtime.getDeviceCount() == 0: + pytest.skip("CUDA device unavailable") + except cp.cuda.runtime.CUDARuntimeError: + pytest.skip("CUDA runtime unavailable") + ext = nvcomp_batch._try_get_cpp_ext() + if ext is None or not all( + hasattr(ext, name) + for name in ( + "plan_native_files", + "NativeBatchBuilder", + "acquire_native_device_buffer", + "batch_deflate_decompress_pooled", + ) + ): + pytest.skip("native xDR extension unavailable") + return cp, ext + + +def test_native_batch_reads_multiple_image_hdus(native, tmp_path): + cp, _ = native + paths, integer_images, float_images = [], [], [] + for index in range(3): + integers = np.arange(63, dtype="i2").reshape(7, 9) - 31 + index + floats = integers.astype("f4") * 0.125 + path = tmp_path / f"images-{index}.fits" + fits.HDUList( + [fits.PrimaryHDU(integers), fits.ImageHDU(floats)] + ).writeto(path) + paths.append(path) + integer_images.append(integers) + float_images.append(floats) + + result = batch_to_device( + paths, + hdu_indices=(0, 1), + native_batcher=True, + decode_batch_files=2, + native_read_threads=2, + native_plan_threads=2, + stream=cp.cuda.Stream(non_blocking=True), + ) + + for actual, expected in zip( + result, (integer_images, float_images), strict=True + ): + np.testing.assert_array_equal(cp.asnumpy(actual), np.stack(expected)) + + +@pytest.mark.parametrize("compression", ["GZIP_1", "GZIP_2"]) +def test_native_batch_reads_compressed_section(native, tmp_path, compression): + cp, _ = native + paths, expected = [], [] + section = (slice(2, 11), slice(3, 13)) + for index in range(3): + pixels = np.arange(13 * 15, dtype="i4").reshape(13, 15) * 7 - index + path = tmp_path / f"compressed-{index}.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.CompImageHDU( + pixels, + compression_type=compression, + tile_shape=(4, 6), + ), + ] + ).writeto(path) + paths.append(path) + expected.append(pixels[section]) + + (actual,) = batch_to_device( + paths, + section=section, + native_batcher=True, + decode_batch_files=2, + native_read_threads=2, + native_plan_threads=2, + stream=cp.cuda.Stream(non_blocking=True), + ) + + np.testing.assert_array_equal(cp.asnumpy(actual), np.stack(expected)) + + +def test_native_pool_retains_views_and_reuses_released_buffer(native): + cp, ext = native + device_id = int(cp.cuda.Device().id) + ext.clear_native_device_pool(device_id) + try: + first = nvcomp_batch._native_device_empty_uint8(1024, ext=ext) + first[:] = cp.arange(1024, dtype=cp.uint8) + pointer = first.data.ptr + view = first[13:29] + del first + ext.clear_native_device_pool(device_id) + + second = nvcomp_batch._native_device_empty_uint8(1024, ext=ext) + assert second.data.ptr != pointer + second.fill(0) + np.testing.assert_array_equal( + cp.asnumpy(view), np.arange(13, 29, dtype=np.uint8) + ) + cp.cuda.get_current_stream().synchronize() + del second + ext.clear_native_device_pool(device_id) + del view + gc.collect() + + reused = nvcomp_batch._native_device_empty_uint8(512, ext=ext) + assert reused.data.ptr == pointer + assert reused.nbytes == 512 + del reused + finally: + cp.cuda.get_current_stream().synchronize() + ext.clear_native_device_pool(device_id) diff --git a/uv.lock b/uv.lock index 1c6b4a02..78c6c5ad 100644 --- a/uv.lock +++ b/uv.lock @@ -206,6 +206,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, ] +[[package]] +name = "coverage" +version = "7.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/7d/8f3b6dc920e3fc6732f7678785a2091db439f186afbec30dbf2214d9b1f7/coverage-7.16.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:724bd0f1e81856b35e59fc98cf7b4e544a3cb662e4e0864dca73d4326ee9d808", size = 255832, upload-time = "2026-09-27T12:26:19.799Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/6c45f15be4eca4ac1062c6a55a323286494c99726a7e58951fe85967ac08/coverage-7.16.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5375ebd99038021b35e99dc88255022912c06565d316212f4a576e4b08d30f5d", size = 258571, upload-time = "2026-09-27T12:26:21.199Z" }, + { url = "https://files.pythonhosted.org/packages/34/fb/b54cbeba3ad89082c2e441278681859e538322cc34b84b2af7ebff00080f/coverage-7.16.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a076277ca9f5750cc230f0f578ebd2620cec60255b25707361699fef6fb465c", size = 259680, upload-time = "2026-09-27T12:26:22.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a2/0dc65ec3d61930e1e4c2e371763b15eb4290896eb343a12d5d3091308116/coverage-7.16.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:58d4a54c6ea672afef66d49be922a2c69826c5ae1a42a9cd94f0c9c2bacdf800", size = 261945, upload-time = "2026-09-27T12:26:24.336Z" }, + { url = "https://files.pythonhosted.org/packages/d6/93/5fad7a61f2c14e08e98946fc31c1c7ffc1195061bf3fdc351db3be77a863/coverage-7.16.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0dcbcfcc059117284c603ff8cb61a65872512882f84a8cf0339241f7f7c2f148", size = 256191, upload-time = "2026-09-27T12:26:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/2d/47/74e5de9227b939ece9f64e729645ddc4296bea10dbfa98721c1333c8be2e/coverage-7.16.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afdf43b72ef3876c1fe66423b91466e37877c9e81e8cec70542b7e8525b9d1b7", size = 257610, upload-time = "2026-09-27T12:26:27.35Z" }, + { url = "https://files.pythonhosted.org/packages/13/fe/2cf28d40b43645d1b72388fe3ee7f7c747533a6a9557bb8c24a7ae74fe1a/coverage-7.16.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9acc7f7ec4a1b5f89bd929fde5b8a714f6fafdc6cc18725413d510aa082b47ad", size = 255757, upload-time = "2026-09-27T12:26:28.949Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3d/7c149fd99fc8bbc39c80db5e688d1d39fd040be2ecb78b8335a51a55b9c0/coverage-7.16.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:80d3f7b48d43ee8fc5e8707a8adb43d743a5a1a85256c25a24f9d6d0e2238fa6", size = 259822, upload-time = "2026-09-27T12:26:30.515Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3f/b283fce09d5995e227bd8e513358dd7471bedc0f78abc85a925ebdb0a2f6/coverage-7.16.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:126d1af8804d7224421fe991ff65d3ce649081560df7a98b1a5ffff07f9923bd", size = 255325, upload-time = "2026-09-27T12:26:32.037Z" }, + { url = "https://files.pythonhosted.org/packages/bf/91/f3325edf0c4223fb1fe1532b8dbef2a1d2f729459a9a7d1a44d073bae534/coverage-7.16.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c19cd6d025c1673f22afcd22c7df8a662d779e05d8e3fa6820c22afb895b0206", size = 257191, upload-time = "2026-09-27T12:26:33.525Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, + { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, + { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/963ff22d3fd27117da3b8cc442f5bdc91196f783321e1a8ff0ec43476772/coverage-7.16.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b", size = 256112, upload-time = "2026-09-27T12:27:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d4/a306940c81c6ae759e82fff27d20b7fdc6896e422b821f51313cce212b6c/coverage-7.16.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03", size = 258727, upload-time = "2026-09-27T12:27:09.927Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a3/d3d99d93b02517087aa05bc0cf2d04d372956b849e5443e059079901429b/coverage-7.16.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda", size = 259909, upload-time = "2026-09-27T12:27:11.55Z" }, + { url = "https://files.pythonhosted.org/packages/08/44/39dd599181726758dd185ae4dc0c0ab3aeabf7ca70e68e145060feeaaa16/coverage-7.16.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c", size = 262481, upload-time = "2026-09-27T12:27:13.17Z" }, + { url = "https://files.pythonhosted.org/packages/99/e8/91ee43f6ded411460c359d7e1aebde4d6fd8f00a2e5394182d9d212eb23c/coverage-7.16.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9", size = 256012, upload-time = "2026-09-27T12:27:14.91Z" }, + { url = "https://files.pythonhosted.org/packages/11/8c/e9499ddc33197bd7eabcb1118ca81756fc874457b324e2b479a4804b2ad2/coverage-7.16.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4", size = 257898, upload-time = "2026-09-27T12:27:16.588Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6e/c081cb5991a0afba99f9c4ad6c74a5fce9513a38ddc64e3e6680c6fed9af/coverage-7.16.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90", size = 255938, upload-time = "2026-09-27T12:27:18.19Z" }, + { url = "https://files.pythonhosted.org/packages/b2/42/1c3d819e8f9b6eb01c2fe90874d67a8882adb9507e0bbb09361ed131ea89/coverage-7.16.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95", size = 260368, upload-time = "2026-09-27T12:27:19.903Z" }, + { url = "https://files.pythonhosted.org/packages/19/4f/d70eac07901fd587b6ab05e659b52afe13959992aa5113bf6cce059cc572/coverage-7.16.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8", size = 255620, upload-time = "2026-09-27T12:27:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/34/5e/6d87af88317d3d9a9b18a9ca1bc1673eb516917f296e579d0d4a55cb3490/coverage-7.16.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6", size = 257521, upload-time = "2026-09-27T12:27:23.358Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c1/980681cd7b33eb66ac835044116ef0a92e11fcc7bdd866cc89d10b1130b9/coverage-7.16.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3", size = 265049, upload-time = "2026-09-27T12:27:34.085Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e3/87679875c33bb2191f0f05544a1cc9adcc940fe0c35443a10f2df753dde5/coverage-7.16.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831", size = 267717, upload-time = "2026-09-27T12:27:36.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/64/5d372776d6eb523d4e93bafba2253f96984e3b18261c4cc56a50863c6d0d/coverage-7.16.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf", size = 270047, upload-time = "2026-09-27T12:27:37.96Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/44082ff0cbf9f97d0043f57970a71204097ec7ba606361a9fd2065393669/coverage-7.16.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681", size = 271360, upload-time = "2026-09-27T12:27:39.766Z" }, + { url = "https://files.pythonhosted.org/packages/b8/17/9a215efe25b5e0ecc87c89dbe525c4a87d14d87c8c0c7316ef140a5f6f3e/coverage-7.16.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0", size = 264736, upload-time = "2026-09-27T12:27:42.072Z" }, + { url = "https://files.pythonhosted.org/packages/a2/da/7f0a31af8e448107d4d32844bd684757f51ea907bc0c68c8fd537b2123ff/coverage-7.16.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb", size = 267600, upload-time = "2026-09-27T12:27:43.85Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/3bfecbd3366b775bacdcb3330394d356cf384b5d8f5b2146ac4b14b252b5/coverage-7.16.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7", size = 264767, upload-time = "2026-09-27T12:27:45.768Z" }, + { url = "https://files.pythonhosted.org/packages/b8/3f/5d62163732d87e4a0c4710a0eab30f0fd6a2d480112abe2029f014fe8c9d/coverage-7.16.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64", size = 269026, upload-time = "2026-09-27T12:27:47.787Z" }, + { url = "https://files.pythonhosted.org/packages/49/4d/8e4579f225426535085a9be371cc75e3b026d058d679b80affbdfb4c3ef0/coverage-7.16.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20", size = 264147, upload-time = "2026-09-27T12:27:49.681Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/ef1f77e2c3f7bb03c2b13b9a2006f88700fdd75535ef158d70049f425c1c/coverage-7.16.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce", size = 266376, upload-time = "2026-09-27T12:27:51.551Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, +] + [[package]] name = "cryptography" version = "50.0.1" @@ -388,6 +437,7 @@ cutile = [ { name = "cupy-cuda13x", extra = ["ctk"] }, ] dev = [ + { name = "coverage" }, { name = "mypy" }, { name = "pre-commit" }, { name = "pytest" }, @@ -438,6 +488,7 @@ viz = [ requires-dist = [ { name = "astropy", specifier = ">=6.1.4" }, { name = "bokeh", marker = "extra == 'viz'", specifier = ">=3.9" }, + { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.10.6,<8" }, { name = "cuda-tile", marker = "sys_platform == 'linux' and extra == 'cutile'", specifier = ">=1.6,<2" }, { name = "cuda-toolkit", extras = ["cufile"], marker = "sys_platform == 'linux' and extra == 'io'", specifier = ">=13,<14" }, { name = "cuphoton", extras = ["io", "photometry"], marker = "extra == 'gpu'" },