From ffc64308fc0ef10a0738046a20f90e0458f54513 Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:50:58 -0700 Subject: [PATCH 1/6] Fix remapping correctness and validate release installation routes --- .github/dependabot.yml | 12 + .github/workflows/build.yml | 6 +- .github/workflows/ci.yml | 14 +- .github/workflows/docker.yml | 63 ++- .github/workflows/docs.yml | 38 +- .github/workflows/ecosystem.yml | 2 +- .github/workflows/release.yml | 443 ++++++++++++++++-- .github/workflows/security.yml | 16 +- .readthedocs.yaml | 1 + .zenodo.json | 27 +- CHANGELOG.md | 12 + CITATION.cff | 2 +- CONTRIBUTING.md | 10 +- Dockerfile | 21 +- Makefile | 21 +- README.md | 13 +- Singularity.def | 28 +- conda-lock.yml | 54 +-- docs/requirements.txt | 7 +- docs/source/api/counting.rst | 13 + docs/source/development.rst | 22 +- docs/source/faq.rst | 2 +- docs/source/installation.rst | 25 +- environment.lock.yml | 8 +- environment.yml | 8 +- galaxy/tools/wasp2/macros.xml | 5 +- galaxy/tools/wasp2/test-data/mapping.bam | Bin 0 -> 250 bytes galaxy/tools/wasp2/test-data/mapping.bam.bai | Bin 0 -> 96 bytes galaxy/tools/wasp2/test-data/mapping.vcf | 6 + .../wasp2/test-data/mapping_remapped.bam | Bin 0 -> 274 bytes .../wasp2/test-data/mapping_remapped.bam.bai | Bin 0 -> 96 bytes galaxy/tools/wasp2/wasp2_count_variants.xml | 13 + galaxy/tools/wasp2/wasp2_filter_remapped.xml | 15 +- galaxy/tools/wasp2/wasp2_find_imbalance.xml | 2 + galaxy/tools/wasp2/wasp2_make_reads.xml | 13 +- .../wasp2_count_variants/environment.yml | 1 + .../wasp2_filter_remapped/environment.yml | 1 + .../wasp2_find_imbalance/environment.yml | 1 + .../local/wasp2_make_reads/environment.yml | 1 + pipelines/nf-atacseq/nextflow.config | 1 + .../modules/wasp2/analyze/environment.yml | 21 +- .../modules/wasp2/count/environment.yml | 24 +- .../nf-modules/modules/wasp2/environment.yml | 29 +- .../modules/wasp2/map/environment.yml | 25 +- .../modules/wasp2/ml_output/environment.yml | 12 +- pipelines/nf-modules/nextflow.config | 1 + .../modules/local/aggregate_counts/main.nf | 2 +- .../modules/local/mae_detect/main.nf | 2 +- .../modules/local/merge_counts/main.nf | 2 +- pipelines/nf-outrider/nextflow.config | 1 + pipelines/nf-rnaseq/nextflow.config | 1 + pipelines/nf-scatac/nextflow.config | 1 + pixi.toml | 26 +- pyproject.toml | 17 +- requirements.txt | 7 +- rust/src/bam_remapper.rs | 146 +++++- rust/src/unified_pipeline.rs | 21 +- scripts/check_sbom.py | 23 + scripts/compare_archive_analysis.py | 5 +- scripts/container_smoke_test.sh | 281 ++++++++++- scripts/create_sanity_data.sh | 64 +-- scripts/prepare_pypi_upload.py | 62 +++ scripts/release_version.py | 29 +- scripts/smoke_test.sh | 157 +------ src/counting/count_alleles.py | 3 + src/wasp2/io/pgen_source.py | 73 ++- src/wasp2/io/variant_source.py | 12 +- tests/io/test_pgen_source.py | 41 ++ tests/regression/README.md | 181 +------ tests/regression/mapping_fixtures.py | 84 ++++ tests/regression/test_pipeline_regression.py | 80 +--- .../test_quickbench_indel_parity.py | 20 +- .../test_quickbench_indel_trim_invariants.py | 18 +- .../regression/test_quickbench_snv_parity.py | 24 +- tests/sanity/conftest.py | 52 +- tests/shared_data/README.md | 2 +- tests/test_release_packaging.py | 104 ++++ tutorials/atac_seq_workflow.ipynb | 142 +++--- tutorials/performance_optimization.ipynb | 94 ++-- tutorials/quickstart_counting.ipynb | 12 +- tutorials/quickstart_mapping.ipynb | 97 ++-- tutorials/rna_seq_workflow.ipynb | 68 +-- tutorials/single_cell_workflow.ipynb | 23 +- 83 files changed, 1955 insertions(+), 1061 deletions(-) create mode 100644 galaxy/tools/wasp2/test-data/mapping.bam create mode 100644 galaxy/tools/wasp2/test-data/mapping.bam.bai create mode 100644 galaxy/tools/wasp2/test-data/mapping.vcf create mode 100644 galaxy/tools/wasp2/test-data/mapping_remapped.bam create mode 100644 galaxy/tools/wasp2/test-data/mapping_remapped.bam.bai create mode 100644 scripts/check_sbom.py create mode 100644 scripts/prepare_pypi_upload.py create mode 100644 tests/io/test_pgen_source.py create mode 100644 tests/regression/mapping_fixtures.py create mode 100644 tests/test_release_packaging.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 675d9951..7f09b442 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -59,3 +59,15 @@ updates: - "rust" commit-message: prefix: "deps(rust)" + + # Keep the tested base-image digests current through normal review. + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "monthly" + groups: + containers: + patterns: ["*"] + labels: ["dependencies", "containers"] + commit-message: + prefix: "deps(docker)" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5de67339..b5d955b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,12 @@ jobs: zlib1g-dev \ libssl-dev \ pkg-config \ - libclang-dev + libclang-dev samtools bcftools bedtools bwa - name: Install system dependencies (macOS) if: runner.os == 'macOS' run: | - brew install htslib xz bzip2 openssl@3 pkg-config llvm@19 + brew install htslib xz bzip2 openssl@3 pkg-config llvm@19 samtools bcftools bedtools bwa - name: Set macOS library paths if: runner.os == 'macOS' @@ -93,6 +93,8 @@ jobs: cd /tmp /tmp/wasp2-wheel/bin/python -c "import wasp2_rust; print(wasp2_rust.__file__)" /tmp/wasp2-wheel/bin/wasp2-count count-cohort --help + export PATH="/tmp/wasp2-wheel/bin:$PATH" + bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" - name: Upload wheel artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66579e85..1993ab0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,9 @@ jobs: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] + include: + - python-version: "3.11" + minimum-dependencies: true steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -82,7 +85,7 @@ jobs: libclang-dev \ samtools \ bcftools \ - bedtools + bedtools bwa - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 @@ -109,7 +112,7 @@ jobs: source .venv/bin/activate uv pip install maturin maturin develop --release - uv pip install -e ".[dev]" --no-build-isolation + uv pip install -e ".[dev,plink,cyvcf2,export]" - name: Verify Rust extension loads run: | @@ -128,6 +131,13 @@ jobs: --ignore=tests/benchmarks \ --ignore=tests/sanity + - name: Check minimum supported scientific dependencies + if: matrix.minimum-dependencies == true + run: | + uv pip install --python .venv/bin/python "numpy==1.26.4" "pandas==1.5.3" "polars==1.0.0" "scipy==1.13.1" + source .venv/bin/activate + pytest tests/counting tests/analysis tests/test_rust_python_counting_parity.py -q --tb=short + - name: Upload Python test results if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6674dd5f..73ee3f3f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,10 +3,10 @@ name: Container CI on: push: branches: [master, dev] - paths: ["Dockerfile", "rust/**", "src/**", "pyproject.toml", "scripts/container_smoke_test.sh", ".github/workflows/docker.yml"] + paths: ["Dockerfile", ".dockerignore", "tests/shared_data/**", "rust/**", "src/**", "pyproject.toml", "scripts/container_smoke_test.sh", "scripts/check_sbom.py", ".github/workflows/docker.yml"] pull_request: branches: [master, dev] - paths: ["Dockerfile", "rust/**", "src/**", "pyproject.toml", "scripts/container_smoke_test.sh", ".github/workflows/docker.yml"] + paths: ["Dockerfile", ".dockerignore", "tests/shared_data/**", "rust/**", "src/**", "pyproject.toml", "scripts/container_smoke_test.sh", "scripts/check_sbom.py", ".github/workflows/docker.yml"] workflow_dispatch: concurrency: @@ -51,8 +51,8 @@ jobs: build-args: | VERSION=${{ steps.version.outputs.value }} REVISION=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=container-${{ strategy.job-index }} + cache-to: type=gha,mode=max,scope=container-${{ strategy.job-index }} - name: Smoke-test image env: PLATFORM: ${{ matrix.platform }} @@ -62,6 +62,61 @@ jobs: test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' wasp2:ci)" = "$EXPECTED_VERSION" test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' wasp2:ci)" = "$EXPECTED_REVISION" docker run --rm --platform "$PLATFORM" -e EXPECTED_WASP2_VERSION="$EXPECTED_VERSION" wasp2:ci /opt/wasp2/scripts/container_smoke_test.sh + - name: Build dependency image for scanning + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + with: + context: . + target: rust-builder + load: true + platforms: ${{ matrix.platform }} + tags: wasp2:builder-scan + cache-from: type=gha,scope=container-${{ strategy.job-index }} + - name: Scan build dependency vulnerabilities + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 + with: + image-ref: wasp2:builder-scan + version: v0.76.0 + format: json + output: builder-vulnerabilities.json + severity: CRITICAL,HIGH + exit-code: '1' + - name: Preserve build dependency scan + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: builder-scan-${{ strategy.job-index }} + path: builder-vulnerabilities.json + if-no-files-found: warn + - name: Inventory runtime image + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + with: + image: wasp2:ci + format: cyclonedx-json + output-file: container-sbom.cdx.json + upload-artifact: false + - name: Verify image inventory + run: python3 scripts/check_sbom.py container-sbom.cdx.json wasp2 numpy scipy samtools bcftools bedtools + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: container-sbom-${{ strategy.job-index }} + path: container-sbom.cdx.json + if-no-files-found: error + - name: Scan runtime image vulnerabilities + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0 + with: + image-ref: wasp2:ci + format: json + output: container-vulnerabilities.json + version: v0.76.0 + severity: CRITICAL,HIGH + exit-code: '1' + - name: Preserve container scan + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: container-scan-${{ strategy.job-index }} + path: container-vulnerabilities.json + if-no-files-found: error required: name: Required / Container diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9279b767..2a94e73c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -33,7 +33,7 @@ jobs: uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-docs-${{ hashFiles('pyproject.toml') }} + key: ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }} restore-keys: ${{ runner.os }}-pip-docs- - name: Install pandoc @@ -44,29 +44,25 @@ jobs: - name: Install documentation dependencies run: | pip install --upgrade pip - # Install docs extras directly (avoids building the Rust extension) - pip install \ - "sphinx>=7.0" \ - "pydata-sphinx-theme>=0.15" \ - "sphinx-autodoc-typehints>=1.25" \ - "nbsphinx>=0.9" \ - "myst-parser>=2.0" \ - "sphinx-copybutton>=0.5" \ - "sphinx-design>=0.5" \ - "ipython>=8.0" \ - "tomli>=2.0; python_version < '3.11'" - # Install core Python deps needed by autodoc (Rust ext is mocked in conf.py) - pip install \ - "numpy>=1.21" \ - "pandas>=1.5,<3.0" \ - "polars>=0.19" \ - "scipy>=1.10" \ - "typer>=0.12" \ - "rich>=13.0" + pip install -r docs/requirements.txt - name: Build Sphinx documentation working-directory: docs - run: make html + run: make html SPHINXOPTS="-W --keep-going" + + - name: Verify API documentation was imported + run: | + python - <<'PYDOCS' + from pathlib import Path + for page, anchors in { + "counting": ["counting.count_alleles.make_count_df", "counting.run_counting_cohort.run_count_cohort"], + "analysis": ["analysis.as_analysis.get_imbalance"], + "mapping": ["mapping.__main__.make_reads"], + }.items(): + html = Path(f"docs/build/html/api/{page}.html").read_text() + for anchor in anchors: + assert f'id="{anchor}"' in html, anchor + PYDOCS - name: Upload artifact if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml index 9edb4763..6259b32b 100644 --- a/.github/workflows/ecosystem.yml +++ b/.github/workflows/ecosystem.yml @@ -80,7 +80,7 @@ jobs: python-version: "3.11" - name: Install single-cell stub dependencies if: matrix.pipeline == 'nf-scatac' - run: python -m pip install "pandas>=1.5,<3.0" "scipy>=1.10" "anndata>=0.8,<0.12" + run: python -m pip install "pandas>=1.5,<3.0" "scipy>=1.13.1" "anndata>=0.8,<0.12" - name: Run pipeline compatibility test run: | test_file=tests/main.nf.test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03f6fbbb..143eac2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,9 @@ on: tags: - "v*" workflow_dispatch: + pull_request: + branches: [master, dev] + paths: ["pyproject.toml", "rust/**", "src/**", "Dockerfile", ".dockerignore", "Singularity.def", "pixi.toml", "environment*.yml", "scripts/**", "tests/shared_data/**", ".github/workflows/release.yml"] concurrency: group: release-${{ github.ref }} @@ -32,6 +35,9 @@ jobs: python-version: "3.12" - name: Validate synchronized metadata run: python scripts/release_version.py --check + - name: Require finalized publication metadata + if: startsWith(github.ref, 'refs/tags/v') + run: python scripts/release_version.py --check --release - name: Read version id: version run: | @@ -40,7 +46,7 @@ jobs: echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Validate manual dry-run source if: github.event_name == 'workflow_dispatch' - run: test "$GITHUB_REF" = refs/heads/master + run: test "${GITHUB_REF#refs/heads/}" != "$GITHUB_REF" - name: Validate signed release tag if: startsWith(github.ref, 'refs/tags/v') env: @@ -91,17 +97,36 @@ jobs: sccache: false before-script-linux: | dnf install -y bzip2-devel xz-devel zlib-devel libcurl-devel openssl-devel clang-devel + - name: Install acceptance tools (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y samtools bcftools bedtools bwa - name: Install and test exact wheel env: PYTHON_VERSION: ${{ matrix.python }} run: | uv venv .wheel-test --python "$PYTHON_VERSION" - uv pip install --python .wheel-test/bin/python dist/*.whl + for wheel in dist/*.whl; do uv pip install --python .wheel-test/bin/python "${wheel}[export,plink,cyvcf2]"; done cd /tmp "$GITHUB_WORKSPACE/.wheel-test/bin/python" -c "import wasp2, wasp2_rust; print(wasp2.__version__, wasp2_rust.__file__)" "$GITHUB_WORKSPACE/.wheel-test/bin/wasp2-count" count-cohort --help "$GITHUB_WORKSPACE/.wheel-test/bin/wasp2-map" --help "$GITHUB_WORKSPACE/.wheel-test/bin/wasp2-analyze" --help + export PATH="$GITHUB_WORKSPACE/.wheel-test/bin:$PATH" + WASP2_TEST_EXPORTS=1 bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" + - name: Inventory installed wheel environment + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + with: + path: ./.wheel-test + format: cyclonedx-json + output-file: runtime-linux-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json + upload-artifact: false + - name: Verify runtime inventory + run: python scripts/check_sbom.py runtime-linux-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json wasp2 numpy scipy + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: runtime-sbom-linux-${{ matrix.arch }}-py${{ matrix.python }} + path: runtime-linux-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json + if-no-files-found: error - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: wheel-linux-${{ matrix.arch }}-py${{ matrix.python }} @@ -139,7 +164,7 @@ jobs: env: DEPLOYMENT_TARGET: ${{ matrix.deployment }} run: | - brew install htslib xz bzip2 openssl@3 pkg-config llvm@19 + brew install htslib xz bzip2 openssl@3 pkg-config llvm@19 samtools bcftools bedtools bwa echo "MACOSX_DEPLOYMENT_TARGET=$DEPLOYMENT_TARGET" >> "$GITHUB_ENV" echo "PKG_CONFIG_PATH=$(brew --prefix htslib)/lib/pkgconfig:$(brew --prefix openssl@3)/lib/pkgconfig" >> "$GITHUB_ENV" echo "LIBRARY_PATH=$(brew --prefix htslib)/lib:$(brew --prefix openssl@3)/lib" >> "$GITHUB_ENV" @@ -156,12 +181,28 @@ jobs: PYTHON_VERSION: ${{ matrix.python }} run: | uv venv .wheel-test --python "$PYTHON_VERSION" - uv pip install --python .wheel-test/bin/python dist/*.whl + for wheel in dist/*.whl; do uv pip install --python .wheel-test/bin/python "${wheel}[export,plink,cyvcf2]"; done cd /tmp "$GITHUB_WORKSPACE/.wheel-test/bin/python" -c "import wasp2, wasp2_rust; print(wasp2.__version__, wasp2_rust.__file__)" "$GITHUB_WORKSPACE/.wheel-test/bin/wasp2-count" count-cohort --help "$GITHUB_WORKSPACE/.wheel-test/bin/wasp2-map" --help "$GITHUB_WORKSPACE/.wheel-test/bin/wasp2-analyze" --help + export PATH="$GITHUB_WORKSPACE/.wheel-test/bin:$PATH" + WASP2_TEST_EXPORTS=1 bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" + - name: Inventory installed wheel environment + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + with: + path: ./.wheel-test + format: cyclonedx-json + output-file: runtime-macos-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json + upload-artifact: false + - name: Verify runtime inventory + run: python scripts/check_sbom.py runtime-macos-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json wasp2 numpy scipy + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: runtime-sbom-macos-${{ matrix.arch }}-py${{ matrix.python }} + path: runtime-macos-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json + if-no-files-found: error - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: wheel-macos-${{ matrix.arch }}-py${{ matrix.python }} @@ -185,7 +226,7 @@ jobs: with: enable-cache: false - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libhts-dev libbz2-dev liblzma-dev zlib1g-dev libssl-dev pkg-config libclang-dev + run: sudo apt-get update && sudo apt-get install -y libhts-dev libbz2-dev liblzma-dev zlib1g-dev libssl-dev pkg-config libclang-dev samtools bcftools bedtools bwa - name: Build and validate sdist run: | uv pip install --system maturin twine @@ -199,15 +240,143 @@ jobs: "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-count" --help "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-map" --help "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-analyze" --help + export PATH="$GITHUB_WORKSPACE/.sdist-test/bin:$PATH" + bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: source-distribution path: dist/*.tar.gz if-no-files-found: error + pixi-candidate: + name: Pixi ${{ matrix.platform }} / ${{ matrix.environment }} + needs: validate + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - {platform: linux-64, runner: ubuntu-24.04, environment: default} + - {platform: linux-64, runner: ubuntu-24.04, environment: dev} + - {platform: linux-aarch64, runner: ubuntu-24.04-arm, environment: default} + - {platform: linux-aarch64, runner: ubuntu-24.04-arm, environment: dev} + - {platform: osx-64, runner: macos-15-intel, environment: default} + - {platform: osx-64, runner: macos-15-intel, environment: dev} + - {platform: osx-arm64, runner: macos-15, environment: default} + - {platform: osx-arm64, runner: macos-15, environment: dev} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - uses: prefix-dev/setup-pixi@ad1da6449d9e506fd127fd67ab2d054a769f6a94 # v0.11.2 + with: + pixi-version: v0.82.0 + environments: ${{ matrix.environment }} + locked: false + cache: false + - name: Verify installed software + env: + PIXI_ENVIRONMENT: ${{ matrix.environment }} + EXPECTED_WASP2_VERSION: ${{ needs.validate.outputs.version }} + run: pixi run -e "$PIXI_ENVIRONMENT" bash scripts/container_smoke_test.sh + - name: Verify development parity task + if: matrix.environment == 'dev' + run: pixi run -e dev test-mapping-parity + + conda-candidate: + name: Conda ${{ matrix.platform }} / Python ${{ matrix.python }} + needs: [validate, sdist] + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - {platform: linux-64, runner: ubuntu-24.04, deployment: "2.28", python: "3.11"} + - {platform: linux-64, runner: ubuntu-24.04, deployment: "2.28", python: "3.12"} + - {platform: linux-aarch64, runner: ubuntu-24.04-arm, deployment: "2.28", python: "3.11"} + - {platform: linux-aarch64, runner: ubuntu-24.04-arm, deployment: "2.28", python: "3.12"} + - {platform: osx-64, runner: macos-15-intel, deployment: "10.13", python: "3.11"} + - {platform: osx-64, runner: macos-15-intel, deployment: "10.13", python: "3.12"} + - {platform: osx-arm64, runner: macos-15, deployment: "11.0", python: "3.11"} + - {platform: osx-arm64, runner: macos-15, deployment: "11.0", python: "3.12"} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: Jaureguy760/bioconda-recipes + ref: 63b7dde03e1898a3aa52cb52943ae9e02d191ebe + sparse-checkout: recipes/wasp2 + path: candidate-bioconda + persist-credentials: false + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: source-distribution + path: candidate-source + - uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 + with: + environment-name: conda-build + create-args: conda-build + condarc: | + channels: + - conda-forge + - bioconda + channel_priority: strict + init-shell: bash + cache-downloads: true + - name: Point candidate recipe at the exact tested source archive + shell: micromamba-shell {0} + env: + RELEASE_VERSION: ${{ needs.validate.outputs.version }} + DEPLOYMENT_TARGET: ${{ matrix.deployment }} + run: | + python - <<'PYCONDA' + import hashlib, os, re, shutil, subprocess, sys + from pathlib import Path + import yaml + shutil.copytree("candidate-bioconda/recipes/wasp2", "candidate-recipe") + archives = list(Path("candidate-source").glob("*.tar.gz")) + assert len(archives) == 1 + source = archives[0].resolve() + recipe = Path("candidate-recipe/meta.yaml") + text = recipe.read_text() + assert f'set version = "{os.environ["RELEASE_VERSION"]}"' in text + text = re.sub(r"^ url: .*$", " url: " + source.as_uri(), text, flags=re.MULTILINE) + text = re.sub(r"^ sha256: .*$", " sha256: " + hashlib.sha256(source.read_bytes()).hexdigest(), text, flags=re.MULTILINE) + recipe.write_text(text) + mac = sys.platform == "darwin" + versions = { + "c_compiler": ["clang" if mac else "gcc"], + "cxx_compiler": ["clangxx" if mac else "gxx"], + "c_compiler_version": ["19" if mac else "14"], + "cxx_compiler_version": ["19" if mac else "14"], + "c_stdlib": ["macosx_deployment_target" if mac else "sysroot"], + "c_stdlib_version": [os.environ["DEPLOYMENT_TARGET"]], + } + if mac: + versions["macosx_deployment_target"] = [os.environ["DEPLOYMENT_TARGET"]] + versions["CONDA_BUILD_SYSROOT"] = [subprocess.check_output(["xcrun", "--show-sdk-path"], text=True).strip()] + Path("candidate-variants.yaml").write_text(yaml.safe_dump(versions)) + PYCONDA + - name: Build and functionally test Conda package + shell: micromamba-shell {0} + env: + CONDA_BLD_PATH: ${{ runner.temp }}/conda-build + CONDA_CHANNEL_PRIORITY: strict + PYTHON_VERSION: ${{ matrix.python }} + run: conda-build candidate-recipe --python "$PYTHON_VERSION" --variant-config-files candidate-variants.yaml --override-channels -c conda-forge -c bioconda --no-anaconda-upload + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: conda-${{ matrix.platform }}-py${{ matrix.python }} + path: ${{ runner.temp }}/conda-build/${{ matrix.platform }}/wasp2-*.conda + if-no-files-found: error + package: name: Verify release bundle - needs: [linux-wheel, macos-wheel, sdist] + needs: [linux-wheel, macos-wheel, sdist, conda-candidate, pixi-candidate] runs-on: ubuntu-latest permissions: contents: read @@ -229,20 +398,31 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: runtime-sbom-* + merge-multiple: true + path: runtime-sboms - name: Validate complete package set run: | test "$(find packages -name '*.whl' -type f | wc -l)" -eq 16 test "$(find packages -name '*.tar.gz' -type f | wc -l)" -eq 1 pip install twine python -m twine check packages/* - sha256sum packages/* > SHA256SUMS.packages - - name: Generate package SBOM + (cd packages && sha256sum *) > SHA256SUMS.packages + - name: Expand source for dependency inventory + run: | + mkdir sbom-source + tar -xzf packages/wasp2-*.tar.gz -C sbom-source + - name: Generate source dependency SBOM uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 with: - path: ./packages + path: ./sbom-source format: cyclonedx-json - output-file: ./wasp2-packages.cdx.json + output-file: ./wasp2-source.cdx.json upload-artifact: false + - name: Verify source SBOM contents + run: python scripts/check_sbom.py wasp2-source.cdx.json wasp2 rust-htslib - name: Attest package provenance if: startsWith(github.ref, 'refs/tags/v') uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 @@ -257,8 +437,9 @@ jobs: with: name: release-metadata path: | - wasp2-packages.cdx.json + wasp2-source.cdx.json SHA256SUMS.packages + runtime-sboms/*.cdx.json if-no-files-found: error container-smoke: @@ -301,6 +482,70 @@ jobs: test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' "$IMAGE")" = "$EXPECTED_VERSION" test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE")" = "$EXPECTED_REVISION" docker run --rm --platform "$PLATFORM" -e EXPECTED_WASP2_VERSION="$EXPECTED_VERSION" "$IMAGE" /opt/wasp2/scripts/container_smoke_test.sh + - name: Build dependency image for scanning + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + with: + context: . + target: rust-builder + load: true + platforms: ${{ matrix.platform }} + tags: wasp2:builder-scan + cache-from: type=gha,scope=release-${{ matrix.tag }} + - name: Scan build dependency vulnerabilities + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 + with: + image-ref: wasp2:builder-scan + version: v0.76.0 + format: json + output: builder-${{ matrix.tag }}-vulnerabilities.json + severity: CRITICAL,HIGH + exit-code: '1' + - name: Preserve build dependency scan + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: builder-scan-${{ matrix.tag }} + path: builder-${{ matrix.tag }}-vulnerabilities.json + if-no-files-found: warn + - name: Inventory runtime image + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + with: + image: wasp2:release-smoke-${{ matrix.tag }} + format: cyclonedx-json + output-file: container-${{ matrix.tag }}.cdx.json + upload-artifact: false + - name: Verify image inventory + run: python3 scripts/check_sbom.py container-${{ matrix.tag }}.cdx.json wasp2 numpy scipy samtools bcftools bedtools + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: release-container-sbom-${{ matrix.tag }} + path: container-${{ matrix.tag }}.cdx.json + if-no-files-found: error + - name: Scan runtime image vulnerabilities + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0 + with: + image-ref: wasp2:release-smoke-${{ matrix.tag }} + format: json + output: container-${{ matrix.tag }}-vulnerabilities.json + version: v0.76.0 + severity: CRITICAL,HIGH + exit-code: '1' + - name: Preserve container scan + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: release-container-scan-${{ matrix.tag }} + path: container-${{ matrix.tag }}-vulnerabilities.json + if-no-files-found: error + - name: Save tested image + env: + IMAGE: wasp2:release-smoke-${{ matrix.tag }} + run: docker save "$IMAGE" | gzip > "${RUNNER_TEMP}/image-${{ matrix.tag }}.tar.gz" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: tested-image-${{ matrix.tag }} + path: ${{ runner.temp }}/image-${{ matrix.tag }}.tar.gz + if-no-files-found: error singularity-smoke: name: Singularity smoke @@ -319,29 +564,33 @@ jobs: apptainer-version: 1.4.5 - name: Verify container runtime run: apptainer --version - - name: Build local Docker archive - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 with: - context: . - platforms: linux/amd64 - tags: wasp2:release-singularity - build-args: | - VERSION=${{ needs.validate.outputs.version }} - REVISION=${{ github.sha }} - outputs: type=docker,dest=${{ runner.temp }}/wasp2-docker.tar - cache-from: type=gha,scope=release-amd64 - cache-to: type=gha,mode=max,scope=release-amd64 + name: tested-image-amd64 + path: ${{ runner.temp }}/tested-image + - name: Unpack tested Docker archive + run: gzip -dc "$RUNNER_TEMP/tested-image/image-amd64.tar.gz" > "$RUNNER_TEMP/wasp2-docker.tar" - name: Build and test local Singularity artifact env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} run: | apptainer build "wasp2_${RELEASE_VERSION}.sif" "docker-archive://${RUNNER_TEMP}/wasp2-docker.tar" - apptainer exec "wasp2_${RELEASE_VERSION}.sif" /opt/wasp2/scripts/container_smoke_test.sh + apptainer exec --env EXPECTED_WASP2_VERSION="$RELEASE_VERSION" "wasp2_${RELEASE_VERSION}.sif" /opt/wasp2/scripts/container_smoke_test.sh + sed -e 's/^Bootstrap: docker$/Bootstrap: localimage/' \ + -e "s|^From: .*|From: $PWD/wasp2_${RELEASE_VERSION}.sif|" \ + Singularity.def > "$RUNNER_TEMP/Singularity.candidate.def" + apptainer build "wasp2_definition_${RELEASE_VERSION}.sif" "$RUNNER_TEMP/Singularity.candidate.def" + apptainer test "wasp2_definition_${RELEASE_VERSION}.sif" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: candidate-sif + path: wasp2_${{ needs.validate.outputs.version }}.sif + if-no-files-found: error publish-pypi: name: Publish PyPI if: startsWith(github.ref, 'refs/tags/v') - needs: [package, singularity] + needs: [validate, package, singularity] runs-on: ubuntu-latest environment: name: pypi @@ -349,17 +598,29 @@ jobs: permissions: id-token: write # Authenticate to PyPI using trusted publishing. steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: + python-version: "3.12" - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: release-packages path: packages + - name: Reconcile immutable PyPI artifacts + id: upload + env: + RELEASE_VERSION: ${{ needs.validate.outputs.version }} + run: python scripts/prepare_pypi_upload.py packages upload "$RELEASE_VERSION" - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 + if: steps.upload.outputs.count != '0' with: - packages-dir: packages/ + packages-dir: upload/ attestations: true container: - name: Publish multi-architecture container + name: Stage tested multi-architecture container if: startsWith(github.ref, 'refs/tags/v') needs: [validate, package, container-smoke, singularity-smoke] runs-on: ubuntu-latest @@ -379,27 +640,47 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and publish image - id: image - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: | - ${{ env.IMAGE_NAME }}:${{ needs.validate.outputs.version }} - ${{ env.IMAGE_NAME }}:latest - build-args: | - VERSION=${{ needs.validate.outputs.version }} - REVISION=${{ github.sha }} - provenance: mode=max - sbom: true - - name: Smoke-test published architectures + pattern: tested-image-* + merge-multiple: true + path: ${{ runner.temp }}/tested-images + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: release-container-sbom-* + merge-multiple: true + path: container-sboms + - name: Publish exact tested candidate images + id: image env: - RELEASE_VERSION: ${{ needs.validate.outputs.version }} + CANDIDATE: candidate-${{ github.sha }}-${{ github.run_attempt }} run: | - docker run --rm --platform linux/amd64 "${IMAGE_NAME}:${RELEASE_VERSION}" wasp2-count count-cohort --help - docker run --rm --platform linux/arm64 "${IMAGE_NAME}:${RELEASE_VERSION}" wasp2-count count-cohort --help + for arch in amd64 arm64; do + gzip -dc "$RUNNER_TEMP/tested-images/image-${arch}.tar.gz" | docker load + docker tag "wasp2:release-smoke-${arch}" "${IMAGE_NAME}:${CANDIDATE}-${arch}" + docker push "${IMAGE_NAME}:${CANDIDATE}-${arch}" + ARCH_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:${CANDIDATE}-${arch}" --format '{{json .Manifest.Digest}}' | tr -d '\"') + echo "${arch}_digest=$ARCH_DIGEST" >> "$GITHUB_OUTPUT" + done + docker buildx imagetools create --tag "${IMAGE_NAME}:${CANDIDATE}" \ + "${IMAGE_NAME}:${CANDIDATE}-amd64" "${IMAGE_NAME}:${CANDIDATE}-arm64" + DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:${CANDIDATE}" --format '{{json .Manifest.Digest}}' | tr -d '\"') + test -n "$DIGEST" + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + - name: Attest amd64 dependency inventory + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 + with: + subject-name: ${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.image.outputs.amd64_digest }} + sbom-path: container-sboms/container-amd64.cdx.json + push-to-registry: true + - name: Attest arm64 dependency inventory + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 + with: + subject-name: ${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.image.outputs.arm64_digest }} + sbom-path: container-sboms/container-arm64.cdx.json + push-to-registry: true - name: Attest container provenance uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 with: @@ -430,14 +711,19 @@ jobs: apptainer-version: 1.4.5 - name: Verify container runtime run: apptainer --version - - name: Build from released image + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: container-metadata + - name: Build from tested candidate digest env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} - run: apptainer build "wasp2_${RELEASE_VERSION}.sif" "docker://${IMAGE_NAME}:${RELEASE_VERSION}" + run: | + DIGEST=$(cat container-digest.txt) + apptainer build "wasp2_${RELEASE_VERSION}.sif" "docker://${IMAGE_NAME}@${DIGEST}" - name: Test Singularity artifact env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} - run: apptainer exec "wasp2_${RELEASE_VERSION}.sif" wasp2-count count-cohort --help + run: apptainer exec --env EXPECTED_WASP2_VERSION="$RELEASE_VERSION" "wasp2_${RELEASE_VERSION}.sif" /opt/wasp2/scripts/container_smoke_test.sh - uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 with: subject-path: "wasp2_${{ needs.validate.outputs.version }}.sif" @@ -447,10 +733,42 @@ jobs: path: "wasp2_${{ needs.validate.outputs.version }}.sif" if-no-files-found: error + promote-container: + name: Promote validated container tags + if: startsWith(github.ref, 'refs/tags/v') + needs: [validate, publish-pypi, singularity, container] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: container-metadata + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Promote digest without rebuilding + env: + RELEASE_VERSION: ${{ needs.validate.outputs.version }} + run: | + DIGEST=$(cat container-digest.txt) + if docker buildx imagetools inspect "${IMAGE_NAME}:${RELEASE_VERSION}" --format '{{json .Manifest.Digest}}' > existing-digest.txt 2> inspect-error.txt; then + EXISTING=$(tr -d '\"\n' < existing-digest.txt) + test "$EXISTING" = "$DIGEST" || { echo "Refusing to replace an existing version tag" >&2; exit 1; } + else + grep -Eiq 'manifest unknown|not found' inspect-error.txt || { cat inspect-error.txt >&2; exit 1; } + docker buildx imagetools create --tag "${IMAGE_NAME}:${RELEASE_VERSION}" "${IMAGE_NAME}@${DIGEST}" + fi + docker buildx imagetools create --tag "${IMAGE_NAME}:latest" "${IMAGE_NAME}@${DIGEST}" + github-release: name: Publish GitHub Release if: startsWith(github.ref, 'refs/tags/v') - needs: [publish-pypi, container, singularity] + needs: [publish-pypi, promote-container, singularity] runs-on: ubuntu-latest permissions: contents: write # Create the GitHub release and upload its assets. @@ -471,6 +789,35 @@ jobs: with: name: singularity-artifact path: release-assets + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: release-container-sbom-* + merge-multiple: true + path: release-assets + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: release-container-scan-* + merge-multiple: true + path: release-assets + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: builder-scan-* + merge-multiple: true + path: release-assets + - name: Flatten release metadata without overwriting assets + run: | + python - <<'PYASSETS' + from pathlib import Path + root = Path("release-assets") + for path in list(root.rglob("*")): + if path.is_file() and path.parent != root: + target = root / path.name + assert not target.exists(), f"Duplicate release asset: {path.name}" + path.rename(target) + for path in sorted(root.rglob("*"), key=lambda p: len(p.parts), reverse=True): + if path.is_dir(): + path.rmdir() + PYASSETS - name: Generate release checksums run: | cd release-assets diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 8ffdfc50..a6fd479f 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -30,11 +30,23 @@ jobs: python-version: "3.12" - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 - name: Resolve auditable dependencies - run: uv pip compile pyproject.toml --extra dev --output-file /tmp/wasp2-audit.txt + run: | + uv pip compile pyproject.toml --extra dev --extra docs --extra plink --extra cyvcf2 --extra export --output-file /tmp/wasp2-audit.txt + python -c 'import tomllib; print("\n".join(tomllib.load(open("pyproject.toml", "rb"))["build-system"]["requires"]))' > /tmp/wasp2-build-audit.in + uv pip compile /tmp/wasp2-build-audit.in --output-file /tmp/wasp2-build-audit.txt - name: Audit Python dependencies - run: uvx pip-audit==2.10.1 --requirement /tmp/wasp2-audit.txt + run: | + uvx pip-audit==2.10.1 --requirement /tmp/wasp2-audit.txt + uvx pip-audit==2.10.1 --requirement /tmp/wasp2-build-audit.txt - name: Run Bandit run: uvx --from 'bandit[toml]==1.9.4' bandit -c pyproject.toml -r src + - name: Preserve resolved audit inputs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: python-audit-requirements + path: /tmp/wasp2*-audit.txt + if-no-files-found: error rust: name: Rust audit diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 19e3fca8..86605e0c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -15,6 +15,7 @@ build: - pip install -r docs/requirements.txt sphinx: + fail_on_warning: true configuration: docs/source/conf.py formats: diff --git a/.zenodo.json b/.zenodo.json index 3d2fbe45..e8499828 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,23 +1,29 @@ { - "title": "WASP2 Sanity Test Dataset - chr21 HG00731", - "description": "Real RNA-seq benchmark data for WASP2 CI sanity testing. Contains chr21 subset of HG00731 sample (~855K reads, ~33K het variants) with expected pipeline outputs for reproducibility validation.", - "upload_type": "dataset", + "title": "WASP2", + "description": "Software for allele-specific analysis of next-generation sequencing data, including mapping bias correction, allele counting, and allelic imbalance inference.", + "upload_type": "software", "access_right": "open", "license": "MIT", "creators": [ + { + "name": "Ho, Aaron" + }, { "name": "Jaureguy, Jeff", "affiliation": "UC San Diego" + }, + { + "name": "McVicker Lab", + "affiliation": "Salk Institute for Biological Studies" } ], "keywords": [ - "WASP2", - "allele-specific expression", - "RNA-seq", "bioinformatics", - "benchmark", - "HG00731", - "1000 Genomes" + "allele-specific analysis", + "RNA-seq", + "ATAC-seq", + "Rust", + "Python" ], "related_identifiers": [ { @@ -25,6 +31,5 @@ "relation": "isSupplementTo", "scheme": "url" } - ], - "notes": "This dataset is used for CI sanity testing of the WASP2 Rust-accelerated pipeline. It validates that allele counting, FASTQ generation, and analysis produce consistent results." + ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d6d0d8c..28082b29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to WASP2 will be documented in this file. ## [Unreleased] ### Fixed +- Resolve numeric mapping genotypes to allele sequences and label swapped FASTQs + with the number of pairs actually written, preserving valid remapped reads. +- Preserve PGEN sample order and phase during BED export and accept the common + variant-selection options used by counting and mapping. +- Correct supported Polars/SciPy dependency floors, position join types, Conda + environments, Pixi tasks and the current Pixi manifest format. +- Fail installation smoke tests on missing inputs and incorrect counts, swapped + sequences, remap decisions or single-cell matrices; reuse them for release artifacts. +- Require complete documentation imports and meaningful SBOM inventories; stage + tested images by digest and reconcile immutable PyPI artifacts before publication. +- Provide optional Zarr/Parquet export dependencies and repair executable tutorial + examples, sample identifiers and Singularity validation. - Restore archived single-cell phasing fallback for mixed phased/unphased genotypes and require the native counter for normal single-cell counting. - Reject overflowing bulk count, threshold, pseudocount and regional-total diff --git a/CITATION.cff b/CITATION.cff index ea8c457f..67a77ad7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -73,7 +73,7 @@ abstract: >- # doi: "10.XXXX/XXXXX" # Zenodo Software DOI - uncomment when a versioned software release is archived -# (Note: .zenodo.json in this repo is for the test dataset, not the software itself) +# Root .zenodo.json describes the software release. # identifiers: # - type: doi # value: "10.5281/zenodo.XXXXXXX" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74cc9446..1b187259 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,9 @@ WASP2 is a hybrid Python/Rust project. You'll need both toolchains to build from | Python | 3.10+ | 3.10, 3.11, 3.12, 3.13 tested (conda env uses 3.11) | | Rust | 1.88+ | Edition 2021 | | Git | 2.0+ | For version control | -| C compiler | gcc/clang | Required for native extensions | +| C/C++ compilers | gcc/g++ or clang | Required for native extensions | +| libclang | System package | Required by bindgen | +| CMake | 3.14+ | Installed by isolated pip builds; needed explicitly for direct Maturin builds | > **Note:** WASP2 is developed and tested on Linux and macOS. Windows is not officially supported. @@ -66,7 +68,6 @@ WASP2 is a hybrid Python/Rust project. You'll need both toolchains to build from ```bash python -m venv .venv source .venv/bin/activate # Linux/macOS - pip install -e ".[dev,docs]" # Includes development and documentation tools ``` 3. **Install the Rust toolchain** (if not already installed): @@ -76,9 +77,12 @@ WASP2 is a hybrid Python/Rust project. You'll need both toolchains to build from source ~/.cargo/env ``` -4. **Build the Rust extension:** +4. **Install the package and development tools, then build the Rust extension:** ```bash + pip install -e ".[dev,docs]" + pip install "cmake>=3.14" + # Development build (faster compile, debug symbols) maturin develop -m rust/Cargo.toml diff --git a/Dockerfile b/Dockerfile index e86fa6ac..52c7906d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,10 @@ # ============================================================================ # Stage 1: Install cargo-chef # ============================================================================ -FROM rust:1.88-bookworm AS chef -RUN cargo install cargo-chef --version 0.1.77 --locked +FROM rust:1.88-bookworm@sha256:af306cfa71d987911a781c37b59d7d67d934f49684058f96cf72079c3626bfe0 AS chef +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && rm -rf /var/lib/apt/lists/* \ + && cargo install cargo-chef --version 0.1.77 --locked WORKDIR /build # ============================================================================ @@ -47,7 +49,7 @@ COPY rust/Cargo.toml rust/Cargo.lock ./ RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/rust/target \ - cargo chef cook --release --recipe-path /build/recipe.json + cargo chef cook --release --locked --recipe-path /build/recipe.json # Copy full source and build wheel WORKDIR /build @@ -60,12 +62,12 @@ COPY LICENSE . RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/build/rust/target \ - maturin build --release -m rust/Cargo.toml -o /wheels + maturin build --release --locked -m rust/Cargo.toml -o /wheels # ============================================================================ # Stage 4: Runtime image # ============================================================================ -FROM python:3.11-slim-bookworm +FROM python:3.11-slim-bookworm@sha256:0a310eeecf4e1f5a0743f9a6520c90c88d089c903ca5fd283f501e3a805f5f89 # Version: keep in sync with rust/Cargo.toml (single source of truth) # Run scripts/check-version-consistency.sh to verify @@ -83,13 +85,15 @@ LABEL maintainer="Jeff Jaureguy " # Install runtime deps + temporary build deps for pybedtools C++ extension # Combined into one RUN to minimize layers; build tools purged at the end -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ # PID 1 init for proper signal handling (Nextflow/HPC) tini \ # Bioinformatics tools samtools \ bcftools \ bedtools \ + bwa \ tabix \ # htslib runtime libs libhts3 \ @@ -107,7 +111,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Copy wheel from builder and install, then purge build tools in same layer COPY --from=rust-builder /wheels/*.whl /tmp/ RUN --mount=type=cache,target=/root/.cache/pip \ - pip install /tmp/*.whl \ + for wheel in /tmp/*.whl; do pip install "${wheel}[export]" || exit; done \ && rm -rf /tmp/*.whl \ && apt-get purge -y --auto-remove g++ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* \ @@ -142,7 +146,8 @@ COPY --chown=wasp2:wasp2 tests/shared_data/chr_test.fa \ COPY --chown=wasp2:wasp2 scripts/container_smoke_test.sh /opt/wasp2/scripts/ # Prevent Python from writing bytecode and ensure output is unbuffered -ENV PYTHONDONTWRITEBYTECODE=1 \ +ENV WASP2_TEST_EXPORTS=1 \ + PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 # Set working directory for Nextflow diff --git a/Makefile b/Makefile index 08bb09a0..1cd04f54 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ SANITY_DATA_DIR := tests/sanity/data # Sanity test data configuration SANITY_VERSION := v1 SANITY_TARBALL := wasp2-sanity-chr21-$(SANITY_VERSION).tar.xz -SANITY_RELEASE_URL := https://github.com/mcvickerlab/WASP2/releases/download/v1.3.0/$(SANITY_TARBALL) +SANITY_RELEASE_URL ?= $(WASP2_SANITY_DATA_URL) # Local sanity data path (for development). Override by exporting # WASP2_SANITY_DATA_DIR or passing LOCAL_SANITY_DATA=... to make. @@ -33,18 +33,19 @@ LOCAL_SANITY_DATA ?= $(or $(WASP2_SANITY_DATA_DIR),$(CURDIR)/benchmarking/sanity all: build ## Build the project (default) -build: rust-build install ## Build Rust extension and install package +build: install ## Build Rust extension and install package install: ## Install package in development mode - $(PYTHON) -m pip install -e ".[dev]" --no-build-isolation -q + $(PYTHON) -m pip install -e ".[dev]" -q # ============================================================================= # Rust build targets # ============================================================================= rust-build: ## Build Rust extension with maturin - $(MATURIN) build --release -m $(RUST_DIR)/Cargo.toml - $(PYTHON) -m pip install $(RUST_DIR)/target/wheels/*.whl --force-reinstall -q + set -eu; wheel_dir=$$(mktemp -d); trap 'rm -rf "$$wheel_dir"' EXIT; \ + $(PYTHON) -m pip wheel --no-deps --wheel-dir "$$wheel_dir" .; \ + $(PYTHON) -m pip install --force-reinstall --no-deps "$$wheel_dir"/wasp2-*.whl rust-dev: ## Build Rust extension in debug mode (faster compile) $(MATURIN) develop -m $(RUST_DIR)/Cargo.toml @@ -70,8 +71,8 @@ test-quick: ## Run quick validation tests only test-rust: ## Run Rust-specific tests $(PYTEST) $(TESTS_DIR) -v --tb=short -m "rust" -test-mapping-parity: ## Run mapping parity tests against legacy and unified paths - $(PYTEST) $(TESTS_DIR)/regression/test_mapping_stage_parity.py -v --tb=short +test-mapping-parity: ## Run mapping regression tests + $(PYTEST) $(TESTS_DIR)/mapping/ $(TESTS_DIR)/regression/test_quickbench*.py -v --tb=short test-integration: ## Run integration tests $(PYTEST) $(TESTS_DIR) -v --tb=short -m "integration" @@ -87,14 +88,16 @@ test-all: ## Run all tests including sanity and slow tests # Sanity data management # ============================================================================= -download-sanity-data: ## Download sanity test data from GitHub release +download-sanity-data: ## Download a configured sanity archive (WASP2_SANITY_DATA_URL and SHA256) + @test -n "$(SANITY_RELEASE_URL)" -a -n "$(WASP2_SANITY_DATA_SHA256)" || { echo "Set WASP2_SANITY_DATA_URL and WASP2_SANITY_DATA_SHA256, or use make sanity-data-local" >&2; exit 1; } @echo "Downloading sanity data from $(SANITY_RELEASE_URL)..." @mkdir -p $(SANITY_DATA_DIR) @if command -v wget > /dev/null; then \ wget -q -O $(SANITY_DATA_DIR)/$(SANITY_TARBALL) $(SANITY_RELEASE_URL); \ else \ - curl -sL -o $(SANITY_DATA_DIR)/$(SANITY_TARBALL) $(SANITY_RELEASE_URL); \ + curl --fail --show-error -sL -o $(SANITY_DATA_DIR)/$(SANITY_TARBALL) $(SANITY_RELEASE_URL); \ fi + @$(PYTHON) -c 'import hashlib, pathlib; p = pathlib.Path("$(SANITY_DATA_DIR)/$(SANITY_TARBALL)"); assert hashlib.sha256(p.read_bytes()).hexdigest() == "$(WASP2_SANITY_DATA_SHA256)", "Sanity archive checksum mismatch"' @echo "Extracting..." @cd $(SANITY_DATA_DIR) && tar -xJf $(SANITY_TARBALL) --strip-components=1 @rm -f $(SANITY_DATA_DIR)/$(SANITY_TARBALL) diff --git a/README.md b/README.md index 8537aeff..259ba86c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Published packages and container tags may lag behind `dev`. This branch targets the unreleased 1.5.0 version. To test development fixes, install from `dev` -using the [source installation instructions](https://mcvickerlab.github.io/WASP2/installation.html#development-installation). +using the [source installation instructions](https://github.com/mcvickerlab/WASP2/blob/dev/docs/source/installation.rst#development-installation). ### Bioconda @@ -70,14 +70,17 @@ backend performs the calculations. ## Quick Start -> **Prerequisite — normalize your VCF first.** WASP2's allelic-imbalance model is biallelic. -> Split multi-allelic records and left-align before running: +> **Variant selection:** the standard workflow uses biallelic SNVs. Ensure the +> VCF and BAM use the same reference and chromosome names. To select these sites: > > ```bash -> bcftools norm -m- -f reference.fa input.vcf.gz -Oz -o input.norm.vcf.gz -> bcftools index -t input.norm.vcf.gz +> bcftools view -m2 -M2 -v snps input.vcf.gz -Oz -o input.snvs.vcf.gz +> bcftools index -t input.snvs.vcf.gz > ``` > +> Splitting multiallelic records beforehand changes which sites pass this filter; +> choose that preprocessing explicitly rather than treating it as equivalent. +> > By default WASP2 keeps **biallelic SNVs only** (multi-allelic sites are dropped, equivalent to > `bcftools view -m2 -M2`). Pass `--include-multiallelic` to `count-variants`, `count-variants-sc`, > or `make-reads` to instead emit one row per ALT at any remaining multi-allelic sites. diff --git a/Singularity.def b/Singularity.def index 3f9dcac5..f554bc0f 100644 --- a/Singularity.def +++ b/Singularity.def @@ -37,32 +37,12 @@ From: ghcr.io/mcvickerlab/wasp2:1.5.0 %environment export LC_ALL=C - export PATH="/app/.venv/bin:/usr/local/bin:$PATH" + export PATH="/usr/local/bin:$PATH" %runscript + if [ "$#" -eq 0 ]; then set -- wasp2-count --help; fi exec "$@" %test - wasp2-count --version - wasp2-map --version - wasp2-analyze --version - python -c "import wasp2_rust; print('Rust extension OK')" - - # Data smoke test (if test data bundled in container) - if [ -f /opt/wasp2/test-data/sample1.bam ]; then - echo "Running data smoke test..." - wasp2-count count-variants \ - /opt/wasp2/test-data/sample1.bam \ - /opt/wasp2/test-data/variants.vcf.gz \ - --samples SAMPLE1 \ - --out /tmp/test_counts.tsv - if [ -s /tmp/test_counts.tsv ]; then - echo "Data smoke test PASSED" - else - echo "Data smoke test FAILED" - exit 1 - fi - rm -f /tmp/test_counts.tsv - fi - - echo "All tests passed!" + set -eu + /opt/wasp2/scripts/container_smoke_test.sh diff --git a/conda-lock.yml b/conda-lock.yml index 9ac3c062..ed4056d9 100644 --- a/conda-lock.yml +++ b/conda-lock.yml @@ -13,8 +13,8 @@ version: 1 metadata: content_hash: - linux-64: bb05ae664cf9071c5d535fcaa891330ef6f0682ccc971f598a0457b78816e150 - osx-64: 88e11e31b2d33d94f6c2834de4ee7908ebd89cd666ee28fd01489c60801ff44e + linux-64: 4f6271d6ae442485f8c64326613758fe51969625c378fae293272672f6b0ee14 + osx-64: 9e848e5c81542c773e40f60a29864f6d26d060accaaf234fa330c1276940d714 channels: - url: conda-forge used_env_vars: [] @@ -3607,62 +3607,62 @@ package: category: main optional: false - name: polars - version: 1.42.1 + version: 1.44.2 manager: conda platform: linux-64 dependencies: - polars-runtime-32: 1.42.1 - python: '>=3.10' - url: https://conda.anaconda.org/conda-forge/noarch/polars-1.42.1-pyh58ad624_0.conda + polars-runtime-32: 1.44.2 + python: '>=3.11' + url: https://conda.anaconda.org/conda-forge/noarch/polars-1.44.2-pyh3138b34_0.conda hash: - md5: 4e315d6adb68e84890cd7a630b2f316f - sha256: 8b55f3021a72b098195ea9fddf125cd5a2e1bd288f8ae8d899e70c1d7b132b18 + md5: 716482997743170f404e0c2a29ab406d + sha256: df9834da83b1eab9296aae08b4b00516be213ba0610fd3592426df2d2e025123 category: main optional: false - name: polars - version: 1.42.1 + version: 1.44.2 manager: conda platform: osx-64 dependencies: - polars-runtime-32: 1.42.1 - python: '>=3.10' - url: https://conda.anaconda.org/conda-forge/noarch/polars-1.42.1-pyh58ad624_0.conda + polars-runtime-32: 1.44.2 + python: '>=3.11' + url: https://conda.anaconda.org/conda-forge/noarch/polars-1.44.2-pyh3138b34_0.conda hash: - md5: 4e315d6adb68e84890cd7a630b2f316f - sha256: 8b55f3021a72b098195ea9fddf125cd5a2e1bd288f8ae8d899e70c1d7b132b18 + md5: 716482997743170f404e0c2a29ab406d + sha256: df9834da83b1eab9296aae08b4b00516be213ba0610fd3592426df2d2e025123 category: main optional: false - name: polars-runtime-32 - version: 1.42.1 + version: 1.44.2 manager: conda platform: linux-64 dependencies: __glibc: '>=2.17,<3.0.a0' _python_abi3_support: 1.* - cpython: '>=3.10' - libgcc: '>=14' - libstdcxx: '>=14' + cpython: '>=3.11' + libgcc: '>=15' + libstdcxx: '>=15' python: '' - url: https://conda.anaconda.org/conda-forge/linux-64/polars-runtime-32-1.42.1-py310h49dadd8_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/polars-runtime-32-1.44.2-py311hb519a9e_0.conda hash: - md5: 393bb50f17571c2e8b129461919296be - sha256: 7fdca7044d5377fbed66864644f52430789876c7571dfe0868e4082dc711b83d + md5: 76e398db25e6e9cd7e09ad16110a0c5e + sha256: b21cb18cbf8bdf706a1b43d0f1228b409105d1d7f29749ad2575b5e96dba2abd category: main optional: false - name: polars-runtime-32 - version: 1.42.1 + version: 1.44.2 manager: conda platform: osx-64 dependencies: __osx: '>=11.0' _python_abi3_support: 1.* - cpython: '>=3.10' - libcxx: '>=19' + cpython: '>=3.11' + libcxx: '>=21' python: '' - url: https://conda.anaconda.org/conda-forge/osx-64/polars-runtime-32-1.42.1-py310h3769acf_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/polars-runtime-32-1.44.2-py311hb25e4a4_0.conda hash: - md5: f1db8c1d8f2b729ae05eecf4aa43f4e5 - sha256: 5a8b1285412ff848ee4e892244a9aedd682c3308a9f54fe9c7c89edfafc355f4 + md5: 49c59abeaf114775439186ccf48d0dac + sha256: ed6cf1f6b81e93ad39603ef8c02a9a409ad50182e8edd4ce7f952a164a0c7079 category: main optional: false - name: psutil diff --git a/docs/requirements.txt b/docs/requirements.txt index f1aef79a..f38c40c8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -21,6 +21,9 @@ sphinx-design>=0.5 # Project dependencies needed for autodoc numpy>=1.24 -pandas>=2.0 -scipy>=1.10 +pandas>=1.5,<3.0 +scipy>=1.13.1 tomli>=2.0; python_version < "3.11" +polars>=1.0,<2.0 +typer>=0.12 +rich>=13.0 diff --git a/docs/source/api/counting.rst b/docs/source/api/counting.rst index f898c6df..971acc30 100644 --- a/docs/source/api/counting.rst +++ b/docs/source/api/counting.rst @@ -67,3 +67,16 @@ CLI Entry Point :members: :undoc-members: :show-inheritance: + +Variant File Interface +---------------------- + +Install ``wasp2[plink]`` for PGEN input or ``wasp2[cyvcf2]`` for the optional +VCF reader. Standard VCF conversion uses the native backend. BED exports keep +numeric genotype indices and their phase separators; the native mapping stage +resolves those indices to the corresponding allele sequences. + +.. automodule:: wasp2.io.variant_source + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/source/development.rst b/docs/source/development.rst index 61937fda..31a3de83 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -4,7 +4,7 @@ Development Guide Setup --------- -WASP2 uses Python 3.10–3.13 and Rust 1.88 or newer. Install a C compiler and +WASP2 uses Python 3.10–3.13 and Rust 1.88 or newer. Install C/C++ compilers, libclang, CMake, and the external tools required by the workflows: samtools, bcftools, and bedtools. .. code-block:: bash @@ -18,8 +18,11 @@ the external tools required by the workflows: samtools, bcftools, and bedtools. The ``dev`` extra provides pytest, Ruff, basedpyright, mypy, build, and security tools. Documentation dependencies are in the separate ``docs`` extra. -Alternatively, use the locked Pixi environment with ``pixi install`` and -``pixi run verify``. +Alternatively, let Pixi resolve the environment with ``pixi install -e dev``. +Use ``pixi run -e dev test`` for tests. No reviewed Pixi lockfile is distributed. +Install Rust 1.88 or newer before the editable pip installation above; pip +provides its isolated Maturin/CMake build dependencies. Direct Maturin commands +also require those tools in the active environment. Checks ---------- @@ -80,8 +83,17 @@ Use descriptive titles and explain behavior changes and validation. Required checks include Python 3.10–3.13, Rust, lint, documentation, and security checks. Version metadata comes from ``rust/Cargo.toml`` via maturin. Before a release, -run ``python scripts/release_version.py --check``, align the changelog and -release metadata, and verify package/container build results. Promotion to +run ``python scripts/release_version.py --check``, then finalize the dated +changelog and citation with ``--check --release``. Development checks allow +unreleased metadata; publication checks require a matching date and version. + +The release candidate workflow tests 16 wheels, an independently installed +sdist, eight Conda platform/Python combinations, native containers, and SIF +conversion. Its Conda recipe is pinned to a reviewed preparation commit; update +that pin when preparing a new recipe. Candidate builds substitute only the +tested source archive URL and checksum. Submit the published archive checksum +to Bioconda after PyPI publication; do not submit an unavailable source URL as +an installable release. Promotion to ``master`` and publication are separate release operations; an updated source version does not establish that registry packages or images exist. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 4c3b03ea..2928aba7 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -157,7 +157,7 @@ to harmonize the VCF. This happens if the wheel was built for a different platform or Python version. Try reinstalling: ``pip install --force-reinstall wasp2``. If building from source, -run ``pixi run verify`` to rebuild. +run ``pixi run build`` to rebuild, then ``pixi run verify`` to check it. **WASP2 runs but produces an empty counts file.** diff --git a/docs/source/installation.rst b/docs/source/installation.rst index b16e3059..8b9bb8e6 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -9,12 +9,21 @@ branch targets the unreleased 1.5.0 version; installing a published package does not necessarily include its changes. Check ``wasp2-count --version`` and use the source installation below when testing development fixes. +Platforms +--------- + +Release wheels are built for Python 3.10–3.13 on Linux x86-64/ARM64 and +macOS Intel/Apple Silicon. The configured wheel tags target Linux glibc 2.28+, +macOS 13+ on Intel, and macOS 11+ on Apple Silicon. CI runs on newer operating +systems; these tag minima do not establish an executed test on every older OS. +Windows and older CPU instruction sets are not currently certified. + Via Bioconda ------------ .. code-block:: bash - mamba install -c conda-forge -c bioconda wasp2 + mamba install --strict-channel-priority -c conda-forge -c bioconda wasp2 Bioconda installs WASP2 together with the external command-line dependencies required by the workflows, including ``samtools``, ``bcftools``, and @@ -69,7 +78,7 @@ Development Installation Source builds require Rust 1.88 or newer, a C/C++ compiler, and libclang. The build backend declares CMake as a build dependency. Pixi supplies these -tools together with the runtime dependencies; published wheels do not need +tools together with the runtime dependencies (Pixi 0.82 or newer); published wheels do not need the compilation toolchain. .. code-block:: bash @@ -79,6 +88,18 @@ the compilation toolchain. pixi install pixi run verify +Optional Export Formats +----------------------- + +Zarr and Parquet exports require additional packages: + +.. code-block:: bash + + pip install "wasp2[export]" + +Source installations can use ``pip install -e ".[export]"``. The Docker +image includes these dependencies. H5AD export uses the base installation. + Verification ------------ diff --git a/environment.lock.yml b/environment.lock.yml index db67cf5f..8b39405a 100644 --- a/environment.lock.yml +++ b/environment.lock.yml @@ -6,8 +6,8 @@ channels: - conda-forge - bioconda -# plink2 and all bioconda bio tools only support linux-64 and osx-64 -# (not osx-arm64, linux-aarch64, or win-64) +# This reviewed runtime lock currently targets Linux and Intel macOS. +# The separate Pixi manifest also resolves Linux ARM64 and Apple Silicon. platforms: - linux-64 - osx-64 @@ -19,8 +19,8 @@ dependencies: # Data processing - numpy>=1.21,<2.0 - pandas>=1.5,<3.0 - - polars>=0.19,<2.0 - - scipy>=1.10 + - polars>=1.0,<2.0 + - scipy>=1.13.1 # Bioinformatics (bioconda) - pysam>=0.21 diff --git a/environment.yml b/environment.yml index 7e3708d9..78e0c428 100644 --- a/environment.yml +++ b/environment.yml @@ -13,8 +13,8 @@ dependencies: # Data processing (aligned with pyproject.toml) - numpy>=1.21,<2.0 # <2.0 for ABI compatibility with pysam, pyarrow - pandas>=1.5,<3.0 - - polars>=0.19,<2.0 - - scipy>=1.10 + - polars>=1.0,<2.0 + - scipy>=1.13.1 # Bioinformatics (bioconda) - pysam>=0.21 @@ -46,8 +46,8 @@ dependencies: # Rust build (PyO3) - rust>=1.88 - cmake>=3.14 - - libclang - - clang + - libclang>=18,<20 + - clang>=18,<20 # Pip-only dependencies - pip diff --git a/galaxy/tools/wasp2/macros.xml b/galaxy/tools/wasp2/macros.xml index 40bdfe87..9a3cc0ac 100644 --- a/galaxy/tools/wasp2/macros.xml +++ b/galaxy/tools/wasp2/macros.xml @@ -52,6 +52,7 @@ + ^([A-Za-z0-9_.-]+(,[A-Za-z0-9_.-]+)*)?$ @@ -60,6 +61,7 @@ + ^([A-Za-z0-9_.-]+(,[A-Za-z0-9_.-]+)*)?$ @@ -82,13 +84,14 @@ help="Process indels in addition to SNPs"/> - + + diff --git a/galaxy/tools/wasp2/test-data/mapping.bam b/galaxy/tools/wasp2/test-data/mapping.bam new file mode 100644 index 0000000000000000000000000000000000000000..79a6c139427f3d59a8cb1a5106041c8e8eaaf367 GIT binary patch literal 250 zcmb2|=3rp}f&Xj_PR>jW(G0~!UsA86CL|PmD0s;8d9%?KA^kigYU|?VZVoxCk1`wNpAqGq`Fx&&vUXvj5=;9zf$N*I@L +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT donor.1 donor1 +chr1 111 . A C . PASS . GT 0|1 0|0 +chr1 211 . A G . PASS . GT 0|0 0|1 diff --git a/galaxy/tools/wasp2/test-data/mapping_remapped.bam b/galaxy/tools/wasp2/test-data/mapping_remapped.bam new file mode 100644 index 0000000000000000000000000000000000000000..187afbbc3fb570473049a608befe2a7ddb17dcb4 GIT binary patch literal 274 zcmb2|=3rp}f&Xj_PR>jW(G0~!UsA86CL|PmD0s;8d9%?KA^kigYU|?VZVoxCk1`wNpAqGq`Fnj~kUXvj5=;9zf$N*I@L + + + + + + + + + + + + + - - - + + + +
+ +
+ + + + + + diff --git a/galaxy/tools/wasp2/wasp2_find_imbalance.xml b/galaxy/tools/wasp2/wasp2_find_imbalance.xml index 47eab757..d97c62f2 100644 --- a/galaxy/tools/wasp2/wasp2_find_imbalance.xml +++ b/galaxy/tools/wasp2/wasp2_find_imbalance.xml @@ -69,6 +69,8 @@ + + diff --git a/galaxy/tools/wasp2/wasp2_make_reads.xml b/galaxy/tools/wasp2/wasp2_make_reads.xml index e8a605a0..ea3b8f1a 100644 --- a/galaxy/tools/wasp2/wasp2_make_reads.xml +++ b/galaxy/tools/wasp2/wasp2_make_reads.xml @@ -60,9 +60,16 @@ - - - + + + + + + + + + + diff --git a/pipelines/nf-atacseq/modules/local/wasp2_count_variants/environment.yml b/pipelines/nf-atacseq/modules/local/wasp2_count_variants/environment.yml index fbf58b57..ad5ccfb6 100644 --- a/pipelines/nf-atacseq/modules/local/wasp2_count_variants/environment.yml +++ b/pipelines/nf-atacseq/modules/local/wasp2_count_variants/environment.yml @@ -3,5 +3,6 @@ channels: - conda-forge - bioconda + - nodefaults dependencies: - bioconda::wasp2=1.5.0 diff --git a/pipelines/nf-atacseq/modules/local/wasp2_filter_remapped/environment.yml b/pipelines/nf-atacseq/modules/local/wasp2_filter_remapped/environment.yml index 9a0c69db..4be9639c 100644 --- a/pipelines/nf-atacseq/modules/local/wasp2_filter_remapped/environment.yml +++ b/pipelines/nf-atacseq/modules/local/wasp2_filter_remapped/environment.yml @@ -3,6 +3,7 @@ channels: - conda-forge - bioconda + - nodefaults dependencies: - bioconda::wasp2=1.5.0 - bioconda::samtools=1.17 diff --git a/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/environment.yml b/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/environment.yml index fbf58b57..ad5ccfb6 100644 --- a/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/environment.yml +++ b/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/environment.yml @@ -3,5 +3,6 @@ channels: - conda-forge - bioconda + - nodefaults dependencies: - bioconda::wasp2=1.5.0 diff --git a/pipelines/nf-atacseq/modules/local/wasp2_make_reads/environment.yml b/pipelines/nf-atacseq/modules/local/wasp2_make_reads/environment.yml index fbf58b57..ad5ccfb6 100644 --- a/pipelines/nf-atacseq/modules/local/wasp2_make_reads/environment.yml +++ b/pipelines/nf-atacseq/modules/local/wasp2_make_reads/environment.yml @@ -3,5 +3,6 @@ channels: - conda-forge - bioconda + - nodefaults dependencies: - bioconda::wasp2=1.5.0 diff --git a/pipelines/nf-atacseq/nextflow.config b/pipelines/nf-atacseq/nextflow.config index 36a17d7e..8949237c 100644 --- a/pipelines/nf-atacseq/nextflow.config +++ b/pipelines/nf-atacseq/nextflow.config @@ -94,6 +94,7 @@ profiles { cleanup = false } conda { + conda.createOptions = '--strict-channel-priority' conda.enabled = true docker.enabled = false singularity.enabled = false diff --git a/pipelines/nf-modules/modules/wasp2/analyze/environment.yml b/pipelines/nf-modules/modules/wasp2/analyze/environment.yml index a7cf6631..8ab29caf 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze/environment.yml +++ b/pipelines/nf-modules/modules/wasp2/analyze/environment.yml @@ -1,20 +1,11 @@ name: wasp2_analyze channels: - - bioconda - conda-forge - - defaults + - bioconda + - nodefaults dependencies: - python=3.11.* - - numpy - - pandas>=2.0.0 - - polars>=0.19 - - scipy - - anndata>=0.8.0 - - scanpy>=1.9.0 - - typer - - rich - - rust - - libclang - - pip - - pip: - - maturin>=1.4 + - bioconda::wasp2=1.5.0 + - samtools>=1.10 + - bcftools>=1.10 + - bedtools>=2.30 diff --git a/pipelines/nf-modules/modules/wasp2/count/environment.yml b/pipelines/nf-modules/modules/wasp2/count/environment.yml index 25d43ac1..80959034 100644 --- a/pipelines/nf-modules/modules/wasp2/count/environment.yml +++ b/pipelines/nf-modules/modules/wasp2/count/environment.yml @@ -1,25 +1,11 @@ name: wasp2_count channels: - - bioconda - conda-forge - - defaults + - bioconda + - nodefaults dependencies: - python=3.11.* - - numpy - - pandas>=2.0.0 - - polars>=0.19 - - scipy - - pysam - - pybedtools - - bedtools - - bcftools + - bioconda::wasp2=1.5.0 - samtools>=1.10 - - htslib>=1.10 - - anndata>=0.8.0 - - typer - - rich - - rust - - libclang - - pip - - pip: - - maturin>=1.4 + - bcftools>=1.10 + - bedtools>=2.30 diff --git a/pipelines/nf-modules/modules/wasp2/environment.yml b/pipelines/nf-modules/modules/wasp2/environment.yml index 95ed8274..c7d6640a 100644 --- a/pipelines/nf-modules/modules/wasp2/environment.yml +++ b/pipelines/nf-modules/modules/wasp2/environment.yml @@ -1,26 +1,13 @@ -# WASP2 Conda Environment -# Version: 1.2.1 (matches biocontainers/wasp2:1.2.1--pyhdfd78af_0) -# -# This environment should be kept in sync with the container version -# defined in ../../../nextflow.config (wasp2_container_version) +# Binary runtime shared by RNA-seq and single-cell modules. name: wasp2 channels: - conda-forge - bioconda - - defaults + - nodefaults dependencies: - - python>=3.10 - - pip - - numpy - - pandas - - polars - - scipy - - pysam - - pybedtools - - samtools - - bcftools - - bedtools - - typer - - rich - - pip: - - wasp2==1.5.0 + - python=3.11.* + - bioconda::wasp2=1.5.0 + - samtools>=1.10 + - bcftools>=1.10 + - bedtools>=2.30 + - bwa>=0.7 diff --git a/pipelines/nf-modules/modules/wasp2/map/environment.yml b/pipelines/nf-modules/modules/wasp2/map/environment.yml index 7e35f6be..3e5c075b 100644 --- a/pipelines/nf-modules/modules/wasp2/map/environment.yml +++ b/pipelines/nf-modules/modules/wasp2/map/environment.yml @@ -1,25 +1,12 @@ name: wasp2_map channels: - - bioconda - conda-forge - - defaults + - bioconda + - nodefaults dependencies: - python=3.11.* - - numpy - - pandas>=2.0.0 - - polars>=0.19 - - scipy - - pysam - - pybedtools - - bedtools - - bcftools + - bioconda::wasp2=1.5.0 - samtools>=1.10 - - htslib>=1.10 - - bwa - - typer - - rich - - rust - - libclang - - pip - - pip: - - maturin>=1.4 + - bcftools>=1.10 + - bedtools>=2.30 + - bwa>=0.7 diff --git a/pipelines/nf-modules/modules/wasp2/ml_output/environment.yml b/pipelines/nf-modules/modules/wasp2/ml_output/environment.yml index 5970c59a..5786adcc 100644 --- a/pipelines/nf-modules/modules/wasp2/ml_output/environment.yml +++ b/pipelines/nf-modules/modules/wasp2/ml_output/environment.yml @@ -1,18 +1,18 @@ name: wasp2_ml_output channels: - - bioconda - conda-forge - - defaults + - bioconda + - nodefaults dependencies: - python=3.11.* - numpy - pandas>=2.0.0 - scipy - - anndata>=0.8.0 - - zarr>=2.16.0 - - pyarrow>=14.0.0 + - anndata>=0.8.0,<0.12 + - zarr>=2.16.0,<3 + - pyarrow>=16.0.0,<26 - typer - rich - pip - pip: - - wasp2 + - wasp2==1.5.0 diff --git a/pipelines/nf-modules/nextflow.config b/pipelines/nf-modules/nextflow.config index 28052d02..78b6e5ba 100644 --- a/pipelines/nf-modules/nextflow.config +++ b/pipelines/nf-modules/nextflow.config @@ -58,6 +58,7 @@ profiles { singularity.enabled = true } conda { + conda.createOptions = '--strict-channel-priority' docker.enabled = false singularity.enabled = false conda.enabled = true diff --git a/pipelines/nf-outrider/modules/local/aggregate_counts/main.nf b/pipelines/nf-outrider/modules/local/aggregate_counts/main.nf index 2a7440f7..da3a2c21 100644 --- a/pipelines/nf-outrider/modules/local/aggregate_counts/main.nf +++ b/pipelines/nf-outrider/modules/local/aggregate_counts/main.nf @@ -2,7 +2,7 @@ process AGGREGATE_COUNTS { tag "$meta.id" label 'process_low' - conda "${moduleDir}/../../../environment.yml" + conda "${moduleDir}/../../../../nf-modules/modules/wasp2/analyze/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'docker://ghcr.io/mcvickerlab/wasp2:1.5.0' : 'ghcr.io/mcvickerlab/wasp2:1.5.0' }" diff --git a/pipelines/nf-outrider/modules/local/mae_detect/main.nf b/pipelines/nf-outrider/modules/local/mae_detect/main.nf index 7b90bef5..a973e9ff 100644 --- a/pipelines/nf-outrider/modules/local/mae_detect/main.nf +++ b/pipelines/nf-outrider/modules/local/mae_detect/main.nf @@ -2,7 +2,7 @@ process MAE_DETECT { tag "$meta.id" label 'process_low' - conda "${moduleDir}/../../../environment.yml" + conda "${moduleDir}/../../../../nf-modules/modules/wasp2/analyze/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'docker://ghcr.io/mcvickerlab/wasp2:1.5.0' : 'ghcr.io/mcvickerlab/wasp2:1.5.0' }" diff --git a/pipelines/nf-outrider/modules/local/merge_counts/main.nf b/pipelines/nf-outrider/modules/local/merge_counts/main.nf index 25037f6d..bd31d227 100644 --- a/pipelines/nf-outrider/modules/local/merge_counts/main.nf +++ b/pipelines/nf-outrider/modules/local/merge_counts/main.nf @@ -2,7 +2,7 @@ process MERGE_COUNTS { tag "merge_counts" label 'process_medium' - conda "${moduleDir}/../../../environment.yml" + conda "${moduleDir}/../../../../nf-modules/modules/wasp2/analyze/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'docker://ghcr.io/mcvickerlab/wasp2:1.5.0' : 'ghcr.io/mcvickerlab/wasp2:1.5.0' }" diff --git a/pipelines/nf-outrider/nextflow.config b/pipelines/nf-outrider/nextflow.config index c8f08f4a..6e53d9fd 100644 --- a/pipelines/nf-outrider/nextflow.config +++ b/pipelines/nf-outrider/nextflow.config @@ -99,6 +99,7 @@ profiles { cleanup = false } conda { + conda.createOptions = '--strict-channel-priority' conda.enabled = true docker.enabled = false singularity.enabled = false diff --git a/pipelines/nf-rnaseq/nextflow.config b/pipelines/nf-rnaseq/nextflow.config index 24d46a55..0bfb73e9 100644 --- a/pipelines/nf-rnaseq/nextflow.config +++ b/pipelines/nf-rnaseq/nextflow.config @@ -71,6 +71,7 @@ profiles { } conda { + conda.createOptions = '--strict-channel-priority' conda.enabled = true docker.enabled = false singularity.enabled = false diff --git a/pipelines/nf-scatac/nextflow.config b/pipelines/nf-scatac/nextflow.config index dbd6add1..b358187d 100644 --- a/pipelines/nf-scatac/nextflow.config +++ b/pipelines/nf-scatac/nextflow.config @@ -93,6 +93,7 @@ profiles { docker.enabled = false } conda { + conda.createOptions = '--strict-channel-priority' conda.enabled = true docker.enabled = false singularity.enabled = false diff --git a/pixi.toml b/pixi.toml index b846d5f6..4fed445e 100644 --- a/pixi.toml +++ b/pixi.toml @@ -2,16 +2,22 @@ # Install: curl -fsSL https://pixi.sh/install.sh | bash && pixi install # Run: pixi run wasp2-count --help # -# pixi resolves ALL dependencies (Python, Rust toolchain, htslib, samtools, etc.) -# in a single lockfile. No system packages required. +# Pixi resolves Python, Rust, and external tools. The generated pixi.lock is +# local to the installation; this repository does not distribute a Pixi lock. -[project] +[workspace] +requires-pixi = ">=0.82.0" name = "wasp2" version = "1.5.0" description = "Allele-specific analysis of next-generation sequencing data with Rust acceleration" channels = ["conda-forge", "bioconda"] platforms = ["linux-64", "linux-aarch64", "osx-arm64", "osx-64"] +# Recognize Bioconda distributions when resolving WASP2's PyPI requirements. +# This avoids a second provider overwriting the same installed Python files. +[workspace.conda-pypi-map] +bioconda = { mapping-mode = "overlay", mapping = { pysam = "pysam", pybedtools = "pybedtools", pgenlib = "pgenlib" } } + [dependencies] # Core Python python = ">=3.10,<3.14" @@ -19,7 +25,7 @@ python = ">=3.10,<3.14" # Bioinformatics tools (resolved by conda) samtools = ">=1.10" bcftools = ">=1.10" -gsl = "2.7" +gsl = "==2.7" bedtools = ">=2.30" htslib = ">=1.10" bwa = ">=0.7" @@ -27,8 +33,8 @@ bwa = ">=0.7" # Data processing numpy = ">=1.21,<2.0" pandas = ">=1.5,<3.0" -polars = ">=0.19,<2.0" -scipy = ">=1.10" +polars = ">=1.0,<2.0" +scipy = ">=1.13.1" pysam = ">=0.21" pybedtools = ">=0.9" anndata = ">=0.8,<0.12" @@ -43,8 +49,8 @@ rich = ">=13.0" # Rust build toolchain rust = ">=1.88" cmake = ">=3.14" -libclang = "*" -clang = "*" +libclang = ">=18,<20" +clang = ">=18,<20" pkg-config = "*" [target.osx-64.dependencies] @@ -70,9 +76,11 @@ dev = { features = ["dev"], solve-group = "default" } verify = "wasp2-count --help && wasp2-map --help && wasp2-analyze --help && python -c 'import wasp2_rust; print(\"OK\")'" # Build Rust extension build = "maturin develop --release -m rust/Cargo.toml" + +[feature.dev.tasks] # Run tests test = "pytest tests/ -v --tb=short --ignore=tests/benchmarks -m 'not benchmark'" -test-mapping-parity = "pytest tests/regression/test_mapping_stage_parity.py -v --tb=short" +test-mapping-parity = "pytest tests/mapping/ tests/regression/test_quickbench*.py -v --tb=short" # Run benchmarks bench = "pytest tests/benchmarks/ -v --tb=short" # Lint diff --git a/pyproject.toml b/pyproject.toml index bd2ee00f..a6737520 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,14 +40,14 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Bio-Informatics", - "Typing :: Typed", ] dependencies = [ "numpy>=1.21.0", "pandas>=1.5.0,<3.0.0", - "polars>=0.19.0,<2.0", - "scipy>=1.10.0", + "polars>=1.0.0,<2.0", + # Native numerical compatibility is validated against SciPy 1.13.1. + "scipy>=1.13.1", "pysam>=0.21.0", "pybedtools>=0.9.0", "anndata>=0.8.0,<0.12.0", @@ -59,6 +59,10 @@ dependencies = [ ] [project.optional-dependencies] +export = [ + "zarr>=2.16,<3", + "pyarrow>=16.0,<26", +] dev = [ "pytest>=7.0", "pytest-cov>=4.0", @@ -129,7 +133,12 @@ module-name = "wasp2_rust" python-packages = ["counting", "mapping", "analysis", "wasp2"] bindings = "pyo3" strip = false -include = ["LICENSE", "README.md", "rust/licenses/SCIPY-LICENSE.txt", "rust/licenses/NUMPY-LICENSE.txt"] +include = [ + "LICENSE", "README.md", "rust/licenses/SCIPY-LICENSE.txt", "rust/licenses/NUMPY-LICENSE.txt", + { path = "scripts/container_smoke_test.sh", format = "sdist" }, + { path = "tests/shared_data/sample1.bam*", format = "sdist" }, + { path = "tests/shared_data/variants.vcf.gz*", format = "sdist" }, +] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/requirements.txt b/requirements.txt index c098ce14..dc9b093f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,15 +6,16 @@ # Data processing numpy>=1.21.0 pandas>=1.5.0,<3.0.0 -polars>=0.19.0,<2.0 -scipy>=1.10.0 +polars>=1.0.0,<2.0 +scipy>=1.13.1 # Bioinformatics pysam>=0.21.0 pybedtools>=0.9.0 anndata>=0.8.0,<0.12.0 scanpy>=1.10.0 -numba>=0.59.0 +numba>=0.59.0,<0.63.0; sys_platform == "darwin" and platform_machine == "x86_64" +numba>=0.59.0; sys_platform != "darwin" or platform_machine != "x86_64" # CLI typer>=0.12.0 diff --git a/rust/src/bam_remapper.rs b/rust/src/bam_remapper.rs index cfcfd9e9..e5d7248f 100644 --- a/rust/src/bam_remapper.rs +++ b/rust/src/bam_remapper.rs @@ -23,6 +23,32 @@ use std::path::Path; use crate::seq_decode::{copy_qual_into, decode_seq_into}; +/// Resolve numeric VCF genotypes or translated nucleotide genotypes at the +/// mapping boundary. Counting retains numeric genotypes for its phase tests. +pub(crate) fn genotype_to_alleles_view<'a>( + genotype: &'a str, + ref_allele: &'a str, + alt_allele: &'a str, +) -> Option<(&'a str, &'a str)> { + let (left, right) = genotype.split_once('|')?; + let resolve = |allele: &'a str| -> Option<&'a str> { + if allele.is_empty() || allele == "." { + return None; + } + if allele.bytes().all(|base| base.is_ascii_digit()) { + let index = allele.parse::().ok()?; + if index == 0 { + Some(ref_allele) + } else { + alt_allele.split(',').nth(index - 1) + } + } else { + Some(allele) + } + }; + Some((resolve(left)?, resolve(right)?)) +} + // ============================================================================ // Data Structures // ============================================================================ @@ -272,12 +298,11 @@ pub fn parse_intersect_bed>( .context("Failed to parse mate number")?; // Parse phased genotype - let gt_parts: Vec<&str> = genotype.split('|').collect(); - if gt_parts.len() != 2 { - continue; // Skip unphased or malformed genotypes - } - let hap1 = gt_parts[0].to_string(); - let hap2 = gt_parts[1].to_string(); + let Some((hap1, hap2)) = genotype_to_alleles_view(genotype, fields[9], fields[10]) else { + continue; // Skip unphased or missing genotypes. + }; + let hap1 = hap1.to_string(); + let hap2 = hap2.to_string(); let span = VariantSpan { chrom, @@ -380,12 +405,11 @@ pub fn parse_intersect_bed_by_chrom>( .parse::() .context("Failed to parse mate number")?; - let gt_parts: Vec<&str> = genotype.split('|').collect(); - if gt_parts.len() != 2 { - continue; - } - let hap1 = gt_parts[0].to_string(); - let hap2 = gt_parts[1].to_string(); + let Some((hap1, hap2)) = genotype_to_alleles_view(genotype, fields[9], fields[10]) else { + continue; // Skip unphased or missing genotypes. + }; + let hap1 = hap1.to_string(); + let hap2 = hap2.to_string(); let span = VariantSpan { chrom: chrom.clone(), @@ -581,21 +605,23 @@ fn process_read_pair( // Only keep pairs where at least one read differs from original let mut haplotype_reads = Vec::new(); - for (hap_idx, ((r1_seq, r1_qual), (r2_seq, r2_qual))) in - r1_haps.iter().zip(r2_haps.iter()).enumerate() + let changed_pairs: Vec<_> = r1_haps + .iter() + .zip(r2_haps.iter()) + .enumerate() + .filter(|(_, ((r1_seq, _), (r2_seq, _)))| r1_seq != &r1_original || r2_seq != &r2_original) + .collect(); + let total_seqs = changed_pairs.len(); + for (write_idx, (hap_idx, ((r1_seq, r1_qual), (r2_seq, r2_qual)))) in + changed_pairs.into_iter().enumerate() { - // Skip if both sequences are unchanged - if r1_seq == &r1_original && r2_seq == &r2_original { - continue; - } - stats.haplotypes_generated += 2; // Count both R1 and R2 // Generate WASP names let r1_pos = read1.pos() as u32; let r2_pos = read2.pos() as u32; - let seq_num = hap_idx + 1; - let total_seqs = 2; // We're generating 2 haplotypes (hap1, hap2) + // The filter expects the number of pairs actually written, as in Python. + let seq_num = write_idx + 1; let base_name = generate_wasp_name(read_name, r1_pos, r2_pos, seq_num, total_seqs); @@ -1970,6 +1996,84 @@ mod tests { use std::io::Write; use tempfile::NamedTempFile; + #[test] + fn test_remap_names_count_only_changed_pairs() { + for (hap1, expected_pairs) in [("A", 1), ("G", 2)] { + let read1 = create_test_record(100, "50M"); + let read2 = create_test_record(200, "50M"); + let mut variants = FxHashMap::default(); + variants.insert( + b"test_read".to_vec(), + vec![VariantSpan { + chrom: "chr1".into(), + start: 100, + stop: 150, + vcf_start: 110, + vcf_stop: 111, + mate: 1, + hap1: hap1.into(), + hap2: "C".into(), + }], + ); + let output = process_read_pair( + &read1, + &read2, + &variants, + &RemapConfig::default(), + &mut RemapStats::default(), + ) + .unwrap() + .unwrap(); + assert_eq!(output.len(), expected_pairs * 2); + for (index, pair) in output.chunks_exact(2).enumerate() { + let base = format!("test_read_WASP_100_200_{}_{}", index + 1, expected_pairs); + assert_eq!(pair[0].name, format!("{base}/1").as_bytes()); + assert_eq!(pair[1].name, format!("{base}/2").as_bytes()); + } + } + } + + #[test] + fn test_numeric_and_translated_genotypes_resolve_identically() { + for (genotype, expected) in [ + ("0|1", Some(("A", "G"))), + ("1|0", Some(("G", "A"))), + ("A|G", Some(("A", "G"))), + ("0/1", None), + (".|1", None), + ("2|0", None), + ] { + assert_eq!(genotype_to_alleles_view(genotype, "A", "G"), expected); + } + assert_eq!( + genotype_to_alleles_view("1|2", "A", "C,G"), + Some(("C", "G")) + ); + assert_eq!( + genotype_to_alleles_view("0|1", "A", "ATT"), + Some(("A", "ATT")) + ); + } + + #[test] + fn test_both_intersection_parsers_translate_numeric_genotypes() { + let mut input = NamedTempFile::new().unwrap(); + writeln!( + input, + "chr1\t100\t150\tread/1\t60\t+\tchr1\t110\t111\tA\tG\t1|0" + ) + .unwrap(); + input.flush().unwrap(); + let parsed = parse_intersect_bed(input.path()).unwrap(); + let grouped = parse_intersect_bed_by_chrom(input.path()).unwrap(); + for spans in [ + parsed.get(b"read".as_slice()).unwrap(), + grouped["chr1"].get(b"read".as_slice()).unwrap(), + ] { + assert_eq!((spans[0].hap1.as_str(), spans[0].hap2.as_str()), ("G", "A")); + } + } + #[test] fn test_parse_intersect_bed() { // Create test BED file diff --git a/rust/src/unified_pipeline.rs b/rust/src/unified_pipeline.rs index 35677d4c..d17a38fe 100644 --- a/rust/src/unified_pipeline.rs +++ b/rust/src/unified_pipeline.rs @@ -39,8 +39,8 @@ fn interval_index(interval: &impl GenericInterval) -> u32 { use crate::bam_intersect::{build_variant_store, VariantStore}; use crate::bam_remapper::{ apply_trim_combination, calculate_indel_delta, classify_variant_location, - generate_haplotype_seqs_view_with_buffers, generate_trim_combinations, IndelConfig, - RemapConfig, VariantLocation, VariantSpanView, + generate_haplotype_seqs_view_with_buffers, generate_trim_combinations, + genotype_to_alleles_view, IndelConfig, RemapConfig, VariantLocation, VariantSpanView, }; use crate::seq_decode::{copy_qual_into, decode_seq_into}; @@ -449,23 +449,6 @@ fn increment_overlap_stats(stats: &mut UnifiedStats, mask: u8) { } } -/// Convert phased genotype to haplotype alleles (borrowed). -/// -/// Supports both 0/1 indexing (ref/alt) and direct allele strings. -fn genotype_to_alleles_view<'a>( - genotype: &'a str, - ref_allele: &'a str, - alt_allele: &'a str, -) -> Option<(&'a str, &'a str)> { - let (left, right) = genotype.split_once('|')?; - let to_allele = |s: &'a str| match s { - "0" => ref_allele, - "1" => alt_allele, - _ => s, - }; - Some((to_allele(left), to_allele(right))) -} - /// Generate haplotype sequences for a read with variants /// /// FIX: Process ALL overlapping variants (not just first) to match Python DEV behavior. diff --git a/scripts/check_sbom.py b/scripts/check_sbom.py new file mode 100644 index 00000000..537e70ff --- /dev/null +++ b/scripts/check_sbom.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Check that a CycloneDX SBOM actually inventories required components.""" + +import argparse +import json +from pathlib import Path + + +def check(path: Path, required: list[str]) -> None: + inventory = json.loads(path.read_text()) + names = {item["name"].lower().replace("_", "-") for item in inventory.get("components", [])} + missing = {name.lower().replace("_", "-") for name in required} - names + if missing: + raise ValueError(f"{path}: missing SBOM components: {', '.join(sorted(missing))}") + print(f"Verified {len(names)} inventoried components in {path.name}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", type=Path) + parser.add_argument("required", nargs="+") + args = parser.parse_args() + check(args.path, args.required) diff --git a/scripts/compare_archive_analysis.py b/scripts/compare_archive_analysis.py index 080aa2ae..1ecbedc5 100644 --- a/scripts/compare_archive_analysis.py +++ b/scripts/compare_archive_analysis.py @@ -302,7 +302,10 @@ def compare_frames(expected: pd.DataFrame, observed: pd.DataFrame, key: str = "r ) equal = valid and left_counts == right_counts deltas[column] = ( - max((abs(a - b) for a, b in zip(left_counts, right_counts, strict=True)), default=0) + max( + (abs(int(a) - int(b)) for a, b in zip(left_counts, right_counts, strict=True)), + default=0, + ) if valid else float("nan") ) diff --git a/scripts/container_smoke_test.sh b/scripts/container_smoke_test.sh index 7a52d13d..2a6ada73 100755 --- a/scripts/container_smoke_test.sh +++ b/scripts/container_smoke_test.sh @@ -23,7 +23,9 @@ set -euo pipefail # Detect data directory: container path or repo path -if [[ -d "/opt/wasp2/test-data" ]]; then +if [[ -n "${WASP2_TEST_DATA_DIR:-}" ]]; then + DATA_DIR="$WASP2_TEST_DATA_DIR" +elif [[ -d "/opt/wasp2/test-data" ]]; then DATA_DIR="/opt/wasp2/test-data" elif [[ -d "tests/shared_data" ]]; then DATA_DIR="tests/shared_data" @@ -53,10 +55,10 @@ echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 1: CLI binaries exist and print version # ───────────────────────────────────────────────────────────────────────────── -echo "[1/5] Version checks..." +echo "[1/7] Version checks..." for cmd in wasp2-count wasp2-map wasp2-analyze; do - if $cmd --version > /dev/null 2>&1; then + if $cmd --version; then echo " PASS: $cmd --version" PASS=$((PASS + 1)) else @@ -70,10 +72,17 @@ from importlib.metadata import version import wasp2 import wasp2_rust +import subprocess +import re + installed = version("wasp2") assert wasp2.__version__ == wasp2_rust.__version__ == installed expected = os.environ.get("EXPECTED_WASP2_VERSION", installed) assert installed == expected, (installed, expected) +for command in ("wasp2-count", "wasp2-map", "wasp2-analyze"): + output = subprocess.check_output([command, "--version"], stderr=subprocess.STDOUT, text=True) + match = re.search(r"WASP2\s*│\s*([^\s│]+)", output) + assert match and match.group(1) == expected, (command, output, expected) print(f" PASS: Python, Rust and distribution versions agree: {installed}") PY PASS=$((PASS + 1)) @@ -82,7 +91,7 @@ echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 2: Python imports # ───────────────────────────────────────────────────────────────────────────── -echo "[2/5] Python imports..." +echo "[2/7] Python imports..." if python -c "import wasp2_rust; print('Rust extension OK')" 2>/dev/null; then echo " PASS: wasp2_rust import" @@ -104,7 +113,7 @@ echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 3: Count variants with the bundled synthetic BAM/VCF # ───────────────────────────────────────────────────────────────────────────── -echo "[3/5] Count variants with bundled BAM/VCF..." +echo "[3/7] Count variants with bundled BAM/VCF..." if [[ -f "$DATA_DIR/sample1.bam" && -f "$DATA_DIR/variants.vcf.gz" ]]; then if wasp2-count count-variants \ @@ -112,31 +121,57 @@ if [[ -f "$DATA_DIR/sample1.bam" && -f "$DATA_DIR/variants.vcf.gz" ]]; then "$DATA_DIR/variants.vcf.gz" \ --samples sample1 \ --out "$TMP_DIR/counts.tsv" \ - 2>/dev/null; then - - if [[ -s "$TMP_DIR/counts.tsv" ]]; then - ROWS=$(wc -l < "$TMP_DIR/counts.tsv") - echo " PASS: count-variants produced $ROWS lines" - PASS=$((PASS + 1)) - else - echo " FAIL: count-variants output is empty" - FAIL=$((FAIL + 1)) - fi + --temp_loc "$TMP_DIR"; then + + python - "$TMP_DIR/counts.tsv" "$DATA_DIR/sample1.bam" "$DATA_DIR/variants.vcf.gz" <<'PYCOUNTS' +import csv +import sys +from collections import Counter +import pysam + +# Independent Python reference: consume each read name at the first fetched SNP, +# with no base-quality filter, matching the original counting contract. +with pysam.AlignmentFile(sys.argv[2]) as bam, pysam.VariantFile(sys.argv[3]) as vcf: + expected = {} + seen = set() + for variant in vcf: + if variant.samples["sample1"]["GT"] not in ((0, 1), (1, 0)): + continue + bases = Counter() + for read in bam.fetch(variant.contig, variant.start, variant.stop): + identity = (variant.contig, read.query_name) + if identity in seen: + continue + seen.add(identity) + for query_pos, reference_pos in read.get_aligned_pairs(matches_only=True): + if reference_pos == variant.start: + bases[read.query_sequence[query_pos]] += 1 + ref, alt = variant.ref, variant.alts[0] + expected[(variant.contig, str(variant.pos), ref, alt)] = (bases[ref], bases[alt], sum(bases.values()) - bases[ref] - bases[alt]) +with open(sys.argv[1]) as handle: + rows = list(csv.DictReader(handle, delimiter="\t")) +actual = {(r["chrom"], r["pos"], r["ref"], r["alt"]): tuple(int(r[k]) for k in ("ref_count", "alt_count", "other_count")) for r in rows} +assert len(rows) == len(actual) == len(expected) > 0 +assert actual == expected, (actual, expected) +print(" PASS: exact keyed allele counts against independent BAM base counts") +PYCOUNTS + PASS=$((PASS + 1)) else echo " FAIL: count-variants exited with error" FAIL=$((FAIL + 1)) fi else - echo " SKIP: Test data not available" + echo " FAIL: Required test BAM/VCF missing" + FAIL=$((FAIL + 1)) fi echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 4: External tools (samtools, bcftools, bedtools) # ───────────────────────────────────────────────────────────────────────────── -echo "[4/5] External tool availability..." +echo "[4/7] External tool availability..." -for tool in samtools bcftools bedtools; do +for tool in samtools bcftools bedtools bwa; do if command -v $tool > /dev/null 2>&1; then echo " PASS: $tool available" PASS=$((PASS + 1)) @@ -150,7 +185,7 @@ echo "" # Exercise the original ATAC failure: reordered columns, nonnumeric region IDs, # and both phase orientations. Compare phased and unphased CLI runs to catch # silently ignored --phased as well as positional parsing and ignored grouping. -echo "[5/5] ATAC region analysis..." +echo "[5/7] ATAC region analysis..." python - "$TMP_DIR/regions.tsv" <<'PY' import csv import sys @@ -171,6 +206,8 @@ python - "$TMP_DIR/phased.tsv" "$TMP_DIR/unphased.tsv" <<'PY' import csv import math import sys +import numpy as np +from scipy.stats import chi2, false_discovery_control def read(path): with open(path) as handle: @@ -184,12 +221,218 @@ for region, counts in [("chr1:100-250", (21, 19)), ("chr1:300-450", (47, 33))]: assert (int(row["ref_count"]), int(row["alt_count"])) == counts assert int(row["snp_count"]) == 2 assert all(math.isfinite(float(row[key])) for key in ["mu", "null_ll", "alt_ll", "pval", "fdr_pval"]) +for results in (phased, unphased): + pvalues = [] + for row in results.values(): + lrt = max(0.0, 2 * (float(row["alt_ll"]) - float(row["null_ll"]))) + assert math.isclose(float(row["lrt"]), lrt, rel_tol=1e-8, abs_tol=1e-8) + pvalues.append(chi2.sf(lrt, 1)) + assert np.allclose([float(row["pval"]) for row in results.values()], pvalues, rtol=1e-8, atol=1e-8) + assert np.allclose([float(row["fdr_pval"]) for row in results.values()], false_discovery_control(pvalues), rtol=1e-8, atol=1e-8) assert any(abs(float(phased[k]["alt_ll"]) - float(unphased[k]["alt_ll"])) > 1e-6 for k in phased) print(" PASS: region grouping, raw counts and phased likelihood through CLI") PY PASS=$((PASS + 1)) echo "" +# Mapping must run through the installed CLI, not merely import successfully. +echo "[6/7] Mapping and remapping..." +# The older shared BAM has no proper-pair flags. Generate a small, uniquely +# mappable proper-pair fixture so this test actually exercises allele swapping. +python - "$TMP_DIR" <<'PYFIXTURE' +import random +import sys +from pathlib import Path +import pysam +root = Path(sys.argv[1]) +rng = random.Random(7341) +sequence = "".join(rng.choices("ACGT", k=3000)) +(root / "reference.fa").write_text(">chr_smoke\n" + sequence + "\n") +pysam.faidx(str(root / "reference.fa")) +header = {"HD": {"VN": "1.6", "SO": "coordinate"}, "SQ": [{"SN": "chr_smoke", "LN": len(sequence)}]} +reads = [] +for name, start in [("overlap", 250), ("unaffected", 800)]: + for mate, position in enumerate((start, start + 200)): + read = pysam.AlignedSegment() + read.query_name = name + read.flag = 99 if mate == 0 else 147 + read.reference_id = read.next_reference_id = 0 + read.reference_start = position + read.next_reference_start = start + 200 if mate == 0 else start + read.template_length = 300 if mate == 0 else -300 + read.mapping_quality = 60 + read.cigarstring = "100M" + read.query_sequence = sequence[position:position + 100] + read.query_qualities = pysam.qualitystring_to_array("I" * 100) + reads.append(read) +with pysam.AlignmentFile(root / "mapping.bam", "wb", header=header) as bam: + for read in sorted(reads, key=lambda r: r.reference_start): + bam.write(read) +pysam.index(str(root / "mapping.bam")) +ref = sequence[300] +alt = next(base for base in "ACGT" if base != ref) +(root / "mapping.vcf").write_text( + '##fileformat=VCFv4.2\n##contig=\n' + '##FORMAT=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n' + f'chr_smoke\t301\t.\t{ref}\t{alt}\t.\tPASS\t.\tGT\t0|1\n') +PYFIXTURE +wasp2-map make-reads "$TMP_DIR/mapping.bam" "$TMP_DIR/mapping.vcf" \ + --samples sample1 --paired --phased --out_dir "$TMP_DIR" \ + --temp_loc "$TMP_DIR" --out_json "$TMP_DIR/mapping.json" +python - "$TMP_DIR/mapping.json" <<'PYMAPPING' +import json +import sys +from collections import Counter +import pysam +with open(sys.argv[1]) as handle: + paths = json.load(handle) +with pysam.AlignmentFile(paths["to_remap_bam"]) as bam: + original = Counter((r.query_name, r.is_read1) for r in bam) +assert original, "fixture must exercise remapping" +root = __import__("pathlib").Path(sys.argv[1]).parent +with pysam.FastaFile(root / "reference.fa") as reference: + sequence = reference.fetch("chr_smoke") +with pysam.VariantFile(root / "mapping.vcf") as variants: + variant = next(variants) +ref_seq = sequence[250:350] +alt_seq = ref_seq[:50] + variant.alts[0] + ref_seq[51:] +mate_seq = sequence[450:550].translate(str.maketrans("ACGT", "TGCA"))[::-1] +expected_sequences = {"remap_fq1": Counter([alt_seq]), "remap_fq2": Counter([mate_seq])} +for key in ("remap_fq1", "remap_fq2"): + with pysam.FastxFile(paths[key]) as fastq: + records = list(fastq) + assert Counter(r.sequence for r in records) == expected_sequences[key], (key, Counter(r.sequence for r in records), expected_sequences[key]) + assert all(r.quality == "I" * 100 for r in records) + assert len({r.name for r in records}) == len(records) +print(" PASS: nonempty paired swapped FASTQs and remap BAM") +PYMAPPING +bwa index "$TMP_DIR/reference.fa" +bwa mem -I 300,10,350,250 "$TMP_DIR/reference.fa" "$TMP_DIR/mapping_swapped_alleles_r1.fq" \ + "$TMP_DIR/mapping_swapped_alleles_r2.fq" | samtools sort -o "$TMP_DIR/remapped.bam" +wasp2-map filter-remapped "$TMP_DIR/remapped.bam" --json "$TMP_DIR/mapping.json" \ + --out_bam "$TMP_DIR/filtered.bam" +python - "$TMP_DIR" <<'PYFILTER' +import sys +from pathlib import Path +from collections import Counter +import pysam +root = Path(sys.argv[1]) +def identities(path): + with pysam.AlignmentFile(path) as bam: + return Counter((r.query_name, r.is_read1) for r in bam if not r.is_secondary and not r.is_supplementary) +original = identities(root / "mapping_keep.bam") + identities(root / "mapping_to_remap.bam") +kept = identities(root / "mapping_keep.bam") +filtered = identities(root / "filtered.bam") +assert filtered == original, (filtered, original) +assert not (kept - filtered), "unaffected reads must survive filtering" +with pysam.AlignmentFile(root / "filtered.bam") as bam: + assert bam.has_index() +print(" PASS: indexed filtered BAM preserves unaffected read identities") +PYFILTER +PASS=$((PASS + 1)) + +# A moved allele-swapped pair and an incomplete pair must both be rejected. +python - "$TMP_DIR" <<'PYNEGATIVE' +import sys +from pathlib import Path +import pysam +root = Path(sys.argv[1]) +with pysam.AlignmentFile(root / "remapped.bam") as bam: + header = bam.header + reads = list(bam) +with pysam.AlignmentFile(root / "missing.bam", "wb", header=header) as bam: + for read in reads: + if read.is_read2: + bam.write(read) +with pysam.AlignmentFile(root / "moved.bam", "wb", header=header) as bam: + for original in reads: + read = pysam.AlignedSegment.fromstring(original.to_string(), header) + if read.is_read1: + read.reference_start += 1 + else: + read.next_reference_start += 1 + bam.write(read) +PYNEGATIVE +for scenario in moved missing; do + wasp2-map filter-remapped "$TMP_DIR/${scenario}.bam" --json "$TMP_DIR/mapping.json" \ + --out_bam "$TMP_DIR/filtered-${scenario}.bam" +done +python - "$TMP_DIR" <<'PYREJECT' +import sys +from pathlib import Path +from collections import Counter +import pysam +root = Path(sys.argv[1]) +for scenario in ("moved", "missing"): + with pysam.AlignmentFile(root / f"filtered-{scenario}.bam") as bam: + kept = Counter((read.query_name, read.is_read1) for read in bam) + assert kept == Counter({("unaffected", True): 1, ("unaffected", False): 1}), (scenario, kept) +print(" PASS: moved and incomplete remap pairs are rejected") +PYREJECT +PASS=$((PASS + 1)) + +echo "[7/7] Single-cell counting..." +python - "$TMP_DIR" <<'PYSCINPUT' +import sys +from pathlib import Path +import pysam +root = Path(sys.argv[1]) +with pysam.AlignmentFile(root / "mapping.bam") as bam: + header = bam.header + reads = [r for r in bam if r.query_name == "overlap"] +with pysam.VariantFile(root / "mapping.vcf") as variants: + variant = next(variants) +records = [] +for cell, alt in [("cellA-1", False), ("cellB-1", True)]: + for original in reads: + read = pysam.AlignedSegment.fromstring(original.to_string(), header) + read.query_name = cell + read.set_tag("CB", cell) + if alt and read.is_read1: + bases = list(read.query_sequence) + bases[50] = variant.alts[0] + read.query_sequence = "".join(bases) + read.query_qualities = pysam.qualitystring_to_array("I" * 100) + records.append(read) +with pysam.AlignmentFile(root / "cells.bam", "wb", header=header) as bam: + for read in sorted(records, key=lambda r: r.reference_start): + bam.write(read) +pysam.index(str(root / "cells.bam")) +(root / "barcodes.tsv").write_text("cellA-1\ncellB-1\n") +(root / "features.bed").write_text("chr_smoke\t250\t550\tpeak1\n") +PYSCINPUT +wasp2-count count-variants-sc "$TMP_DIR/cells.bam" "$TMP_DIR/mapping.vcf" \ + "$TMP_DIR/barcodes.tsv" --samples sample1 --feature "$TMP_DIR/features.bed" \ + --out "$TMP_DIR/cells.h5ad" --temp_loc "$TMP_DIR" +python - "$TMP_DIR" <<'PYSC' +import os +import sys +from pathlib import Path +import anndata as ad +import numpy as np +import pandas as pd +root = Path(sys.argv[1]) +data = ad.read_h5ad(root / "cells.h5ad") +assert data.shape == (2, 1), data.shape +assert list(data.obs_names) == ["cellA-1", "cellB-1"] +assert np.array_equal(data.layers["ref"].toarray(), [[1], [0]]) +assert np.array_equal(data.layers["alt"].toarray(), [[0], [1]]) +assert np.array_equal(data.X.toarray(), [[1], [1]]) +assert "feature" in data.uns +if os.environ.get("WASP2_TEST_EXPORTS") == "1": + data.write_zarr(root / "cells.zarr") + reopened = ad.read_zarr(root / "cells.zarr") + assert reopened.shape == data.shape + assert np.array_equal(reopened.X.toarray(), data.X.toarray()) + frame = pd.read_csv(root / "phased.tsv", sep="\t") + frame.to_parquet(root / "results.parquet", index=False) + pd.testing.assert_frame_equal(pd.read_parquet(root / "results.parquet"), frame) + print(" PASS: Zarr and Parquet export round trips") +print(" PASS: exact single-cell counts, barcodes, axes and feature metadata") +PYSC +PASS=$((PASS + 1)) + # ───────────────────────────────────────────────────────────────────────────── # Summary # ───────────────────────────────────────────────────────────────────────────── diff --git a/scripts/create_sanity_data.sh b/scripts/create_sanity_data.sh index 260e5c5a..1194eb65 100755 --- a/scripts/create_sanity_data.sh +++ b/scripts/create_sanity_data.sh @@ -50,7 +50,7 @@ log_error() { check_prerequisites() { log_info "Checking prerequisites..." - for cmd in samtools bcftools tabix bgzip; do + for cmd in samtools bcftools tabix bgzip wasp2-count wasp2-map wasp2-analyze; do if ! command -v "$cmd" &> /dev/null; then log_error "$cmd is required but not installed." exit 1 @@ -112,51 +112,21 @@ generate_expected_outputs() { local temp_dir="$OUT_DIR/temp" mkdir -p "$temp_dir" - # Generate expected counts - log_info "Running counting pipeline..." - WASP2 counting count-variants \ - "$bam" \ - "$vcf" \ - --out_file "$OUT_DIR/expected_counts.tsv" \ - --temp_loc "$temp_dir" \ - 2>&1 | tee "$OUT_DIR/counting.log" || { - log_warn "Counting pipeline may have warnings, check log" - } - - # Generate expected remapping outputs - log_info "Running mapping pipeline..." - WASP2 mapping make-reads \ - "$bam" \ - "$vcf" \ - --out_dir "$OUT_DIR/wasp_output" \ - --temp_loc "$temp_dir" \ - 2>&1 | tee "$OUT_DIR/mapping.log" || { - log_warn "Mapping pipeline may have warnings, check log" - } - - # Move FASTQ outputs to expected locations - if [[ -d "$OUT_DIR/wasp_output" ]]; then - # Find and copy R1/R2 FASTQ files - find "$OUT_DIR/wasp_output" -name "*_R1*.fq*" -o -name "*_1.fq*" | head -1 | while read -r f; do - if [[ -n "$f" ]]; then - if [[ "$f" == *.gz ]]; then - cp "$f" "$OUT_DIR/expected_r1.fq.gz" - else - gzip -c "$f" > "$OUT_DIR/expected_r1.fq.gz" - fi - fi - done - - find "$OUT_DIR/wasp_output" -name "*_R2*.fq*" -o -name "*_2.fq*" | head -1 | while read -r f; do - if [[ -n "$f" ]]; then - if [[ "$f" == *.gz ]]; then - cp "$f" "$OUT_DIR/expected_r2.fq.gz" - else - gzip -c "$f" > "$OUT_DIR/expected_r2.fq.gz" - fi - fi - done - fi + local sample + sample="${WASP2_SANITY_SAMPLE:-$(bcftools query -l "$vcf" | head -1)}" + test -n "$sample" + wasp2-count count-variants "$bam" "$vcf" --samples "$sample" \ + --out "$OUT_DIR/expected_counts.tsv" --temp_loc "$temp_dir" \ + 2>&1 | tee "$OUT_DIR/counting.log" + wasp2-map make-reads "$bam" "$vcf" --samples "$sample" --paired \ + --out_dir "$OUT_DIR/wasp_output" --temp_loc "$temp_dir" \ + 2>&1 | tee "$OUT_DIR/mapping.log" + gzip -c "$OUT_DIR/wasp_output/chr21_swapped_alleles_r1.fq" > "$OUT_DIR/expected_r1.fq.gz" + gzip -c "$OUT_DIR/wasp_output/chr21_swapped_alleles_r2.fq" > "$OUT_DIR/expected_r2.fq.gz" + wasp2-analyze find-imbalance "$OUT_DIR/expected_counts.tsv" \ + --out "$OUT_DIR/expected_analysis.tsv" + # These are generated candidates, not an independent scientific reference. + log_warn "Review these outputs independently before distributing them as baselines." # Cleanup temp rm -rf "$temp_dir" @@ -198,7 +168,7 @@ create_metadata() { "lines": $counts_lines } }, - "wasp2_version": "$(WASP2 --version 2>&1 | head -1 || echo 'unknown')" + "wasp2_version": "$(python -c 'from importlib.metadata import version; print(version("wasp2"))')" } EOF diff --git a/scripts/prepare_pypi_upload.py b/scripts/prepare_pypi_upload.py new file mode 100644 index 00000000..68931e63 --- /dev/null +++ b/scripts/prepare_pypi_upload.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Stage missing PyPI artifacts, refusing to overwrite different published bytes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import urllib.error +import urllib.request +from pathlib import Path + + +def stage(packages: Path, destination: Path, version: str) -> int: + try: + with urllib.request.urlopen( + f"https://pypi.org/pypi/wasp2/{version}/json", timeout=30 + ) as response: + published = json.load(response)["urls"] + except urllib.error.HTTPError as error: + if error.code != 404: + raise + published = [] + hashes = {item["filename"]: item["digests"]["sha256"] for item in published} + artifacts = sorted(p for p in packages.iterdir() if p.name.endswith((".whl", ".tar.gz"))) + if not artifacts: + raise ValueError("No package artifacts supplied") + missing = [] + for artifact in artifacts: + expected = hashes.get(artifact.name) + if expected is None: + missing.append(artifact) + elif hashlib.sha256(artifact.read_bytes()).hexdigest() != expected: + raise ValueError( + f"Published artifact differs: {artifact.name}; do not overwrite a release" + ) + # Check all hashes before creating any upload payload. + destination.mkdir(parents=True, exist_ok=False) + for artifact in missing: + shutil.copy2(artifact, destination / artifact.name) + return len(missing) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("packages", type=Path) + parser.add_argument("destination", type=Path) + parser.add_argument("version") + args = parser.parse_args() + count = stage(args.packages, args.destination, args.version) + print(f"Staged {count} missing artifacts") + # GitHub Actions reads this output; stdout remains useful locally. + import os + + if output := os.environ.get("GITHUB_OUTPUT"): + with open(output, "a") as handle: + handle.write(f"count={count}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/release_version.py b/scripts/release_version.py index 0f67dce8..1a5eed88 100755 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -10,7 +10,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -SEMVER = r"[0-9]+\.[0-9]+\.[0-9]+(?:[a-z0-9.-]+)?" +SEMVER = r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)" def read(path: str) -> str: @@ -63,6 +63,9 @@ def set_version(version: str, release_date: str) -> None: if re.fullmatch(SEMVER, version) is None: raise RuntimeError(f"invalid release version: {version}") + if dt.date.fromisoformat(release_date).isoformat() != release_date: + raise ValueError("Release dates must use YYYY-MM-DD") + replace_one("rust/Cargo.toml", rf'^(version\s*=\s*)"{SEMVER}"', rf'\1"{version}"') replace_one( "rust/Cargo.lock", @@ -120,7 +123,7 @@ def check_version() -> list[str]: if lock_match is None or lock_match.group(1) != expected: errors.append("rust/Cargo.lock root package version differs from rust/Cargo.toml") - pixi_version = toml_section_version("pixi.toml", "project") + pixi_version = toml_section_version("pixi.toml", "workspace") if pixi_version != expected: errors.append(f"pixi.toml: {pixi_version} != {expected}") @@ -165,11 +168,31 @@ def check_version() -> list[str]: return errors +def check_release_metadata() -> list[str]: + """Require finalized metadata only when publishing a stable release.""" + expected = cargo_version() + citation = read("CITATION.cff") + match = re.search(r'^date-released:\s*"(.*?)"\s*$', citation, re.MULTILINE) + if match is None: + return ["CITATION.cff: add date-released before publishing"] + date = match.group(1) + try: + if dt.date.fromisoformat(date).isoformat() != date: + raise ValueError(date) + except ValueError: + return ["CITATION.cff: date-released must use YYYY-MM-DD"] + heading = rf"^## \[?{re.escape(expected)}\]?\s*[-–]\s*{re.escape(date)}\s*$" + if not re.search(heading, read("CHANGELOG.md"), re.MULTILINE): + return [f"CHANGELOG.md: require '## [{expected}] - {date}' matching the citation"] + return [] + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--check", action="store_true", help="validate all active version pins") group.add_argument("--set", metavar="VERSION", help="update all active version pins") + parser.add_argument("--release", action="store_true", help="require finalized release metadata") parser.add_argument("--date", default=dt.date.today().isoformat(), help="release date") args = parser.parse_args() @@ -177,6 +200,8 @@ def main() -> int: set_version(args.set, args.date) errors = check_version() + if args.release: + errors.extend(check_release_metadata()) if errors: print("Version consistency failed:", file=sys.stderr) for error in sorted(set(errors)): diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index fa6e1448..d772635b 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -1,156 +1,5 @@ -#!/bin/bash -# ============================================================================= -# WASP2 CLI Smoke Test -# ============================================================================= -# Validates all WASP2 CLI subcommands with real test data. -# Uses tests/shared_data/ as input. -# -# Prerequisites: WASP2 installed (conda activate WASP2_dev2) -# Shared core data generated (tests/shared_data/generate_core_data.sh) -# -# Usage: -# bash scripts/smoke_test.sh -# -# Exit codes: -# 0 = all tests passed -# 1 = one or more tests failed -# ============================================================================= - +#!/usr/bin/env bash +# Run the same strict acceptance checks used by packaged containers. set -euo pipefail - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DATA_DIR="$REPO_ROOT/tests/shared_data" -TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/wasp2_smoke_XXXXXX") - -PASS=0 -FAIL=0 - -cleanup() { - rm -rf "$TMP_DIR" -} -trap cleanup EXIT - -echo "===================================================================" -echo " WASP2 CLI Smoke Test" -echo "===================================================================" -echo "Data dir: $DATA_DIR" -echo "Temp dir: $TMP_DIR" -echo "" - -# Validate shared data exists -if [[ ! -f "$DATA_DIR/sample1.bam" ]]; then - echo "ERROR: Shared core data not found at $DATA_DIR" - echo " Run: cd tests/shared_data && bash generate_core_data.sh" - exit 1 -fi - -assert_file_not_empty() { - local filepath=$1 - local label=$2 - if [[ -f "$filepath" ]] && [[ -s "$filepath" ]]; then - echo " PASS: $label ($(du -h "$filepath" | cut -f1))" - PASS=$((PASS + 1)) - else - echo " FAIL: $label - file missing or empty" - FAIL=$((FAIL + 1)) - fi -} - -assert_exit_zero() { - local label=$1 - shift - if "$@" > /dev/null 2>&1; then - echo " PASS: $label" - PASS=$((PASS + 1)) - else - echo " FAIL: $label (exit code $?)" - FAIL=$((FAIL + 1)) - fi -} - -# ───────────────────────────────────────────────────────────────────────────── -# Test 1: Version checks -# ───────────────────────────────────────────────────────────────────────────── -echo "[1/5] Version checks..." -assert_exit_zero "wasp2-count --version" wasp2-count --version -assert_exit_zero "wasp2-map --version" wasp2-map --version -assert_exit_zero "wasp2-analyze --version" wasp2-analyze --version -echo "" - -# ───────────────────────────────────────────────────────────────────────────── -# Test 2: Count variants -# ───────────────────────────────────────────────────────────────────────────── -echo "[2/5] Count variants..." -wasp2-count count-variants \ - "$DATA_DIR/sample1.bam" \ - "$DATA_DIR/variants.vcf.gz" \ - --samples SAMPLE1 \ - --out "$TMP_DIR/counts.tsv" \ - 2>/dev/null || true -assert_file_not_empty "$TMP_DIR/counts.tsv" "count-variants output" -echo "" - -# ───────────────────────────────────────────────────────────────────────────── -# Test 3: Count variants with BED region -# ───────────────────────────────────────────────────────────────────────────── -echo "[3/5] Count variants with region..." -wasp2-count count-variants \ - "$DATA_DIR/sample1.bam" \ - "$DATA_DIR/variants.vcf.gz" \ - --samples SAMPLE1 \ - --region "$DATA_DIR/regions.bed" \ - --out "$TMP_DIR/counts_region.tsv" \ - 2>/dev/null || true -assert_file_not_empty "$TMP_DIR/counts_region.tsv" "count-variants with region" -echo "" - -# ───────────────────────────────────────────────────────────────────────────── -# Test 4: Make remap reads -# ───────────────────────────────────────────────────────────────────────────── -echo "[4/5] Make remap reads..." -mkdir -p "$TMP_DIR/remap_out" -wasp2-map make-reads \ - "$DATA_DIR/sample1.bam" \ - "$DATA_DIR/variants.vcf.gz" \ - --samples SAMPLE1 \ - --out_dir "$TMP_DIR/remap_out" \ - --out_json "$TMP_DIR/remap_out/wasp_data.json" \ - --paired \ - --phased \ - 2>/dev/null || true -assert_file_not_empty "$TMP_DIR/remap_out/wasp_data.json" "make-reads WASP data JSON" -echo "" - -# ───────────────────────────────────────────────────────────────────────────── -# Test 5: Find imbalance (using generated counts) -# ───────────────────────────────────────────────────────────────────────────── -echo "[5/5] Find imbalance..." -if [[ -s "$TMP_DIR/counts.tsv" ]]; then - wasp2-analyze find-imbalance \ - "$TMP_DIR/counts.tsv" \ - --out "$TMP_DIR/analysis.tsv" \ - --min 1 \ - 2>/dev/null || true - assert_file_not_empty "$TMP_DIR/analysis.tsv" "find-imbalance output" -else - echo " SKIP: No counts output to analyze" - FAIL=$((FAIL + 1)) -fi -echo "" - -# ───────────────────────────────────────────────────────────────────────────── -# Summary -# ───────────────────────────────────────────────────────────────────────────── -TOTAL=$((PASS + FAIL)) -echo "===================================================================" -echo " Results: $PASS/$TOTAL passed" -echo "===================================================================" - -if [[ $FAIL -gt 0 ]]; then - echo " WARNING: $FAIL test(s) failed" - exit 1 -else - echo " All smoke tests passed!" - exit 0 -fi +exec bash "$SCRIPT_DIR/container_smoke_test.sh" "$@" diff --git a/src/counting/count_alleles.py b/src/counting/count_alleles.py index 85965851..63dcd9be 100644 --- a/src/counting/count_alleles.py +++ b/src/counting/count_alleles.py @@ -161,6 +161,9 @@ def make_count_df(bam_file: str, df: pl.DataFrame, use_rust: bool = True) -> pl. orient="row", ) + # Older supported Polars releases require identical join-key dtypes. + # Preserve the caller's position dtype rather than relying on coercion. + count_df = count_df.with_columns(pl.col("pos").cast(df.schema["pos"])) df = df.with_columns([pl.col("chrom").cast(chrom_enum)]).join( count_df, on=["chrom", "pos"], how="left" ) diff --git a/src/wasp2/io/pgen_source.py b/src/wasp2/io/pgen_source.py index bc1d48ab..3bacfdb5 100644 --- a/src/wasp2/io/pgen_source.py +++ b/src/wasp2/io/pgen_source.py @@ -412,34 +412,61 @@ def to_bed( samples: list[str] | None = None, het_only: bool = True, include_genotypes: bool = True, + include_indels: bool = False, + max_indel_len: int = 10, + biallelic_only: bool = True, ) -> Path: - """Export variants to BED format file. + """Export one BED row per variant, preserving sample order and phase. - BED format uses 0-based start, 1-based end coordinates. - - Args: - output: Output BED file path - samples: Optional list of sample IDs to include - het_only: If True, only export heterozygous variants - include_genotypes: If True, include genotype column - - Returns: - Path to the created BED file + Genotypes use VCF allele indices, as in VCFSource. Unphased calls retain + ``/``; they must not acquire invented haplotype orientation on export. + With no sample selection, emit sites without genotype columns. """ output_path = Path(output) - - with open(output_path, "w") as f: - for vg in self.iter_variants(samples=samples, het_only=het_only): - # Write BED line: chrom, start (0-based), end (1-based), ref, alt - line = vg.variant.to_bed_line() - - # Add genotype if requested + selected = [self.get_sample_idx(sample) for sample in samples] if samples else [] + subset = sorted(set(selected)) + if samples is not None and not selected: + raise ValueError("Select at least one PGEN sample") + if subset: + self._reader.change_sample_subset(np.asarray(subset, dtype=np.uint32)) + positions = {sample: index for index, sample in enumerate(subset)} + alleles = np.empty(2 * len(subset), dtype=np.int32) + phase = np.empty(len(subset), dtype=np.uint8) + with output_path.open("w") as handle: + for variant_idx in range(self.variant_count): + row = self._pvar_df.iloc[variant_idx] + ref, alt = str(row["REF"]), str(row["ALT"]) + alternatives = alt.split(",") + if biallelic_only and len(alternatives) != 1: + continue + if not include_indels and any(len(base) != 1 for base in [ref, *alternatives]): + continue + if include_indels and any( + abs(len(ref) - len(base)) > max_indel_len for base in alternatives + ): + continue + genotype_columns = [] + if selected: + self._reader.read_alleles_and_phasepresent(variant_idx, alleles, phase) + if het_only and not any( + alleles[2 * i] >= 0 + and alleles[2 * i + 1] >= 0 + and alleles[2 * i] != alleles[2 * i + 1] + for i in range(len(subset)) + ): + continue + for sample in selected: + index = positions[sample] + left, right = alleles[2 * index : 2 * index + 2] + separator = "|" if phase[index] else "/" + genotype_columns.append( + f"{left}{separator}{right}" if left >= 0 and right >= 0 else "./." + ) + pos = int(row["POS"]) + fields = [str(row["CHROM"]), str(pos - 1), str(pos - 1 + len(ref)), ref, alt] if include_genotypes: - gt_str = self._genotype_to_string(vg.genotype) - line += f"\t{gt_str}" - - f.write(line + "\n") - + fields.extend(genotype_columns) + handle.write("\t".join(fields) + "\n") return output_path def _normalize_chrom(self, chrom: str) -> str: diff --git a/src/wasp2/io/variant_source.py b/src/wasp2/io/variant_source.py index bd98bcb0..17071096 100644 --- a/src/wasp2/io/variant_source.py +++ b/src/wasp2/io/variant_source.py @@ -122,7 +122,8 @@ class VariantSource(ABC): - Abstract methods: iter_variants, get_genotype, query_region, to_bed - Optional: close() for cleanup - Usage: + Usage:: + # Factory pattern with automatic format detection with VariantSource.open("variants.vcf.gz") as source: for vg in source.iter_variants(het_only=True): @@ -130,15 +131,18 @@ class VariantSource(ABC): # Direct subclass instantiation from wasp2.io.vcf_source import VCFSource + source = VCFSource("variants.vcf.gz") samples = source.samples source.close() - Registering a new format handler: + Registering a new format handler:: + @VariantSource.register("vcf", "bcf") class VCFSource(VariantSource): def __init__(self, path: str): self.path = path + # ... implement abstract methods """ @@ -160,11 +164,13 @@ def register(cls, *extensions: str): Returns: Decorator function that registers the subclass and returns it unchanged. - Example: + Example:: + @VariantSource.register("vcf", "bcf") class VCFSource(VariantSource): pass + @VariantSource.register(".pgen") class PGENSource(VariantSource): pass diff --git a/tests/io/test_pgen_source.py b/tests/io/test_pgen_source.py new file mode 100644 index 00000000..4a502192 --- /dev/null +++ b/tests/io/test_pgen_source.py @@ -0,0 +1,41 @@ +"""Exercise the installed optional PGEN reader through the public BED interface.""" + +from pathlib import Path + +import numpy as np +import pytest + +from wasp2.io.compat import variants_to_bed + +pgenlib = pytest.importorskip("pgenlib") + + +@pytest.fixture +def pgen(tmp_path: Path): + path = tmp_path / "variants.pgen" + path.with_suffix(".psam").write_text("#IID\ndonor.1\ndonor1\n") + path.with_suffix(".pvar").write_text( + "#CHROM\tPOS\tID\tREF\tALT\n1\t100\trs1\tA\tG\n1\t200\trs2\tC\tT\n1\t300\trs3\tA\tATT\n" + ) + with pgenlib.PgenWriter(bytes(path), 2, variant_ct=3, hardcall_phase_present=True) as writer: + writer.append_alleles(np.array([1, 0, 0, 0], dtype=np.int32), all_phased=True) + writer.append_alleles(np.array([0, 1, 1, 1], dtype=np.int32), all_phased=False) + writer.append_alleles(np.array([0, 1, 0, 0], dtype=np.int32), all_phased=True) + return path + + +@pytest.mark.parametrize("samples", [["donor.1"], ["donor1", "donor.1"]]) +def test_pgen_bed_keeps_phase_orientation_and_sample_columns(pgen, tmp_path, samples): + output = variants_to_bed(pgen, tmp_path / "variants.bed", samples=samples) + rows = [line.split("\t") for line in output.read_text().splitlines()] + assert len(rows) == 2 + assert all(len(row) == 5 + len(samples) for row in rows) + assert rows[0][5 + samples.index("donor.1")] == "1|0" + assert rows[1][5 + samples.index("donor.1")] == "0/1" + + +def test_pgen_export_indel_flag(pgen, tmp_path): + output = variants_to_bed( + pgen, tmp_path / "variants.bed", samples=["donor.1"], include_indels=True + ) + assert len(output.read_text().splitlines()) == 3 diff --git a/tests/regression/README.md b/tests/regression/README.md index a1fd1733..0346d0f2 100644 --- a/tests/regression/README.md +++ b/tests/regression/README.md @@ -1,165 +1,16 @@ -# Regression Test Suite - -**Purpose:** Validate that code changes don't break functionality or degrade performance. - -## Quick Start - -```bash -# Run all regression tests -pytest tests/regression/ -v - -# Run specific test class -pytest tests/regression/test_pipeline_regression.py::TestCountingRegression -v - -# Run with performance tests (slow) -pytest tests/regression/ -v -m slow -``` - -## What Gets Tested - -### ✅ Output Correctness -- **MD5 checksums** - Outputs must match baseline exactly -- **File structure** - Column names, data types, row counts -- **Statistical validity** - Values in correct ranges (p-values [0,1], etc.) - -### ⚡ Performance -- **Memory usage** - Must not exceed baseline × 1.20 (20% tolerance) -- **Execution time** - Must not exceed baseline × 1.30 (30% tolerance) -- **WASP filter rate** - Must keep >95% of reads - -### 📊 Baselines Used - -From `baselines/` directory (committed): -``` -Counting: 9.26s, 639 MB, MD5: 127a81810a43db3cc6924a26f591cc7a -Analysis: 2.97s, 340 MB, MD5: 394e1a7dbf14220079c3142c5b15bad8 -Mapping: 8s, 488 MB, 125,387 reads kept (99%) -``` - -## Usage Workflow - -### Before Refactoring -```bash -# Ensure all tests pass -pytest tests/regression/ -v - -# If any fail, investigate before starting -``` - -### During Refactoring -```bash -# Run tests frequently (after each logical change) -pytest tests/regression/ -v - -# Run fast tests only (skip full pipeline) -pytest tests/regression/ -v -m "not slow" -``` - -### After Refactoring -```bash -# Run full test suite including slow E2E tests -pytest tests/regression/ -v -m slow - -# If MD5 changed but output is correct, update baseline: -# 1. Manually verify new output is correct -# 2. Update MD5 in test_pipeline_regression.py:BASELINE_EXPECTATIONS -# 3. Commit new baseline files -``` - -## Test Categories - -| Test Class | Speed | What It Tests | -|------------|-------|---------------| -| `TestCountingRegression` | Fast (1s) | Counting output, memory, performance | -| `TestAnalysisRegression` | Fast (1s) | Analysis output, memory, performance | -| `TestMappingRegression` | Fast (1s) | WASP filtering, read counts | -| `TestFullPipelineIntegration` | Slow (20s) | End-to-end reproducibility | - -## Continuous Integration - -Add to `.github/workflows/regression.yml`: - -```yaml -name: Regression Tests -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.11' - - name: Install dependencies - run: | - pip install -e . - pip install pytest pandas - - name: Run regression tests - run: pytest tests/regression/ -v -``` - -## Updating Baselines - -When you **intentionally** change outputs: - -1. **Verify change is correct** - ```bash - # Compare old vs new output - diff baselines/counting/counts.tsv new_output/counts.tsv - ``` - -2. **Update baseline files** - ```bash - # Run pipeline to regenerate baselines - ./scripts/run_full_pipeline_baseline.sh - ``` - -3. **Update expected MD5s** - ```bash - # Calculate new checksums - md5sum baselines/counting/counts.tsv - md5sum baselines/analysis/ai_results.tsv - - # Update BASELINE_EXPECTATIONS in test_pipeline_regression.py - ``` - -4. **Commit changes** - ```bash - git add baselines/ tests/regression/test_pipeline_regression.py - git commit -m "Update baselines after [description of change]" - ``` - -## Troubleshooting - -### Test fails with MD5 mismatch -**Cause:** Output has changed -**Fix:** Compare outputs to verify correctness, then update baseline - -### Test fails with memory regression -**Cause:** Code now uses more memory -**Fix:** Investigate memory leak or optimize, OR increase tolerance if justified - -### Test fails with performance regression -**Cause:** Code is slower -**Fix:** Profile and optimize hot paths, OR increase tolerance if complexity trade-off - -### Test skipped -**Cause:** Baseline files not found -**Fix:** Run `./scripts/run_full_pipeline_baseline.sh` to generate baselines - -## Philosophy - -> **"Tests are a safety net, not a straightjacket"** - -- ✅ Tests should **enable** refactoring, not prevent it -- ✅ Tolerances exist to avoid flaky tests (±20-30%) -- ✅ Update baselines when outputs **intentionally** change -- ❌ Don't disable tests just because they fail -- ❌ Don't increase tolerances to paper over problems - -## See Also - -- `baselines/pipeline_metadata.txt` - Detailed benchmark data -- `docs/modules/COUNTING_MODULE.md` - Module documentation +# Regression tests + +Run `pytest tests/regression/ -v` for the mapping regressions. The deterministic +SNV and indel fixtures live alongside the tests and require the native extension. +The installed CLI acceptance test also requires samtools, bcftools, bedtools and +BWA. It checks exact counts and swapped sequences, remapping/filtering, analysis +identities, and single-cell matrices through `scripts/smoke_test.sh`. + +Historical benchmark checks in `test_pipeline_regression.py` require optional +local `baselines/` measurements and skip explicitly when those are absent. Their +MD5 and timing checks are historical evidence, not the release acceptance gate. + +Bulk and single-cell statistical reference comparisons are under `tests/analysis` +and `tests/counting`. They execute the pinned Python archive and distinguish its +narrow-integer limitations from explicitly widened reference inputs. Keep exact +count comparisons separate from statistical tolerances. diff --git a/tests/regression/mapping_fixtures.py b/tests/regression/mapping_fixtures.py new file mode 100644 index 00000000..51511e25 --- /dev/null +++ b/tests/regression/mapping_fixtures.py @@ -0,0 +1,84 @@ +"""Small deterministic mapping fixtures; no external benchmark checkout required.""" + +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +import pysam + + +@dataclass(frozen=True) +class Variant: + start: int + ref: str + alt: str + + +def quickbench_snv_variants(): + return [Variant(110, "A", "C"), Variant(210, "A", "G"), Variant(220, "A", "T")] + + +def quickbench_indel_variants(): + return [Variant(110, "A", "ATT")] + + +def write_bed(path: Path, variants): + path.write_text( + "".join( + f"chr1\t{v.start}\t{v.start + len(v.ref)}\t{v.ref}\t{v.alt}\t{v.ref}|{v.alt}\n" + for v in variants + ) + ) + + +def write_pair(path: Path, name: str, reverse_mate: bool): + header = {"HD": {"VN": "1.6", "SO": "coordinate"}, "SQ": [{"SN": "chr1", "LN": 1000}]} + with pysam.AlignmentFile(path, "wb", header=header) as bam: + for mate, position in enumerate((100, 200)): + read = pysam.AlignedSegment() + read.query_name = name + read.flag = (99 if mate == 0 else 147) if reverse_mate else (67 if mate == 0 else 131) + read.reference_id = read.next_reference_id = 0 + read.reference_start = position + read.next_reference_start = 200 if mate == 0 else 100 + read.template_length = 150 if mate == 0 else -150 + read.mapping_quality = 60 + read.cigarstring = "50M" + read.query_sequence = "A" * 50 + read.query_qualities = list(range(50)) + bam.write(read) + pysam.index(str(path)) + + +def write_synthetic_bam(path): + write_pair(path, "pairR", True) + + +def write_synthetic_bam_indel(path): + write_pair(path, "pairI", False) + + +class CanonicalFastqRecord(NamedTuple): + name: str + mate: int + sequence: str + quality: str + + +def iter_fastq(path): + with pysam.FastxFile(str(path)) as records: + for record in records: + yield record.name, record.sequence, record.quality + + +def fastq_counter(*paths): + result = Counter() + for mate, path in enumerate(paths, start=1): + for name, sequence, quality in iter_fastq(path): + result[CanonicalFastqRecord(name.split("_WASP_", 1)[0], mate, sequence, quality)] += 1 + return result + + +def counter_diff(left, right): + return list((left - right).elements()), list((right - left).elements()) diff --git a/tests/regression/test_pipeline_regression.py b/tests/regression/test_pipeline_regression.py index 29b8a6a9..c7740d4b 100644 --- a/tests/regression/test_pipeline_regression.py +++ b/tests/regression/test_pipeline_regression.py @@ -323,75 +323,27 @@ def test_mapping_wasp_filter_rate(self): class TestFullPipelineIntegration: - """End-to-end pipeline integration tests.""" + """Exercise the supported installed CLI with the shared acceptance suite.""" @pytest.mark.slow - def test_full_pipeline_reproducibility(self, tmp_path): - """Run full pipeline and verify output matches baseline exactly. - - This is a slow test (20+ seconds) but provides strongest guarantee. - """ - # Create temp output directory - temp_baseline = tmp_path / "baseline_test" - temp_baseline.mkdir() - - # Run pipeline script - script = ROOT / "scripts" / "run_full_pipeline_baseline.sh" - - if not script.exists(): - pytest.skip("Pipeline script not found") - - # Require external deps that the script needs; skip if unavailable - missing = [cmd for cmd in ["bcftools", "bedtools", "samtools"] if shutil.which(cmd) is None] + def test_full_pipeline_reproducibility(self): + required = [ + "bcftools", + "bedtools", + "samtools", + "bwa", + "wasp2-count", + "wasp2-map", + "wasp2-analyze", + ] + missing = [command for command in required if shutil.which(command) is None] if missing: pytest.skip(f"Pipeline prerequisites missing: {', '.join(missing)}") - - env = dict(subprocess.os.environ) - env_prefix = env.get("CONDA_PREFIX_2", env.get("CONDA_PREFIX", "")) - env["PYTHONPATH"] = str(ROOT / "src") - env["PATH"] = f"{Path(env_prefix) / 'bin'}:{env.get('PATH', '')}" - env["LD_LIBRARY_PATH"] = f"{Path(env_prefix) / 'lib'}:{env.get('LD_LIBRARY_PATH', '')}" - - # Ensure test data exists - required_files = [ - ROOT / "test_data" / "CD4_ATACseq_Day1_merged_filtered.sort.bam", - ROOT / "test_data" / "filter_chr10.vcf", - ROOT / "test_data" / "NA12878_snps_chr10.bed", - ] - for fpath in required_files: - if not fpath.exists(): - pytest.skip(f"Required test data missing: {fpath}") - - # Run with temp output result = subprocess.run( - [str(script)], - env={**env, "BASELINE_DIR": str(temp_baseline)}, - cwd=str(ROOT), + ["bash", str(ROOT / "scripts" / "smoke_test.sh")], + cwd=ROOT, capture_output=True, text=True, + check=False, ) - - if result.returncode != 0: - pytest.fail(f"Pipeline failed:\n{result.stderr}") - - # Compare outputs - temp_counts = temp_baseline / "counting" / "counts.tsv" - baseline_counts = BASELINE_DIR / "counting" / "counts.tsv" - - if temp_counts.exists() and baseline_counts.exists(): - assert md5_file(temp_counts) == md5_file(baseline_counts), ( - "Counting output not reproducible!" - ) - - temp_analysis = temp_baseline / "analysis" / "ai_results.tsv" - baseline_analysis = BASELINE_DIR / "analysis" / "ai_results.tsv" - - if temp_analysis.exists() and baseline_analysis.exists(): - assert md5_file(temp_analysis) == md5_file(baseline_analysis), ( - "Analysis output not reproducible!" - ) - - -if __name__ == "__main__": - # Run tests with verbose output - pytest.main([__file__, "-v", "--tb=short"]) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/regression/test_quickbench_indel_parity.py b/tests/regression/test_quickbench_indel_parity.py index b9d67f07..d2c76a0b 100644 --- a/tests/regression/test_quickbench_indel_parity.py +++ b/tests/regression/test_quickbench_indel_parity.py @@ -3,6 +3,14 @@ import pytest +from tests.regression.mapping_fixtures import ( + counter_diff, + fastq_counter, + quickbench_indel_variants, + write_bed, + write_synthetic_bam_indel, +) + ROOT = Path(__file__).resolve().parents[2] SRC = ROOT / "src" @@ -16,17 +24,6 @@ def test_quickbench_indel_parity(tmp_path: Path) -> None: """Unified make-reads matches the multi-pass path on a simple INDEL dataset (no trim combos).""" wasp2_rust = pytest.importorskip("wasp2_rust") - # Skip if benchmarking module not available (not included in release) - try: - from benchmarking.quickbench.fastq_utils import counter_diff, fastq_counter - from benchmarking.quickbench.synthetic_dataset import ( - quickbench_indel_variants, - write_bed, - write_synthetic_bam_indel, - ) - except ImportError: - pytest.skip("benchmarking module not available (not included in release)") - from mapping.intersect_variant_data import intersect_reads, process_bam from mapping.make_remap_reads import write_remap_bam @@ -88,6 +85,7 @@ def test_quickbench_indel_parity(tmp_path: Path) -> None: baseline_counter = fastq_counter(baseline_r1, baseline_r2) unified_counter = fastq_counter(unified_r1, unified_r2) + assert baseline_counter and unified_counter only_baseline, only_unified = counter_diff(baseline_counter, unified_counter) assert only_baseline == [] and only_unified == [], ( diff --git a/tests/regression/test_quickbench_indel_trim_invariants.py b/tests/regression/test_quickbench_indel_trim_invariants.py index 4333759d..ab67952a 100644 --- a/tests/regression/test_quickbench_indel_trim_invariants.py +++ b/tests/regression/test_quickbench_indel_trim_invariants.py @@ -3,6 +3,13 @@ import pytest +from tests.regression.mapping_fixtures import ( + iter_fastq, + quickbench_indel_variants, + write_bed, + write_synthetic_bam_indel, +) + ROOT = Path(__file__).resolve().parents[2] SRC = ROOT / "src" @@ -23,17 +30,6 @@ def test_quickbench_indel_trim_invariants(tmp_path: Path) -> None: """INDEL-mode produces N+1 trim-combos for a +2bp insertion and preserves read length.""" wasp2_rust = pytest.importorskip("wasp2_rust") - # Skip if benchmarking module not available (not included in release) - try: - from benchmarking.quickbench.fastq_utils import iter_fastq - from benchmarking.quickbench.synthetic_dataset import ( - quickbench_indel_variants, - write_bed, - write_synthetic_bam_indel, - ) - except ImportError: - pytest.skip("benchmarking module not available (not included in release)") - import pysam bam = tmp_path / "synthetic_indel.bam" diff --git a/tests/regression/test_quickbench_snv_parity.py b/tests/regression/test_quickbench_snv_parity.py index c576864a..ecc1ba92 100644 --- a/tests/regression/test_quickbench_snv_parity.py +++ b/tests/regression/test_quickbench_snv_parity.py @@ -3,10 +3,18 @@ import pytest +from tests.regression.mapping_fixtures import ( + counter_diff, + fastq_counter, + quickbench_snv_variants, + write_bed, + write_synthetic_bam, +) + ROOT = Path(__file__).resolve().parents[2] SRC = ROOT / "src" -# Allow importing `benchmarking.quickbench.*` and `mapping.*` +# Resolve project modules for source-checkout test runs. for p in (ROOT, SRC): if str(p) not in sys.path: sys.path.insert(0, str(p)) @@ -17,17 +25,6 @@ def test_quickbench_snv_parity(tmp_path: Path) -> None: """Unified make-reads matches the established multi-pass path on SNVs.""" wasp2_rust = pytest.importorskip("wasp2_rust") - # Skip if benchmarking module not available (not included in release) - try: - from benchmarking.quickbench.fastq_utils import counter_diff, fastq_counter - from benchmarking.quickbench.synthetic_dataset import ( - quickbench_snv_variants, - write_bed, - write_synthetic_bam, - ) - except ImportError: - pytest.skip("benchmarking module not available (not included in release)") - from mapping.intersect_variant_data import intersect_reads, process_bam from mapping.make_remap_reads import write_remap_bam @@ -89,6 +86,7 @@ def test_quickbench_snv_parity(tmp_path: Path) -> None: baseline_counter = fastq_counter(baseline_r1, baseline_r2) unified_counter = fastq_counter(unified_r1, unified_r2) + assert baseline_counter and unified_counter only_baseline, only_unified = counter_diff(baseline_counter, unified_counter) assert only_baseline == [] and only_unified == [], ( @@ -99,7 +97,7 @@ def test_quickbench_snv_parity(tmp_path: Path) -> None: # Strand sanity check: `pairR` has R2 flagged reverse in the BAM and should be # written to FASTQ in the original read orientation (rev-comp + qual reversal). - from benchmarking.quickbench.fastq_utils import CanonicalFastqRecord + from tests.regression.mapping_fixtures import CanonicalFastqRecord hap2_aligned = ["A"] * 50 hap2_aligned[10] = "G" diff --git a/tests/sanity/conftest.py b/tests/sanity/conftest.py index 48406e4d..0409ea23 100644 --- a/tests/sanity/conftest.py +++ b/tests/sanity/conftest.py @@ -8,6 +8,8 @@ from __future__ import annotations +import hashlib +import os import shutil import subprocess import tarfile @@ -22,18 +24,11 @@ # Sanity data version and paths SANITY_DATA_VERSION = "v1" SANITY_TARBALL_NAME = f"wasp2-sanity-chr21-{SANITY_DATA_VERSION}.tar.xz" -SANITY_DATA_DIR = Path(__file__).parent / "data" - -# Data hosting URLs (primary: GitHub Releases, backup: Zenodo) -# GitHub Releases: Fast, CI-integrated, cached by Actions -# Zenodo: DOI-backed archival for academic citation -GITHUB_RELEASE_URL = ( - f"https://github.com/mcvickerlab/WASP2/releases/download/v1.3.0/{SANITY_TARBALL_NAME}" -) -# Zenodo DOI URL for archival citation (update after Zenodo deposit) -# To activate: replace None with the direct file URL from Zenodo, e.g.: -# "https://zenodo.org/records/14538902/files/wasp2-sanity-chr21-v1.tar.xz" -# See: https://github.com/mcvickerlab/WASP2/issues/246 +SANITY_DATA_DIR = Path(os.environ.get("WASP2_SANITY_DATA_DIR", Path(__file__).parent / "data")) + +# No public archive is currently published. Use a reviewed local dataset or +# configure an archive URL and its independently verified SHA-256 explicitly. +GITHUB_RELEASE_URL = os.environ.get("WASP2_SANITY_DATA_URL") ZENODO_DOI_URL: str | None = None # Expected files in sanity dataset @@ -101,10 +96,18 @@ def download_sanity_data( if release_url is not None: urls_to_try.append(release_url) else: - urls_to_try.append(GITHUB_RELEASE_URL) + if GITHUB_RELEASE_URL: + urls_to_try.append(GITHUB_RELEASE_URL) if ZENODO_DOI_URL is not None: urls_to_try.append(ZENODO_DOI_URL) + if not urls_to_try: + raise RuntimeError( + "No public sanity archive configured. Set WASP2_SANITY_DATA_DIR for local data, or WASP2_SANITY_DATA_URL and WASP2_SANITY_DATA_SHA256." + ) + expected_hash = os.environ.get("WASP2_SANITY_DATA_SHA256") + if not expected_hash or len(expected_hash) != 64: + raise RuntimeError("Set WASP2_SANITY_DATA_SHA256 to the reviewed archive digest") tarball_path = SANITY_DATA_DIR / SANITY_TARBALL_NAME # Try each URL until one succeeds @@ -119,7 +122,7 @@ def download_sanity_data( ) elif shutil.which("curl"): subprocess.run( - ["curl", "-sL", "-o", str(tarball_path), url], + ["curl", "--fail", "--show-error", "-sL", "-o", str(tarball_path), url], check=True, capture_output=True, ) @@ -137,7 +140,10 @@ def download_sanity_data( f"Failed to download sanity data from any source. Last error: {last_error}" ) - # Extract tarball + # Verify before opening the archive; only regular files/directories are allowed. + if hashlib.sha256(tarball_path.read_bytes()).hexdigest() != expected_hash: + tarball_path.unlink(missing_ok=True) + raise RuntimeError("Sanity archive checksum mismatch") try: with tarfile.open(tarball_path, "r:xz") as tar: # Extract to data directory, stripping top-level dir @@ -146,13 +152,25 @@ def download_sanity_data( parts = Path(member.name).parts if len(parts) > 1: member.name = str(Path(*parts[1:])) - tar.extract(member, SANITY_DATA_DIR) + destination = (SANITY_DATA_DIR / member.name).resolve() + if not destination.is_relative_to(SANITY_DATA_DIR.resolve()): + raise RuntimeError("Sanity archive contains an unsafe path") + if member.isdir(): + destination.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + destination.parent.mkdir(parents=True, exist_ok=True) + with tar.extractfile(member) as source, destination.open("wb") as target: + shutil.copyfileobj(source, target) + else: + raise RuntimeError("Sanity archive contains a non-regular file") except (tarfile.TarError, OSError) as e: raise RuntimeError(f"Failed to extract sanity data: {e}") from e finally: # Clean up tarball tarball_path.unlink(missing_ok=True) + if not is_sanity_data_available(): + raise RuntimeError("Sanity archive is missing required expected outputs") return SANITY_DATA_DIR @@ -165,7 +183,7 @@ def sanity_data_dir() -> Path: if not is_sanity_data_available(): pytest.skip( "Sanity data not available. Run 'make download-sanity-data' " - "or download from GitHub releases." + "after configuring a reviewed archive URL and SHA-256." ) return SANITY_DATA_DIR diff --git a/tests/shared_data/README.md b/tests/shared_data/README.md index eb242eab..6e803185 100644 --- a/tests/shared_data/README.md +++ b/tests/shared_data/README.md @@ -15,7 +15,7 @@ bash generate_core_data.sh | File | Description | Size | |------|-------------|------| | `chr_test.fa` + `.fai` | 20kb synthetic reference genome (single contig `chr_test`) | ~20K | -| `variants.vcf` + `.gz` + `.tbi` | 10 het SNPs across 2 samples (SAMPLE1, SAMPLE2) | ~2K | +| `variants.vcf` + `.gz` + `.tbi` | 10 het SNPs across 2 samples (sample1, sample2) | ~2K | | `annotation.gtf` | 2 genes, 6 exons (INTGENE001 + strand, INTGENE002 - strand) | ~1.5K | | `regions.bed` | Exonic regions from GTF | ~300B | | `sample{1,2,3}.bam` + `.bai` | BWA-aligned wgsim reads (500 pairs each, seeds 42/43/44) | ~50K each | diff --git a/tests/test_release_packaging.py b/tests/test_release_packaging.py new file mode 100644 index 00000000..d3edd61a --- /dev/null +++ b/tests/test_release_packaging.py @@ -0,0 +1,104 @@ +"""Publication guards must fail before changing files or uploading artifacts.""" + +import hashlib +import importlib.util +import io +import json +from pathlib import Path +from urllib.error import HTTPError + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("version", ["1.5.0banana", "1.5.0..", "1.5.0-rc.1", "01.5.0"]) +def test_invalid_version_does_not_write(monkeypatch, version): + release = load_script("release_version") + monkeypatch.setattr(release, "write", lambda *args: pytest.fail("invalid version wrote files")) + with pytest.raises(RuntimeError, match="invalid release version"): + release.set_version(version, "2026-10-16") + + +def test_invalid_release_date_does_not_write(monkeypatch): + release = load_script("release_version") + monkeypatch.setattr(release, "write", lambda *args: pytest.fail("invalid date wrote files")) + with pytest.raises(ValueError): + release.set_version("1.5.0", "2026-02-30") + + +def test_publication_requires_matching_dated_metadata(monkeypatch): + release = load_script("release_version") + monkeypatch.setattr(release, "cargo_version", lambda: "1.5.0") + files = {"CITATION.cff": 'version: "1.5.0"\n', "CHANGELOG.md": "## [Unreleased]\n"} + monkeypatch.setattr(release, "read", files.__getitem__) + assert release.check_release_metadata() + files["CITATION.cff"] += 'date-released: "2026-10-16"\n' + assert release.check_release_metadata() + files["CHANGELOG.md"] = "## [1.5.0] - 2026-10-16\n" + assert release.check_release_metadata() == [] + + +@pytest.mark.parametrize("published", [False, True]) +def test_pypi_resume_only_stages_missing_identical_artifacts(tmp_path, monkeypatch, published): + upload = load_script("prepare_pypi_upload") + packages = tmp_path / "packages" + packages.mkdir() + (packages / "wasp2-1.5.0.tar.gz").write_bytes(b"source") + (packages / "wasp2-1.5.0.whl").write_bytes(b"wheel") + if published: + response = { + "urls": [ + { + "filename": "wasp2-1.5.0.tar.gz", + "digests": {"sha256": hashlib.sha256(b"source").hexdigest()}, + } + ] + } + monkeypatch.setattr( + upload.urllib.request, + "urlopen", + lambda *a, **k: io.BytesIO(json.dumps(response).encode()), + ) + else: + + def absent(*args, **kwargs): + raise HTTPError("https://pypi.org", 404, "missing", {}, None) + + monkeypatch.setattr(upload.urllib.request, "urlopen", absent) + destination = tmp_path / "upload" + assert upload.stage(packages, destination, "1.5.0") == (1 if published else 2) + assert (destination / "wasp2-1.5.0.whl").read_bytes() == b"wheel" + assert (destination / "wasp2-1.5.0.tar.gz").exists() != published + + +def test_pypi_resume_refuses_different_bytes_before_staging(tmp_path, monkeypatch): + upload = load_script("prepare_pypi_upload") + packages = tmp_path / "packages" + packages.mkdir() + (packages / "wasp2.whl").write_bytes(b"different") + response = {"urls": [{"filename": "wasp2.whl", "digests": {"sha256": "0" * 64}}]} + monkeypatch.setattr( + upload.urllib.request, "urlopen", lambda *a, **k: io.BytesIO(json.dumps(response).encode()) + ) + destination = tmp_path / "upload" + with pytest.raises(ValueError, match="Published artifact differs"): + upload.stage(packages, destination, "1.5.0") + assert not destination.exists() + + +def test_sbom_cannot_pass_with_an_empty_inventory(tmp_path): + inventory = load_script("check_sbom") + path = tmp_path / "sbom.json" + path.write_text('{"components": []}') + with pytest.raises(ValueError, match="missing SBOM components"): + inventory.check(path, ["wasp2", "rust-htslib"]) + path.write_text('{"components": [{"name": "wasp2"}, {"name": "rust-htslib"}]}') + inventory.check(path, ["wasp2", "rust-htslib"]) diff --git a/tutorials/atac_seq_workflow.ipynb b/tutorials/atac_seq_workflow.ipynb index 27e016ca..a3d93bf9 100644 --- a/tutorials/atac_seq_workflow.ipynb +++ b/tutorials/atac_seq_workflow.ipynb @@ -33,7 +33,26 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Prerequisites\n\n### Software\n\n- **WASP2** (`pip install wasp2`)\n- **Python 3.10+** with pandas, matplotlib, numpy\n- **samtools** (for BAM operations)\n- **tabix** (for VCF indexing)\n\n### Input Data\n\n| File | Description | Format |\n|------|-------------|--------|\n| `sample.bam` | ATAC-seq aligned reads | BAM (indexed) |\n| `variants.vcf.gz` | Phased heterozygous variants | VCF (indexed) |\n| `peaks.bed` | ATAC-seq peaks from MACS2/SEACR | BED or narrowPeak |\n\n**Note:** For best results, use WASP-filtered BAM files to correct reference mapping bias. See the [mapping documentation](../user_guide/mapping.rst) for details." + "source": [ + "## Prerequisites\n", + "\n", + "### Software\n", + "\n", + "- **WASP2** (`pip install wasp2`)\n", + "- **Python 3.10+** with pandas, matplotlib, numpy\n", + "- **samtools** (for BAM operations)\n", + "- **tabix** (for VCF indexing)\n", + "\n", + "### Input Data\n", + "\n", + "| File | Description | Format |\n", + "|------|-------------|--------|\n", + "| `sample.bam` | ATAC-seq aligned reads | BAM (indexed) |\n", + "| `variants.vcf.gz` | Phased heterozygous variants | VCF (indexed) |\n", + "| `peaks.bed` | ATAC-seq peaks from MACS2/SEACR | BED or narrowPeak |\n", + "\n", + "**Note:** For best results, use WASP-filtered BAM files to correct reference mapping bias. See the [mapping documentation](../user_guide/mapping.rst) for details." + ] }, { "cell_type": "markdown", @@ -93,7 +112,7 @@ "source": [ "# Load and inspect peaks\n", "peak_columns = [\n", - " \"chr\",\n", + " \"chrom\",\n", " \"start\",\n", " \"end\",\n", " \"name\",\n", @@ -233,11 +252,11 @@ "# Simulate realistic ATAC-seq allele counts\n", "counts_df = pd.DataFrame(\n", " {\n", - " \"chr\": np.random.choice([\"chr1\", \"chr2\", \"chr3\", \"chr4\", \"chr5\"], n_snps),\n", + " \"chrom\": np.random.choice([\"chr1\", \"chr2\", \"chr3\", \"chr4\", \"chr5\"], n_snps),\n", " \"pos\": np.random.randint(1e6, 2e8, n_snps),\n", " \"ref\": np.random.choice([\"A\", \"C\", \"G\", \"T\"], n_snps),\n", " \"alt\": np.random.choice([\"A\", \"C\", \"G\", \"T\"], n_snps),\n", - " \"region_id\": [f\"peak_{i}\" for i in np.random.randint(0, 1500, n_snps)],\n", + " \"region\": [f\"peak_{i}\" for i in np.random.randint(0, 1500, n_snps)],\n", " }\n", ")\n", "\n", @@ -251,9 +270,15 @@ "counts_df[\"ref_count\"] = np.random.binomial(total_depth, imbalance_prob)\n", "counts_df[\"alt_count\"] = total_depth - counts_df[\"ref_count\"]\n", "counts_df[\"other_count\"] = 0\n", + "peak_index = counts_df[\"region\"].str.removeprefix(\"peak_\").astype(int)\n", + "counts_df[\"chrom\"] = \"chr\" + ((peak_index % 5) + 1).astype(str)\n", + "counts_df[\"pos\"] = 1_000_000 + peak_index * 1000 + counts_df.groupby(\"region\").cumcount() * 2\n", + "counts_df[\"start\"] = counts_df[\"pos\"] - 1\n", + "# Keep the illustrative variant alleles distinct.\n", + "counts_df[\"alt\"] = counts_df[\"ref\"].map({\"A\": \"G\", \"G\": \"A\", \"C\": \"T\", \"T\": \"C\"})\n", "\n", "print(f\"Total SNPs counted: {len(counts_df):,}\")\n", - "print(f\"Unique peaks with SNPs: {counts_df['region_id'].nunique():,}\")\n", + "print(f\"Unique peaks with SNPs: {counts_df['region'].nunique():,}\")\n", "print(\"\\nCount statistics:\")\n", "print(counts_df[[\"ref_count\", \"alt_count\"]].describe())\n", "counts_df.head(10)" @@ -349,6 +374,7 @@ " {COUNTS_FILE} \\\\\n", " --min 5 \\\\\n", " --model single \\\\\n", + " --region-col region \\\\\n", " --output {IMBALANCE_FILE}\n", "\"\"\"\n", "\n", @@ -361,7 +387,17 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "%%bash -s \"$COUNTS_FILE\" \"$IMBALANCE_FILE\"\n# Uncomment to run (requires actual count file)\n# wasp2-analyze find-imbalance \\\n# \"$1\" \\\n# --min 5 \\\n# --model single \\\n# --output \"$2\"\n\necho \"Note: Uncomment the command above to run with your data\"" + "source": [ + "%%bash -s \"$COUNTS_FILE\" \"$IMBALANCE_FILE\"\n", + "# Uncomment to run (requires actual count file)\n", + "# wasp2-analyze find-imbalance \\\n", + "# \"$1\" \\\n", + "# --min 5 \\\n", + "# --model single \\\n", + "# --output \"$2\"\n", + "\n", + "echo \"Note: Uncomment the command above to run with your data\"" + ] }, { "cell_type": "markdown", @@ -378,57 +414,26 @@ "metadata": {}, "outputs": [], "source": [ - "# Aggregate counts by peak (region)\n", - "peak_counts = (\n", - " counts_df.groupby(\"region_id\")\n", - " .agg(\n", - " {\n", - " \"chr\": \"first\",\n", - " \"pos\": [\"min\", \"max\"],\n", - " \"ref_count\": \"sum\",\n", - " \"alt_count\": \"sum\",\n", - " }\n", - " )\n", - " .reset_index()\n", - ")\n", - "peak_counts.columns = [\"region\", \"chr\", \"start\", \"end\", \"ref_count\", \"alt_count\"]\n", - "\n", - "# Calculate statistics\n", - "peak_counts[\"total\"] = peak_counts[\"ref_count\"] + peak_counts[\"alt_count\"]\n", - "peak_counts[\"mu\"] = peak_counts[\"ref_count\"] / peak_counts[\"total\"]\n", - "# Add pseudocount (+1) to avoid log(0) and stabilize ratios for low-count regions\n", - "peak_counts[\"effect_size\"] = np.log2(\n", - " (peak_counts[\"ref_count\"] + 1) / (peak_counts[\"alt_count\"] + 1)\n", - ")\n", - "\n", - "# Simulate p-values (truly imbalanced peaks get low p-values)\n", - "# Note: This simulation uses binomial for simplicity. Real ATAC-seq data exhibits\n", - "# overdispersion, which is why WASP2 uses the beta-binomial model.\n", - "np.random.seed(42)\n", - "is_imbalanced = np.abs(peak_counts[\"mu\"] - 0.5) > 0.15\n", - "peak_counts[\"p_value\"] = np.where(\n", - " is_imbalanced,\n", - " 10 ** (-np.random.uniform(2, 10, len(peak_counts))), # Significant\n", - " np.random.uniform(0.05, 1, len(peak_counts)), # Not significant\n", + "# Infer statistics using the installed WASP2 model, rather than simulated p-values.\n", + "import subprocess\n", + "counts_df.to_csv(COUNTS_FILE, sep=\"\\t\", index=False)\n", + "subprocess.run([\n", + " \"wasp2-analyze\", \"find-imbalance\", str(COUNTS_FILE),\n", + " \"--region-col\", \"region\", \"--min\", \"5\", \"--model\", \"single\",\n", + " \"--out\", str(IMBALANCE_FILE),\n", + "], check=True)\n", + "results_df = pd.read_csv(IMBALANCE_FILE, sep=\"\\t\")\n", + "coordinates = counts_df.groupby(\"region\").agg(\n", + " chrom=(\"chrom\", \"first\"), start=(\"start\", \"min\"), end=(\"pos\", \"max\")\n", + ").reset_index()\n", + "results_df = results_df.merge(coordinates, on=\"region\")\n", + "# A descriptive ratio for plotting; it is not an additional inference pseudocount.\n", + "results_df[\"effect_size\"] = np.log2(\n", + " (results_df[\"ref_count\"] + 1) / (results_df[\"alt_count\"] + 1)\n", ")\n", - "\n", - "# FDR correction (Benjamini-Hochberg)\n", - "# Note: This manual BH implementation is for demonstration.\n", - "# WASP2 internally uses scipy.stats.false_discovery_control()\n", - "peak_counts = peak_counts.sort_values(\"p_value\")\n", - "n_tests = len(peak_counts)\n", - "peak_counts[\"rank\"] = range(1, n_tests + 1)\n", - "peak_counts[\"fdr_pval\"] = np.minimum(peak_counts[\"p_value\"] * n_tests / peak_counts[\"rank\"], 1.0)\n", - "peak_counts[\"fdr_pval\"] = peak_counts[\"fdr_pval\"][::-1].cummin()[::-1]\n", - "\n", - "# Filter to testable peaks\n", - "results_df = peak_counts[peak_counts[\"total\"] >= 5].copy()\n", - "results_df = results_df.drop(\"rank\", axis=1)\n", - "\n", "print(f\"Peaks tested: {len(results_df):,}\")\n", "print(f\"Significant (FDR < 0.05): {(results_df['fdr_pval'] < 0.05).sum():,}\")\n", - "print(f\"Significant (FDR < 0.01): {(results_df['fdr_pval'] < 0.01).sum():,}\")\n", - "results_df.head(10)" + "results_df.head(10)\n" ] }, { @@ -451,7 +456,7 @@ "fig, ax = plt.subplots(figsize=(10, 8))\n", "\n", "# Calculate -log10(p-value) for plotting\n", - "results_df[\"neg_log10_pval\"] = -np.log10(results_df[\"p_value\"].clip(lower=1e-50))\n", + "results_df[\"neg_log10_pval\"] = -np.log10(results_df[\"pval\"].clip(lower=1e-50))\n", "\n", "# Define significance thresholds\n", "sig_mask = results_df[\"fdr_pval\"] < 0.05\n", @@ -571,12 +576,12 @@ "print(\"=\" * 80)\n", "display_cols = [\n", " \"region\",\n", - " \"chr\",\n", + " \"chrom\",\n", " \"ref_count\",\n", " \"alt_count\",\n", " \"mu\",\n", " \"effect_size\",\n", - " \"p_value\",\n", + " \"pval\",\n", " \"fdr_pval\",\n", "]\n", "top_hits[display_cols].round(4)" @@ -601,7 +606,7 @@ "source": [ "# Export significant peaks as BED for overlap analysis\n", "sig_peaks = results_df[results_df[\"fdr_pval\"] < 0.05][\n", - " [\"chr\", \"start\", \"end\", \"region\", \"effect_size\", \"fdr_pval\"]\n", + " [\"chrom\", \"start\", \"end\", \"region\", \"effect_size\", \"fdr_pval\"]\n", "].copy()\n", "sig_peaks.to_csv(RESULTS_DIR / \"significant_peaks.bed\", sep=\"\\t\", header=False, index=False)\n", "\n", @@ -653,11 +658,13 @@ "metadata": {}, "outputs": [], "source": [ - "# Simulate overlap results for demonstration\n", + "# Illustrative synthetic overlap data; replace with measured overlaps for interpretation\n", "np.random.seed(123)\n", "\n", "# Create simulated eQTL overlap data\n", - "n_overlap = 150\n", + "n_sig_peaks = int(sig_mask.sum())\n", + "n_total_peaks = len(results_df)\n", + "n_overlap = min(150, n_sig_peaks)\n", "overlap_df = pd.DataFrame(\n", " {\n", " \"peak\": [f\"peak_{i}\" for i in np.random.choice(range(1500), n_overlap, replace=False)],\n", @@ -687,7 +694,11 @@ " [n_eqtl_overlap, n_sig_peaks - n_eqtl_overlap],\n", " [expected_overlap, n_sig_peaks - expected_overlap],\n", "]\n", - "odds_ratio, p_value = fisher_exact(contingency)\n", + "if n_sig_peaks > 0:\n", + " odds_ratio, p_value = fisher_exact(contingency)\n", + "else:\n", + " odds_ratio, p_value = np.nan, np.nan\n", + " print(\"No significant peaks: no enrichment test is performed.\")\n", "\n", "print(\"\\nEnrichment Analysis:\")\n", "print(f\" Imbalanced peaks: {n_sig_peaks}\")\n", @@ -716,7 +727,10 @@ "overlap_counts = [n_eqtl_overlap, n_sig_peaks - n_eqtl_overlap]\n", "labels = [\"Overlap with eQTL\", \"No eQTL overlap\"]\n", "colors = [\"#e74c3c\", \"#95a5a6\"]\n", - "ax.pie(overlap_counts, labels=labels, colors=colors, autopct=\"%1.1f%%\", startangle=90)\n", + "if n_sig_peaks:\n", + " ax.pie(overlap_counts, labels=labels, colors=colors, autopct=\"%1.1f%%\", startangle=90)\n", + "else:\n", + " ax.text(0.5, 0.5, \"No significant peaks\", ha=\"center\", va=\"center\")\n", "ax.set_title(\"Imbalanced Peaks Overlapping eQTLs\")\n", "\n", "# Bar chart: Overlap by tissue\n", @@ -835,7 +849,7 @@ "wasp2-analyze compare-imbalance \\\\\n", " scatac_counts.h5ad \\\\\n", " barcode_celltype_map.tsv \\\\\n", - " --groups \"excitatory,inhibitory\" \\\\\n", + " --groups excitatory --groups inhibitory \\\\\n", " --sample SAMPLE_ID\n", "\"\"\"\n", "\n", @@ -859,7 +873,7 @@ "\n", "### Key Takeaways\n", "\n", - "- ATAC-seq has **lower coverage per peak** than RNA-seq; use `--min-count 5` instead of 10\n", + "- ATAC-seq has **lower coverage per peak** than RNA-seq; use `--min 5` instead of 10\n", "- **FDR correction** is essential for multiple testing across thousands of peaks\n", "- Consider **effect size** alongside significance for biological relevance\n", "- **QTL overlap** helps validate findings and identify causal variants\n", diff --git a/tutorials/performance_optimization.ipynb b/tutorials/performance_optimization.ipynb index 9f257ac2..9ccf9992 100644 --- a/tutorials/performance_optimization.ipynb +++ b/tutorials/performance_optimization.ipynb @@ -240,7 +240,7 @@ "\n", " # Use WASP2's Rust-powered VCF-to-BED conversion\n", " start = time.perf_counter()\n", - " n_variants = wasp2_rust.vcf_to_bed(\n", + " n_variants = wasp2_rust.vcf_to_bed_py(\n", " str(vcf_file),\n", " str(bed_out),\n", " samples=[\"sample1\"], # Filter to one sample\n", @@ -328,7 +328,7 @@ " out_file = Path(tmpdir) / \"intersect.bed\"\n", "\n", " # Create BED from VCF\n", - " n_variants = wasp2_rust.vcf_to_bed(str(vcf_file), str(bed_file))\n", + " n_variants = wasp2_rust.vcf_to_bed_py(str(vcf_file), str(bed_file))\n", "\n", " if n_variants == 0:\n", " print(\"WARNING: No variants extracted from VCF\")\n", @@ -546,25 +546,22 @@ "\n", "# Validate inputs\n", "validate_inputs \"$BAM\" \"$VCF\"\n", - "check_disk_space \"$OUTDIR\"\n", - "\n", - "# Create output directory\n", "mkdir -p \"$OUTDIR\"\n", + "check_disk_space \"$OUTDIR\"\n", "\n", "# Run WASP2 pipeline with explicit thread count\n", "log \"Starting WASP2 pipeline...\"\n", "wasp2-map make-reads \\\\\n", - " --bam \"$BAM\" \\\\\n", - " --vcf \"$VCF\" \\\\\n", - " --sample \"$SAMPLE\" \\\\\n", + " \"$BAM\" \"$VCF\" \\\\\n", + " --samples \"$SAMPLE\" \\\\\n", " --threads ${SLURM_CPUS_PER_TASK} \\\\\n", " --out_dir \"$OUTDIR\"\n", "\n", "# Verify output\n", - "if [[ -f \"${OUTDIR}/remap_r1.fq.gz\" ]]; then\n", + "if [[ -f \"${OUTDIR}/$(basename \"$BAM\" .bam)_swapped_alleles_r1.fq\" ]]; then\n", " log \"SUCCESS: WASP2 completed successfully\"\n", " log \"Output files:\"\n", - " ls -lh \"${OUTDIR}/\"*.fq.gz 2>/dev/null || true\n", + " ls -lh \"${OUTDIR}/\"*.fq 2>/dev/null || true\n", "else\n", " log \"WARNING: Expected output files not found\"\n", " exit 1\n", @@ -671,7 +668,7 @@ "outputs": [], "source": [ "singularity_usage = \"\"\"# Pull the WASP2 container\n", - "singularity pull wasp2.sif docker://ghcr.io/your-org/wasp2:latest\n", + "singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG\n", "\n", "# Run WASP2 via Singularity\n", "singularity exec --bind /data:/data wasp2.sif \\\n", @@ -817,40 +814,26 @@ "outputs": [], "source": [ "chunked_script = \"\"\"#!/bin/bash\n", - "# Process BAM chromosome-by-chromosome to reduce memory\n", - "\n", + "# Count chromosomes separately; do not split a global dispersion fit.\n", + "set -euo pipefail\n", "BAM=$1\n", "VCF=$2\n", "OUTDIR=$3\n", - "\n", - "# Get chromosome list from BAM\n", - "CHROMS=$(samtools view -H $BAM | grep \"^@SQ\" | cut -f2 | sed 's/SN://')\n", - "\n", - "# Process each chromosome separately\n", - "for CHR in $CHROMS; do\n", - " echo \"Processing $CHR...\"\n", - " \n", - " # Extract chromosome\n", - " samtools view -b $BAM $CHR > ${OUTDIR}/${CHR}.bam\n", - " samtools index ${OUTDIR}/${CHR}.bam\n", - " \n", - " # Run WASP2 on subset\n", - " wasp2-map make-reads \\\n", - " --bam ${OUTDIR}/${CHR}.bam \\\n", - " --vcf $VCF \\\n", - " --region $CHR \\\n", - " --out_dir ${OUTDIR}/${CHR}/\n", - " \n", - " # Clean up intermediate file\n", - " rm ${OUTDIR}/${CHR}.bam*\n", - "done\n", - "\n", - "# Merge results\n", - "cat ${OUTDIR}/*/counts.tsv > ${OUTDIR}/all_counts.tsv\n", + "mkdir -p \"$OUTDIR\"\n", + "samtools idxstats \"$BAM\" | awk '$1 != \"*\" {print $1}' > \"$OUTDIR/chromosomes.txt\"\n", + "while IFS= read -r CHR; do\n", + " mkdir -p \"$OUTDIR/$CHR\"\n", + " samtools view -b \"$BAM\" \"$CHR\" > \"$OUTDIR/$CHR/input.bam\"\n", + " samtools index \"$OUTDIR/$CHR/input.bam\"\n", + " bcftools view -r \"$CHR\" \"$VCF\" -Oz -o \"$OUTDIR/$CHR/input.vcf.gz\"\n", + " bcftools index -t \"$OUTDIR/$CHR/input.vcf.gz\"\n", + " wasp2-count count-variants \"$OUTDIR/$CHR/input.bam\" \"$OUTDIR/$CHR/input.vcf.gz\" --out \"$OUTDIR/$CHR/counts.tsv\"\n", + "done < \"$OUTDIR/chromosomes.txt\"\n", + "# Merge one header, then fit dispersion once on the combined counts.\n", + "awk 'FNR == 1 && NR != 1 {next} {print}' \"$OUTDIR\"/*/counts.tsv > \"$OUTDIR/all_counts.tsv\"\n", + "wasp2-analyze find-imbalance \"$OUTDIR/all_counts.tsv\" --out \"$OUTDIR/imbalance.tsv\"\n", "\"\"\"\n", - "\n", - "print(\"Chunked processing script:\")\n", - "print(chunked_script)" + "print(chunked_script)\n" ] }, { @@ -872,13 +855,13 @@ "pip install memory_profiler\n", "\n", "# Run with memory profiling\n", - "mprof run wasp2-map make-reads --bam input.bam --vcf variants.vcf.gz\n", + "mprof run wasp2-map make-reads input.bam variants.vcf.gz\n", "\n", "# View memory plot\n", "mprof plot\n", "\n", "# Or use peak memory reporting\n", - "/usr/bin/time -v wasp2-map make-reads --bam input.bam --vcf variants.vcf.gz 2>&1 | \\\n", + "/usr/bin/time -v wasp2-map make-reads input.bam variants.vcf.gz 2>&1 | \\\n", " grep \"Maximum resident set size\"\n", "\"\"\"\n", "\n", @@ -1025,7 +1008,30 @@ "cell_type": "markdown", "id": "fneppl9hsmm", "metadata": {}, - "source": "### Debugging Tips for HPC Environments\n\n```bash\n# 1. Check if Rust extension loads correctly\npython -c \"import wasp2_rust; print('OK')\"\n\n# 2. Verify BAM is sorted and indexed\nsamtools quickcheck input.bam && echo \"BAM OK\" || echo \"BAM corrupt\"\nsamtools idxstats input.bam | head -5\n\n# 3. Check VCF is valid\nbcftools stats input.vcf.gz | head -20\n\n# 4. Monitor memory during run\nwatch -n 5 'ps -o pid,rss,vsz,comm -p $(pgrep -f wasp2)'\n\n# 5. Check for stalled processes\nstrace -p -e trace=read,write 2>&1 | head -20\n\n# 6. Verify output files are being written\nwatch -n 10 'ls -lh output_dir/*.fq.gz 2>/dev/null'\n```" + "source": [ + "### Debugging Tips for HPC Environments\n", + "\n", + "```bash\n", + "# 1. Check if Rust extension loads correctly\n", + "python -c \"import wasp2_rust; print('OK')\"\n", + "\n", + "# 2. Verify BAM is sorted and indexed\n", + "samtools quickcheck input.bam && echo \"BAM OK\" || echo \"BAM corrupt\"\n", + "samtools idxstats input.bam | head -5\n", + "\n", + "# 3. Check VCF is valid\n", + "bcftools stats input.vcf.gz | head -20\n", + "\n", + "# 4. Monitor memory during run\n", + "watch -n 5 'ps -o pid,rss,vsz,comm -p $(pgrep -f wasp2)'\n", + "\n", + "# 5. Check for stalled processes\n", + "strace -p -e trace=read,write 2>&1 | head -20\n", + "\n", + "# 6. Verify output files are being written\n", + "watch -n 10 'ls -lh output_dir/*.fq 2>/dev/null'\n", + "```" + ] } ], "metadata": { diff --git a/tutorials/quickstart_counting.ipynb b/tutorials/quickstart_counting.ipynb index 7e7bf5f6..6166568d 100644 --- a/tutorials/quickstart_counting.ipynb +++ b/tutorials/quickstart_counting.ipynb @@ -268,7 +268,17 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Next Steps\n\nNow that you have allele counts, you can:\n\n1. **Analyze allelic imbalance** using `wasp2-analyze find-imbalance`\n2. **Compare between conditions** using `wasp2-analyze compare-imbalance`\n3. **Correct mapping bias** using `wasp2-map` (for WASP-filtered BAMs)\n\nSee the documentation for detailed guides on [counting](https://wasp2.readthedocs.io/en/latest/user_guide/counting.html), [single-cell analysis](https://wasp2.readthedocs.io/en/latest/tutorials/scrna_seq.html), and [comparative imbalance](https://wasp2.readthedocs.io/en/latest/tutorials/comparative_imbalance.html)." + "source": [ + "## Next Steps\n", + "\n", + "Now that you have allele counts, you can:\n", + "\n", + "1. **Analyze allelic imbalance** using `wasp2-analyze find-imbalance`\n", + "2. **Compare between conditions** using `wasp2-analyze compare-imbalance`\n", + "3. **Correct mapping bias** using `wasp2-map` (for WASP-filtered BAMs)\n", + "\n", + "See the documentation for detailed guides on [counting](https://mcvickerlab.github.io/WASP2/user_guide/counting.html), [single-cell analysis](https://mcvickerlab.github.io/WASP2/user_guide/single_cell.html), and [comparative imbalance](https://mcvickerlab.github.io/WASP2/user_guide/analysis.html)." + ] }, { "cell_type": "markdown", diff --git a/tutorials/quickstart_mapping.ipynb b/tutorials/quickstart_mapping.ipynb index 510837ec..f4d929ef 100644 --- a/tutorials/quickstart_mapping.ipynb +++ b/tutorials/quickstart_mapping.ipynb @@ -120,7 +120,24 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Step 2: Create Reads for Remapping\n\nThe first step identifies reads overlapping heterozygous SNPs and generates allele-swapped versions.\n\n```bash\nwasp2-map make-reads sample.bam variants.vcf.gz \\\n --samples SAMPLE1 \\\n --out-dir wasp_output/\n```\n\nThis produces (where `sample` is your BAM file prefix):\n- `wasp_output/sample_to_remap.bam`: Original reads needing remapping\n- `wasp_output/sample_keep.bam`: Reads not overlapping variants (kept as-is)\n- `wasp_output/sample_swapped_alleles_r1.fq`: Allele-swapped read 1\n- `wasp_output/sample_swapped_alleles_r2.fq`: Allele-swapped read 2\n- `wasp_output/sample_wasp_data_files.json`: Metadata for filter step" + "source": [ + "## Step 2: Create Reads for Remapping\n", + "\n", + "The first step identifies reads overlapping heterozygous SNPs and generates allele-swapped versions.\n", + "\n", + "```bash\n", + "wasp2-map make-reads sample.bam variants.vcf.gz \\\n", + " --samples SAMPLE1 \\\n", + " --out_dir wasp_output/\n", + "```\n", + "\n", + "This produces (where `sample` is your BAM file prefix):\n", + "- `wasp_output/sample_to_remap.bam`: Original reads needing remapping\n", + "- `wasp_output/sample_keep.bam`: Reads not overlapping variants (kept as-is)\n", + "- `wasp_output/sample_swapped_alleles_r1.fq`: Allele-swapped read 1\n", + "- `wasp_output/sample_swapped_alleles_r2.fq`: Allele-swapped read 2\n", + "- `wasp_output/sample_wasp_data_files.json`: Metadata for filter step" + ] }, { "cell_type": "code", @@ -132,7 +149,7 @@ "make_reads_cmd = f\"\"\"\n", "wasp2-map make-reads {BAM_FILE} {VCF_FILE} \\\\\n", " --samples {SAMPLE_ID} \\\\\n", - " --out-dir {WASP_DIR}/\n", + " --out_dir {WASP_DIR}/\n", "\"\"\"\n", "print(\"Step 2 command:\")\n", "print(make_reads_cmd)" @@ -150,7 +167,8 @@ "outputs": [], "source": [ "# Command to remap swapped reads (using BAM prefix)\n", - "BAM_PREFIX = BAM_FILE.replace(\".bam\", \"\")\n", + "from pathlib import Path\n", + "BAM_PREFIX = Path(BAM_FILE).stem\n", "\n", "remap_cmd = f\"\"\"\n", "bwa mem -M -t 8 genome.fa \\\\\n", @@ -167,7 +185,18 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Step 4: Filter Remapped Reads\n\nThe WASP filter compares original and remapped positions. Reads that map to a different location after allele swapping are removed.\n\n```bash\nwasp2-map filter-remapped \\\n wasp_output/sample_to_remap.bam \\\n wasp_output/sample_remapped.bam \\\n wasp_output/sample_wasp_filtered.bam\n```" + "source": [ + "## Step 4: Filter Remapped Reads\n", + "\n", + "The WASP filter compares original and remapped positions. Reads that map to a different location after allele swapping are removed.\n", + "\n", + "```bash\n", + "wasp2-map filter-remapped \\\n", + " wasp_output/sample_remapped.bam \\\n", + " --json wasp_output/sample_wasp_data_files.json \\\n", + " --out_bam wasp_output/sample_wasp_filtered.bam\n", + "```" + ] }, { "cell_type": "code", @@ -178,9 +207,9 @@ "# Command to filter remapped reads\n", "filter_cmd = f\"\"\"\n", "wasp2-map filter-remapped \\\\\n", - " {WASP_DIR}/{BAM_PREFIX}_to_remap.bam \\\\\n", " {WASP_DIR}/{BAM_PREFIX}_remapped.bam \\\\\n", - " {WASP_DIR}/{BAM_PREFIX}_wasp_filtered.bam\n", + " --json {WASP_DIR}/{BAM_PREFIX}_wasp_data_files.json \\\\\n", + " --out_bam {WASP_DIR}/{BAM_PREFIX}_wasp_filtered.bam\n", "\"\"\"\n", "print(\"Step 4 command:\")\n", "print(filter_cmd)" @@ -350,7 +379,7 @@ "echo \"Step 1: Creating swapped reads...\"\n", "wasp2-map make-reads $BAM $VCF \\\\\n", " --samples $SAMPLE \\\\\n", - " --out-dir $OUTDIR/\n", + " --out_dir $OUTDIR/\n", "\n", "# Step 2: Remap with same aligner (BWA example)\n", "echo \"Step 2: Remapping swapped reads...\"\n", @@ -360,19 +389,11 @@ " samtools sort -o $OUTDIR/${PREFIX}_remapped.bam -\n", "samtools index $OUTDIR/${PREFIX}_remapped.bam\n", "\n", - "# Step 3: Filter biased reads\n", - "echo \"Step 3: Filtering biased reads...\"\n", + "# Step 3: Filter and merge with unaffected reads\n", "wasp2-map filter-remapped \\\\\n", - " $OUTDIR/${PREFIX}_to_remap.bam \\\\\n", - " $OUTDIR/${PREFIX}_remapped.bam \\\\\n", - " $OUTDIR/${PREFIX}_wasp_filtered.bam\n", - "\n", - "# Step 4: Merge with non-overlapping reads\n", - "echo \"Step 4: Merging final BAM...\"\n", - "samtools merge -f $OUTDIR/${PREFIX}_final.bam \\\\\n", - " $OUTDIR/${PREFIX}_wasp_filtered.bam \\\\\n", - " $OUTDIR/${PREFIX}_keep.bam\n", - "samtools index $OUTDIR/${PREFIX}_final.bam\n", + " \"$OUTDIR/${PREFIX}_remapped.bam\" \\\\\n", + " --json \"$OUTDIR/${PREFIX}_wasp_data_files.json\" \\\\\n", + " --out_bam \"$OUTDIR/${PREFIX}_final.bam\"\n", "\n", "echo \"Done! WASP-filtered BAM: $OUTDIR/${PREFIX}_final.bam\"\n", "\"\"\"\n", @@ -386,15 +407,9 @@ "source": [ "## Rust Acceleration\n", "\n", - "WASP2 includes a high-performance Rust backend that accelerates the filter step by 10-15x:\n", - "\n", - "| Dataset Size | Python | Rust |\n", - "|-------------|--------|------|\n", - "| 1M reads | ~5 min | ~30 sec |\n", - "| 10M reads | ~50 min | ~5 min |\n", - "| 100M reads | ~8 hours | ~50 min |\n", - "\n", - "The Rust backend is used automatically when available." + "WASP2 installs its native extension with the Python package. Use the CLI to\n", + "preserve its input validation, metadata handling, and final BAM merge/index steps.\n", + "Performance depends on the data, hardware, and thread count.\n" ] }, { @@ -403,26 +418,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Check Rust backend status\n", - "if RUST_AVAILABLE:\n", - " print(\"Using Rust-accelerated WASP filter (10-15x faster)\")\n", - "\n", - " # Example of direct Rust API usage\n", - " print(\"\\nRust API example:\")\n", - " print(\"\"\"\n", - "from wasp2_rust import filter_bam_wasp\n", - "\n", - "kept, removed_moved, removed_missing = filter_bam_wasp(\n", - " to_remap_bam=\"to_remap.bam\",\n", - " remapped_bam=\"remapped.bam\",\n", - " remap_keep_bam=\"filtered.bam\",\n", - " threads=4,\n", - ")\n", - "print(f\"Kept: {kept}, Removed: {removed_moved + removed_missing}\")\n", - "\"\"\")\n", - "else:\n", - " print(\"Using pure Python WASP filter\")\n", - " print(\"Install wasp2_rust for 10-15x speedup: pip install wasp2[rust]\")" + "# Verify the installed native extension\n", + "import wasp2_rust\n", + "assert wasp2_rust.__version__ == wasp2.__version__\n", + "print(f\"Native extension version: {wasp2_rust.__version__}\")\n" ] }, { @@ -447,7 +446,7 @@ "\n", "- [User Guide: Mapping](../docs/source/user_guide/mapping.rst) - Detailed mapping module documentation\n", "- [Methods: WASP Algorithm](../docs/source/methods/mapping_filter.rst) - Algorithm details and math\n", - "- [Tutorial: 10X scRNA-seq](../docs/source/tutorials/scrna_seq.rst) - Single-cell workflow" + "- [Tutorial: Single-cell ATAC-seq](../docs/source/tutorials/single_cell_workflow.rst) - Single-cell workflow" ] }, { diff --git a/tutorials/rna_seq_workflow.ipynb b/tutorials/rna_seq_workflow.ipynb index 0fe329d4..cb115e2b 100644 --- a/tutorials/rna_seq_workflow.ipynb +++ b/tutorials/rna_seq_workflow.ipynb @@ -292,7 +292,7 @@ "# Example count data structure\n", "example_counts = pd.DataFrame(\n", " {\n", - " \"chr\": [\"chr1\", \"chr1\", \"chr1\", \"chr2\", \"chr2\"],\n", + " \"chrom\": [\"chr1\", \"chr1\", \"chr1\", \"chr2\", \"chr2\"],\n", " \"pos\": [100000, 150000, 200000, 50000, 75000],\n", " \"ref\": [\"A\", \"G\", \"C\", \"T\", \"A\"],\n", " \"alt\": [\"G\", \"A\", \"T\", \"C\", \"G\"],\n", @@ -387,6 +387,7 @@ " --min 10 \\\n", " --pseudocount 1 \\\n", " --phased \\\n", + " --region-col gene_id \\\n", " --out_file {AI_RESULTS_FILE}\n", "\"\"\"\n", "\n", @@ -415,16 +416,9 @@ "source": [ "### Understanding the Output\n", "\n", - "The output includes:\n", - "\n", - "| Column | Description |\n", - "|--------|-------------|\n", - "| `region` | Gene or region identifier |\n", - "| `ref_count` | Total reference allele counts |\n", - "| `alt_count` | Total alternate allele counts |\n", - "| `p_value` | Likelihood ratio test p-value |\n", - "| `fdr_pval` | FDR-corrected p-value |\n", - "| `effect_size` | Log2 fold change (ref/alt) |" + "WASP2 writes `region`, `ref_count`, `alt_count`, `N`, `snp_count`, `mu`,\n", + "`null_ll`, `alt_ll`, `lrt`, `pval`, and `fdr_pval`. The plotting examples\n", + "below derive a descriptive log2 count ratio; `effect_size` is not a CLI output.\n" ] }, { @@ -433,24 +427,25 @@ "metadata": {}, "outputs": [], "source": [ - "# Example analysis results\n", - "# Replace with: results_df = pd.read_csv(AI_RESULTS_FILE, sep='\\t')\n", - "\n", - "example_results = pd.DataFrame(\n", - " {\n", - " \"region\": [\"ENSG00000001\", \"ENSG00000002\", \"ENSG00000003\", \"ENSG00000004\", \"ENSG00000005\"],\n", - " \"gene_name\": [\"SNRPN\", \"H19\", \"KCNQ1OT1\", \"ACTB\", \"GAPDH\"],\n", - " \"ref_count\": [245, 15, 8, 156, 189],\n", - " \"alt_count\": [12, 234, 287, 148, 195],\n", - " \"p_value\": [1.2e-45, 3.4e-38, 5.6e-52, 0.65, 0.82],\n", - " \"fdr_pval\": [6.0e-44, 8.5e-37, 5.6e-50, 0.75, 0.85],\n", - " \"effect_size\": [4.35, -3.96, -5.16, 0.08, -0.05],\n", - " \"dispersion\": [0.02, 0.03, 0.01, 0.05, 0.04],\n", - " }\n", + "# Run the installed software on the illustrative counts above.\n", + "# For real data, load the count table from the counting step instead.\n", + "import subprocess\n", + "example_counts[\"start\"] = example_counts[\"pos\"] - 1\n", + "example_counts[\"GT\"] = example_counts[SAMPLE_ID]\n", + "example_counts.to_csv(COUNTS_FILE, sep=\"\\t\", index=False)\n", + "subprocess.run([\n", + " \"wasp2-analyze\", \"find-imbalance\", str(COUNTS_FILE),\n", + " \"--region-col\", \"gene_id\", \"--phased\", \"--min\", \"10\",\n", + " \"--pseudocount\", \"1\", \"--out\", str(AI_RESULTS_FILE),\n", + "], check=True)\n", + "example_results = pd.read_csv(AI_RESULTS_FILE, sep=\"\\t\")\n", + "annotations = example_counts[[\"gene_id\", \"gene_name\"]].drop_duplicates()\n", + "example_results = example_results.merge(annotations, left_on=\"region\", right_on=\"gene_id\")\n", + "# Descriptive plotting ratio, not the fitted effect parameter or an extra model pseudocount.\n", + "example_results[\"effect_size\"] = np.log2(\n", + " (example_results[\"ref_count\"] + 1) / (example_results[\"alt_count\"] + 1)\n", ")\n", - "\n", - "print(\"Allelic imbalance results:\")\n", - "print(example_results)" + "print(example_results)\n" ] }, { @@ -503,7 +498,7 @@ "\n", " # Calculate -log10 p-value\n", " results = results.copy()\n", - " results[\"-log10_pval\"] = -np.log10(results[\"p_value\"] + 1e-300)\n", + " results[\"-log10_pval\"] = -np.log10(results[\"pval\"] + 1e-300)\n", "\n", " # Classify points\n", " is_sig = results[\"fdr_pval\"] < fdr_thresh\n", @@ -879,7 +874,18 @@ { "cell_type": "markdown", "metadata": {}, - "source": "### Concordance Analysis\n\nCheck if ASE direction agrees with eQTL effect direction.\n\n**Understanding Direction Concordance:**\n- **ASE effect_size > 0**: Reference allele is MORE expressed\n- **eQTL slope > 0**: Alternate allele INCREASES expression (standard eQTL convention)\n- **Concordance**: These directions should be OPPOSITE (ref high in ASE = alt low in eQTL)\n\nThis counterintuitive relationship occurs because ASE measures relative allele expression while eQTL slopes measure how the alternate allele affects total expression." + "source": [ + "### Concordance Analysis\n", + "\n", + "Check if ASE direction agrees with eQTL effect direction.\n", + "\n", + "**Understanding Direction Concordance:**\n", + "- **ASE effect_size > 0**: Reference allele is MORE expressed\n", + "- **eQTL slope > 0**: Alternate allele INCREASES expression (standard eQTL convention)\n", + "- **Concordance**: These directions should be OPPOSITE (ref high in ASE = alt low in eQTL)\n", + "\n", + "This counterintuitive relationship occurs because ASE measures relative allele expression while eQTL slopes measure how the alternate allele affects total expression." + ] }, { "cell_type": "code", @@ -906,7 +912,7 @@ "\n", " if len(concordance_data) == 0:\n", " print(\"No genes with both significant ASE and eQTL data.\")\n", - " return data\n", + " return concordance_data.assign(concordant=pd.Series(dtype=bool))\n", "\n", " # Determine effect directions\n", " concordance_data[\"ase_direction\"] = np.sign(concordance_data[\"effect_size\"])\n", diff --git a/tutorials/single_cell_workflow.ipynb b/tutorials/single_cell_workflow.ipynb index fbd60e2b..e6db8531 100644 --- a/tutorials/single_cell_workflow.ipynb +++ b/tutorials/single_cell_workflow.ipynb @@ -371,7 +371,7 @@ " Parameters\n", " ----------\n", " results : DataFrame\n", - " WASP2 results with 'effect_size' and 'fdr_pval' columns.\n", + " WASP2 results with 'mu' and 'fdr_pval' columns.\n", " fdr_threshold : float\n", " FDR threshold for significance.\n", "\n", @@ -380,6 +380,8 @@ " Figure\n", " Matplotlib figure.\n", " \"\"\"\n", + " results = results.copy()\n", + " results[\"effect_size\"] = results[\"mu\"] - 0.5\n", " fig, ax = plt.subplots(figsize=(8, 6))\n", "\n", " ns = results[\"fdr_pval\"] >= fdr_threshold\n", @@ -387,7 +389,7 @@ "\n", " ax.scatter(\n", " results.loc[ns, \"effect_size\"],\n", - " -np.log10(results.loc[ns, \"fdr_pval\"]),\n", + " -np.log10(results.loc[ns, \"fdr_pval\"].clip(lower=1e-300)),\n", " alpha=0.5,\n", " s=10,\n", " c=\"gray\",\n", @@ -395,7 +397,7 @@ " )\n", " ax.scatter(\n", " results.loc[sig, \"effect_size\"],\n", - " -np.log10(results.loc[sig, \"fdr_pval\"]),\n", + " -np.log10(results.loc[sig, \"fdr_pval\"].clip(lower=1e-300)),\n", " alpha=0.7,\n", " s=20,\n", " c=\"red\",\n", @@ -404,7 +406,7 @@ "\n", " ax.axhline(-np.log10(fdr_threshold), color=\"black\", linestyle=\"--\", alpha=0.5)\n", " ax.axvline(0, color=\"black\", linestyle=\"-\", alpha=0.3)\n", - " ax.set_xlabel(\"Effect Size (Log2 Ref/Alt)\")\n", + " ax.set_xlabel(\"Fitted Allele Frequency Minus 0.5\")\n", " ax.set_ylabel(\"-Log10(FDR)\")\n", " ax.legend()\n", " plt.tight_layout()\n", @@ -440,7 +442,7 @@ " regions = set()\n", " for df in results_dict.values():\n", " regions.update(df[df[\"fdr_pval\"] < 0.05][\"region\"].head(top_n))\n", - " regions = list(regions)[:top_n]\n", + " regions = sorted(regions)[:top_n]\n", "\n", " # Build matrix\n", " matrix = pd.DataFrame(index=regions, columns=list(results_dict.keys()))\n", @@ -448,11 +450,14 @@ " df_idx = df.set_index(\"region\")\n", " for r in regions:\n", " if r in df_idx.index:\n", - " matrix.loc[r, ct] = df_idx.loc[r, \"effect_size\"]\n", + " matrix.loc[r, ct] = df_idx.loc[r, \"mu\"] - 0.5\n", "\n", " fig, ax = plt.subplots(figsize=(10, 8))\n", + " if matrix.empty:\n", + " ax.text(0.5, 0.5, \"No significant regions\", ha=\"center\", va=\"center\")\n", + " return fig\n", " sns.heatmap(\n", - " matrix.astype(float), cmap=\"RdBu_r\", center=0, ax=ax, cbar_kws={\"label\": \"Effect Size\"}\n", + " matrix.astype(float), cmap=\"RdBu_r\", center=0, ax=ax, cbar_kws={\"label\": \"Fitted Allele Frequency Minus 0.5\"}\n", " )\n", " ax.set_title(\"Cell-Type-Specific Allelic Imbalance\")\n", " plt.tight_layout()\n", @@ -524,7 +529,7 @@ " --phased --min 10 -z 3\n", "```\n", "\n", - "**Output**: `ai_results_.tsv` per cell type with columns: region, ref_count, alt_count, p_value, fdr_pval, effect_size\n", + "**Output**: `ai_results_.tsv` per cell type with columns: region, ref_count, alt_count, mu, null_ll, alt_ll, pval, fdr_pval\n", "\n", "**Step 2: Compare between cell types**\n", "\n", @@ -533,7 +538,7 @@ " allele_counts.h5ad \\\n", " barcodes_celltype.tsv \\\n", " --sample SAMPLE_ID \\\n", - " --groups \"CellTypeA,CellTypeB\" \\\n", + " --groups CellTypeA --groups CellTypeB \\\n", " --phased --min 15\n", "```\n", "\n", From b16c12e18a961ff500f885a511f2ca95041fe41a Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:56:13 -0700 Subject: [PATCH 2/6] Use array chunks in remapping regression for current Clippy --- rust/src/bam_remapper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/bam_remapper.rs b/rust/src/bam_remapper.rs index e5d7248f..aed2a348 100644 --- a/rust/src/bam_remapper.rs +++ b/rust/src/bam_remapper.rs @@ -2025,7 +2025,7 @@ mod tests { .unwrap() .unwrap(); assert_eq!(output.len(), expected_pairs * 2); - for (index, pair) in output.chunks_exact(2).enumerate() { + for (index, pair) in output.as_chunks::<2>().0.iter().enumerate() { let base = format!("test_read_WASP_100_200_{}_{}", index + 1, expected_pairs); assert_eq!(pair[0].name, format!("{base}/1").as_bytes()); assert_eq!(pair[1].name, format!("{base}/2").as_bytes()); From 253bb4dc9509a551f868261f8e7c23b9475e46ac Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:05:19 -0700 Subject: [PATCH 3/6] Cancel superseded PR validation without interrupting releases --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 143eac2c..3488c68f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ on: concurrency: group: release-${{ github.ref }} - cancel-in-progress: false + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read From 80c502ae54278aa26a1cbd40480beeeac27bfe94 Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:22:14 -0700 Subject: [PATCH 4/6] Refresh HTS bindings and collect independent container validation evidence --- .github/workflows/docker.yml | 13 ++++++++-- .github/workflows/release.yml | 20 +++++++++++++--- CHANGELOG.md | 2 ++ rust/Cargo.lock | 45 +++++++++-------------------------- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 73ee3f3f..e429f4c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -42,6 +42,7 @@ jobs: print("value=" + tomllib.load(handle)["package"]["version"]) PY - name: Build local test image + id: build_image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . @@ -54,6 +55,7 @@ jobs: cache-from: type=gha,scope=container-${{ strategy.job-index }} cache-to: type=gha,mode=max,scope=container-${{ strategy.job-index }} - name: Smoke-test image + id: smoke env: PLATFORM: ${{ matrix.platform }} EXPECTED_VERSION: ${{ steps.version.outputs.value }} @@ -87,7 +89,10 @@ jobs: name: builder-scan-${{ strategy.job-index }} path: builder-vulnerabilities.json if-no-files-found: warn + # Collect runtime evidence even when the build-image scan fails; both scans remain gates. - name: Inventory runtime image + id: inventory + if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 with: image: wasp2:ci @@ -95,13 +100,17 @@ jobs: output-file: container-sbom.cdx.json upload-artifact: false - name: Verify image inventory + if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} run: python3 scripts/check_sbom.py container-sbom.cdx.json wasp2 numpy scipy samtools bcftools bedtools - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: name: container-sbom-${{ strategy.job-index }} path: container-sbom.cdx.json if-no-files-found: error - name: Scan runtime image vulnerabilities + id: runtime_scan + if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0 with: image-ref: wasp2:ci @@ -111,7 +120,7 @@ jobs: severity: CRITICAL,HIGH exit-code: '1' - name: Preserve container scan - if: always() + if: ${{ !cancelled() && steps.runtime_scan.outcome != 'skipped' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: name: container-scan-${{ strategy.job-index }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3488c68f..bd1e80af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -461,6 +461,7 @@ jobs: persist-credentials: false - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build local release image + id: build_image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . @@ -473,6 +474,7 @@ jobs: cache-from: type=gha,scope=release-${{ matrix.tag }} cache-to: type=gha,mode=max,scope=release-${{ matrix.tag }} - name: Smoke-test local release image + id: smoke env: IMAGE: wasp2:release-smoke-${{ matrix.tag }} PLATFORM: ${{ matrix.platform }} @@ -507,7 +509,10 @@ jobs: name: builder-scan-${{ matrix.tag }} path: builder-${{ matrix.tag }}-vulnerabilities.json if-no-files-found: warn + # Collect runtime evidence even when the build-image scan fails; both scans remain gates. - name: Inventory runtime image + id: inventory + if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 with: image: wasp2:release-smoke-${{ matrix.tag }} @@ -515,13 +520,17 @@ jobs: output-file: container-${{ matrix.tag }}.cdx.json upload-artifact: false - name: Verify image inventory + if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} run: python3 scripts/check_sbom.py container-${{ matrix.tag }}.cdx.json wasp2 numpy scipy samtools bcftools bedtools - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: name: release-container-sbom-${{ matrix.tag }} path: container-${{ matrix.tag }}.cdx.json if-no-files-found: error - name: Scan runtime image vulnerabilities + id: runtime_scan + if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0 with: image-ref: wasp2:release-smoke-${{ matrix.tag }} @@ -531,17 +540,20 @@ jobs: severity: CRITICAL,HIGH exit-code: '1' - name: Preserve container scan - if: always() + if: ${{ !cancelled() && steps.runtime_scan.outcome != 'skipped' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: name: release-container-scan-${{ matrix.tag }} path: container-${{ matrix.tag }}-vulnerabilities.json if-no-files-found: error - name: Save tested image + id: save_image + if: ${{ !cancelled() && steps.smoke.outcome == 'success' }} env: IMAGE: wasp2:release-smoke-${{ matrix.tag }} run: docker save "$IMAGE" | gzip > "${RUNNER_TEMP}/image-${{ matrix.tag }}.tar.gz" - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - if: ${{ !cancelled() && steps.save_image.outcome == 'success' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: name: tested-image-${{ matrix.tag }} path: ${{ runner.temp }}/image-${{ matrix.tag }}.tar.gz @@ -550,6 +562,8 @@ jobs: singularity-smoke: name: Singularity smoke needs: [validate, package, container-smoke] + # Test conversion independently; publication still requires successful container security gates. + if: ${{ !cancelled() && needs.validate.result == 'success' && needs.package.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 60 permissions: diff --git a/CHANGELOG.md b/CHANGELOG.md index 28082b29..b43fa2e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to WASP2 will be documented in this file. ## [Unreleased] ### Fixed +- Refresh Rust HTSlib wrapper/binding patch releases and collect independent runtime + scans and SIF checks when a build-image security gate fails. - Resolve numeric mapping genotypes to allele sequences and label swapped FASTQs with the number of pairs actually written, preserving valid remapped reads. - Preserve PGEN sample order and phase during BED export and accept the common diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b7a2c179..e799b16c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -61,20 +61,18 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "bindgen" -version = "0.69.5" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", + "itertools", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex", "syn", ] @@ -281,7 +279,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", - "itertools 0.13.0", + "itertools", "num-traits", "oorandom", "page_size", @@ -301,7 +299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", - "itertools 0.13.0", + "itertools", ] [[package]] @@ -578,9 +576,9 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hts-sys" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e38d7f1c121cd22aa214cb4dadd4277dc5447391eac518b899b29ba6356fbbb2" +checksum = "fc7e68eb880b02c80cfb41e8dc7904062a3ea7e27b7c4556e88d648dd2f038da" dependencies = [ "bindgen", "cc", @@ -715,15 +713,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -765,12 +754,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128fmt" version = "0.1.0" @@ -1364,9 +1347,9 @@ checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] name = "rust-htslib" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22161678c3d72e6434c5f3383325dbf88c3cacce665f0c7b4b077fc6e957ba9" +checksum = "2386a28563a1c0713d7901e5c6e07732ca46a844ec8ee08cd17c631e581988ad" dependencies = [ "bio-types", "byteorder", @@ -1384,12 +1367,6 @@ dependencies = [ "url", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -1849,7 +1826,7 @@ dependencies = [ "pyo3-build-config", "rayon", "rust-htslib", - "rustc-hash 2.1.1", + "rustc-hash", "smallvec", "statrs", "tempfile", From 8fd587ebb9117c5df83d6e85edd93c76c747fe6f Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:48:07 -0700 Subject: [PATCH 5/6] Refresh supported CI action releases and remove unused runtime build helpers --- .github/workflows/build.yml | 10 +- .github/workflows/ci.yml | 20 ++-- .github/workflows/codeql.yml | 8 +- .github/workflows/docker.yml | 22 ++--- .github/workflows/docs.yml | 10 +- .github/workflows/ecosystem.yml | 30 +++--- .github/workflows/release.yml | 156 ++++++++++++++++---------------- .github/workflows/security.yml | 16 ++-- CHANGELOG.md | 1 + Dockerfile | 2 + 10 files changed, 139 insertions(+), 136 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5d955b8..4db23263 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -62,7 +62,7 @@ jobs: uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable - name: Set up sccache - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Configure sccache run: | @@ -70,12 +70,12 @@ jobs: echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install uv - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install build tools run: uv pip install --system maturin twine @@ -97,7 +97,7 @@ jobs: bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" - name: Upload wheel artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wheel-${{ matrix.os }} path: dist/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1993ab0a..4101e9d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,17 +27,17 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install ruff run: uv pip install --system ruff @@ -66,7 +66,7 @@ jobs: - python-version: "3.11" minimum-dependencies: true steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # Runtime reference tests read the pinned pre-Rust archive commit. @@ -88,7 +88,7 @@ jobs: bedtools bwa - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -96,7 +96,7 @@ jobs: uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable - name: Set up sccache - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Configure sccache run: | @@ -104,7 +104,7 @@ jobs: echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - name: Install uv - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Create virtualenv and install dependencies run: | @@ -140,7 +140,7 @@ jobs: - name: Upload Python test results if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: pytest-${{ matrix.python-version }} path: test-results.xml @@ -154,7 +154,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -176,7 +176,7 @@ jobs: components: clippy - name: Set up sccache - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Configure sccache run: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 309bfad9..d6075736 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -32,20 +32,20 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@5049b573e2cbf31c2dbde702a60c24fe476f0766 # v4 + uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 with: languages: python queries: security-extended - name: Autobuild - uses: github/codeql-action/autobuild@5049b573e2cbf31c2dbde702a60c24fe476f0766 # v4 + uses: github/codeql-action/autobuild@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@5049b573e2cbf31c2dbde702a60c24fe476f0766 # v4 + uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 with: category: "/language:python" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e429f4c5..95550782 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,11 +28,11 @@ jobs: - {runner: ubuntu-24.04, platform: linux/amd64} - {runner: ubuntu-24.04-arm, platform: linux/arm64} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3 - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Read package version id: version run: | @@ -43,7 +43,7 @@ jobs: PY - name: Build local test image id: build_image - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . load: true @@ -65,7 +65,7 @@ jobs: test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' wasp2:ci)" = "$EXPECTED_REVISION" docker run --rm --platform "$PLATFORM" -e EXPECTED_WASP2_VERSION="$EXPECTED_VERSION" wasp2:ci /opt/wasp2/scripts/container_smoke_test.sh - name: Build dependency image for scanning - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . target: rust-builder @@ -74,7 +74,7 @@ jobs: tags: wasp2:builder-scan cache-from: type=gha,scope=container-${{ strategy.job-index }} - name: Scan build dependency vulnerabilities - uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: wasp2:builder-scan version: v0.76.0 @@ -84,7 +84,7 @@ jobs: exit-code: '1' - name: Preserve build dependency scan if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: builder-scan-${{ strategy.job-index }} path: builder-vulnerabilities.json @@ -93,7 +93,7 @@ jobs: - name: Inventory runtime image id: inventory if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: image: wasp2:ci format: cyclonedx-json @@ -103,7 +103,7 @@ jobs: if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} run: python3 scripts/check_sbom.py container-sbom.cdx.json wasp2 numpy scipy samtools bcftools bedtools - if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: container-sbom-${{ strategy.job-index }} path: container-sbom.cdx.json @@ -111,7 +111,7 @@ jobs: - name: Scan runtime image vulnerabilities id: runtime_scan if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} - uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: wasp2:ci format: json @@ -121,7 +121,7 @@ jobs: exit-code: '1' - name: Preserve container scan if: ${{ !cancelled() && steps.runtime_scan.outcome != 'skipped' }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: container-scan-${{ strategy.job-index }} path: container-vulnerabilities.json diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2a94e73c..228f1fce 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,17 +20,17 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Cache pip dependencies - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }} @@ -66,7 +66,7 @@ jobs: - name: Upload artifact if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: docs/build/html @@ -85,4 +85,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml index 6259b32b..61849021 100644 --- a/.github/workflows/ecosystem.yml +++ b/.github/workflows/ecosystem.yml @@ -20,14 +20,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable - - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Install build and Galaxy dependencies run: | sudo apt-get update @@ -46,7 +46,7 @@ jobs: planemo test galaxy/tools/wasp2 --no_dependency_resolution --test_output_json galaxy-test-results.json - name: Upload Galaxy results if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: galaxy-test-results path: galaxy-test-results.json @@ -64,18 +64,18 @@ jobs: run: working-directory: pipelines/${{ matrix.pipeline }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: nf-core/setup-nextflow@6c2e22b4d901f0c42ca66c5069f8026df026d165 # v2 + - uses: nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914 # v3.0.1 with: version: "24.10.4" - - uses: nf-core/setup-nf-test@f2db585128ae681918440f6c074dededeceadce1 # v1 + - uses: nf-core/setup-nf-test@4069fbbaabe94c08faba4ad261bfa88225ba133f # v2.1.0 with: version: "0.9.2" - name: Set up Python for single-cell H5AD stubs if: matrix.pipeline == 'nf-scatac' - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - name: Install single-cell stub dependencies @@ -90,7 +90,7 @@ jobs: nf-test test "$test_file" --tag ci_stub --profile=test_stub --ci --tap=ci-stub.tap --verbose - name: Upload nf-test result if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: nf-test-${{ matrix.pipeline }} path: pipelines/${{ matrix.pipeline }}/ci-stub.tap @@ -101,18 +101,18 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.11" - uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable - - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 - - uses: nf-core/setup-nextflow@6c2e22b4d901f0c42ca66c5069f8026df026d165 # v2 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + - uses: nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914 # v3.0.1 with: version: "24.10.4" - - uses: nf-core/setup-nf-test@f2db585128ae681918440f6c074dededeceadce1 # v1 + - uses: nf-core/setup-nf-test@4069fbbaabe94c08faba4ad261bfa88225ba133f # v2.1.0 with: version: "0.9.2" - name: Build matching native software @@ -153,7 +153,7 @@ jobs: pipelines/nf-scatac/tests/subworkflows/wasp_allelic_sc.nf.test - name: Upload native process results if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: nf-test-native-processes path: native-process.tap diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd1e80af..003113b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,11 +26,11 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Validate synchronized metadata @@ -79,17 +79,17 @@ jobs: - {runner: ubuntu-24.04-arm, arch: aarch64, python: "3.12"} - {runner: ubuntu-24.04-arm, arch: aarch64, python: "3.13"} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python }} - - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false - name: Build wheel - uses: PyO3/maturin-action@3b5f400f9576c9934eaeed862ce6281fef121b67 # v1 + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.arch }} args: --release --locked --out dist -m rust/Cargo.toml -i python${{ matrix.python }} @@ -114,7 +114,7 @@ jobs: export PATH="$GITHUB_WORKSPACE/.wheel-test/bin:$PATH" WASP2_TEST_EXPORTS=1 bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" - name: Inventory installed wheel environment - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: path: ./.wheel-test format: cyclonedx-json @@ -122,12 +122,12 @@ jobs: upload-artifact: false - name: Verify runtime inventory run: python scripts/check_sbom.py runtime-linux-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json wasp2 numpy scipy - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: runtime-sbom-linux-${{ matrix.arch }}-py${{ matrix.python }} path: runtime-linux-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json if-no-files-found: error - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wheel-linux-${{ matrix.arch }}-py${{ matrix.python }} path: dist/*.whl @@ -151,13 +151,13 @@ jobs: - {runner: macos-15, arch: arm64, target: aarch64-apple-darwin, deployment: "11.0", python: "3.12"} - {runner: macos-15, arch: arm64, target: aarch64-apple-darwin, deployment: "11.0", python: "3.13"} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python }} - - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false - name: Install htslib @@ -171,7 +171,7 @@ jobs: echo "CPATH=$(brew --prefix htslib)/include" >> "$GITHUB_ENV" echo "LIBCLANG_PATH=$(brew --prefix llvm@19)/lib" >> "$GITHUB_ENV" - name: Build wheel - uses: PyO3/maturin-action@3b5f400f9576c9934eaeed862ce6281fef121b67 # v1 + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.target }} args: --release --locked --out dist -m rust/Cargo.toml -i python${{ matrix.python }} @@ -190,7 +190,7 @@ jobs: export PATH="$GITHUB_WORKSPACE/.wheel-test/bin:$PATH" WASP2_TEST_EXPORTS=1 bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" - name: Inventory installed wheel environment - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: path: ./.wheel-test format: cyclonedx-json @@ -198,12 +198,12 @@ jobs: upload-artifact: false - name: Verify runtime inventory run: python scripts/check_sbom.py runtime-macos-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json wasp2 numpy scipy - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: runtime-sbom-macos-${{ matrix.arch }}-py${{ matrix.python }} path: runtime-macos-${{ matrix.arch }}-py${{ matrix.python }}.cdx.json if-no-files-found: error - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wheel-macos-${{ matrix.arch }}-py${{ matrix.python }} path: dist/*.whl @@ -215,14 +215,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable - - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: false - name: Install system dependencies @@ -242,7 +242,7 @@ jobs: "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-analyze" --help export PATH="$GITHUB_WORKSPACE/.sdist-test/bin:$PATH" bash "$GITHUB_WORKSPACE/scripts/container_smoke_test.sh" - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: source-distribution path: dist/*.tar.gz @@ -266,7 +266,7 @@ jobs: - {platform: osx-arm64, runner: macos-15, environment: default} - {platform: osx-arm64, runner: macos-15, environment: dev} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: prefix-dev/setup-pixi@ad1da6449d9e506fd127fd67ab2d054a769f6a94 # v0.11.2 @@ -302,21 +302,21 @@ jobs: - {platform: osx-arm64, runner: macos-15, deployment: "11.0", python: "3.11"} - {platform: osx-arm64, runner: macos-15, deployment: "11.0", python: "3.12"} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: Jaureguy760/bioconda-recipes ref: 63b7dde03e1898a3aa52cb52943ae9e02d191ebe sparse-checkout: recipes/wasp2 path: candidate-bioconda persist-credentials: false - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: source-distribution path: candidate-source - - uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 + - uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1 with: environment-name: conda-build create-args: conda-build @@ -368,7 +368,7 @@ jobs: CONDA_CHANNEL_PRIORITY: strict PYTHON_VERSION: ${{ matrix.python }} run: conda-build candidate-recipe --python "$PYTHON_VERSION" --variant-config-files candidate-variants.yaml --override-channels -c conda-forge -c bioconda --no-anaconda-upload - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: conda-${{ matrix.platform }}-py${{ matrix.python }} path: ${{ runner.temp }}/conda-build/${{ matrix.platform }}/wasp2-*.conda @@ -383,22 +383,22 @@ jobs: id-token: write # Authenticate package provenance attestations. attestations: write # Publish package provenance attestations. steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: path: packages pattern: "wheel-*" merge-multiple: true - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: source-distribution path: packages - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: runtime-sbom-* merge-multiple: true @@ -415,7 +415,7 @@ jobs: mkdir sbom-source tar -xzf packages/wasp2-*.tar.gz -C sbom-source - name: Generate source dependency SBOM - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: path: ./sbom-source format: cyclonedx-json @@ -425,15 +425,15 @@ jobs: run: python scripts/check_sbom.py wasp2-source.cdx.json wasp2 rust-htslib - name: Attest package provenance if: startsWith(github.ref, 'refs/tags/v') - uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: "packages/*" - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-packages path: packages/* if-no-files-found: error - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-metadata path: | @@ -456,13 +456,13 @@ jobs: - {runner: ubuntu-24.04, platform: linux/amd64, tag: amd64} - {runner: ubuntu-24.04-arm, platform: linux/arm64, tag: arm64} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Build local release image id: build_image - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . load: true @@ -485,7 +485,7 @@ jobs: test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE")" = "$EXPECTED_REVISION" docker run --rm --platform "$PLATFORM" -e EXPECTED_WASP2_VERSION="$EXPECTED_VERSION" "$IMAGE" /opt/wasp2/scripts/container_smoke_test.sh - name: Build dependency image for scanning - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . target: rust-builder @@ -494,7 +494,7 @@ jobs: tags: wasp2:builder-scan cache-from: type=gha,scope=release-${{ matrix.tag }} - name: Scan build dependency vulnerabilities - uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: wasp2:builder-scan version: v0.76.0 @@ -504,7 +504,7 @@ jobs: exit-code: '1' - name: Preserve build dependency scan if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: builder-scan-${{ matrix.tag }} path: builder-${{ matrix.tag }}-vulnerabilities.json @@ -513,7 +513,7 @@ jobs: - name: Inventory runtime image id: inventory if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} - uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 + uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3 with: image: wasp2:release-smoke-${{ matrix.tag }} format: cyclonedx-json @@ -523,7 +523,7 @@ jobs: if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} run: python3 scripts/check_sbom.py container-${{ matrix.tag }}.cdx.json wasp2 numpy scipy samtools bcftools bedtools - if: ${{ !cancelled() && steps.inventory.outcome == 'success' }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-container-sbom-${{ matrix.tag }} path: container-${{ matrix.tag }}.cdx.json @@ -531,7 +531,7 @@ jobs: - name: Scan runtime image vulnerabilities id: runtime_scan if: ${{ !cancelled() && steps.build_image.outcome == 'success' }} - uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: image-ref: wasp2:release-smoke-${{ matrix.tag }} format: json @@ -541,7 +541,7 @@ jobs: exit-code: '1' - name: Preserve container scan if: ${{ !cancelled() && steps.runtime_scan.outcome != 'skipped' }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-container-scan-${{ matrix.tag }} path: container-${{ matrix.tag }}-vulnerabilities.json @@ -553,7 +553,7 @@ jobs: IMAGE: wasp2:release-smoke-${{ matrix.tag }} run: docker save "$IMAGE" | gzip > "${RUNNER_TEMP}/image-${{ matrix.tag }}.tar.gz" - if: ${{ !cancelled() && steps.save_image.outcome == 'success' }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: tested-image-${{ matrix.tag }} path: ${{ runner.temp }}/image-${{ matrix.tag }}.tar.gz @@ -569,16 +569,16 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 with: apptainer-version: 1.4.5 - name: Verify container runtime run: apptainer --version - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: tested-image-amd64 path: ${{ runner.temp }}/tested-image @@ -595,7 +595,7 @@ jobs: Singularity.def > "$RUNNER_TEMP/Singularity.candidate.def" apptainer build "wasp2_definition_${RELEASE_VERSION}.sif" "$RUNNER_TEMP/Singularity.candidate.def" apptainer test "wasp2_definition_${RELEASE_VERSION}.sif" - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: candidate-sif path: wasp2_${{ needs.validate.outputs.version }}.sif @@ -612,13 +612,13 @@ jobs: permissions: id-token: write # Authenticate to PyPI using trusted publishing. steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-packages path: packages @@ -627,7 +627,7 @@ jobs: env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} run: python scripts/prepare_pypi_upload.py packages upload "$RELEASE_VERSION" - - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 if: steps.upload.outputs.count != '0' with: packages-dir: upload/ @@ -644,22 +644,22 @@ jobs: id-token: write # Authenticate container provenance attestations. attestations: write # Publish container provenance attestations. steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3 - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: tested-image-* merge-multiple: true path: ${{ runner.temp }}/tested-images - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: release-container-sbom-* merge-multiple: true @@ -682,21 +682,21 @@ jobs: test -n "$DIGEST" echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" - name: Attest amd64 dependency inventory - uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-name: ${{ env.IMAGE_NAME }} subject-digest: ${{ steps.image.outputs.amd64_digest }} sbom-path: container-sboms/container-amd64.cdx.json push-to-registry: true - name: Attest arm64 dependency inventory - uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-name: ${{ env.IMAGE_NAME }} subject-digest: ${{ steps.image.outputs.arm64_digest }} sbom-path: container-sboms/container-arm64.cdx.json push-to-registry: true - name: Attest container provenance - uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-name: ${{ env.IMAGE_NAME }} subject-digest: ${{ steps.image.outputs.digest }} @@ -705,7 +705,7 @@ jobs: env: CONTAINER_DIGEST: ${{ steps.image.outputs.digest }} run: printf '%s\n' "$CONTAINER_DIGEST" > container-digest.txt - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: container-metadata path: container-digest.txt @@ -725,7 +725,7 @@ jobs: apptainer-version: 1.4.5 - name: Verify container runtime run: apptainer --version - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: container-metadata - name: Build from tested candidate digest @@ -738,10 +738,10 @@ jobs: env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} run: apptainer exec --env EXPECTED_WASP2_VERSION="$RELEASE_VERSION" "wasp2_${RELEASE_VERSION}.sif" /opt/wasp2/scripts/container_smoke_test.sh - - uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 + - uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: "wasp2_${{ needs.validate.outputs.version }}.sif" - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: singularity-artifact path: "wasp2_${{ needs.validate.outputs.version }}.sif" @@ -756,11 +756,11 @@ jobs: contents: read packages: write steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: container-metadata - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -787,33 +787,33 @@ jobs: permissions: contents: write # Create the GitHub release and upload its assets. steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-packages path: release-assets - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-metadata path: release-assets - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: container-metadata path: release-assets - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: singularity-artifact path: release-assets - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: release-container-sbom-* merge-multiple: true path: release-assets - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: release-container-scan-* merge-multiple: true path: release-assets - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: builder-scan-* merge-multiple: true @@ -838,7 +838,7 @@ jobs: find . -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > "$RUNNER_TEMP/SHA256SUMS" mv "$RUNNER_TEMP/SHA256SUMS" SHA256SUMS sha256sum --check SHA256SUMS - - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: files: release-assets/* generate_release_notes: true diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index a6fd479f..bb3cc5b2 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -22,13 +22,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - name: Resolve auditable dependencies run: | uv pip compile pyproject.toml --extra dev --extra docs --extra plink --extra cyvcf2 --extra export --output-file /tmp/wasp2-audit.txt @@ -42,7 +42,7 @@ jobs: run: uvx --from 'bandit[toml]==1.9.4' bandit -c pyproject.toml -r src - name: Preserve resolved audit inputs if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: python-audit-requirements path: /tmp/wasp2*-audit.txt @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -94,10 +94,10 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4 + - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 required: name: Required / Security diff --git a/CHANGELOG.md b/CHANGELOG.md index b43fa2e7..78d908e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to WASP2 will be documented in this file. ## [Unreleased] ### Fixed +- Refresh maintained CI actions and remove unused Python build helpers from runtime images. - Refresh Rust HTSlib wrapper/binding patch releases and collect independent runtime scans and SIF checks when a build-image security gate fails. - Resolve numeric mapping genotypes to allele sequences and label swapped FASTQs diff --git a/Dockerfile b/Dockerfile index 52c7906d..468d3298 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,6 +112,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ COPY --from=rust-builder /wheels/*.whl /tmp/ RUN --mount=type=cache,target=/root/.cache/pip \ for wheel in /tmp/*.whl; do pip install "${wheel}[export]" || exit; done \ + # Runtime dependencies do not require setuptools or its vendored build helpers. + && pip uninstall -y setuptools \ && rm -rf /tmp/*.whl \ && apt-get purge -y --auto-remove g++ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* \ From 92b58216e457b9640c6359db88cdd954f66e8c36 Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:29:21 -0700 Subject: [PATCH 6/6] Reserve macOS extension header space for package relocation --- .github/workflows/release.yml | 17 +++++++++++++++++ CHANGELOG.md | 1 + rust/build.rs | 4 ++++ 3 files changed, 22 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 003113b1..5cc0ce4e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -176,6 +176,23 @@ jobs: target: ${{ matrix.target }} args: --release --locked --out dist -m rust/Cargo.toml -i python${{ matrix.python }} sccache: false + - name: Verify macOS wheel relocation + run: | + python - <<'PY' + from pathlib import Path + import subprocess, tempfile, zipfile + wheels = list(Path("dist").glob("*.whl")) + assert len(wheels) == 1 + with tempfile.TemporaryDirectory() as directory, zipfile.ZipFile(wheels[0]) as archive: + extensions = [name for name in archive.namelist() if name.endswith(".so")] + assert len(extensions) == 1 + library = Path(directory) / Path(extensions[0]).name + library.write_bytes(archive.read(extensions[0])) + rpath = "@loader_path/" + "/".join(["relocation_prefix" * 5] * 6) + subprocess.run(["xcrun", "install_name_tool", "-add_rpath", rpath, str(library)], check=True) + commands = subprocess.check_output(["xcrun", "otool", "-l", str(library)], text=True) + assert rpath in commands + PY - name: Install and test exact wheel env: PYTHON_VERSION: ${{ matrix.python }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 78d908e5..5d388cab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to WASP2 will be documented in this file. ## [Unreleased] ### Fixed +- Reserve macOS extension header space for Conda library-path relocation. - Refresh maintained CI actions and remove unused Python build helpers from runtime images. - Refresh Rust HTSlib wrapper/binding patch releases and collect independent runtime scans and SIF checks when a build-image security gate fails. diff --git a/rust/build.rs b/rust/build.rs index 258e36c8..a72e7d44 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,4 +1,8 @@ fn main() { pyo3_build_config::add_extension_module_link_args(); pyo3_build_config::use_pyo3_cfgs(); + if matches!(std::env::var("CARGO_CFG_TARGET_OS").as_deref(), Ok("macos")) { + // Reserve space for library-path relocation when wheels are repackaged by Conda. + println!("cargo:rustc-link-arg-cdylib=-Wl,-headerpad_max_install_names"); + } }