From 412486be32752071d3266aa2ba0001e7394b358e Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:58:14 -0700 Subject: [PATCH 1/2] Restore analysis compatibility and validate integrated workflows Combine single-cell and RNA annotation fixes with checked count arithmetic, workflow input corrections, native process regressions, documentation cleanup, and consistent source-build and release validation requirements. --- .github/ISSUE_TEMPLATE/audit-finding.yml | 111 --- .github/ISSUE_TEMPLATE/bug-report.yml | 31 + .github/ISSUE_TEMPLATE/config.yml | 16 +- .../ISSUE_TEMPLATE/scientific-validation.yml | 2 +- .github/workflows/ci.yml | 10 +- .github/workflows/ecosystem.yml | 74 +- .github/workflows/release.yml | 33 +- CHANGELOG.md | 41 +- CITATION.cff | 3 +- CONTRIBUTING.md | 8 +- README.md | 30 +- conda-lock.yml | 58 +- docs/AUDITOR_GUIDE.md | 240 ----- docs/AUDIT_CHECKLIST.md | 184 ---- docs/CONTAINER_USAGE.md | 39 +- docs/PLINK2_INTEGRATION_DESIGN.md | 881 ------------------ docs/VCF_PERFORMANCE.md | 334 ------- docs/WASP2_ECOSYSTEM.md | 157 ---- docs/source/changelog.rst | 42 +- docs/source/choosing_workflow.rst | 25 +- docs/source/development.rst | 154 ++- docs/source/faq.rst | 48 +- docs/source/index.rst | 8 +- docs/source/installation.rst | 40 +- docs/source/tutorials/bulk_workflow.rst | 46 +- .../source/tutorials/single_cell_workflow.rst | 189 ++-- docs/source/user_guide/analysis.rst | 40 +- docs/source/user_guide/counting.rst | 15 +- docs/source/user_guide/single_cell.rst | 63 +- environment.lock.yml | 4 +- environment.yml | 5 +- galaxy/tools/wasp2/wasp2_count_variants.xml | 2 +- galaxy/tools/wasp2/wasp2_find_imbalance.xml | 4 +- pipelines/README.md | 64 +- pipelines/nf-atacseq/docs/output.md | 19 +- .../modules/local/wasp2_find_imbalance.nf | 4 +- .../local/wasp2_find_imbalance/main.nf | 28 +- .../local/wasp2_find_imbalance.nf.test | 24 + .../nf-modules/modules/wasp2/analyze/main.nf | 6 +- .../modules/wasp2/analyze/tests/main.nf.test | 22 + .../wasp2/analyze/tests/main.nf.test.snap | 4 +- .../wasp2/analyze/tests/min_count.config | 5 +- .../modules/wasp2/analyze_imbalance/main.nf | 6 +- .../analyze_imbalance/tests/main.nf.test | 26 + pipelines/nf-rnaseq/README.md | 2 +- pipelines/nf-rnaseq/docs/output.md | 28 +- pipelines/nf-rnaseq/main.nf | 6 +- .../local/wasp2_analyze_imbalance/main.nf | 6 +- pipelines/nf-rnaseq/nextflow_schema.json | 2 +- pipelines/nf-rnaseq/tests/main.nf.test | 12 +- .../local/wasp2_analyze_imbalance.nf.test | 26 + .../wasp2_analyze_imbalance.nf.test.snap | 4 +- pipelines/nf-scatac/README.md | 44 +- pipelines/nf-scatac/conf/modules.config | 14 +- pipelines/nf-scatac/docs/output.md | 220 +---- pipelines/nf-scatac/docs/usage.md | 45 +- pipelines/nf-scatac/main.nf | 4 +- .../local/scatac_count_alleles/main.nf | 22 +- .../local/scatac_count_alleles/meta.yml | 2 +- .../local/scatac_create_anndata/main.nf | 20 +- .../local/scatac_create_anndata/meta.yml | 10 +- .../modules/local/scatac_pseudobulk/main.nf | 37 +- .../modules/local/scatac_pseudobulk/meta.yml | 10 +- pipelines/nf-scatac/nextflow.config | 8 +- pipelines/nf-scatac/nextflow_schema.json | 2 +- .../local/utils_nfscatac_pipeline.nf | 10 +- .../local/wasp_allelic_sc/main.nf | 26 +- .../local/wasp_allelic_sc/meta.yml | 9 +- pipelines/nf-scatac/tests/main.nf.test | 65 +- .../local/scatac_count_alleles.nf.test.snap | 12 +- .../local/scatac_create_anndata.nf.test.snap | 8 +- .../modules/local/scatac_pseudobulk.nf.test | 37 + .../local/scatac_pseudobulk.nf.test.snap | 4 +- pipelines/nf-scatac/tests/pipeline.nf.test | 65 ++ .../subworkflows/wasp_allelic_sc.nf.test | 36 + pipelines/nf-scatac/workflows/scatac.nf | 4 +- pixi.toml | 6 +- pyproject.toml | 2 +- rust/Cargo.lock | 169 +--- rust/Cargo.toml | 23 +- rust/src/analysis.rs | 201 ++-- rust/src/bam_counter.rs | 15 +- rust/src/bam_counter_sc.rs | 20 +- rust/src/bam_filter.rs | 2 +- rust/src/bam_intersect.rs | 18 +- rust/src/bam_remapper.rs | 32 +- rust/src/cigar_utils.rs | 2 +- rust/src/lib.rs | 49 +- rust/src/mapping_filter.rs | 26 +- rust/src/multi_sample.rs | 9 +- rust/src/read_pairer.rs | 34 - rust/src/unified_pipeline.rs | 20 +- rust/src/vcf_to_bed.rs | 22 +- scripts/compare_archive_analysis.py | 180 +++- scripts/release_version.py | 20 +- src/analysis/__main__.py | 10 +- src/analysis/as_analysis.py | 13 + src/analysis/as_analysis_sc.py | 4 +- src/analysis/compare_ai.py | 3 +- src/analysis/run_analysis.py | 18 +- src/analysis/run_analysis_sc.py | 4 +- src/counting/__main__.py | 2 +- src/counting/count_alleles_sc.py | 27 +- src/counting/parse_gene_data.py | 102 +- src/counting/run_counting_sc.py | 26 +- tests/analysis/fixtures/public_atac/README.md | 8 +- tests/analysis/test_archive_parity.py | 184 +++- tests/analysis/test_public_atac_matrix.py | 41 +- tests/analysis/test_reference_adapters.py | 133 +++ tests/analysis/test_region_col_modes.py | 29 +- tests/analysis/test_scipy_numerical_kernel.py | 149 +++ tests/counting/test_gene_data.py | 371 ++++++++ tests/counting/test_single_cell_pipeline.py | 301 ++++++ tests/test_extension_version.py | 69 ++ 114 files changed, 2819 insertions(+), 3470 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/audit-finding.yml create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml delete mode 100644 docs/AUDITOR_GUIDE.md delete mode 100644 docs/AUDIT_CHECKLIST.md delete mode 100644 docs/PLINK2_INTEGRATION_DESIGN.md delete mode 100644 docs/VCF_PERFORMANCE.md delete mode 100644 docs/WASP2_ECOSYSTEM.md create mode 100644 pipelines/nf-scatac/tests/pipeline.nf.test create mode 100644 pipelines/nf-scatac/tests/subworkflows/wasp_allelic_sc.nf.test create mode 100644 tests/counting/test_gene_data.py create mode 100644 tests/counting/test_single_cell_pipeline.py diff --git a/.github/ISSUE_TEMPLATE/audit-finding.yml b/.github/ISSUE_TEMPLATE/audit-finding.yml deleted file mode 100644 index 0e92766b..00000000 --- a/.github/ISSUE_TEMPLATE/audit-finding.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: "Audit Finding" -description: "Report an issue found during human review of AI-generated code" -title: "[AUDIT] " -labels: ["audit", "human-review", "needs-triage"] -body: - - type: markdown - attributes: - value: | - ## AI Code Audit Finding - Use this template to report issues discovered during human review of AI-generated code. - Please be as specific as possible about the location and nature of the concern. - - - type: dropdown - id: severity - attributes: - label: Severity - description: "How critical is this finding?" - options: - - Critical (blocks release) - - High (must fix before release) - - Medium (should fix) - - Low (nice to have) - validations: - required: true - - - type: dropdown - id: category - attributes: - label: Category - description: "What type of issue is this?" - options: - - Scientific Accuracy - - Code Quality - - Security Concern - - Documentation - - Test Coverage Gap - - Performance Issue - - User Experience - validations: - required: true - - - type: dropdown - id: reviewer-role - attributes: - label: Reviewer Role - description: "Your role on the audit team" - options: - - PI - - Software Engineer - - Bioinformatician - - Staff Research Scientist - - Other - validations: - required: true - - - type: textarea - id: location - attributes: - label: Location - description: "File path and line numbers where the issue was found" - placeholder: "src/analysis/as_analysis.py:150-175" - validations: - required: true - - - type: textarea - id: description - attributes: - label: Description - description: "What did you find? Be specific about the problem." - placeholder: "Describe the issue in detail..." - validations: - required: true - - - type: textarea - id: expected - attributes: - label: Expected Behavior - description: "What should happen instead?" - placeholder: "Describe the correct behavior..." - validations: - required: true - - - type: textarea - id: evidence - attributes: - label: Evidence/References - description: "Literature citations, benchmark comparisons, test results, or other supporting evidence" - placeholder: | - - Paper: Author et al., 2024, DOI: ... - - Benchmark: GATK ASEReadCounter shows r² = 0.95 vs expected 0.99 - - Test output: ... - - - type: textarea - id: suggested-fix - attributes: - label: Suggested Fix - description: "If you have ideas on how to resolve this, please share them" - placeholder: "Optional: Describe potential solutions..." - - - type: checkboxes - id: verification - attributes: - label: Verification - description: "Confirm you've done the following" - options: - - label: I have read the relevant code section - required: true - - label: I have checked if this issue already exists - required: true - - label: I have provided enough detail for someone else to understand and fix this - required: true diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..07fcb32d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,31 @@ +name: Bug report +description: Report a reproducible software or documentation problem +title: "[BUG] " +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Describe what happened and what you expected. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Include the exact command, relevant flags, and a small input example when possible. Do not attach sensitive data. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Installation and environment + description: Include the WASP2 version, installation method or container tag, operating system, and Python version. + validations: + required: true + - type: textarea + id: output + attributes: + label: Error output + description: Paste the relevant error or traceback. + render: text diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 6bd2164d..4cfc2391 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,17 +1,5 @@ -blank_issues_enabled: false +blank_issues_enabled: true contact_links: - - name: Security Vulnerability + - name: Security vulnerability url: https://github.com/mcvickerlab/WASP2/security/advisories/new about: Report security issues privately via GitHub Security Advisories - - - name: General Question - url: https://github.com/mcvickerlab/WASP2/discussions - about: Ask questions or discuss WASP2 usage in Discussions - - - name: Feature Request - url: https://github.com/mcvickerlab/WASP2/discussions/categories/ideas - about: Suggest new features or improvements in Discussions - - - name: Bug Report (Non-Audit) - url: https://github.com/mcvickerlab/WASP2/issues/new - about: For general bug reports outside of release audits, create a blank issue diff --git a/.github/ISSUE_TEMPLATE/scientific-validation.yml b/.github/ISSUE_TEMPLATE/scientific-validation.yml index b3d5556f..7346dd8f 100644 --- a/.github/ISSUE_TEMPLATE/scientific-validation.yml +++ b/.github/ISSUE_TEMPLATE/scientific-validation.yml @@ -1,7 +1,7 @@ name: "Scientific Validation" description: "Report scientific accuracy concerns (for bioinformaticians/scientists)" title: "[SCIENCE] " -labels: ["scientific-accuracy", "audit", "needs-domain-expert"] +labels: ["scientific-accuracy", "needs-domain-expert"] body: - type: markdown attributes: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6277af57..66579e85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,11 +123,19 @@ jobs: - name: Run tests run: | source .venv/bin/activate - pytest tests/ -v --tb=short \ + pytest tests/ -v --tb=short --junitxml=test-results.xml -o junit_family=xunit1 \ -m "not benchmark and not sanity" \ --ignore=tests/benchmarks \ --ignore=tests/sanity + - name: Upload Python test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: pytest-${{ matrix.python-version }} + path: test-results.xml + if-no-files-found: warn + # =========================================================================== # Rust unit tests and clippy lints # =========================================================================== diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml index 3855f208..67287ed1 100644 --- a/.github/workflows/ecosystem.yml +++ b/.github/workflows/ecosystem.yml @@ -74,7 +74,12 @@ jobs: with: version: "0.9.2" - name: Run pipeline compatibility test - run: nf-test test tests/main.nf.test --tag ci_stub --profile=test_stub --ci --tap=ci-stub.tap --verbose + run: | + test_file=tests/main.nf.test + if [ "${{ matrix.pipeline }}" = "nf-scatac" ]; then + test_file=tests/pipeline.nf.test + fi + nf-test test "$test_file" --tag ci_stub --profile=test_stub --ci --tap=ci-stub.tap --verbose - name: Upload nf-test result if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 @@ -83,16 +88,81 @@ jobs: path: pipelines/${{ matrix.pipeline }}/ci-stub.tap if-no-files-found: warn + nextflow-native: + name: Nextflow native process regressions + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.11" + - uses: dtolnay/rust-toolchain@4be9e76fd7c4901c61fb841f559994984270fce7 # stable + - uses: astral-sh/setup-uv@7eb50e6e20e1e2009087999ba91242e80253875f # v7 + - uses: nf-core/setup-nextflow@6c2e22b4d901f0c42ca66c5069f8026df026d165 # v2 + with: + version: "24.10.4" + - uses: nf-core/setup-nf-test@f2db585128ae681918440f6c074dededeceadce1 # v1 + with: + version: "0.9.2" + - name: Build matching native software + run: | + sudo apt-get update + sudo apt-get install -y libhts-dev libbz2-dev liblzma-dev zlib1g-dev libssl-dev pkg-config libclang-dev + uv venv .venv + source .venv/bin/activate + uv pip install "maturin==1.14.1" + maturin develop --release --uv + - name: Run actual analysis and coverage regressions + run: | + source .venv/bin/activate + cat > "$RUNNER_TEMP/nf-native.config" < "$RUNNER_TEMP/nextflow-native.config" <<'EOF' + docker.enabled = false + singularity.enabled = false + conda.enabled = false + process.executor = 'local' + process.cpus = 1 + process.memory = '1.GB' + params.min_count = 0 + params.pseudocount = 0 + params.min_cells_per_snp = 1 + EOF + nf-test test -c "$RUNNER_TEMP/nf-native.config" --ci --tag correctness_regression --tap=native-process.tap \ + pipelines/nf-atacseq/tests/modules/local/wasp2_find_imbalance.nf.test \ + pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test \ + pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test \ + pipelines/nf-modules/modules/wasp2/analyze_imbalance/tests/main.nf.test \ + pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test \ + pipelines/nf-scatac/tests/subworkflows/wasp_allelic_sc.nf.test + - name: Upload native process results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: nf-test-native-processes + path: native-process.tap + if-no-files-found: warn + required: name: Required / Ecosystem if: always() - needs: [galaxy, nextflow] + needs: [galaxy, nextflow, nextflow-native] runs-on: ubuntu-latest steps: - name: Verify ecosystem jobs env: GALAXY_RESULT: ${{ needs.galaxy.result }} NEXTFLOW_RESULT: ${{ needs.nextflow.result }} + NATIVE_RESULT: ${{ needs.nextflow-native.result }} run: | test "$GALAXY_RESULT" = success test "$NEXTFLOW_RESULT" = success + test "$NATIVE_RESULT" = success diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12150483..03f6fbbb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -193,7 +193,12 @@ jobs: python -m twine check dist/* uv venv .sdist-test uv pip install --python .sdist-test/bin/python dist/*.tar.gz - .sdist-test/bin/python -c "import wasp2, wasp2_rust; print(wasp2.__version__)" + SDIST_PYTHON="$GITHUB_WORKSPACE/.sdist-test/bin/python" + cd /tmp + "$SDIST_PYTHON" -c "import wasp2, wasp2_rust; print(wasp2.__version__)" + "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-count" --help + "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-map" --help + "$GITHUB_WORKSPACE/.sdist-test/bin/wasp2-analyze" --help - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: source-distribution @@ -309,9 +314,11 @@ jobs: with: persist-credentials: false - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 + - uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 with: - singularity-version: 3.11.4 + apptainer-version: 1.4.5 + - name: Verify container runtime + run: apptainer --version - name: Build local Docker archive uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: @@ -328,8 +335,8 @@ jobs: env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} run: | - singularity build "wasp2_${RELEASE_VERSION}.sif" "docker-archive://${RUNNER_TEMP}/wasp2-docker.tar" - singularity exec "wasp2_${RELEASE_VERSION}.sif" /opt/wasp2/scripts/container_smoke_test.sh + apptainer build "wasp2_${RELEASE_VERSION}.sif" "docker-archive://${RUNNER_TEMP}/wasp2-docker.tar" + apptainer exec "wasp2_${RELEASE_VERSION}.sif" /opt/wasp2/scripts/container_smoke_test.sh publish-pypi: name: Publish PyPI @@ -418,17 +425,19 @@ jobs: id-token: write # Authenticate Singularity provenance attestations. attestations: write # Publish Singularity provenance attestations. steps: - - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 + - uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 with: - singularity-version: 3.11.4 + apptainer-version: 1.4.5 + - name: Verify container runtime + run: apptainer --version - name: Build from released image env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} - run: singularity build "wasp2_${RELEASE_VERSION}.sif" "docker://${IMAGE_NAME}:${RELEASE_VERSION}" + run: apptainer build "wasp2_${RELEASE_VERSION}.sif" "docker://${IMAGE_NAME}:${RELEASE_VERSION}" - name: Test Singularity artifact env: RELEASE_VERSION: ${{ needs.validate.outputs.version }} - run: singularity exec "wasp2_${RELEASE_VERSION}.sif" wasp2-count count-cohort --help + run: apptainer exec "wasp2_${RELEASE_VERSION}.sif" wasp2-count count-cohort --help - uses: actions/attest-build-provenance@96b4a1ef7235a096b17240c259729fdd70c83d45 # v2 with: subject-path: "wasp2_${{ needs.validate.outputs.version }}.sif" @@ -463,7 +472,11 @@ jobs: name: singularity-artifact path: release-assets - name: Generate release checksums - run: sha256sum release-assets/* > release-assets/SHA256SUMS + run: | + cd release-assets + find . -maxdepth 1 -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > "$RUNNER_TEMP/SHA256SUMS" + mv "$RUNNER_TEMP/SHA256SUMS" SHA256SUMS + sha256sum --check SHA256SUMS - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: files: release-assets/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aa426b3..2d6d0d8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,39 @@ All notable changes to WASP2 will be documented in this file. ## [Unreleased] ### Fixed +- Restore archived single-cell phasing fallback for mixed phased/unphased + genotypes and require the native counter for normal single-cell counting. +- Reject overflowing bulk count, threshold, pseudocount and regional-total + arithmetic instead of wrapping UInt32 values. +- Preserve feature membership and zero-valued analysis parameters in Nextflow + wrappers; restore RNA annotation input. Fragment-only scATAC outputs represent + coverage and no longer enter allelic inference without allele observations. +- Run native Nextflow process regressions and restore actual single-cell + pipeline test discovery in CI. +- Correct tutorials, output schemas and installation guidance; remove obsolete + planning/audit reports and empty test placeholders. +- Align the declared Rust minimum and environment locks with supported builds, + validate the root Cargo lockfile version, declare CMake for source builds, + and update release SIF setup. +- Compare validation count columns exactly before floating-point conversion, + including UInt64 values above 2**53. Distinguish archived narrow CSV input + from explicitly widened reference counts; retain strict legacy discrepancies. +- Document Rust counting/bulk inference and Python/SciPy single-cell inference, + and correct the single-cell AnnData axes. Precision regressions cover balanced + likelihoods, overflow and conditional unphased symmetry. +- Single-cell sparse counts and totals use UInt64 so H5AD output preserves values + above 65,535. Metadata conversion no longer requires an undeclared PyArrow install. +- Automatic single-sample selection now applies the same heterozygote filtering + and feature reindexing as explicit selection, correcting inherited Python behavior. +- Single-cell counting normalizes repeated, comma-separated and file-based sample options. +- Shared-SNP differential analysis applies minimum depth per region, as archived + Python did. Unphased Python likelihoods preserve its sequential arithmetic while + retaining log-space fallback for underflow. Expanded comparisons explicitly widen + the archive's UInt16 accumulators; this also avoids incidental float32 dispatch. +- Restore archived exon/transcript/gene annotation priority and parent-grouping + CLI aliases through the Rust bulk backend. Parse final GFF3 attributes without + requiring a trailing semicolon and reject missing feature or parent identifiers + before writing BED intervals. - Donor feature deduplication uses the parsed first GT allele, as Python does. Original GT tokens remain available for the backend's pre-dedup phasing decision; fitting weights, raw counts, QC and provenance reflect retained rows. @@ -15,8 +48,8 @@ All notable changes to WASP2 will be documented in this file. checks, parses numeric first alleles consistently, and rejects malformed mixed phased input instead of silently changing a region's likelihood. Interior dispersion values are no longer clipped to the endpoint regularization range. -- Full 32-run public ATAC comparisons execute archived Python in pinned Linux - CI, including CLI inference and donor routes. Heterogeneous donor checks cover +- Full 32-run public ATAC comparisons execute archived Python on wide-integer + DataFrames in pinned Linux CI; narrow CSV precision is checked separately, including CLI inference and donor routes. Heterogeneous donor checks cover both dispersion models, phasing modes and pseudocounts with independent fits. - Container checks exercise reordered ATAC headers, region grouping, phased likelihoods and raw counts. Images carry package-version and source-revision @@ -57,7 +90,7 @@ All notable changes to WASP2 will be documented in this file. - Integrate the raw-count and single-cell layout compatibility fixes from #135. Exact-SNV requests are rejected rather than switching the p-value method. -## [1.5.0] - 2026-07-21 +### Additional changes planned for 1.5.0 ### Added - `wasp2-count count-cohort` for one-final-BAM-per-donor bulk counting with explicit @@ -69,7 +102,7 @@ All notable changes to WASP2 will be documented in this file. correction, manifest pinning, and non-overwriting result, dispersion, QC, and provenance files. - `--per-variant` / `--snv-solo` for independent SNV tests and explicit `--region_col` support for feature- or peak-level aggregation. -- A Rust single-cell ATAC allele counter with chromosome-parallel execution and Python fallback. +- A Rust single-cell ATAC allele counter with chromosome-parallel execution. ### Changed - VCF conversion now selects biallelic SNVs by default; multi-allelic records require the diff --git a/CITATION.cff b/CITATION.cff index 6d023466..ea8c457f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ # CITATION.cff - Citation File Format metadata for WASP2 # Specification: https://citation-file-format.github.io/ # Validates with: cffconvert --validate -# Update when: version changes in pyproject.toml, authors change, or paper is published +# Update when: version changes in rust/Cargo.toml, authors change, or paper is published cff-version: 1.2.0 title: "WASP2" @@ -23,7 +23,6 @@ authors: affiliation: "Salk Institute for Biological Studies" version: "1.5.0" -date-released: "2026-07-21" license: MIT repository-code: "https://github.com/mcvickerlab/WASP2" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58965e82..74cc9446 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,8 +38,8 @@ WASP2 is a hybrid Python/Rust project. You'll need both toolchains to build from | Requirement | Version | Notes | |-------------|---------|-------| -| Python | 3.10+ | 3.10, 3.11, 3.12 supported (conda env uses 3.11) | -| Rust | 1.70+ | Edition 2021 | +| Python | 3.10+ | 3.10, 3.11, 3.12, 3.13 tested (conda env uses 3.11) | +| Rust | 1.88+ | Edition 2021 | | Git | 2.0+ | For version control | | C compiler | gcc/clang | Required for native extensions | @@ -50,7 +50,7 @@ WASP2 is a hybrid Python/Rust project. You'll need both toolchains to build from 1. **Clone the repository:** ```bash - git clone https://github.com/mcvickerlab/WASP2.git + git clone --branch dev https://github.com/mcvickerlab/WASP2.git cd WASP2 ``` @@ -66,7 +66,7 @@ WASP2 is a hybrid Python/Rust project. You'll need both toolchains to build from ```bash python -m venv .venv source .venv/bin/activate # Linux/macOS - pip install -e ".[dev]" # Includes maturin and dev tools + pip install -e ".[dev,docs]" # Includes development and documentation tools ``` 3. **Install the Rust toolchain** (if not already installed): diff --git a/README.md b/README.md index a42e994d..8537aeff 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ ## Installation +Published packages and container tags may lag behind `dev`. This branch targets +the unreleased 1.5.0 version. To test development fixes, install from `dev` +using the [source installation instructions](https://mcvickerlab.github.io/WASP2/installation.html#development-installation). + ### Bioconda ```bash @@ -36,15 +40,18 @@ The PyPI package does not install external tools such as `samtools`, ### Docker +Choose an existing tag from the [container registry](https://github.com/mcvickerlab/WASP2/pkgs/container/wasp2) +and replace `PUBLISHED_TAG` below. A matching source version does not imply a published image. + ```bash -docker pull ghcr.io/mcvickerlab/wasp2:1.5.0 -docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-count --help +docker pull ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG +docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-count --help ``` ### Singularity/Apptainer ```bash -singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:1.5.0 +singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG singularity exec wasp2.sif wasp2-count --help ``` @@ -56,6 +63,11 @@ WASP2 installs three command-line entry points: - `wasp2-count` - `wasp2-analyze` +The Python CLI coordinates compiled Rust counting and bulk statistical inference. +Single-cell counting uses Rust; single-cell imbalance and differential inference +currently use Python/SciPy. A Python command entry point does not change which +backend performs the calculations. + ## Quick Start > **Prerequisite — normalize your VCF first.** WASP2's allelic-imbalance model is biallelic. @@ -177,15 +189,21 @@ Valid interior dispersion values are retained without clipping. Exact rho endpoints and probability-product underflow use documented finite extensions because archived Python is undefined there. -## Single-Cell Example +## Single-Cell ATAC-seq Example + +Counting uses a one-column barcode whitelist; analysis uses a separate +two-column barcode-to-group mapping. Single-cell phased inference is selected +only when the retained genotype set is exactly `{0|1, 1|0}`. Mixed phasing +or a single observed orientation selects unphased inference with a warning, +including when `--phased` was requested. ```bash wasp2-count count-variants-sc \ - cellranger.bam \ + sc_atac.bam \ variants.vcf.gz \ barcodes.tsv \ --samples sample1 \ - --feature genes.gtf \ + --feature peaks.bed \ --out_file allele_counts.h5ad wasp2-analyze find-imbalance-sc \ diff --git a/conda-lock.yml b/conda-lock.yml index baee9eca..9ac3c062 100644 --- a/conda-lock.yml +++ b/conda-lock.yml @@ -13,8 +13,8 @@ version: 1 metadata: content_hash: - linux-64: 2881080b4f148b9c6bda70b6ed38695b20edd70e3d7d8abc465d218ceb78ef1f - osx-64: 9ceab0a4b044cdc738244ea38c22799aa223511023039a8757680246fae841ef + linux-64: bb05ae664cf9071c5d535fcaa891330ef6f0682ccc971f598a0457b78816e150 + osx-64: 88e11e31b2d33d94f6c2834de4ee7908ebd89cd666ee28fd01489c60801ff44e channels: - url: conda-forge used_env_vars: [] @@ -2816,38 +2816,38 @@ package: category: main optional: false - name: llvmlite - version: 0.47.0 + version: 0.45.1 manager: conda platform: linux-64 dependencies: __glibc: '>=2.17,<3.0.a0' libgcc: '>=14' libstdcxx: '>=14' - libzlib: '>=1.3.2,<2.0a0' + libzlib: '>=1.3.1,<2.0a0' python: '>=3.11,<3.12.0a0' python_abi: 3.11.* zstd: '>=1.5.7,<1.6.0a0' - url: https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.47.0-py311h41a00d4_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.45.1-py311h41a00d4_0.conda hash: - md5: 62d9d69ab00067a2b3cdf60407f4c491 - sha256: 689cc23dc0f80c008094931aaa54ec229dd9dbabd169e825e13ea262d5e6dade + md5: 8f1e63415ecf9e21530b825dbcd843f4 + sha256: 5529ee8dea19299f574991f39d1995334e1c2979c15736618cd2db0d6fb499e0 category: main optional: false - name: llvmlite - version: 0.47.0 + version: 0.45.1 manager: conda platform: osx-64 dependencies: - __osx: '>=11.0' + __osx: '>=10.13' libcxx: '>=19' - libzlib: '>=1.3.2,<2.0a0' + libzlib: '>=1.3.1,<2.0a0' python: '>=3.11,<3.12.0a0' python_abi: 3.11.* zstd: '>=1.5.7,<1.6.0a0' - url: https://conda.anaconda.org/conda-forge/osx-64/llvmlite-0.47.0-py311h2a7c09c_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/llvmlite-0.45.1-py311hb26b958_0.conda hash: - md5: 5777950d9751f0f58b20bcb72d4bd327 - sha256: 307c38ebb38f9436d68ad71a56536c5f1d10e4cc3a388eabb724c9f9524f5ce2 + md5: 5ee443cfd08cf3f81feda6167204f43a + sha256: 0740b6589efe049d93a767779457ff32e189b94238472d1a109acbb0f6feb967 category: main optional: false - name: markdown-it-py @@ -3147,7 +3147,7 @@ package: category: main optional: false - name: numba - version: 0.65.1 + version: 0.62.1 manager: conda platform: linux-64 dependencies: @@ -3155,32 +3155,32 @@ package: _openmp_mutex: '>=4.5' libgcc: '>=14' libstdcxx: '>=14' - llvmlite: '>=0.47.0,<0.48.0a0' - numpy: '>=1.22.3,<2.5,>=1.23,<3' + llvmlite: '>=0.45.1,<0.46.0a0' + numpy: '>=1.23,<3,>=1.24,<2.4' python: '>=3.11,<3.12.0a0' python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/linux-64/numba-0.65.1-py311h3c884d5_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/numba-0.62.1-py311hcdb8226_1.conda hash: - md5: 89bb99267eee107ffb38f37fba4c973f - sha256: 5f6eacaba962cadba6da07756138492291c837cc83ac4d6d51224db6ac596953 + md5: 1241ccc9ea3f6f8271554ba33a455084 + sha256: 73b0692e95b677e4bafe9a0a8e09ddf966896c1be1810ec3ffbedf12ec0eb4ac category: main optional: false - name: numba - version: 0.65.1 + version: 0.62.1 manager: conda platform: osx-64 dependencies: - __osx: '>=11.0' + __osx: '>=10.13' libcxx: '>=19' - llvm-openmp: '>=19.1.7' - llvmlite: '>=0.47.0,<0.48.0a0' - numpy: '>=1.22.3,<2.5,>=1.23,<3' + llvm-openmp: '>=19.1.7,>=21.1.7' + llvmlite: '>=0.45.1,<0.46.0a0' + numpy: '>=1.23,<3,>=1.24,<2.4' python: '>=3.11,<3.12.0a0' python_abi: 3.11.* - url: https://conda.anaconda.org/conda-forge/osx-64/numba-0.65.1-py311h7fce02c_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/numba-0.62.1-py311h70add3f_1.conda hash: - md5: 801327e8542545ab784f7e804750906d - sha256: 5196c330f8c20ac465627504718dc11f78a52d6beaaabafa77c7583d8ae8cf82 + md5: fcfeac40673d6187d02dc77d239490a3 + sha256: 732bf09c4c4565a42da729131d6152fb2963e08a2cbd93a993cde8caea97dd8b category: main optional: false - name: numpy @@ -4819,7 +4819,7 @@ package: manager: conda platform: linux-64 dependencies: - typing_extensions: ==4.16.0 + typing_extensions: 4.16.0 url: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.16.0-h69aa097_0.conda hash: md5: c680b5747e8c4c8f23dca0bb7042a8fc @@ -4831,7 +4831,7 @@ package: manager: conda platform: osx-64 dependencies: - typing_extensions: ==4.16.0 + typing_extensions: 4.16.0 url: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.16.0-h69aa097_0.conda hash: md5: c680b5747e8c4c8f23dca0bb7042a8fc diff --git a/docs/AUDITOR_GUIDE.md b/docs/AUDITOR_GUIDE.md deleted file mode 100644 index c68eef38..00000000 --- a/docs/AUDITOR_GUIDE.md +++ /dev/null @@ -1,240 +0,0 @@ -# WASP2 Release Audit Guide - -## For Human Reviewers of AI-Generated Code - -This guide helps audit team members review WASP2 code for the v1.4.0 release. WASP2 contains AI-generated code that requires human validation for scientific accuracy, code quality, and security. - ---- - -## Getting Started - -### 1. Accept GitHub Invitation -Check your email for a collaborator invitation to `mcvickerlab/WASP2`. - -### 2. Clone the Repository -```bash -git clone https://github.com/mcvickerlab/WASP2.git -cd WASP2 -``` - -### 3. Set Up Your Environment -```bash -# Option A: Using conda (recommended) -conda env create -f environment.yml -conda activate wasp2 - -# Option B: Using pip -pip install -e ".[dev]" -``` - -### 4. Verify Installation -```bash -# Run quick sanity check -pytest tests/sanity/ -v --tb=short - -# Check the CLI works -wasp2 --help -``` - ---- - -## How to Report Findings - -### Creating an Issue - -1. Go to **Issues** → **New Issue** -2. Select the appropriate template: - - **Audit Finding**: General code quality, security, documentation issues - - **Scientific Validation**: Domain-specific scientific concerns -3. Fill in all required fields -4. Add appropriate labels if needed - -### Issue Templates Quick Reference - -| Template | Use When | Who Should Use | -|----------|----------|----------------| -| Audit Finding | Code bugs, security issues, test gaps | All team members | -| Scientific Validation | Statistical accuracy, biological correctness | Bioinformatician, Scientist | - ---- - -## Severity Guidelines - -| Severity | Definition | Examples | -|----------|------------|----------| -| **Critical** | Blocks release, data corruption risk | Statistical model produces wrong p-values; security vulnerability | -| **High** | Must fix before release, significant impact | Missing edge case that affects 10%+ of use cases | -| **Medium** | Should fix, moderate impact | Poor error messages; documentation unclear | -| **Low** | Nice to have, minor impact | Code style improvements; minor documentation typos | - ---- - -## Your Review Focus (by Role) - -### PI (Admin) -- Final release authorization -- CHANGELOG.md completeness and accuracy -- CITATION.cff verification -- Documentation quality review -- Security audit sign-off - -**Key Files:** -- `CHANGELOG.md` -- `CITATION.cff` -- `SECURITY_AUDIT.md` -- `README.md` -- `docs/source/installation.rst` - -### Software Engineer (Maintain) -- CI/CD validation -- Test coverage analysis -- Rust/Python integration correctness -- Build system and packaging -- Performance optimization - -**Key Files:** -- `.github/workflows/` -- `tests/` (coverage gaps from audit #200) -- `rust/` (Rust implementation) -- `pyproject.toml` -- `Dockerfile`, `Singularity.def` - -**Commands to Run:** -```bash -# Full test suite -pytest tests/ -v --tb=short - -# Test coverage report -pytest tests/ --cov=wasp2 --cov-report=html - -# Rust tests -cd rust && cargo test - -# Version consistency -scripts/check-version-consistency.sh - -# Security scan -pip-audit -bandit -r src/ -``` - -### Bioinformatician (Write) -- Statistical model validation -- Benchmark concordance (vs GATK) -- Output format correctness -- Scientific documentation accuracy - -**Key Files:** -- `src/analysis/as_analysis.py` (beta-binomial model) -- `src/counting/count_alleles.py` (allele counting logic) -- `tests/sanity/` (validation tests) -- `tests/test_indel_correctness.py` -- `benchmarking/` - -**Validation Steps:** -```bash -# Run sanity tests with chr21 data -pytest tests/sanity/ -v - -# Compare with GATK ASEReadCounter -# Expected: r² > 0.99 -python benchmarking/compare_gatk.py - -# Check dispersion clamping (Issue #228) -python -c "from wasp2.analysis import as_analysis; print(as_analysis.DISPERSION_BOUNDS)" -``` - -### Staff Research Scientist (Write) -- Real-world data testing -- VCF/PGEN input handling -- Output interpretability -- Documentation clarity for non-computational users - -**Key Files:** -- `docs/source/tutorials/` -- `tests/data/` (test data validation) -- `src/io/` (input handling) -- `tutorials/` - -**Testing Focus:** -- Run with real CRISPR/ASE data from lab -- Verify VCF handling with lab-generated genotypes -- Check Docker container on lab computing resources -- Validate outputs are usable for wet lab follow-up - ---- - -## Key Files to Review - -| Area | Files | What to Check | -|------|-------|---------------| -| **Statistics** | `src/analysis/as_analysis.py` | Beta-binomial model correctness | -| **Counting** | `src/counting/count_alleles.py` | Allele counting accuracy | -| **Tests** | `tests/` | Coverage gaps (see audit #200) | -| **Rust Core** | `rust/src/` | Memory safety, performance | -| **CI/CD** | `.github/workflows/` | All platforms pass | -| **Docs** | `docs/source/` | Clarity for new users | -| **Security** | `SECURITY_AUDIT.md` | Known issues addressed | - ---- - -## Common Issues to Look For - -### Scientific Accuracy -- [ ] Statistical assumptions match published methods -- [ ] Edge cases handled (zero counts, missing data) -- [ ] Dispersion parameters within valid bounds -- [ ] Output values in expected ranges - -### Code Quality -- [ ] Error messages are informative -- [ ] Input validation catches invalid data -- [ ] No hardcoded paths or magic numbers -- [ ] Logging provides useful debugging info - -### Security -- [ ] No sensitive data in test fixtures -- [ ] Input sanitization for file paths -- [ ] Dependencies have no known vulnerabilities - -### Documentation -- [ ] Installation instructions work -- [ ] Examples are accurate and runnable -- [ ] Parameters are documented -- [ ] Edge cases explained - ---- - -## Communication - -### Questions During Review -- Create a Discussion thread for questions -- Tag relevant team members using @username -- Use the issue templates for actual bugs - -### Progress Updates -- Update your assigned issues as you work -- Mark issues as "in progress" when investigating -- Close issues when resolved with a summary - ---- - -## Timeline - -| Milestone | Date | Description | -|-----------|------|-------------| -| Audit Start | TBD | All team members have access | -| Initial Review | TBD +1 week | First pass complete, critical issues identified | -| Fix Period | TBD +2 weeks | Address critical and high severity issues | -| Final Review | TBD +3 weeks | Verify fixes, sign-off | -| Release | TBD +4 weeks | v1.4.0 published | - ---- - -## Resources - -- [WASP2 Documentation](https://wasp2.readthedocs.io/) -- [Original WASP Paper](https://doi.org/10.1038/nmeth.3582) -- [GitHub Issue Templates Guide](https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests) -- [SECURITY_AUDIT.md](../SECURITY_AUDIT.md) - Known security considerations -- [CHANGELOG.md](../CHANGELOG.md) - Version history diff --git a/docs/AUDIT_CHECKLIST.md b/docs/AUDIT_CHECKLIST.md deleted file mode 100644 index 671b14fb..00000000 --- a/docs/AUDIT_CHECKLIST.md +++ /dev/null @@ -1,184 +0,0 @@ -# WASP2 v1.4.0 Release Audit Checklist - -## Pre-Release Verification - -Use this checklist to track audit progress. Each team member should complete their assigned section. - ---- - -## Code Quality (Software Engineer) - -### CI/CD & Build -- [ ] All CI checks pass on main branch -- [ ] CI passes on all platforms (Linux, macOS, Windows) -- [ ] Docker build completes successfully -- [ ] Singularity build completes successfully -- [ ] Wheel builds for all platforms (manylinux, macos, windows) - -### Test Coverage -- [ ] Test coverage meets threshold (≥30%) -- [ ] Audit #200 test coverage gaps reviewed and prioritized -- [ ] No critical paths without test coverage -- [ ] Integration tests pass with real data - -### Code Quality -- [ ] No security vulnerabilities (`bandit -r src/`) -- [ ] No dependency vulnerabilities (`pip-audit`) -- [ ] Version consistency verified (`scripts/check-version-consistency.sh`) -- [ ] Pre-commit hooks pass (`pre-commit run --all-files`) - -### Rust Component -- [ ] `cargo test` passes all tests -- [ ] `cargo clippy` shows no warnings -- [ ] Rust-Python binding works correctly -- [ ] Memory safety verified (no unsafe blocks without justification) - -**Sign-off:** __________________ Date: __________ - ---- - -## Scientific Accuracy (Bioinformatician) - -### Statistical Model -- [ ] Beta-binomial model in `as_analysis.py` validated -- [ ] Dispersion clamping works correctly (Issue #228) -- [ ] P-value calculations verified -- [ ] Effect size calculations verified - -### Benchmark Concordance -- [ ] GATK ASEReadCounter comparison (r² > 0.99) -- [ ] Sanity tests pass with chr21 data -- [ ] Edge cases tested (zero counts, single allele) - -### Variant Handling -- [ ] SNP handling verified -- [ ] INDEL handling verified (`test_indel_correctness.py`) -- [ ] Multi-allelic variant handling verified -- [ ] VCF parsing robust - -### Single-Cell Analysis -- [ ] Single-cell output format correct -- [ ] Cell barcode handling verified -- [ ] Aggregation logic validated - -**Sign-off:** __________________ Date: __________ - ---- - -## Documentation (PI) - -### Release Documentation -- [ ] CHANGELOG.md accurately reflects all changes -- [ ] README.md is up-to-date -- [ ] CITATION.cff has correct version and date -- [ ] Release notes drafted - -### User Documentation -- [ ] Installation instructions work -- [ ] Tutorials are runnable -- [ ] API documentation complete -- [ ] Error messages documented - -### Security & Compliance -- [ ] SECURITY_AUDIT.md reviewed and approved -- [ ] No sensitive data in repository -- [ ] License file present and correct -- [ ] Code of conduct present - -### Final Approval -- [ ] All critical issues resolved -- [ ] All high severity issues resolved or deferred with justification -- [ ] Release candidate tested end-to-end - -**Sign-off:** __________________ Date: __________ - ---- - -## Real-World Testing (Staff Research Scientist) - -### Data Compatibility -- [ ] Tested with real CRISPR/ASE data -- [ ] VCF input handling verified with lab genotypes -- [ ] PGEN input handling verified (if applicable) -- [ ] BAM/CRAM handling verified - -### Container Testing -- [ ] Docker container runs on lab systems -- [ ] Singularity container runs on HPC -- [ ] Memory usage acceptable -- [ ] Runtime acceptable - -### Output Usability -- [ ] Output files are interpretable -- [ ] Output can be used for downstream analysis -- [ ] Visualization recommendations work -- [ ] Results match expectations from known samples - -### Documentation Clarity -- [ ] Non-computational users can follow tutorials -- [ ] Error messages are actionable -- [ ] Help text is clear -- [ ] Examples are relevant to real use cases - -**Sign-off:** __________________ Date: __________ - ---- - -## Issue Summary - -### Critical Issues (Must Fix) -| Issue # | Description | Status | Assignee | -|---------|-------------|--------|----------| -| | | | | - -### High Severity Issues (Should Fix) -| Issue # | Description | Status | Assignee | -|---------|-------------|--------|----------| -| | | | | - -### Deferred Issues (Document for Future) -| Issue # | Description | Reason for Deferral | -|---------|-------------|---------------------| -| | | | - ---- - -## Final Release Checklist - -### Pre-Release -- [ ] All critical issues closed -- [ ] All high severity issues closed or documented -- [ ] Audit sign-offs complete from all team members -- [ ] Release candidate tag created -- [ ] Final test pass on release candidate - -### Release -- [ ] Version number updated -- [ ] CHANGELOG finalized -- [ ] Git tag created and pushed -- [ ] PyPI release published -- [ ] Conda release published (bioconda PR) -- [ ] GitHub release created -- [ ] Documentation deployed - -### Post-Release -- [ ] Announcement posted -- [ ] Known issues documented -- [ ] Next version roadmap updated - ---- - -## Approval Signatures - -| Role | Name | Signature | Date | -|------|------|-----------|------| -| PI | | | | -| Software Engineer | | | | -| Bioinformatician | | | | -| Staff Research Scientist | | | | - -**Release Approved:** Yes / No - -**Release Version:** v1.4.0 - -**Release Date:** __________ diff --git a/docs/CONTAINER_USAGE.md b/docs/CONTAINER_USAGE.md index b30f3e2e..54c72f0b 100644 --- a/docs/CONTAINER_USAGE.md +++ b/docs/CONTAINER_USAGE.md @@ -1,63 +1,58 @@ # WASP2 Container Usage -## Verified Docker Flow +## Docker -The Docker image validated for this update is: - -```bash -ghcr.io/mcvickerlab/wasp2:1.5.0 -``` +Choose an existing tag from the [container registry](https://github.com/mcvickerlab/WASP2/pkgs/container/wasp2) +and replace `PUBLISHED_TAG` below. Development source versions do not imply +that a matching image has been published. Pull and inspect the available CLI tools: ```bash -docker pull ghcr.io/mcvickerlab/wasp2:1.5.0 +docker pull ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG -docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-count --help -docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-map --help -docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-analyze --help +docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-count --help +docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-map --help +docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-analyze --help ``` Mount local data when running workflows: ```bash -docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:1.5.0 \ +docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG \ wasp2-count count-variants /data/sample.bam /data/variants.vcf.gz -o /data/counts.tsv ``` -## Intended Singularity / Apptainer Flow +## Singularity / Apptainer For HPC environments using SIF images: ```bash -singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:1.5.0 +singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG singularity exec wasp2.sif wasp2-count --help ``` or: ```bash -apptainer pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:1.5.0 +apptainer pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG apptainer exec wasp2.sif wasp2-count --help ``` -These commands are the intended container path, but they were not executed in -this development environment because neither `singularity` nor `apptainer` was -installed locally. - ## Mapping Example ```bash -docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:1.5.0 \ +docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG \ wasp2-map make-reads /data/sample.bam /data/variants.vcf.gz \ --samples sample1 \ --out_dir /data/remap_dir ``` -After realigning the swapped FASTQ reads with your aligner of choice: +After realigning the swapped FASTQ reads with the same aligner, version, +reference, and parameters as the original BAM: ```bash -docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:1.5.0 \ +docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG \ wasp2-map filter-remapped /data/remapped.bam \ --wasp_data_json /data/remap_dir/sample_wasp_data_files.json \ --out_bam /data/filtered.bam @@ -66,7 +61,7 @@ docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:1.5.0 \ ## Counting Example ```bash -docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:1.5.0 \ +docker run --rm -v "$PWD":/data ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG \ wasp2-count count-variants /data/filtered.bam /data/variants.vcf.gz \ --samples sample1 \ --region /data/genes.gtf \ diff --git a/docs/PLINK2_INTEGRATION_DESIGN.md b/docs/PLINK2_INTEGRATION_DESIGN.md deleted file mode 100644 index 0612ee66..00000000 --- a/docs/PLINK2_INTEGRATION_DESIGN.md +++ /dev/null @@ -1,881 +0,0 @@ -# WASP2 Multi-Format Variant Support: Design Document - -## Executive Summary - -This document outlines the design for integrating PLINK2 (PGEN/PVAR/PSAM) format support into WASP2, alongside existing VCF support. The design follows software engineering best practices using the **Strategy + Factory + Registry** pattern to enable extensible, maintainable, and testable multi-format support. - ---- - -## 1. Current State Analysis - -### 1.1 Existing VCF Handling in WASP2-exp - -| Module | File | VCF Handling | Issues | -|--------|------|--------------|--------| -| mapping | `intersect_variant_data.py` | `vcf_to_bed()` via bcftools subprocess | Duplicated in counting module | -| mapping | `make_remap_reads.py` | Uses BED output from above | Tightly coupled to VCF | -| counting | `filter_variant_data.py` | `vcf_to_bed()` (duplicate) | Code duplication | - -### 1.2 Key Problems with Current Architecture - -1. **Code Duplication**: `vcf_to_bed()` exists in both mapping and counting modules -2. **Format Lock-in**: Direct bcftools subprocess calls hardcode VCF format -3. **No Abstraction Layer**: Business logic mixed with file format handling -4. **Subprocess Dependency**: Relies on external bcftools binary -5. **No Format Auto-detection**: User must know and specify format - -### 1.3 Existing PLINK2 Implementation (WASP2-improved-new) - -The `WASP2-improved-new` repo has substantial PLINK2 support: - -| File | Status | Quality | -|------|--------|---------| -| `pgen_utils.py` | Complete | Good - handles VCF→PGEN conversion, normalization | -| `pgen_genotype_reader.py` | Complete | Good - reads genotypes via pgenlib | -| `variant_reader.py` | Complete | Good - ABC pattern already implemented | - -**What's Good:** -- Abstract `VariantReader` base class -- `VcfVariantReader` and `PgenVariantReader` implementations -- `open_variant_reader()` factory function -- Chunked reading for memory efficiency - -**What Needs Improvement:** -- No registry pattern (can't easily add new formats) -- Missing `to_bed()` method for bedtools compatibility -- Not integrated with WASP2-exp's `WaspDataFiles` -- Lacks heterozygous site filtering at the source level - ---- - -## 2. Proposed Architecture - -### 2.1 Design Pattern: Strategy + Factory + Registry - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ User / CLI Layer │ -│ wasp2 mapping --variants data.pgen --bam reads.bam │ -└─────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────┐ -│ VariantSourceFactory │ -│ ┌─────────────────────────────────────────────────────────────┐ │ -│ │ Registry: {'.vcf': VCFSource, '.pgen': PGENSource, ...} │ │ -│ └─────────────────────────────────────────────────────────────┘ │ -│ • Auto-detect format from extension/magic bytes │ -│ • Return appropriate VariantSource implementation │ -│ • @register decorator for extensibility │ -└─────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────┐ -│ VariantSource (Abstract Base Class) │ -│ ═══════════════════════════════════════════════════════════════ │ -│ Properties: │ -│ • samples: List[str] │ -│ • variant_count: int │ -│ • sample_count: int │ -│ │ -│ Abstract Methods: │ -│ • iter_variants(samples?) -> Iterator[Variant] │ -│ • get_het_sites(sample) -> Iterator[Variant] │ -│ • get_genotype(sample, chrom, pos) -> Genotype │ -│ • query_region(chrom, start, end) -> Iterator[Variant] │ -│ • to_bed(output, samples?, het_only?) -> Path │ -│ │ -│ Concrete Methods: │ -│ • get_sample_idx(sample_id) -> int │ -│ • validate() -> bool │ -└─────────────────────────────────────────────────────────────────────┘ - │ │ │ - ▼ ▼ ▼ -┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ -│ VCFSource │ │ PGENSource │ │ Future Formats │ -│ ───────────── │ │ ──────────── │ │ ───────────── │ -│ • pysam/cyvcf2 │ │ • pgenlib │ │ • BCF │ -│ • bcftools query │ │ • Direct binary │ │ • BGEN │ -│ • Indexed access │ │ • Chunked read │ │ • Zarr │ -└───────────────────┘ └───────────────────┘ └───────────────────┘ -``` - -### 2.2 Core Data Structures - -```python -from dataclasses import dataclass -from typing import Optional, Tuple -from enum import Enum - -class Genotype(Enum): - """Standardized genotype representation.""" - HOM_REF = 0 # 0/0 - HET = 1 # 0/1 or 1/0 - HOM_ALT = 2 # 1/1 - MISSING = -1 # ./. - -@dataclass(frozen=True, slots=True) -class Variant: - """Immutable variant representation.""" - chrom: str - pos: int # 1-based position - ref: str - alt: str - id: Optional[str] = None - - @property - def pos0(self) -> int: - """0-based position for BED format.""" - return self.pos - 1 - - def to_bed_line(self) -> str: - """Convert to BED format line.""" - return f"{self.chrom}\t{self.pos0}\t{self.pos}\t{self.ref}\t{self.alt}" - -@dataclass -class VariantGenotype: - """Variant with genotype information.""" - variant: Variant - genotype: Genotype - allele1: Optional[str] = None # For phased data - allele2: Optional[str] = None - - @property - def is_het(self) -> bool: - return self.genotype == Genotype.HET -``` - -### 2.3 Abstract Base Class - -```python -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Iterator, List, Optional, Dict, Any - -class VariantSource(ABC): - """ - Abstract interface for variant data sources. - - Implementations handle format-specific reading while exposing - a unified API for WASP2's mapping and counting modules. - """ - - # Class-level registry for format handlers - _registry: Dict[str, type] = {} - - @classmethod - def register(cls, *extensions: str): - """Decorator to register format handlers.""" - def decorator(subclass): - for ext in extensions: - cls._registry[ext.lower().lstrip('.')] = subclass - return subclass - return decorator - - @classmethod - def open(cls, path: Path, **kwargs) -> 'VariantSource': - """Factory method with auto-detection.""" - path = Path(path) - ext = cls._detect_format(path) - if ext not in cls._registry: - raise ValueError(f"Unsupported format: {ext}. " - f"Supported: {list(cls._registry.keys())}") - return cls._registry[ext](path, **kwargs) - - @classmethod - def _detect_format(cls, path: Path) -> str: - """Detect format from extension, handling compression.""" - suffixes = path.suffixes - if suffixes[-1] in ('.gz', '.bgz', '.zst'): - return suffixes[-2].lstrip('.') if len(suffixes) > 1 else '' - return suffixes[-1].lstrip('.') if suffixes else '' - - # ───────────────────────────────────────────────────────────── - # Abstract Properties - # ───────────────────────────────────────────────────────────── - - @property - @abstractmethod - def samples(self) -> List[str]: - """List of sample IDs in the file.""" - ... - - @property - @abstractmethod - def variant_count(self) -> int: - """Total number of variants.""" - ... - - @property - @abstractmethod - def sample_count(self) -> int: - """Total number of samples.""" - ... - - # ───────────────────────────────────────────────────────────── - # Abstract Methods - Must be implemented by subclasses - # ───────────────────────────────────────────────────────────── - - @abstractmethod - def iter_variants(self, - samples: Optional[List[str]] = None, - het_only: bool = False) -> Iterator[VariantGenotype]: - """ - Iterate over variants, optionally filtered by sample/het status. - - Args: - samples: Sample IDs to include (None = all) - het_only: If True, only yield heterozygous sites - - Yields: - VariantGenotype objects - """ - ... - - @abstractmethod - def get_genotype(self, sample: str, chrom: str, pos: int) -> Genotype: - """Get genotype for a specific sample at a position.""" - ... - - @abstractmethod - def query_region(self, - chrom: str, - start: int, - end: int, - samples: Optional[List[str]] = None) -> Iterator[VariantGenotype]: - """Query variants in a genomic region (1-based, inclusive).""" - ... - - @abstractmethod - def to_bed(self, - output: Path, - samples: Optional[List[str]] = None, - het_only: bool = True, - include_genotypes: bool = True) -> Path: - """ - Export variants to BED format for bedtools intersection. - - This is the key method for WASP2 integration - it replaces - the current vcf_to_bed() subprocess calls. - - Args: - output: Output BED file path - samples: Samples to include - het_only: Only include heterozygous sites - include_genotypes: Include genotype columns - - Returns: - Path to output BED file - """ - ... - - # ───────────────────────────────────────────────────────────── - # Concrete Methods - Shared implementation - # ───────────────────────────────────────────────────────────── - - def get_sample_idx(self, sample_id: str) -> int: - """Get 0-based index for a sample ID.""" - try: - return self.samples.index(sample_id) - except ValueError: - raise ValueError(f"Sample '{sample_id}' not found. " - f"Available: {self.samples[:5]}...") - - def validate(self) -> bool: - """Validate the variant source is readable.""" - try: - _ = self.variant_count - _ = self.sample_count - return True - except Exception: - return False - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def close(self): - """Clean up resources. Override in subclasses if needed.""" - pass -``` - -### 2.4 VCF Implementation - -```python -@VariantSource.register('vcf', 'vcf.gz', 'bcf') -class VCFSource(VariantSource): - """VCF/BCF variant source using pysam.""" - - def __init__(self, path: Path, **kwargs): - import pysam - self.path = Path(path) - self._vcf = pysam.VariantFile(str(self.path)) - self._samples = list(self._vcf.header.samples) - self._variant_count = None # Lazy computation - - @property - def samples(self) -> List[str]: - return self._samples - - @property - def variant_count(self) -> int: - if self._variant_count is None: - # Use tabix index if available - if self.path.suffix == '.gz': - try: - import subprocess - result = subprocess.run( - ['bcftools', 'index', '--nrecords', str(self.path)], - capture_output=True, text=True - ) - self._variant_count = int(result.stdout.strip()) - except: - self._variant_count = sum(1 for _ in self._vcf) - self._vcf.reset() - else: - self._variant_count = sum(1 for _ in self._vcf) - self._vcf.reset() - return self._variant_count - - @property - def sample_count(self) -> int: - return len(self._samples) - - def iter_variants(self, samples=None, het_only=False): - self._vcf.reset() - sample_indices = None - if samples: - sample_indices = [self.get_sample_idx(s) for s in samples] - - for record in self._vcf: - variant = Variant( - chrom=record.contig, - pos=record.pos, - ref=record.ref, - alt=record.alts[0] if record.alts else '.', - id=record.id - ) - - # Get genotypes for requested samples - for idx, sample in enumerate(samples or self._samples): - gt = record.samples[sample].get('GT', (None, None)) - genotype = self._parse_gt(gt) - - if het_only and genotype != Genotype.HET: - continue - - alleles = self._get_alleles(record, gt) - yield VariantGenotype( - variant=variant, - genotype=genotype, - allele1=alleles[0], - allele2=alleles[1] - ) - - def to_bed(self, output, samples=None, het_only=True, include_genotypes=True): - """Export to BED using bcftools for efficiency.""" - import subprocess - - # Build bcftools pipeline - view_cmd = ['bcftools', 'view', str(self.path), - '-m2', '-M2', '-v', 'snps', '-Ou'] - - if samples: - view_cmd.extend(['-s', ','.join(samples)]) - if het_only and len(samples) == 1: - # Filter het genotypes - view_proc = subprocess.run(view_cmd, capture_output=True) - het_cmd = ['bcftools', 'view', '--genotype', 'het', '-Ou'] - view_proc = subprocess.run(het_cmd, input=view_proc.stdout, - capture_output=True) - view_output = view_proc.stdout - else: - view_proc = subprocess.run(view_cmd, capture_output=True) - view_output = view_proc.stdout - else: - view_cmd.append('--drop-genotypes') - view_proc = subprocess.run(view_cmd, capture_output=True) - view_output = view_proc.stdout - - # Query to BED format - fmt = '%CHROM\t%POS0\t%END\t%REF\t%ALT' - if include_genotypes and samples: - fmt += r'[\t%TGT]' - fmt += '\n' - - query_cmd = ['bcftools', 'query', '-f', fmt, '-o', str(output)] - subprocess.run(query_cmd, input=view_output, check=True) - - return Path(output) - - def _parse_gt(self, gt) -> Genotype: - if None in gt: - return Genotype.MISSING - if sum(gt) == 0: - return Genotype.HOM_REF - if all(a == gt[0] for a in gt): - return Genotype.HOM_ALT - return Genotype.HET - - def close(self): - if self._vcf: - self._vcf.close() -``` - -### 2.5 PGEN Implementation - -```python -@VariantSource.register('pgen') -class PGENSource(VariantSource): - """PLINK2 PGEN variant source using pgenlib.""" - - def __init__(self, path: Path, **kwargs): - import pgenlib - import pandas as pd - - self.path = Path(path) - self.pvar_path = self.path.with_suffix('.pvar') - self.psam_path = self.path.with_suffix('.psam') - - # Validate files exist - for p in [self.path, self.pvar_path, self.psam_path]: - if not p.exists(): - raise FileNotFoundError(f"Required file not found: {p}") - - # Read sample info - self._psam_df = self._read_psam() - self._samples = self._psam_df['IID'].tolist() - - # Read variant info - self._pvar_df = self._read_pvar() - - # Initialize pgenlib reader with multiallelic support - allele_counts = self._pvar_df['ALT'].str.count(',') + 2 - self._allele_idx_offsets = np.zeros(len(self._pvar_df) + 1, dtype=np.uintp) - self._allele_idx_offsets[1:] = np.cumsum(allele_counts) - - self._reader = pgenlib.PgenReader( - bytes(str(self.path), 'utf-8'), - allele_idx_offsets=self._allele_idx_offsets - ) - - @property - def samples(self) -> List[str]: - return self._samples - - @property - def variant_count(self) -> int: - return self._reader.get_variant_ct() - - @property - def sample_count(self) -> int: - return self._reader.get_raw_sample_ct() - - def iter_variants(self, samples=None, het_only=False): - sample_indices = None - if samples: - sample_indices = np.array([self.get_sample_idx(s) for s in samples], - dtype=np.uint32) - self._reader.change_sample_subset(sample_indices) - - genotype_buf = np.empty(2, dtype=np.int32) - - for var_idx in range(self.variant_count): - row = self._pvar_df.iloc[var_idx] - variant = Variant( - chrom=str(row['CHROM']), - pos=int(row['POS']), - ref=row['REF'], - alt=row['ALT'].split(',')[0], # First alt for biallelic - id=row.get('ID', '.') - ) - - # Read genotype - self._reader.read_alleles(var_idx, genotype_buf) - genotype = self._parse_alleles(genotype_buf) - - if het_only and genotype != Genotype.HET: - continue - - yield VariantGenotype( - variant=variant, - genotype=genotype, - allele1=self._allele_to_base(genotype_buf[0], variant), - allele2=self._allele_to_base(genotype_buf[1], variant) - ) - - def to_bed(self, output, samples=None, het_only=True, include_genotypes=True): - """Export to BED format directly from PGEN.""" - with open(output, 'w') as f: - for vg in self.iter_variants(samples=samples, het_only=het_only): - line = vg.variant.to_bed_line() - if include_genotypes: - line += f"\t{vg.allele1}|{vg.allele2}" - f.write(line + '\n') - return Path(output) - - def _read_psam(self) -> pd.DataFrame: - """Read PSAM file with standard column detection.""" - df = pd.read_csv(self.psam_path, sep='\t', dtype=str) - df.columns = [c.lstrip('#') for c in df.columns] - return df - - def _read_pvar(self) -> pd.DataFrame: - """Read PVAR file skipping header comments.""" - return pd.read_csv(self.pvar_path, sep='\t', comment='#', - names=['CHROM', 'POS', 'ID', 'REF', 'ALT'], - dtype={'CHROM': str, 'POS': int, 'ID': str, - 'REF': str, 'ALT': str}) - - def _parse_alleles(self, buf) -> Genotype: - if buf[0] < 0 or buf[1] < 0: - return Genotype.MISSING - if buf[0] == 0 and buf[1] == 0: - return Genotype.HOM_REF - if buf[0] == buf[1]: - return Genotype.HOM_ALT - return Genotype.HET - - def _allele_to_base(self, allele_idx: int, variant: Variant) -> str: - if allele_idx < 0: - return '.' - if allele_idx == 0: - return variant.ref - alts = variant.alt.split(',') - return alts[allele_idx - 1] if allele_idx <= len(alts) else '.' - - def close(self): - if self._reader: - self._reader.close() -``` - ---- - -## 3. Integration Plan - -### 3.1 File Structure - -``` -src/ -├── wasp2/ -│ ├── __init__.py -│ ├── io/ # NEW: I/O abstraction layer -│ │ ├── __init__.py -│ │ ├── variant_source.py # ABC and factory -│ │ ├── vcf_source.py # VCF implementation -│ │ ├── pgen_source.py # PGEN implementation -│ │ └── formats/ # Future formats -│ │ └── __init__.py -│ ├── mapping/ -│ │ ├── intersect_variant_data.py # UPDATED: Use VariantSource -│ │ ├── make_remap_reads.py -│ │ └── ... -│ └── counting/ -│ ├── filter_variant_data.py # UPDATED: Use VariantSource -│ └── ... -``` - -### 3.2 Migration Steps - -| Phase | Task | Changes | -|-------|------|---------| -| 1 | Create `io/` module | New files, no breaking changes | -| 2 | Implement `VCFSource` | Port existing bcftools logic | -| 3 | Implement `PGENSource` | Port from WASP2-improved-new | -| 4 | Update `intersect_variant_data.py` | Replace `vcf_to_bed()` with `source.to_bed()` | -| 5 | Update `filter_variant_data.py` | Remove duplicate `vcf_to_bed()` | -| 6 | Update CLI | Add `--variant-format` auto-detection | -| 7 | Add tests | Unit + integration tests | - -### 3.3 Backward Compatibility - -```python -# Old code (still works): -from mapping.intersect_variant_data import vcf_to_bed -vcf_to_bed(vcf_file, out_bed, samples) - -# New code: -from wasp2.io import VariantSource -with VariantSource.open(variant_file) as source: - source.to_bed(out_bed, samples=samples, het_only=True) - -# The old vcf_to_bed becomes a thin wrapper: -def vcf_to_bed(vcf_file, out_bed, samples=None): - """Deprecated: Use VariantSource.to_bed() instead.""" - warnings.warn("vcf_to_bed is deprecated, use VariantSource", DeprecationWarning) - with VariantSource.open(vcf_file) as source: - return source.to_bed(out_bed, samples=samples, het_only=True) -``` - ---- - -## 4. Benchmarking Plan - -### 4.1 Metrics to Measure - -| Metric | Description | Tool | -|--------|-------------|------| -| **Wall time** | End-to-end execution time | `time` / `timeit` | -| **Peak memory** | Maximum RSS during execution | `/usr/bin/time -v` / `memory_profiler` | -| **I/O throughput** | Variants processed per second | Custom logging | -| **CPU utilization** | User vs system time | `time` | - -### 4.2 Test Datasets - -| Dataset | Size | Variants | Samples | Source | -|---------|------|----------|---------|--------| -| Small | ~10MB | 100K | 1 | Synthetic | -| Medium | ~500MB | 5M | 10 | 1000 Genomes subset | -| Large | ~5GB | 50M | 100 | CVPC subset | -| WGS | ~50GB | 500M | 1 | Full WGS sample | - -### 4.3 Benchmark Scenarios - -```python -# benchmark_config.py -BENCHMARKS = { - "vcf_to_bed_single_sample": { - "description": "Export het sites for single sample to BED", - "formats": ["vcf", "vcf.gz", "pgen"], - "samples": [1], - "het_only": True, - }, - "vcf_to_bed_multi_sample": { - "description": "Export het sites for multiple samples", - "formats": ["vcf", "vcf.gz", "pgen"], - "samples": [1, 10, 100], - "het_only": True, - }, - "full_pipeline_mapping": { - "description": "Complete WASP mapping pipeline", - "formats": ["vcf.gz", "pgen"], - "samples": [1], - "include": ["vcf_to_bed", "intersect", "remap"], - }, - "genotype_lookup": { - "description": "Random access genotype queries", - "formats": ["vcf.gz", "pgen"], - "queries": [100, 1000, 10000], - }, -} -``` - -### 4.4 Benchmark Script Structure - -```python -# benchmarks/run_benchmarks.py -import time -import tracemalloc -from pathlib import Path -from dataclasses import dataclass -from typing import List, Dict, Any -import json - -@dataclass -class BenchmarkResult: - name: str - format: str - dataset: str - wall_time_sec: float - peak_memory_mb: float - variants_processed: int - throughput_variants_per_sec: float - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - -class VariantSourceBenchmark: - """Benchmark suite for VariantSource implementations.""" - - def __init__(self, output_dir: Path): - self.output_dir = Path(output_dir) - self.results: List[BenchmarkResult] = [] - - def benchmark_to_bed(self, - source_path: Path, - samples: List[str], - het_only: bool = True, - n_runs: int = 3) -> BenchmarkResult: - """Benchmark the to_bed() operation.""" - from wasp2.io import VariantSource - - times = [] - memories = [] - - for _ in range(n_runs): - tracemalloc.start() - start = time.perf_counter() - - with VariantSource.open(source_path) as source: - out_bed = self.output_dir / "bench_output.bed" - source.to_bed(out_bed, samples=samples, het_only=het_only) - variant_count = source.variant_count - - elapsed = time.perf_counter() - start - current, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - - times.append(elapsed) - memories.append(peak / 1024 / 1024) # MB - - avg_time = sum(times) / len(times) - avg_memory = sum(memories) / len(memories) - - return BenchmarkResult( - name="to_bed", - format=source_path.suffix, - dataset=source_path.stem, - wall_time_sec=avg_time, - peak_memory_mb=avg_memory, - variants_processed=variant_count, - throughput_variants_per_sec=variant_count / avg_time - ) - - def run_all(self, datasets: Dict[str, Path]) -> None: - """Run all benchmarks on all datasets.""" - for name, path in datasets.items(): - # Test different scenarios - for n_samples in [1, 10]: - samples = [f"sample_{i}" for i in range(n_samples)] - result = self.benchmark_to_bed(path, samples) - self.results.append(result) - - # Save results - with open(self.output_dir / "benchmark_results.json", "w") as f: - json.dump([r.to_dict() for r in self.results], f, indent=2) - - def generate_report(self) -> str: - """Generate markdown benchmark report.""" - # ... generate comparison tables and charts -``` - -### 4.5 Expected Performance Comparison - -| Operation | VCF (bcftools) | VCF (pysam) | PGEN (pgenlib) | Expected Winner | -|-----------|----------------|-------------|----------------|-----------------| -| Load metadata | Fast | Medium | Fast | Tie | -| Single sample het export | Medium | Slow | Fast | PGEN (2-3x) | -| Multi-sample het export | Medium | Slow | Fast | PGEN (5-10x) | -| Random access query | Fast (indexed) | Fast | Fast | Tie | -| Memory (large file) | Low (streaming) | High | Low | VCF/PGEN | -| Full pipeline | Baseline | - | TBD | TBD | - -### 4.6 Validation Tests - -```python -def validate_output_equivalence(vcf_path: Path, pgen_path: Path, sample: str): - """Ensure VCF and PGEN produce identical BED output.""" - from wasp2.io import VariantSource - - with VariantSource.open(vcf_path) as vcf_source: - vcf_source.to_bed(Path("/tmp/vcf.bed"), samples=[sample]) - - with VariantSource.open(pgen_path) as pgen_source: - pgen_source.to_bed(Path("/tmp/pgen.bed"), samples=[sample]) - - # Compare outputs - import filecmp - assert filecmp.cmp("/tmp/vcf.bed", "/tmp/pgen.bed"), \ - "VCF and PGEN outputs differ!" -``` - ---- - -## 5. Testing Strategy - -### 5.1 Unit Tests - -```python -# tests/test_variant_source.py -import pytest -from wasp2.io import VariantSource, VCFSource, PGENSource - -class TestVariantSourceFactory: - def test_auto_detect_vcf(self, vcf_file): - source = VariantSource.open(vcf_file) - assert isinstance(source, VCFSource) - - def test_auto_detect_pgen(self, pgen_file): - source = VariantSource.open(pgen_file) - assert isinstance(source, PGENSource) - - def test_unsupported_format(self, tmp_path): - bad_file = tmp_path / "data.xyz" - bad_file.touch() - with pytest.raises(ValueError, match="Unsupported format"): - VariantSource.open(bad_file) - -class TestVCFSource: - def test_samples(self, vcf_file): - with VCFSource(vcf_file) as source: - assert len(source.samples) > 0 - - def test_iter_het_only(self, vcf_file): - with VCFSource(vcf_file) as source: - het_sites = list(source.iter_variants(het_only=True)) - for site in het_sites: - assert site.genotype == Genotype.HET - -class TestPGENSource: - def test_samples(self, pgen_file): - with PGENSource(pgen_file) as source: - assert len(source.samples) > 0 - - def test_to_bed_matches_vcf(self, vcf_file, pgen_file, tmp_path): - """Ensure PGEN and VCF produce equivalent BED output.""" - # ... comparison test -``` - -### 5.2 Integration Tests - -```python -# tests/test_integration.py -class TestMappingPipeline: - def test_full_pipeline_vcf(self, vcf_file, bam_file): - """Test complete mapping pipeline with VCF input.""" - # ... end-to-end test - - def test_full_pipeline_pgen(self, pgen_file, bam_file): - """Test complete mapping pipeline with PGEN input.""" - # ... end-to-end test - - def test_pipeline_equivalence(self, vcf_file, pgen_file, bam_file): - """Ensure VCF and PGEN produce identical WASP results.""" - # ... comparison test -``` - ---- - -## 6. Timeline and Milestones - -| Week | Milestone | Deliverables | -|------|-----------|--------------| -| 1 | Core architecture | `VariantSource` ABC, factory, data classes | -| 2 | VCF implementation | `VCFSource` with full test coverage | -| 3 | PGEN implementation | `PGENSource` ported and tested | -| 4 | Integration | Update mapping/counting modules | -| 5 | Benchmarking | Run benchmarks, generate report | -| 6 | Documentation | Update docs, examples, migration guide | - ---- - -## 7. Risks and Mitigations - -| Risk | Impact | Mitigation | -|------|--------|------------| -| pgenlib API changes | High | Pin version, add compatibility layer | -| Performance regression | Medium | Benchmark at each phase | -| bcftools dependency | Low | Keep as fallback option | -| Memory issues with large files | Medium | Ensure streaming/chunked processing | - ---- - -## 8. References - -- [Stack Overflow: Design patterns for multiple file formats](https://stackoverflow.com/questions/35139016/which-design-pattern-to-use-to-process-different-files-in-java) -- [Hail Import/Export](https://hail.is/docs/0.2/methods/impex.html) -- [scikit-allel I/O utilities](https://scikit-allel.readthedocs.io/en/stable/io.html) -- [pgenlib Python API](https://github.com/chrchang/plink-ng/tree/master/2.0/Python) -- [PLINK2 file formats](https://www.cog-genomics.org/plink/2.0/formats) diff --git a/docs/VCF_PERFORMANCE.md b/docs/VCF_PERFORMANCE.md deleted file mode 100644 index bb680365..00000000 --- a/docs/VCF_PERFORMANCE.md +++ /dev/null @@ -1,334 +0,0 @@ -# VCF Performance Optimization with cyvcf2 - -This document describes the high-performance VCF parsing integration using cyvcf2, which provides **6.9x faster** VCF parsing compared to the baseline pysam implementation. - -## Overview - -WASP2 now supports multiple VCF parsing backends: - -| Backend | Library | Performance | Use Case | -|---------|---------|-------------|----------| -| **VCFSource** | pysam | Baseline (1x) | Default, stable, well-tested | -| **CyVCF2Source** | cyvcf2 | **6.9x faster** | Production workloads, large files | -| **PGENSource** | pgenlib | **~25x faster** | Genotype-only data (PLINK2 format) | - -## Installation - -### Install cyvcf2 Support - -```bash -# Option 1: Install with pip -pip install wasp2[cyvcf2] - -# Option 2: Install from source with optional dependencies -pip install -e ".[cyvcf2]" - -# Option 3: Install cyvcf2 directly -pip install cyvcf2>=0.31.0 -``` - -### Install All Performance Enhancements - -```bash -# Install cyvcf2 + pgenlib + other optional dependencies -pip install wasp2[cyvcf2,plink] -``` - -## Usage - -### Automatic Detection (Recommended) - -The unified `VariantSource` interface automatically uses the best available backend: - -```python -from wasp2.io import VariantSource - -# Automatically uses CyVCF2Source if cyvcf2 is installed -with VariantSource.open("data.vcf.gz") as source: - for variant in source.iter_variants(het_only=True): - print(f"{variant.variant.chrom}:{variant.variant.pos}") -``` - -### Explicit Backend Selection - -Force a specific backend by direct instantiation: - -```python -from wasp2.io.cyvcf2_source import CyVCF2Source -from wasp2.io.vcf_source import VCFSource - -# Force cyvcf2 (high performance) -with CyVCF2Source("data.vcf.gz") as source: - variants = list(source.iter_variants()) - -# Force pysam (maximum compatibility) -with VCFSource("data.vcf.gz") as source: - variants = list(source.iter_variants()) -``` - -## Performance Benchmarks - -### Expected Performance Improvements - -Based on published cyvcf2 benchmarks and our testing: - -| Operation | pysam (baseline) | cyvcf2 | Speedup | -|-----------|------------------|--------|---------| -| **VCF Parsing** | 1.0x | **6.9x** | 6.9x faster | -| **Iteration** | 1.0x | **6.9x** | 6.9x faster | -| **Het Filtering** | 1.0x | **~7x** | ~7x faster | -| **Memory Usage** | Baseline | Similar | No increase | - -### Running Benchmarks - -Use the included benchmark script to measure performance on your data: - -```bash -# Basic benchmark (VCF only) -python benchmarks/benchmark_vcf_performance.py data.vcf.gz - -# Compare VCF vs PGEN -python benchmarks/benchmark_vcf_performance.py data.vcf.gz --pgen data.pgen - -# Specify sample for filtering -python benchmarks/benchmark_vcf_performance.py data.vcf.gz --sample sample1 -``` - -### Real-World Example - -```bash -$ python benchmarks/benchmark_vcf_performance.py large_cohort.vcf.gz - -================================================================================ -Benchmarking Multi-Format Variant I/O Performance -================================================================================ - -VCF file: large_cohort.vcf.gz -VCF file size: 2500.00 MB - -================================================================================ -Benchmark 1: Variant Counting Speed -================================================================================ -pysam VCFSource: 45.2341s (1,000,000 variants) [baseline] -cyvcf2 CyVCF2Source: 6.5432s (1,000,000 variants) - └─ Speedup vs pysam: 6.91x faster - -================================================================================ -Benchmark 2: Full Iteration Performance -================================================================================ -pysam VCFSource: 52.1234s (19,186 variants/s, +156.2 MB) [baseline] -cyvcf2 CyVCF2Source: 7.6543s (130,679 variants/s, +158.1 MB) - └─ Speedup vs pysam: 6.81x faster (6.81x throughput) - -================================================================================ -SUMMARY -================================================================================ - -Performance Improvements (cyvcf2 vs pysam): --------------------------------------------------------------------------------- -Counting............................................. 6.91x faster -Iteration............................................ 6.81x faster -Het Filtering........................................ 7.05x faster -Average Speedup...................................... 6.92x faster - -✅ Recommendation: Use CyVCF2Source for production workloads - Expected performance gain: ~5-7x faster VCF parsing -``` - -## Technical Details - -### How It Works - -**cyvcf2** is a Cython wrapper around htslib that provides: - -1. **Zero-copy numpy arrays**: Genotype data exposed directly from htslib memory -2. **Optimized parsing**: Cython-compiled code with minimal Python overhead -3. **Direct memory access**: Bypasses Python object creation for genotype arrays - -### Key Differences from pysam - -| Feature | pysam | cyvcf2 | -|---------|-------|--------| -| **Performance** | Baseline | 6.9x faster | -| **Memory** | Python objects | Zero-copy numpy | -| **API** | VariantRecord | Variant (similar) | -| **Genotypes** | Dict lookup | numpy array | -| **Stability** | Mature | Stable (v0.31+) | - -### Compatibility - -- **Formats**: VCF, VCF.gz (bgzip), BCF -- **Indexing**: Supports .tbi and .csi indexes -- **Region queries**: Yes (requires indexed files) -- **Multi-allelic**: Yes (same as pysam) -- **Missing data**: Yes (./. handled correctly) - -## Migration Guide - -### From pysam VCFSource to CyVCF2Source - -No code changes required! Both implement the same `VariantSource` interface: - -```python -# Before: Using pysam VCFSource -from wasp2.io.vcf_source import VCFSource - -with VCFSource("data.vcf.gz") as source: - for vg in source.iter_variants(het_only=True): - process(vg) - -# After: Using cyvcf2 CyVCF2Source -from wasp2.io.cyvcf2_source import CyVCF2Source - -with CyVCF2Source("data.vcf.gz") as source: - for vg in source.iter_variants(het_only=True): - process(vg) # Same API, 6.9x faster! -``` - -### Gradual Migration Strategy - -1. **Install cyvcf2**: `pip install wasp2[cyvcf2]` -2. **Benchmark your data**: Run `benchmark_vcf_performance.py` -3. **Test with your workflow**: Use `CyVCF2Source` directly for testing -4. **Verify results**: Compare outputs with pysam baseline -5. **Deploy**: Switch to cyvcf2 or rely on automatic detection - -### Fallback Behavior - -If cyvcf2 is not installed: -- `CyVCF2Source` will raise `ImportError` with installation instructions -- `VariantSource.open()` will automatically fall back to `VCFSource` (pysam) -- No code changes required - -## Troubleshooting - -### cyvcf2 Installation Issues - -**Issue**: `pip install cyvcf2` fails to compile - -**Solution**: Install htslib development headers first - -```bash -# Ubuntu/Debian -sudo apt-get install libhtslib-dev - -# macOS -brew install htslib - -# Then retry -pip install cyvcf2 -``` - -### Performance Not as Expected - -**Issue**: cyvcf2 not showing 6.9x improvement - -**Possible causes**: - -1. **Small files**: Overhead dominates for <1000 variants - - Use cyvcf2 for large files (>100k variants) - -2. **I/O bottleneck**: Network filesystem or slow disk - - Test on local SSD for accurate results - -3. **Old cyvcf2 version**: Earlier versions have bugs - - Ensure cyvcf2 >= 0.31.0 - -### Verification Test - -```python -# Quick test to verify cyvcf2 is working -import sys -try: - from wasp2.io.cyvcf2_source import CyVCF2Source, CYVCF2_AVAILABLE - print(f"✅ cyvcf2 available: {CYVCF2_AVAILABLE}") - if CYVCF2_AVAILABLE: - import cyvcf2 - print(f" Version: {cyvcf2.__version__}") -except ImportError as e: - print(f"❌ cyvcf2 not available: {e}") - sys.exit(1) -``` - -## Best Practices - -### When to Use cyvcf2 - -✅ **Use cyvcf2 for**: -- Large VCF files (>100k variants) -- Production pipelines -- Performance-critical workflows -- Batch processing many files - -❌ **Stick with pysam for**: -- Small test files (<1000 variants) -- Maximum compatibility requirements -- Debugging/development (more mature tooling) - -### Optimizing Performance - -1. **Use indexed files** for region queries: - ```bash - bcftools index data.vcf.gz # Creates .tbi index - ``` - -2. **Use BCF format** for best performance: - ```bash - bcftools view -O b data.vcf.gz > data.bcf - bcftools index data.bcf - # BCF is 5-8x faster than VCF.gz - ``` - -3. **Enable libdeflate** in htslib for 2x compression speedup: - ```bash - # Rebuild htslib with libdeflate support - # See: https://github.com/samtools/htslib#building-htslib - ``` - -## Multi-allelic policy - -VCF→BED conversion is **biallelic-only by default** across all backends. Multi-allelic sites are -dropped at the VCF→BED stage (before allele counting; the read counter itself has no allele-count -concept). The Rust `noodles` path does this **natively** (`alt_alleles.len() > 1` → skip; -`vcf_to_bed.rs`), which is *equivalent to* `bcftools -m2 -M2` but uses no subprocess; the -pysam/cyvcf2 + bcftools fallbacks use the **literal** `bcftools view -m2 -M2`. This keeps the -biallelic reference/alternate imbalance model consistent and avoids near-duplicate per-ALT SNVs -that lose reads to per-chromosome read deduplication and undercount their neighbors. - -**Provenance — this restores the original WASP2 behavior.** The pre-refactor release -(`archive/master-pre-v1.3.0`) hard-coded `bcftools view -m2 -M2 -v snps` in both -`src/counting/filter_variant_data.py` and `src/mapping/intersect_variant_data.py`, so biallelic-only -was the published, paper-consistent default. The v1.2.0 migration to the `wasp2.io` layer **removed -`-m2 -M2`** (commented "to include multi-allelic het sites"), silently introducing multi-allelic -inclusion. Defaulting `biallelic_only=True` restores the original behavior — across the Rust path -(native gate) and the bcftools fallbacks (literal flag) alike. - -Pass `--include-multiallelic` (CLI: `count-variants`, `count-variants-sc`, `make-reads`) or -`biallelic_only=False` (Python: `variants_to_bed`, `VCFSource.to_bed`) to opt back into the -post-v1.2.0 multi-allelic behavior, which emits one BED row per heterozygous ALT at multi-allelic -sites. - -We recommend normalizing inputs with `bcftools norm -m-` so that genuine multi-allelic calls are -split into biallelic records and retained, rather than dropped. - -## References - -- **cyvcf2 Paper**: Pedersen BS, Quinlan AR (2017). cyvcf2: fast, flexible variant analysis with Python. *Bioinformatics* 33(12):1867-1869. [doi:10.1093/bioinformatics/btx057](https://academic.oup.com/bioinformatics/article/33/12/1867/2971439) -- **cyvcf2 GitHub**: https://github.com/brentp/cyvcf2 -- **Performance Benchmarks**: https://github.com/brentp/vcf-bench -- **htslib**: http://www.htslib.org/ -- **VCF Specification**: https://samtools.github.io/hts-specs/VCFv4.2.pdf - -## Version History - -- **1.2.0** (2025): Initial cyvcf2 integration with CyVCF2Source -- **1.1.0** (2024): PLINK2 PGEN support added -- **1.0.0** (2023): Original pysam-only implementation - ---- - -**Next Steps**: Try running the benchmark on your data and see the performance improvements! - -```bash -python benchmarks/benchmark_vcf_performance.py your_data.vcf.gz -``` diff --git a/docs/WASP2_ECOSYSTEM.md b/docs/WASP2_ECOSYSTEM.md deleted file mode 100644 index 653774b0..00000000 --- a/docs/WASP2_ECOSYSTEM.md +++ /dev/null @@ -1,157 +0,0 @@ -# WASP2 Nextflow Pipeline Ecosystem - -> Tracking document for [EPIC #25](https://github.com/mcvickerlab/WASP2/issues/25) - -## Status Matrix - -### Core Pipelines - -| Component | Issue | Status | Infrastructure | -|-----------|-------|--------|----------------| -| wasp2-nf-modules | [#29](../../issues/29) | ✅ Complete | 9 modules, nf-test | -| wasp2-nf-rnaseq | [#30](../../issues/30) | ✅ Complete | docs, tests, assets | -| wasp2-nf-atacseq | [#31](../../issues/31) | ✅ Complete | docs, tests, assets, bin | -| wasp2-nf-scatac | [#32](../../issues/32) | ✅ Complete | docs, tests, assets, bin | -| wasp2-nf-outrider | [#35](../../issues/35) | ✅ Complete | docs, tests, assets, bin | - -### Integrations - -| Component | Issue | Status | -|-----------|-------|--------| -| ML Output Formats | [#36](../../issues/36) | ✅ Complete | -| GenVarLoader | [#37](../../issues/37) | ✅ Complete | -| nf-core Compliance | [#38](../../issues/38) | ✅ Complete | - -## Module Inventory - -| Module | Function | Implementation | -|--------|----------|----------------| -| WASP2_COUNT | Allelic read counting | Rust-accelerated | -| WASP2_MAP | Read remapping/filtering | Rust-accelerated | -| WASP2_ANALYZE | Statistical analysis | Rust-backed | -| WASP2_COUNT_ALLELES | Single-cell counting | Rust | -| WASP2_ANALYZE_IMBALANCE | SC imbalance | Rust | -| WASP2_ML_OUTPUT | ML format conversion | Zarr, Parquet, AnnData | -| VCF_TO_BED | VCF conversion | Rust: 7-25× faster | -| STAR_ALIGN | STAR 2-pass | Native | - -## Pipeline Directory Structure - -All pipelines follow a consistent nf-core-inspired structure: - -``` -pipelines/ -├── nf-modules/ # Shared DSL2 modules -│ └── modules/wasp2/ # WASP2-specific modules -├── nf-rnaseq/ # RNA-seq allelic imbalance -├── nf-atacseq/ # ATAC-seq allelic imbalance -├── nf-scatac/ # Single-cell ATAC-seq AI -│ ├── main.nf -│ ├── nextflow.config -│ ├── workflows/ -│ ├── subworkflows/ -│ ├── modules/local/ -│ ├── conf/ -│ ├── assets/ # samplesheet schema, multiqc config -│ ├── bin/ # helper scripts -│ ├── docs/ # usage.md, output.md -│ └── tests/ # nf-test, stub data -└── nf-outrider/ # OUTRIDER aberrant expression - ├── main.nf - ├── nextflow.config - ├── workflows/ - ├── subworkflows/ - ├── modules/local/ - ├── conf/ - ├── assets/ - ├── bin/ - ├── docs/ - └── tests/ -``` - -## Dependency Graph - -``` - wasp2-nf-modules (#29) ✅ - │ - ┌───────────────┼───────────────┐ - ▼ ▼ ▼ - nf-rnaseq ✅ nf-atacseq ✅ ML Formats ✅ - │ │ │ - ▼ ▼ ▼ - nf-outrider ✅ nf-scatac ✅ GenVarLoader ✅ - │ - ▼ - nf-core Compliance ✅ -``` - -## Implementation Roadmap - -### Phase 1: Foundation ✅ -- [x] Core DSL2 modules (9 modules) -- [x] nf-rnaseq pipeline -- [x] nf-atacseq pipeline -- [x] Docker builds, nf-test infrastructure - -### Phase 2: Expansion ✅ -- [x] nf-scatac (#32) - Single-cell ATAC-seq allelic imbalance -- [x] nf-outrider (#35) - OUTRIDER aberrant expression + MAE -- [x] ML output formats (#36) - Zarr, Parquet, AnnData - -### Phase 3: Integration ✅ -- [x] GenVarLoader integration (#37) - Via Zarr output format -- [x] nf-core compliance (#38) - Pipeline structure compliance - -## ML Output Formats - -All pipelines support optional ML-ready output formats via the `--output_format` parameter: - -```bash -# Single format -nextflow run . --output_format zarr - -# Multiple formats (comma-separated) -nextflow run . --output_format zarr,parquet,anndata -``` - -### Available Formats - -| Format | Description | Ecosystem | -|--------|-------------|-----------| -| **Zarr** | Chunked cloud-native arrays | GenVarLoader, xarray | -| **Parquet** | Columnar analytics format | Polars, DuckDB, pandas | -| **AnnData** | H5AD with layers | Scanpy, ArchR, scverse | - -### GenVarLoader Compatibility - -Zarr outputs are directly compatible with [GenVarLoader](https://genvarloader.readthedocs.io/) for ML training: - -```python -import genvarloader as gvl -loader = gvl.VariantLoader(zarr_path="sample.zarr") -``` - -## Testing - -All pipelines support: -- **Stub tests**: Fast CI/CD validation with `-profile test_stub -stub-run` -- **Integration tests**: Real data with `-profile test_real` or `-profile test` -- **nf-test framework**: Modular testing at workflow, subworkflow, and module levels - -Run stub tests: -```bash -cd pipelines/nf-scatac -nextflow run . -profile test_stub -stub-run - -cd pipelines/nf-outrider -nextflow run . -profile test_stub -stub-run -``` - -## References - -- [nf-core/drop](https://nf-co.re/drop/dev/) - Reference OUTRIDER implementation -- [GenVarLoader](https://genvarloader.readthedocs.io/) - ML variant loading - ---- -*Milestone: v1.3.0 - Pipeline Ecosystem* -*Last updated: 2026-02-03* diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 6877a9b8..5f5b3b8b 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,41 +1,9 @@ Changelog ========= -Version 1.0.0 (2025-11-17) --------------------------- +The `canonical changelog `_ +records development changes and release history. Unreleased source changes may +not yet be available from PyPI, Bioconda, or the container registry. -Initial Release -~~~~~~~~~~~~~~~ - -**Features:** - -* Complete type hint coverage (24 files, 5,500 lines) -* PyPI package available (pip install wasp2) -* CI/CD pipeline with GitHub Actions -* Pre-commit hooks for code quality -* Comprehensive documentation on ReadTheDocs - -**Modules:** - -* **Counting**: Allele-specific read counting from BAM files -* **Mapping**: WASP algorithm for unbiased read remapping -* **Analysis**: Statistical detection of allelic imbalance - -**Type Hints:** - -* TH-1: Counting module (7 files) -* TH-2: Analysis module (10 files) -* TH-3: Mapping module (7 files) - -**Testing:** - -* Regression tests (memory, performance) -* Full pipeline validation with real genomic data -* All tests passing in CI - -**Documentation:** - -* API documentation auto-generated from type hints -* User guides for each module -* Installation and quickstart guides -* Development and contributing guides +For published versions and release assets, see +`GitHub Releases `_. diff --git a/docs/source/choosing_workflow.rst b/docs/source/choosing_workflow.rst index f55f2785..23f643eb 100644 --- a/docs/source/choosing_workflow.rst +++ b/docs/source/choosing_workflow.rst @@ -1,7 +1,8 @@ Choosing the Right Workflow =========================== -WASP2 supports four major data types. Use this guide to find your workflow. +WASP2 supports bulk RNA-seq, bulk ATAC-seq, and barcode-based scATAC-seq. +Use this guide to find your workflow. .. list-table:: :header-rows: 1 @@ -12,19 +13,15 @@ WASP2 supports four major data types. Use this guide to find your workflow. - Goal - Start Here * - **Bulk RNA-seq** - - BAM + phased VCF + - BAM + donor VCF - Allele-specific expression (ASE) - :doc:`tutorials/bulk_workflow` * - **Bulk ATAC-seq** - - BAM + phased VCF + - BAM + donor VCF - Allele-specific chromatin accessibility - :doc:`tutorials/bulk_workflow` - * - **scRNA-seq (10x)** - - Cell Ranger BAM + VCF + barcodes - - Per-cell or per-cell-type ASE - - :doc:`tutorials/single_cell_workflow` * - **scATAC-seq (10x)** - - Fragments/BAM + VCF + barcodes + - CB-tagged BAM + VCF + barcodes - Single-cell allelic imbalance in ATAC peaks - :doc:`tutorials/single_cell_workflow` @@ -39,8 +36,8 @@ Decision Flowchart **Step 2: Bulk or single-cell RNA-seq?** * Bulk RNA-seq → :doc:`tutorials/bulk_workflow` -* 10x Chromium scRNA-seq → :doc:`tutorials/single_cell_workflow` -* Other single-cell protocol → see :doc:`user_guide/single_cell` +* Single-cell RNA-seq → the counter does not implement UMI-aware molecule + counting; the scATAC tutorial is not an scRNA-seq workflow. **Step 3: Bulk or single-cell ATAC-seq?** @@ -71,8 +68,9 @@ See :doc:`user_guide/mapping` for the full remapping workflow. What VCF Do I Need? -------------------- -WASP2 requires a **phased VCF** with heterozygous variants for the sample(s) -you are analyzing. Supported formats: +WASP2 uses heterozygous variants for the selected samples. Phasing is +optional: use the phased likelihood only with suitable phased genotypes. +Supported formats: * VCF/BCF (bgzip + tabix indexed) * PLINK2 PGEN files (with ``.pvar`` + ``.psam``) @@ -88,7 +86,8 @@ and output publishing, use the bundled Nextflow pipelines instead of the CLI: * **nf-rnaseq** — bulk RNA-seq allele-specific expression * **nf-atacseq** — bulk ATAC-seq allele-specific chromatin accessibility -* **nf-scatac** — single-cell ATAC-seq allelic imbalance +* **nf-scatac** — scATAC coverage workflow; see its documentation for the + current BAM/inference limitations * **nf-outrider** — outlier expression detection with allele-aware correction See the pipeline-specific documentation for samplesheet format and parameter diff --git a/docs/source/development.rst b/docs/source/development.rst index 4a4b71e3..61937fda 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -1,124 +1,90 @@ Development Guide -================= - -Contributions are welcome. This page covers the WASP2-specific bits; for -generic Python tooling (pytest, black, pre-commit) refer to each project's -own docs. +===================== Setup ------ +--------- + +WASP2 uses Python 3.10–3.13 and Rust 1.88 or newer. Install a C compiler and +the external tools required by the workflows: samtools, bcftools, and bedtools. .. code-block:: bash - git clone https://github.com/mcvickerlab/WASP2 + git clone --branch dev https://github.com/mcvickerlab/WASP2.git cd WASP2 - pip install -e ".[dev]" + python -m venv .venv + source .venv/bin/activate + pip install -e ".[dev,docs]" pre-commit install -The ``[dev]`` extra pulls in pytest, mypy, black, flake8, pre-commit, and -Sphinx. Pre-commit runs formatting + linting + mypy + a quick test subset -on every commit. +The ``dev`` extra provides pytest, Ruff, basedpyright, mypy, build, and +security tools. Documentation dependencies are in the separate ``docs`` extra. +Alternatively, use the locked Pixi environment with ``pixi install`` and +``pixi run verify``. -Code standards --------------- +Checks +---------- -- **Type hints are required** on new public API. WASP2 maintains full - mypy coverage on ``src/counting``, ``src/mapping``, and ``src/analysis``. -- **Line length 100**, enforced by black. -- **Ruff** runs as the primary lint tool; flake8 is retained for - docstring-specific rules. -- **Tests** live in ``tests/``. All new features need a test. Maintain - ≥80% line coverage. +Ruff handles Python linting and formatting with a 100-character line limit. +Pre-commit also runs file checks, Bandit, Gitleaks, and basedpyright. Add type +annotations to new public APIs and meaningful regressions for behavior changes. .. code-block:: bash - pytest tests/ -v # full suite - pytest tests/ -v -m "not slow" # fast subset - pytest tests/ --cov=src # coverage - - mypy src/counting/ src/mapping/ src/analysis/ + ruff check src tests scripts + ruff format --check src tests scripts + basedpyright + pytest tests/ -v + pytest tests/ -v -m "not slow" + cargo fmt --manifest-path rust/Cargo.toml --check + cargo clippy --manifest-path rust/Cargo.toml -- -D warnings + cargo test --manifest-path rust/Cargo.toml -Branching ---------- - -- ``master`` is the release branch. Tagged versions (e.g., ``v1.4.1``) - trigger PyPI + Docker publish via GitHub Actions. -- ``dev`` is the integration branch. Feature PRs target ``dev``, not - ``master``. -- Use short, kebab-case feature branches off ``dev``: ``feat/...``, - ``fix/...``, ``docs/...``. -- After a batch of features merges to ``dev``, a ``dev → master`` PR - promotes them; CI runs on both sides of that merge. - -Pull requests -------------- - -1. Branch off ``dev``. -2. Commit changes with descriptive messages; the squash-merge commit on - ``dev`` is what ends up in the changelog, so the PR title and body - should be clean. -3. CI must be green: ruff + Rust tests + pytest (Python 3.10 / 3.11 / - 3.12) + CodeQL. -4. Code review looks at correctness, type safety, test coverage, docs, - and API surface. - -Rust layer ----------- - -The ``wasp2_rust`` extension is built with maturin against Python 3.10+. +Rebuild the extension after Rust changes: .. code-block:: bash - cd rust/ - cargo fmt --check - cargo clippy -- -D warnings - cargo test - maturin develop --release --manifest-path Cargo.toml - -Changes to Rust code must keep the parity test -(``tests/test_rust_python_counting_parity.py``) green — the Rust counter -must produce byte-identical output to the Python reference implementation -on the shared chr21 fixture. + maturin develop --release --manifest-path rust/Cargo.toml -Project layout --------------- +Parity tests compare native counting and bulk inference against explicit +Python references. Preserve the statistical contract and keep legacy precision +limitations distinct from corrected wide-integer reference results. See the +existing counting and analysis tests for fixture requirements and tolerances. -.. code-block:: text +Project Layout +------------------ - WASP2/ - ├── src/ - │ ├── counting/ # allele counting (Python + Rust bridge) - │ ├── mapping/ # WASP remap-and-filter - │ └── analysis/ # beta-binomial LRT, FDR - ├── rust/ # Rust extension (bam filter, counter) - ├── tests/ # pytest + regression - ├── docs/ # Sphinx source - └── pipelines/ # Nextflow DSL2 workflows +* ``src/counting`` — allele counting and the Rust bridge. +* ``src/mapping`` — WASP remap-and-filter commands. +* ``src/analysis`` — bulk Rust dispatch and Python/SciPy single-cell inference. +* ``rust`` — native extension and numerical routines. +* ``tests`` — unit, regression, and integration tests. +* ``docs`` — Sphinx documentation. +* ``pipelines`` — Nextflow workflows and their documentation. -Docs ----- +Documentation +----------------- .. code-block:: bash - cd docs && make html + make -C docs html -The live site is built and deployed by GitHub Pages on every push to -``master``. Docs must build without warnings; cross-references are -checked by ``sphinx-build -n`` in CI. +The documentation workflow builds pushes and pull requests to ``dev`` and +``master``. GitHub Pages deployment uses ``master``. Update examples alongside +public interface changes and check generated pages and cross-references. -Releases --------- +Pull Requests and Releases +------------------------------ -1. Update ``pyproject.toml`` and ``rust/Cargo.toml`` versions (they share - one source of truth — ``pyproject.toml`` reads from Cargo). -2. Update ``CHANGELOG.md``. -3. Merge ``dev → master``. -4. Tag: ``git tag vX.Y.Z && git push origin vX.Y.Z``. -5. The ``release.yml`` workflow builds wheels and publishes to PyPI via - OIDC trusted publishing; ``docker.yml`` publishes the ghcr.io image. +Branch from ``dev`` and target ``dev`` for feature and fix pull requests. +Use descriptive titles and explain behavior changes and validation. Required +checks include Python 3.10–3.13, Rust, lint, documentation, and security checks. -Getting help ------------- +Version metadata comes from ``rust/Cargo.toml`` via maturin. Before a release, +run ``python scripts/release_version.py --check``, align the changelog and +release metadata, and verify package/container build results. Promotion to +``master`` and publication are separate release operations; an updated source +version does not establish that registry packages or images exist. -- Issues and feature requests: https://github.com/mcvickerlab/WASP2/issues -- Discussions: GitHub Discussions +See `CONTRIBUTING.md `_ +for contribution details. Report problems using the +`issue forms `_. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 007245cf..4c3b03ea 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -16,8 +16,8 @@ Use Docker or Singularity on HPC clusters or when you need full reproducibility. **What Python versions are supported?** -Python 3.10, 3.11, 3.12, and 3.13. Pre-built wheels are available for all four -on Linux (x86_64, aarch64) and macOS (Intel, Apple Silicon). +The development branch tests Python 3.10, 3.11, 3.12, and 3.13. Available +published wheels depend on the release and platform; see the release files. **Why do I get an error about missing samtools/bcftools/bedtools?** @@ -30,9 +30,10 @@ Input Data **Do I need phased genotypes?** -Yes. WASP2 assigns reads to haplotypes using phased heterozygous variants. Without -phase information, WASP2 cannot distinguish which allele a read came from. Use -WhatsHap, SHAPEIT4, or Eagle2 to phase your VCF before running WASP2. +No. Counting uses bases at heterozygous variant positions and does not require +phasing. Bulk analysis supports phased and unphased likelihoods; select +``--phased`` only when using suitable phased genotypes. Single-cell analysis +checks genotypes automatically unless explicitly set to ``--unphased``. **What VCF formats does WASP2 support?** @@ -43,9 +44,9 @@ Multi-sample VCFs are supported; use ``-s SAMPLE_ID`` to specify the target samp **Can I use an unphased VCF?** -The counting step (``wasp2-count``) will still run but the allele assignments will -be arbitrary. The statistical results will have reduced power and increased false -positives. Always use phased genotypes when possible. +Yes. REF and ALT counts remain defined by the variant alleles. Use unphased +inference when haplotype phase is unavailable; this marginalizes possible +orientations rather than treating arbitrary assignments as known phase. **My BAM doesn't have read groups. Will WASP2 work?** @@ -76,27 +77,25 @@ parallelization automatically. **What is the ``--region`` flag for?** -Restrict counting to a specific genomic region (e.g., ``chr1:1000000-2000000``). -Useful for testing on a subset of data or for chromosome-level parallelization. +Supply a region **file**, such as ``peaks.bed`` or ``genes.gtf``, to restrict +and annotate overlapping variants. ``--region`` is not a ``chr:start-end`` +interval string. Single-cell region input uses BED peaks. Single-Cell ----------- **What single-cell chemistries are supported?** -All 10x Genomics Chromium chemistries (scRNA v1/v2/v3, scATAC v1/v2) and any -other protocol with a cell barcode tag in the BAM (CB tag by default). See -:doc:`user_guide/single_cell` for barcode format details. +The validated workflow is scATAC-seq with a barcode-tagged BAM. The current +counter does not implement UMI-aware molecule counting for scRNA-seq. +See :doc:`user_guide/single_cell` for barcode files and supported outputs. **Do I need Cell Ranger output?** -No, but it is the most common input. WASP2 needs: - -* A BAM with cell barcodes in a BAM tag (default: ``CB``) -* A whitelist of valid barcodes (optional but recommended) -* A phased VCF - -Any aligner that produces CB-tagged BAMs will work (STARsolo, Alevin-fry, etc.). +No. Counting requires a sorted, indexed BAM with cell barcodes in a tag +(default ``CB``), donor variants, and a one-column barcode whitelist. +Analysis uses a separate two-column barcode-to-group mapping. Fragment +coordinates alone cannot provide REF/ALT base evidence. **How do I get per-cell-type results?** @@ -110,8 +109,8 @@ Output and Results **What does the p-value in the output represent?** The p-value comes from a likelihood ratio test comparing the beta-binomial model -under allelic imbalance vs. the null model of balanced expression. The test is -calibrated for the overdispersion typical of RNA-seq count data. +under allelic imbalance vs. the null model of balanced expression. The model +estimates overdispersion from the supplied counts. **What FDR threshold should I use?** @@ -123,10 +122,9 @@ See :doc:`user_guide/analysis` for the BH procedure and the NaN-propagation warn Common causes: -* Low coverage at heterozygous sites (increase ``--min_count``) +* Low coverage at heterozygous sites (inspect depth and the minimum-count threshold) * Too few heterozygous variants in the VCF * VCF and BAM use different chromosome naming conventions (``chr1`` vs ``1``) -* VCF is not phased **My output has too many significant sites (inflated FDR).** @@ -139,7 +137,7 @@ Yes. WASP2 counting applies only the unmapped filter (see :doc:`methods/mapping_ "Canonical Filter Contract"); it does **not** correct reference mapping bias on its own. You must run ``wasp2-map make-reads`` + re-alignment + ``wasp2-map filter-remapped`` first, then pass the resulting -``*_wasp_filt_rmdup.bam`` to ``wasp2-count``. Counting on raw BWA output +final filtered BAM to ``wasp2-count``. Counting on raw BWA output leaves reference bias uncorrected — reads carrying the alt allele are systematically under-represented. diff --git a/docs/source/index.rst b/docs/source/index.rst index f1d23c9f..963eaba6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,7 @@ WASP2: Allele-Specific Analysis :target: https://pypi.org/project/wasp2/ :alt: PyPI -.. image:: https://github.com/mcvickerlab/WASP2/workflows/WASP2%20Tests/badge.svg +.. image:: https://github.com/mcvickerlab/WASP2/actions/workflows/ci.yml/badge.svg :target: https://github.com/mcvickerlab/WASP2/actions :alt: Tests @@ -19,8 +19,8 @@ Features * **Allele Counting**: Count allele-specific reads from BAM files * **Statistical Analysis**: Beta-binomial likelihood-ratio tests for allelic imbalance -* **Single-Cell Support**: Specialized tools for single-cell RNA-seq and - scATAC-seq +* **Single-Cell Support**: Barcode-aware counting and group-level scATAC-seq + inference * **Rust-Accelerated**: Rust-backed BAM filtering and counting for large cohorts @@ -37,7 +37,7 @@ Count alleles from a BAM file: .. code-block:: bash - wasp2-count count-variants sample.bam variants.vcf + wasp2-count count-variants sample.bam variants.vcf --out_file counts.tsv Analyze allelic imbalance: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 48809214..b16e3059 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,6 +1,14 @@ Installation ============ +Published Packages +------------------ + +Package registries and container tags may lag behind ``dev``. This source +branch targets the unreleased 1.5.0 version; installing a published package +does not necessarily include its changes. Check ``wasp2-count --version`` +and use the source installation below when testing development fixes. + Via Bioconda ------------ @@ -26,16 +34,18 @@ does not install external tools such as ``samtools``, ``bcftools``, or Via Docker ---------- -The Docker image is the most reproducible fully bundled option available in -this environment. +Select an existing published tag from the +`container registry `_. +Replace ``PUBLISHED_TAG`` below with that tag. A source version does not +guarantee that a matching container has been published. .. code-block:: bash - docker pull ghcr.io/mcvickerlab/wasp2:1.5.0 + docker pull ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG - docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-count --help - docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-map --help - docker run --rm ghcr.io/mcvickerlab/wasp2:1.5.0 wasp2-analyze --help + docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-count --help + docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-map --help + docker run --rm ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG wasp2-analyze --help Via Singularity/Apptainer ------------------------- @@ -44,29 +54,27 @@ For HPC environments that require SIF images: .. code-block:: bash - singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:1.5.0 + singularity pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG singularity exec wasp2.sif wasp2-count --help or with Apptainer: .. code-block:: bash - apptainer pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:1.5.0 + apptainer pull wasp2.sif docker://ghcr.io/mcvickerlab/wasp2:PUBLISHED_TAG apptainer exec wasp2.sif wasp2-count --help -.. note:: - - Docker was validated in this development environment. ``apptainer`` / - ``singularity`` binaries were not available locally during this doc update, - so those examples reflect the intended pull/exec workflow but were not - executed here. - Development Installation ------------------------ +Source builds require Rust 1.88 or newer, a C/C++ compiler, and libclang. +The build backend declares CMake as a build dependency. Pixi supplies these +tools together with the runtime dependencies; published wheels do not need +the compilation toolchain. + .. code-block:: bash - git clone https://github.com/mcvickerlab/WASP2.git + git clone --branch dev https://github.com/mcvickerlab/WASP2.git cd WASP2 pixi install pixi run verify diff --git a/docs/source/tutorials/bulk_workflow.rst b/docs/source/tutorials/bulk_workflow.rst index efe9e4b7..91772a6c 100644 --- a/docs/source/tutorials/bulk_workflow.rst +++ b/docs/source/tutorials/bulk_workflow.rst @@ -60,12 +60,6 @@ position-dependent (see :doc:`/methods/mapping_filter` and [vandeGeijn2015]_). the original alignment. A different scoring function invalidates the filter contract. -Filter rates are **approximately** 1–10% of SNP-overlapping reads dropped -(developer experience; varies by sequencing platform, aligner, and variant -density). Outlier rates should be investigated — they typically signal -CNVs, aligner-version mismatch, or variant-call errors rather than a WASP -failure. - Step 2 — Count alleles ---------------------- @@ -115,38 +109,32 @@ Benjamini–Hochberg (:doc:`/user_guide/analysis`). ``--phased`` applies only when the VCF GT field uses ``0|1`` / ``1|0``; pass it off for unphased data. -Output columns: ``region``, ``num_snps``, aggregated ``ref_count`` / -``alt_count``, ``pval``, ``fdr_pval``, ``mu``, ``effect_size`` -(:math:`\log_2(\text{ref/alt})`). +Output columns: ``region``, ``snp_count``, aggregated ``ref_count`` / +``alt_count``, ``pval``, ``fdr_pval``, ``mu``, ``null_ll``, and ``alt_ll``. +The bulk route also writes total count ``N`` and likelihood-ratio statistic +``lrt``. Cohort outputs add donor and provenance information. Filtering and interpretation ---------------------------- -.. code-block:: bash - - # FDR < 0.05 and |log2FC| > 1 - awk -F'\t' 'NR==1 || ($5 < 0.05 && ($6 > 1 || $6 < -1))' \ - ai_results.tsv > significant_ase.tsv - -Monoallelic expression (ref fraction > 0.9 or < 0.1) is a common hallmark of -genomic imprinting on RNA-seq and of strong cis-regulatory variants on ATAC. - -eQTL integration (RNA-seq only) -------------------------------- - .. code-block:: python import pandas as pd - ase = pd.read_csv('ai_results.tsv', sep='\t') - eqtl = pd.read_csv('gtex_eqtl.tsv', sep='\t') + results = pd.read_csv('ai_results.tsv', sep='\t') + significant = results.loc[results['fdr_pval'] < 0.05] + significant.to_csv('significant_ai.tsv', sep='\t', index=False) - merged = ase.merge(eqtl, left_on='region', right_on='gene_id') +``mu`` is a fitted proportion in the model's regional orientation. In the +unphased model, the first SNP anchors that orientation; subsequent SNPs +are averaged over possible orientations. Phased orientation also depends +on the selected genotypes. Regional ``mu`` does not identify the REF or +ALT allele of every constituent SNP. - # ASE effect_size > 0 => REF allele more expressed. - # eQTL slope > 0 => ALT allele increases expression. - # Concordance = opposite signs. - merged['concordant'] = (merged['effect_size'] > 0) != (merged['slope'] > 0) +Inspect coverage, genotypes, mapping bias, and copy-number changes before +drawing biological conclusions. Integrating eQTL results requires matching +variant identities and explicitly harmonizing effect alleles and haplotypes. +The sign of ``mu - 0.5`` alone does not establish effect concordance. Troubleshooting --------------- @@ -169,4 +157,4 @@ See Also - :doc:`/user_guide/analysis` — analysis CLI reference - :doc:`/methods/mapping_filter` — canonical WASP filter contract - :doc:`/user_guide/analysis` — analysis CLI + defaults + contracts -- :doc:`/tutorials/single_cell_workflow` — single-cell RNA-seq / ATAC-seq +- :doc:`/tutorials/single_cell_workflow` — single-cell ATAC-seq diff --git a/docs/source/tutorials/single_cell_workflow.rst b/docs/source/tutorials/single_cell_workflow.rst index 25461f35..637181fa 100644 --- a/docs/source/tutorials/single_cell_workflow.rst +++ b/docs/source/tutorials/single_cell_workflow.rst @@ -1,164 +1,119 @@ -Single-Cell Workflow (scRNA-seq / scATAC-seq) -============================================== +Single-Cell ATAC-seq Workflow +================================= -End-to-end allele-specific workflow for single-cell data — 10X Chromium -scRNA-seq and 10X scATAC-seq. Pipeline is the same in both cases; the -data-type difference shows up as GTF (for scRNA-seq genes) vs. BED (for -scATAC-seq peaks) in the ``--feature`` argument. +Count alleles in a barcode-tagged scATAC-seq BAM and test imbalance after +aggregating cells into annotated groups. The counter does not perform +UMI-aware molecule counting for scRNA-seq. Inputs ------- +---------- -- Cell Ranger BAM with cell barcodes in the ``CB:Z:...`` tag + index -- Phased VCF/BCF/PGEN for the donor -- Barcode-to-group TSV (cell type or other assignment — see - :doc:`/user_guide/single_cell` for Seurat/Scanpy export code and format) -- **scRNA-seq**: GTF gene annotation -- **scATAC-seq**: BED peak file (usually from Cell Ranger - ``filtered_peak_bc_matrix`` or a consensus peak set) +* Coordinate-sorted, indexed BAM with cell barcodes in the ``CB`` tag. +* Donor VCF with heterozygous SNPs and matching reference coordinates. +* BED peak file. +* One-column barcode whitelist for counting, without a header. +* Two-column barcode-to-group TSV for analysis, without a header: -Step 1 — Count alleles per cell --------------------------------- +.. code-block:: text + + AAACCCAAGAAACACT-1 B_cell + AAACCCAAGAAAGCGA-1 T_cell -**scRNA-seq (genes):** +Derive the counting whitelist from this mapping: .. code-block:: bash - wasp2-count count-variants-sc \ - cellranger_output/outs/possorted_genome_bam.bam \ - phased_variants.vcf.gz \ - barcodes_celltype.tsv \ - --feature genes.gtf \ - --samples SAMPLE_ID \ - --out_file allele_counts.h5ad + cut -f1 barcode_groups.tsv > barcodes.txt + +Use a BAM that has undergone appropriate mapping-bias correction; see +:doc:`/user_guide/mapping`. Fragment-coordinate files alone do not contain +base-level evidence for REF and ALT allele counts. -**scATAC-seq (peaks):** +Step 1 — Count alleles per cell +----------------------------------- .. code-block:: bash wasp2-count count-variants-sc \ - cellranger_output/outs/possorted_bam.bam \ - phased_variants.vcf.gz \ - barcodes_celltype.tsv \ + sample.bam variants.vcf.gz barcodes.txt \ --feature peaks.bed \ --samples SAMPLE_ID \ --out_file allele_counts.h5ad -Output: an AnnData ``.h5ad`` with ``ref`` / ``alt`` / ``other`` layers, -genotype columns in ``.obs``, and cell-type assignments in ``.var``. See -:doc:`/user_guide/single_cell` for the full schema. +AnnData has cells in ``obs``, SNPs and genotypes in ``var``, and ``ref``, +``alt``, and ``other`` layers of shape ``cells × SNPs``. Peak membership is +stored in ``uns['feature']``. -Step 2 — Per-group imbalance ----------------------------- +Step 2 — Test imbalance within each group +--------------------------------------------- .. code-block:: bash wasp2-analyze find-imbalance-sc \ - allele_counts.h5ad \ - barcodes_celltype.tsv \ + allele_counts.h5ad barcode_groups.tsv \ --sample SAMPLE_ID \ - --phased --min 10 -z 3 \ - --out_file imbalance_by_celltype.tsv + --phased --min 10 \ + --out_file imbalance.tsv -Output columns: ``region``, ``cell_type``, aggregated ``ref_count`` / -``alt_count``, ``pval``, ``fdr_pval``, ``effect_size`` (log₂ ref/alt). +This writes one file per group, such as ``imbalance_B_cell.tsv``. Columns +are ``region``, ``num_snps``, ``mu``, ``null_ll``, ``alt_ll``, ``pval``, and +``fdr_pval``. The filename identifies the group; there is no ``cell_type`` +column or reported ``effect_size``. -Step 3 — Compare groups (optional) ------------------------------------ +Phasing is optional. Omit ``--phased`` to use automatic genotype checks, +or specify ``--unphased``. A phased request falls back to the unphased +model when the selected genotypes do not satisfy the phase checks. + +Step 3 — Compare groups +--------------------------- .. code-block:: bash wasp2-analyze compare-imbalance \ - allele_counts.h5ad \ - barcodes_celltype.tsv \ - --groups "CD4_T_cell,CD8_T_cell" \ + allele_counts.h5ad barcode_groups.tsv \ + --sample SAMPLE_ID \ + --groups B_cell --groups T_cell \ --phased \ - --out_file differential_imbalance.tsv - -Omit ``--groups`` to compare all available groups pairwise. See -:doc:`/user_guide/analysis` for the full CLI reference and output columns. - -Per-cell vs. pseudo-bulk ------------------------- - -Single-cell ATAC data is especially sparse — most cells contribute zero -reads to most peaks. Two analysis modes are common: + --out_file comparison.tsv -.. list-table:: - :header-rows: 1 - :widths: 20 40 40 +Omit ``--groups`` to compare all available groups pairwise. One file is +written per pair, such as ``comparison_B_cell_T_cell.tsv``. Columns include +``region``, ``num_snps``, ``combined_mu``, ``mu1``, ``mu2``, ``null_ll``, +``alt_ll``, ``pval``, and ``fdr_pval``. - * - Aspect - - Per-cell - - Pseudo-bulk (per-cell-type) - * - Resolution - - Single cell - - Cell population - * - Power - - Low (sparse) - - High (aggregated) - * - Use case - - Outlier cells - - Population-level imbalance - -Pseudo-bulk (the default, via the barcode-to-group TSV) is the right -starting point for most scATAC experiments. Per-cell analysis is useful -when investigating rare subpopulations or outlier effects. - -Interpreting results --------------------- +Interpretation +------------------ .. code-block:: python import pandas as pd - results = pd.read_csv('imbalance_by_celltype.tsv', sep='\t') - sig = results[results['fdr_pval'] < 0.05] + results = pd.read_csv('imbalance_B_cell.tsv', sep='\t') + significant = results.loc[results['fdr_pval'] < 0.05] + print(significant[['region', 'mu', 'fdr_pval']]) - top = (sig.groupby('cell_type') - .apply(lambda x: x.nsmallest(10, 'fdr_pval')) - .reset_index(drop=True)) - - print(top[['region', 'cell_type', 'effect_size', 'fdr_pval']]) +``mu`` is a fitted allele/haplotype proportion in the model's regional +orientation. It is not a gene-wide REF expression fraction or a log-fold +change. Comparing direction across regions or datasets requires consistent +SNP and haplotype orientation. Troubleshooting ---------------- - -**Zero barcodes matched.** Confirm barcode format in the BAM vs. the TSV — -the ``CB:Z:...`` tag often has a ``-1`` suffix that your export must match: - -.. code-block:: bash - - samtools view your.bam | head -10000 | grep -o 'CB:Z:[^[:space:]]*' \ - | cut -d: -f3 | sort -u > bam_bc.txt - cut -f1 barcodes.tsv | sort -u > file_bc.txt - comm -12 bam_bc.txt file_bc.txt | wc -l # should be > 0 +------------------- -Fix a missing suffix: +**No matching barcodes.** Match whitelist and annotation barcodes to BAM +``CB`` tags exactly, including suffixes such as ``-1``. -.. code-block:: bash - - awk -F'\t' '{print $1"-1\t"$2}' barcodes_no_suffix.tsv > barcodes.tsv - -**Sparse counts / low power.** Aggregate to pseudo-bulk by cell type, -lower ``--min`` / ``--min_count``, or focus on highly expressed genes -(scRNA-seq) / high-coverage peaks (scATAC-seq). - -**Memory.** For large cohorts, split the feature file by chromosome and -process chunks: - -.. code-block:: bash +**Sparse counts.** Aggregate cells into biologically meaningful groups and +inspect coverage before choosing a minimum-count threshold. The CLI tests +group-level aggregate counts; it has no separate per-cell model. - for chr in chr{1..22}; do - grep "^${chr}\s" peaks.bed > peaks_${chr}.bed - wasp2-count count-variants-sc sample.bam variants.vcf.gz barcodes.tsv \ - --feature peaks_${chr}.bed --out_file counts_${chr}.h5ad - done +**Coverage outliers.** The optional ``-z`` filter removes SNPs and associated +regions with extreme coverage. Apply a consistent filtering policy when +comparing groups. See Also --------- +------------ -- :doc:`/user_guide/single_cell` — barcode format, Seurat/Scanpy export -- :doc:`/user_guide/analysis` — analysis CLI reference -- :doc:`/methods/mapping_filter` — canonical WASP filter contract -- :doc:`bulk_workflow` — sibling tutorial for bulk RNA-seq / ATAC-seq +* :doc:`/user_guide/single_cell` — barcode files and AnnData schema +* :doc:`/user_guide/analysis` — models and command options +* :doc:`/methods/mapping_filter` — mapping-bias filter contract diff --git a/docs/source/user_guide/analysis.rst b/docs/source/user_guide/analysis.rst index 1b11b7e0..a854cd13 100644 --- a/docs/source/user_guide/analysis.rst +++ b/docs/source/user_guide/analysis.rst @@ -5,7 +5,14 @@ Overview -------- ``wasp2-analyze`` runs allelic imbalance analysis on bulk or single-cell count -outputs. +outputs. The Python CLI calls Rust for bulk inference. Single-cell imbalance +and differential inference currently run in Python/SciPy; single-cell counting +uses Rust. + +Numerical compatibility retains the archived statistical procedures with wide +counts and consistent precision. Untouched legacy file readers and explicitly +widened references are evaluated separately; the latter do not claim to reproduce +historical UInt16 overflow or mixed scalar/array precision. Commands: @@ -31,7 +38,10 @@ Useful options: * ``--model``: dispersion model (currently ``single`` or ``linear`` input) * ``--output`` / ``--out_file`` / ``-o``: output TSV path * ``--region_col``: explicit region column name if auto-detection is not desired -* ``--groupby``: group on an alternate annotation column, such as a parent gene column +* ``--groupby`` (aliases ``--group``, ``--parent_col``, ``--parent``): group on + an alternate annotation column, such as a parent gene column. The Rust backend + groups SNP observations by this column before deduplication and fitting; + this is equivalent to ``--region_col ``. Donor-Local Bulk Analysis ------------------------- @@ -139,7 +149,7 @@ Comparative Single-Cell Analysis allele_counts.h5ad \ barcode_groups.tsv \ --sample SAMPLE1 \ - --groups B_cell T_cell \ + --groups B_cell --groups T_cell \ --out_file compare_ai.tsv This compares allelic imbalance between the requested groups and writes one TSV @@ -171,16 +181,14 @@ sets the reference orientation; the first SNP is not symmetrized. - Beta-binomial LRT: :math:`\rho` is held at its **null-model MLE** while maximizing the alternative likelihood over :math:`\mu` (profile likelihood, df = 1). :math:`\rho` is not jointly re-estimated under :math:`H_1`. -- Scalar dispersion is fit on :math:`(0,1)` with boundary values clamped for - numerical stability. The linear predictor retains the :math:`\pm 10` guard. The - linear model uses raw-depth standardization, - :math:`\rho(N)=\operatorname{expit}(d_1 + d_2(N-c)/s)`, and persists the fitted - center :math:`c` and positive scale :math:`s` with :math:`d_1` and :math:`d_2`. -- FDR correction uses :func:`scipy.stats.false_discovery_control` with - ``method='bh'`` (Benjamini–Hochberg). ``scipy`` raises on NaN p-values, - but a hand-written BH loop using ``np.minimum.accumulate`` silently - propagates NaN through the cumulative minimum — always drop or impute - NaN p-values before BH correction. +- Scalar dispersion is fit on :math:`(0,1)`; exact endpoints receive numerical + regularization. The linear fit uses pseudocounted depth and a :math:`\pm 10` + predictor guard during fitting. Stored coefficients preserve that predictor + using :math:`\rho(N)=\operatorname{expit}(d_1+d_2(N-c)/s)` with raw depth + :math:`N`, center :math:`c=-2\,\mathrm{pseudocount}` and scale :math:`s=1`. +- Both backends use Benjamini–Hochberg FDR correction. Bulk uses the Rust + implementation; single-cell analysis uses + :func:`scipy.stats.false_discovery_control` with ``method='bh'``. See the upstream mapping-filter docs (:doc:`/methods/mapping_filter`) for the WASP2 canonical filter contract that applies to both remapping and counting @@ -195,8 +203,10 @@ Typical bulk outputs include: * aggregated ``ref_count`` and ``alt_count`` * p-values and FDR-adjusted p-values -Typical single-cell outputs include the same statistics stratified by barcode -group. +Single-cell outputs contain ``region``, ``num_snps``, ``mu``, ``null_ll``, +``alt_ll``, ``pval``, and ``fdr_pval`` in one file per group. They do not +include aggregated count columns. Differential outputs replace ``mu`` with +``combined_mu``, ``mu1``, and ``mu2``. Next Steps ---------- diff --git a/docs/source/user_guide/counting.rst b/docs/source/user_guide/counting.rst index 8daec6d3..c41c3ac1 100644 --- a/docs/source/user_guide/counting.rst +++ b/docs/source/user_guide/counting.rst @@ -41,7 +41,9 @@ Supported region files: * GFF3 For GTF/GFF3 inputs, WASP2 derives interval annotations from feature rows and -defaults to ``gene`` features when present. +selects ``exon``, then ``transcript``, then ``gene`` features by default. +Use ``--gene_feature`` to select explicitly. Feature and parent identifiers must +be present and nonempty on every selected annotation row. Useful options: @@ -130,10 +132,17 @@ Important points: The output is an AnnData ``.h5ad`` file with: * sparse count layers for ``ref``, ``alt``, and ``other`` -* variant metadata in ``adata.obs`` -* barcode names in ``adata.var_names`` +* variant metadata and selected sample genotypes in ``adata.var`` +* cell barcode names in ``adata.obs_names`` +* count layers shaped ``cells × SNPs`` * feature-to-variant mapping in ``adata.uns["feature"]`` when annotations are used +For downstream single-cell analysis, automatic or requested phased inference +uses the archived selection rule: the retained genotype set must be exactly +``{0|1, 1|0}``. Mixed phased/unphased genotypes or a single observed phase +orientation select unphased inference with a warning. Explicit ``--unphased`` +always selects unphased inference. + Examples -------- diff --git a/docs/source/user_guide/single_cell.rst b/docs/source/user_guide/single_cell.rst index a243cdc7..06e95994 100644 --- a/docs/source/user_guide/single_cell.rst +++ b/docs/source/user_guide/single_cell.rst @@ -4,12 +4,22 @@ Single-Cell Analysis Overview -------- -WASP2 provides specialized tools for allele-specific analysis in single-cell RNA-seq (scRNA-seq) data. This guide covers the barcode file format requirements and single-cell-specific workflows. +WASP2 supports barcode-based allele counting and group-level imbalance analysis. +The validated workflow is scATAC-seq. Rust performs allele counting; Python/SciPy +performs single-cell imbalance and differential inference through the Python CLI. +The current counter does not implement UMI-aware molecule counting for scRNA-seq. + +Counting uses wide integer storage and inference uses wide count sums. Statistical +compatibility follows the archived likelihood and testing procedures with consistent +precision; it does not reproduce historical UInt16 overflow or mixed-precision +rounding. The original and explicitly widened archive references are tested separately. Barcode File Format ------------------- -WASP2 uses a two-column TSV (tab-separated values) format for barcode files. This format maps cell barcodes to cell type annotations. +Counting accepts a one-column barcode whitelist without a header. Analysis +uses a two-column TSV mapping barcodes to groups. The format below describes +the analysis mapping; derive a whitelist with ``cut -f1 barcodes.tsv``. Format Specification ~~~~~~~~~~~~~~~~~~~~ @@ -73,22 +83,11 @@ Format Specification AAACCCAAGAACCAAG-1 Monocyte AAACCCAAGAACGATA-1 NK_cell -**Multi-Sample Aggregated Example:** - -When using Cell Ranger ``aggr`` to combine multiple samples, barcodes are distinguished by suffix: - -.. code-block:: text - - AAACCCAAGAAACACT-1 B_cell sample1 - AAACCCAAGAAACTGT-1 B_cell sample1 - AAACCCAAGAAACACT-2 B_cell sample2 - AAACCCAAGAAACTGT-2 B_cell sample2 - AAACCCAAGAAACACT-3 CD4_T_cell sample3 - -.. note:: +**Aggregated samples:** - For multi-sample experiments, WASP2 uses only the first two columns (barcode, cell_type). - The third column (sample origin) is optional metadata for your reference. +Preserve sample-specific barcode suffixes in both the whitelist and group +mapping. The analysis mapping must contain exactly the barcode and group +columns; keep additional sample metadata in a separate file. Barcode Format Validation ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -123,7 +122,7 @@ Before running WASP2, validate your barcode file format: import re def validate_10x_barcode_file(filepath): - """Validate 10X scRNA-seq barcode file format.""" + """Validate 10X barcode syntax.""" pattern = re.compile(r'^[ACGT]{16}-\d+$') errors = [] i = 0 @@ -168,10 +167,12 @@ When using Cell Ranger output, barcodes can be found in: cellranger_output/ └── outs/ - └── filtered_feature_bc_matrix/ + └── filtered_peak_bc_matrix/ └── barcodes.tsv.gz -This file contains only the barcode column. To create a WASP2-compatible barcode file, you need to add cell type annotations from your downstream analysis. +Use this one-column file as the counting whitelist after decompression. +For group-level analysis, create a separate two-column mapping with cell +type annotations from downstream analysis. Generating Barcode Files ------------------------ @@ -244,7 +245,7 @@ Common Format Variations .. code-block:: bash # Extract filtered barcodes (single-column, add cell types later) - zcat cellranger_output/outs/filtered_feature_bc_matrix/barcodes.tsv.gz > barcodes_raw.txt + zcat cellranger_output/outs/filtered_peak_bc_matrix/barcodes.tsv.gz > barcodes_raw.txt **Barcode Suffix Handling:** @@ -270,8 +271,8 @@ Count Alleles wasp2-count count-variants-sc \ sample.bam \ variants.vcf.gz \ - barcodes.tsv \ - --region peaks.bed \ + barcodes.txt \ + --feature peaks.bed \ --samples NA12878 \ --out_file allele_counts.h5ad @@ -300,12 +301,14 @@ The single-cell counting module outputs an AnnData (``.h5ad``) file containing: **Observations (obs):** -* SNP information (chrom, pos, ref, alt) -* Aggregate counts per SNP +* Cell barcodes in ``obs_names``; groups are added during analysis. **Variables (var):** -* Cell barcodes +* SNP information (chrom, pos, ref, alt) and selected sample genotypes +* Aggregate counts per SNP + +Count layers have shape ``cells × SNPs``. **Unstructured (uns):** @@ -321,7 +324,8 @@ Quality Filtering * Filter low-quality cells before generating barcode file * Remove doublets and dead cells -* Use cells with sufficient UMI counts (>1000 for most protocols) +* Use scATAC quality metrics, such as fragment depth and transcription-start-site + enrichment; the counter does not deduplicate UMIs. Cell Type Annotation ~~~~~~~~~~~~~~~~~~~~ @@ -364,7 +368,7 @@ regulatory variation. wasp2-analyze compare-imbalance \ allele_counts.h5ad \ barcode_celltype_map.tsv \ - --groups "excitatory_neurons,inhibitory_neurons" \ + --groups excitatory_neurons --groups inhibitory_neurons \ --sample SAMPLE_ID \ --phased @@ -376,9 +380,8 @@ For comprehensive coverage of comparative analysis, see: * :doc:`analysis` - Statistical methods for comparative imbalance See Also - -------- -* :doc:`/tutorials/single_cell_workflow` - scRNA-seq / scATAC-seq workflow +* :doc:`/tutorials/single_cell_workflow` - scATAC-seq workflow * :doc:`analysis` - Statistical analysis methods * :doc:`counting` - General allele counting diff --git a/environment.lock.yml b/environment.lock.yml index 43dfa92f..db67cf5f 100644 --- a/environment.lock.yml +++ b/environment.lock.yml @@ -19,7 +19,7 @@ dependencies: # Data processing - numpy>=1.21,<2.0 - pandas>=1.5,<3.0 - - polars>=0.19 + - polars>=0.19,<2.0 - scipy>=1.10 # Bioinformatics (bioconda) @@ -34,7 +34,7 @@ dependencies: - bwa>=0.7 - anndata>=0.8,<0.12 - scanpy>=1.10 - - numba>=0.59 + - numba>=0.59,<0.63 - plink2 - pgenlib>=0.90 diff --git a/environment.yml b/environment.yml index 9e358df7..7e3708d9 100644 --- a/environment.yml +++ b/environment.yml @@ -28,7 +28,7 @@ dependencies: - bwa>=0.7 - anndata>=0.8,<0.12 - scanpy>=1.10 - - numba>=0.59 + - numba>=0.59,<0.63 - plink2 - pgenlib>=0.90 @@ -44,7 +44,8 @@ dependencies: - ruff>=0.9 # Rust build (PyO3) - - rust>=1.70 + - rust>=1.88 + - cmake>=3.14 - libclang - clang diff --git a/galaxy/tools/wasp2/wasp2_count_variants.xml b/galaxy/tools/wasp2/wasp2_count_variants.xml index 1859d906..fad19eda 100644 --- a/galaxy/tools/wasp2/wasp2_count_variants.xml +++ b/galaxy/tools/wasp2/wasp2_count_variants.xml @@ -109,7 +109,7 @@ Counts allele-specific reads at heterozygous variant positions from aligned BAM **Usage Notes** For best results, use phased VCF files from population imputation (e.g., SHAPEIT, Eagle). -For single-cell data, use the companion tool "WASP2 Count Variants SC". +Single-cell counting is available through the WASP2 command line; no single-cell Galaxy wrapper is included. ]]> diff --git a/galaxy/tools/wasp2/wasp2_find_imbalance.xml b/galaxy/tools/wasp2/wasp2_find_imbalance.xml index 33a3bc41..47eab757 100644 --- a/galaxy/tools/wasp2/wasp2_find_imbalance.xml +++ b/galaxy/tools/wasp2/wasp2_find_imbalance.xml @@ -27,7 +27,7 @@ label="Allele counts file" help="Output from 'WASP2 Count Variants'"/> - @@ -111,7 +111,7 @@ WASP2 uses a beta-binomial model to test for allelic imbalance: **Usage Notes** -For single-cell data, use "WASP2 Find Imbalance SC" with barcode groupings. +Single-cell analysis with barcode groupings is available through the WASP2 command line; no single-cell Galaxy wrapper is included. For comparing imbalance between conditions/cell types, use "WASP2 Compare Imbalance". diff --git a/pipelines/README.md b/pipelines/README.md index d9d0e8d8..c7b858c4 100644 --- a/pipelines/README.md +++ b/pipelines/README.md @@ -1,47 +1,37 @@ -# WASP2 Nextflow Pipelines +# WASP2 workflow integrations -Modular Nextflow DSL2 pipelines for allele-specific analysis. +The repository includes Nextflow DSL2 pipelines and Galaxy wrappers. Pipeline presence and stub tests do not establish biological validation or nf-core certification. -## Pipelines +| Component | Purpose | Current boundary | +|-----------|---------|------------------| +| [nf-rnaseq](nf-rnaseq/) | STAR alignment, WASP correction, allele counting and bulk inference | Optional GTF exon/gene grouping is preserved | +| [nf-atacseq](nf-atacseq/) | ATAC alignment, WASP correction, peak-associated counting and bulk inference | Original region membership is passed to analysis | +| [nf-scatac](nf-scatac/) | Per-cell BAM allele counts or fragment overlap coverage; AnnData output | Fragment coverage has no observed alleles; downstream imbalance inference is not wired for either input path | +| [nf-outrider](nf-outrider/) | OUTRIDER aberrant-expression workflow | Separate expression-outlier model, not an allelic imbalance backend | +| [nf-modules](nf-modules/) | Shared WASP2 and alignment modules | Bulk analysis and single-cell counting are separate modules | +| [Galaxy wrappers](../galaxy/tools/wasp2/) | Mapping, bulk counting and bulk inference | No single-cell Galaxy wrappers are included | -| Pipeline | Description | Status | -|----------|-------------|--------| -| **nf-rnaseq** | RNA-seq allele-specific expression (ASE) | 🚧 Planned | -| **nf-atacseq** | ATAC-seq allelic imbalance (AI) | 🚧 Planned | -| **nf-scatac** | Single-cell ATAC-seq AI | 🚧 Planned | -| **nf-modules** | Shared DSL2 modules | 🚧 Planned | +The Python command line intentionally calls native Rust for supported counting and bulk inference. Single-cell inference is implemented in Python/SciPy; the bulk `analyze_imbalance` workflow module does not provide single-cell inference. -## Architecture +## Running workflows -``` -pipelines/ -├── nf-modules/ # Shared modules (WASP2 counting, filtering) -│ └── modules/ -│ ├── wasp2_count/ -│ ├── wasp2_filter/ -│ └── vcf_processing/ -├── nf-rnaseq/ # RNA-seq ASE pipeline -│ ├── main.nf -│ ├── nextflow.config -│ └── conf/ -├── nf-atacseq/ # ATAC-seq AI pipeline -└── nf-scatac/ # Single-cell ATAC pipeline -``` - -## Usage +Run from the repository root using the profile appropriate for the installed runtime: ```bash -# RNA-seq ASE -nextflow run pipelines/nf-rnaseq -profile docker --input samplesheet.csv - -# ATAC-seq AI -nextflow run pipelines/nf-atacseq -profile singularity --input samplesheet.csv +nextflow run pipelines/nf-rnaseq -profile docker --input samplesheet.csv --vcf variants.vcf.gz --star_index star_index --gtf genes.gtf +nextflow run pipelines/nf-atacseq -profile docker --input samplesheet.csv --vcf variants.vcf.gz --fasta reference.fa ``` -## nf-core Compatibility +Read each pipeline's usage documentation for its required reference files, samplesheet format and supported profiles. Container tags in source describe the intended release; development changes require an image built from the matching source. + +## Shared module inventory + +- `wasp2/count`, `wasp2/count_alleles`: bulk allele counting. +- `wasp2/count_sc`: per-cell allele counting from BAM. +- `wasp2/analyze`, `wasp2/analyze_imbalance`: bulk inference. +- `wasp2/unified_make_reads`, `wasp2/filter_remapped`: mapping-bias correction. +- `wasp2/vcf_to_bed`: variant-coordinate conversion. +- `wasp2/ml_output`: format conversion; it does not change the statistical model. +- Alignment and BAM-processing modules integrate tools such as STAR, BWA and samtools. -These pipelines follow nf-core standards where practical: -- DSL2 modules with meta maps -- MultiQC integration -- Conda/Docker/Singularity support -- Tower compatibility +Modules use DSL2 channels and metadata maps. Meaningful validation includes actual process execution and schema/parameter assertions in addition to stub tests. Detailed output formats are documented beside each pipeline. diff --git a/pipelines/nf-atacseq/docs/output.md b/pipelines/nf-atacseq/docs/output.md index b101caa4..a819e86b 100644 --- a/pipelines/nf-atacseq/docs/output.md +++ b/pipelines/nf-atacseq/docs/output.md @@ -87,11 +87,14 @@ Allelic imbalance statistical results: | region | Peak/region ID | | ref_count | Total reference reads | | alt_count | Total alternate reads | -| total_count | Total reads | -| pval | Beta-binomial p-value | +| N | Total reference + alternate reads | +| snp_count | Number of SNPs in region | +| null_ll | Null log-likelihood | +| alt_ll | Alternative log-likelihood | +| mu | Fitted allelic proportion anchored to the first SNP | +| lrt | Likelihood-ratio statistic | +| pval | Chi-square likelihood-ratio p-value (one degree of freedom) | | fdr_pval | FDR-corrected p-value | -| log2_ratio | log2(ref/alt) ratio | -| dispersion | Estimated dispersion | ### MultiQC @@ -111,7 +114,7 @@ Allelic imbalance statistical results: Regions with significant allelic imbalance (AI) have: - `fdr_pval < 0.05`: Statistically significant after FDR correction -- `|log2_ratio| > 0.5`: At least 1.4-fold difference between alleles +The fitted `mu` describes the model orientation anchored to the first SNP; it does not establish a shared reference/alternate direction across a multi-SNP region. ### WASP Filtering Statistics @@ -135,13 +138,14 @@ Check the MultiQC report for: ```r library(readr) +library(dplyr) # Load allelic imbalance results ai_results <- read_tsv("results/analysis/sample_ai_results.tsv") # Filter significant regions sig_ai <- ai_results %>% - filter(fdr_pval < 0.05, abs(log2_ratio) > 0.5) + filter(fdr_pval < 0.05) ``` ### Loading Results in Python @@ -157,8 +161,7 @@ ai_results = pd.read_csv("results/analysis/sample_ai_results.tsv", sep="\t") # Filter significant sig_ai = ai_results[ - (ai_results['fdr_pval'] < 0.05) & - (abs(ai_results['log2_ratio']) > 0.5) + ai_results['fdr_pval'] < 0.05 ] ``` diff --git a/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance.nf b/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance.nf index 20e6ef89..ffb268d8 100644 --- a/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance.nf +++ b/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance.nf @@ -39,8 +39,8 @@ process WASP2_FIND_IMBALANCE { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - echo -e "region\\tref_count\\talt_count\\tpval\\tfdr_pval\\tlog2_ratio" > ${prefix}_ai_results.tsv - echo -e "peak_1\\t10\\t8\\t0.05\\t0.1\\t0.322" >> ${prefix}_ai_results.tsv + echo -e "region\\tref_count\\talt_count\\tN\\tsnp_count\\tnull_ll\\talt_ll\\tmu\\tlrt\\tpval\\tfdr_pval" > ${prefix}_ai_results.tsv + echo -e "peak_1\\t10\\t8\\t18\\t1\\t-2.5\\t-2.4\\t0.55\\t0.2\\t0.655\\t0.655" >> ${prefix}_ai_results.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/main.nf b/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/main.nf index 28843f3b..47c9aa9b 100644 --- a/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/main.nf +++ b/pipelines/nf-atacseq/modules/local/wasp2_find_imbalance/main.nf @@ -23,30 +23,8 @@ process WASP2_FIND_IMBALANCE { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - # WASP2 v1.4.0 workaround: normalize count-variants output to - # the 9-column format expected by find-imbalance Rust backend: - # chrom pos0 pos ref alt GT ref_count alt_count other_count - # count-variants outputs (with --region): - # chrom pos ref alt GT region ref_count alt_count other_count - # We strip 'region' and insert 'pos0' (= pos - 1). - if head -1 ${counts} | grep -q 'region'; then - # Strip region column - REGION_COL=\$(head -1 ${counts} | tr '\\t' '\\n' | grep -n 'region' | cut -d: -f1) - cut -f1-\$((REGION_COL-1)),\$((REGION_COL+1))- ${counts} > ${prefix}_counts_stripped.tsv - else - cp ${counts} ${prefix}_counts_stripped.tsv - fi - - # Insert pos0 column if missing (count-variants lacks it) - if ! head -1 ${prefix}_counts_stripped.tsv | grep -q 'pos0'; then - awk -F'\\t' 'BEGIN{OFS="\\t"} NR==1{print \$1,"pos0",\$0; next} {print \$1,\$2-1,\$0}' ${prefix}_counts_stripped.tsv | cut -f1,2,4- > ${prefix}_counts_fixed.tsv - COUNTS_INPUT=${prefix}_counts_fixed.tsv - else - COUNTS_INPUT=${prefix}_counts_stripped.tsv - fi - wasp2-analyze find-imbalance \\ - \$COUNTS_INPUT \\ + ${counts} \\ --min ${min_count} \\ --pseudocount ${pseudocount} \\ ${args} \\ @@ -61,8 +39,8 @@ process WASP2_FIND_IMBALANCE { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - echo -e "region\\tref_count\\talt_count\\tpval\\tfdr_pval\\tlog2_ratio" > ${prefix}_ai_results.tsv - echo -e "peak_1\\t10\\t8\\t0.05\\t0.1\\t0.322" >> ${prefix}_ai_results.tsv + echo -e "region\\tref_count\\talt_count\\tN\\tsnp_count\\tnull_ll\\talt_ll\\tmu\\tlrt\\tpval\\tfdr_pval" > ${prefix}_ai_results.tsv + echo -e "peak_1\\t10\\t8\\t18\\t1\\t-2.5\\t-2.4\\t0.55\\t0.2\\t0.655\\t0.655" >> ${prefix}_ai_results.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/pipelines/nf-atacseq/tests/modules/local/wasp2_find_imbalance.nf.test b/pipelines/nf-atacseq/tests/modules/local/wasp2_find_imbalance.nf.test index 222e5935..5e4fd06e 100644 --- a/pipelines/nf-atacseq/tests/modules/local/wasp2_find_imbalance.nf.test +++ b/pipelines/nf-atacseq/tests/modules/local/wasp2_find_imbalance.nf.test @@ -57,4 +57,28 @@ nextflow_process { assert process.out.versions } } + test("Preserves peak membership and zero pseudocount in actual analysis") { + tag "correctness_regression" + when { + process { + """ + def counts_file = file("${workDir}/peak_counts.tsv") + counts_file.text = '''chrom\tpos\tref\talt\tGT\tregion\tref_count\talt_count\tother_count\nchr1\t100\tA\tG\t0|1\tpeak_shared\t5\t3\t0\nchr1\t120\tC\tT\t0|1\tpeak_shared\t4\t6\t0\n''' + input[0] = [[id:'peak_test'], counts_file] + input[1] = 0 + input[2] = 0 + """ + } + } + then { + assert process.success + def rows = path(process.out.results[0][1]).readLines() + assert rows.size() == 2 + def result = [rows[0].split('\t'), rows[1].split('\t')].transpose().collectEntries() + assert result.region == 'peak_shared' + assert result.snp_count == '2' + assert result.N == '18' + } + } + } diff --git a/pipelines/nf-modules/modules/wasp2/analyze/main.nf b/pipelines/nf-modules/modules/wasp2/analyze/main.nf index 6f210cee..4846e9b0 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze/main.nf +++ b/pipelines/nf-modules/modules/wasp2/analyze/main.nf @@ -21,8 +21,8 @@ process WASP2_ANALYZE { def args = task.ext.args ?: '' // Sanitize inputs to prevent shell injection def prefix = (task.ext.prefix ?: "${meta.id}").replaceAll(/[^a-zA-Z0-9._-]/, '_') - def min_count = task.ext.min_count ?: 10 - def pseudocount = task.ext.pseudocount ?: 1 + def min_count = task.ext.min_count != null ? task.ext.min_count : 10 + def pseudocount = task.ext.pseudocount != null ? task.ext.pseudocount : 1 def phased_arg = meta.phased ? '--phased' : '' """ set -euo pipefail @@ -60,7 +60,7 @@ process WASP2_ANALYZE { def prefix = (task.ext.prefix ?: "${meta.id}").replaceAll(/[^a-zA-Z0-9._-]/, '_') """ cat <<-END_HEADER > ${prefix}.stats.tsv - region ref_count alt_count ratio log2fc pval fdr_pval dispersion + region ref_count alt_count N snp_count null_ll alt_ll mu lrt pval fdr_pval END_HEADER cat <<-END_VERSIONS > versions.yml diff --git a/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test b/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test index 5e52d3d1..580d0245 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test +++ b/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test @@ -82,4 +82,26 @@ nextflow_process { ) } } + test("Preserves zero task-extension thresholds in actual analysis") { + tag "correctness_regression" + config "./min_count.config" + when { + process { + """ + def counts_file = file("${workDir}/zero_counts.tsv") + counts_file.text = '''chrom\tpos\tref\talt\tGT\tregion\tref_count\talt_count\tother_count\nchr1\t100\tA\tG\t0|1\tlow_depth\t1\t1\t0\n''' + input[0] = [[id:'zero_test', phased:false], counts_file] + """ + } + } + then { + assert process.success + def rows = path(process.out.stats[0][1]).readLines() + assert rows.size() == 2 + def result = [rows[0].split('\t'), rows[1].split('\t')].transpose().collectEntries() + assert result.region == 'low_depth' + assert Math.abs(result.null_ll.toDouble() - Math.log(0.5)) < 0.02 + } + } + } diff --git a/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test.snap b/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test.snap index e176a70c..96636182 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test.snap +++ b/pipelines/nf-modules/modules/wasp2/analyze/tests/main.nf.test.snap @@ -8,7 +8,7 @@ "id": "test_phased", "phased": true }, - "test_phased.stats.tsv:md5,7c26e78eb635001b288dec206c4f980f" + "test_phased.stats.tsv:md5,351e14240535962ec58c2cb8424dd122" ] ], "1": [ @@ -20,7 +20,7 @@ "id": "test_phased", "phased": true }, - "test_phased.stats.tsv:md5,7c26e78eb635001b288dec206c4f980f" + "test_phased.stats.tsv:md5,351e14240535962ec58c2cb8424dd122" ] ], "versions": [ diff --git a/pipelines/nf-modules/modules/wasp2/analyze/tests/min_count.config b/pipelines/nf-modules/modules/wasp2/analyze/tests/min_count.config index d161a6d9..718ab1f4 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze/tests/min_count.config +++ b/pipelines/nf-modules/modules/wasp2/analyze/tests/min_count.config @@ -1,4 +1,5 @@ -// Override min_count for testing with small datasets (2+2 per SNP < default 10) +// Preserve explicitly supplied zero-valued analysis options. process { - ext.min_count = 1 + ext.min_count = 0 + ext.pseudocount = 0 } diff --git a/pipelines/nf-modules/modules/wasp2/analyze_imbalance/main.nf b/pipelines/nf-modules/modules/wasp2/analyze_imbalance/main.nf index 18dfeb9c..3384673b 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze_imbalance/main.nf +++ b/pipelines/nf-modules/modules/wasp2/analyze_imbalance/main.nf @@ -20,8 +20,8 @@ process WASP2_ANALYZE_IMBALANCE { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def min_count = params.min_count ?: 10 - def pseudocount = params.pseudocount ?: 1 + def min_count = params.min_count != null ? params.min_count : 10 + def pseudocount = params.pseudocount != null ? params.pseudocount : 1 """ wasp2-analyze find-imbalance \\ ${counts} \\ @@ -39,7 +39,7 @@ process WASP2_ANALYZE_IMBALANCE { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - echo -e "region\\tsnp_count\\tref_sum\\talt_sum\\tmu\\tnull_ll\\talt_ll\\tLRT\\tpvalue\\tfdr" > ${prefix}_ai_results.tsv + echo -e "region\\tref_count\\talt_count\\tN\\tsnp_count\\tnull_ll\\talt_ll\\tmu\\tlrt\\tpval\\tfdr_pval" > ${prefix}_ai_results.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/pipelines/nf-modules/modules/wasp2/analyze_imbalance/tests/main.nf.test b/pipelines/nf-modules/modules/wasp2/analyze_imbalance/tests/main.nf.test index 43b77b48..31b310a7 100644 --- a/pipelines/nf-modules/modules/wasp2/analyze_imbalance/tests/main.nf.test +++ b/pipelines/nf-modules/modules/wasp2/analyze_imbalance/tests/main.nf.test @@ -59,4 +59,30 @@ nextflow_process { ) } } + test("Preserves zero minimum count and zero pseudocount") { + tag "correctness_regression" + when { + params { + min_count = 0 + pseudocount = 0 + } + process { + """ + def counts_file = file("${workDir}/zero_counts.tsv") + counts_file.text = '''chrom\tpos\tref\talt\tGT\tregion\tref_count\talt_count\tother_count\nchr1\t100\tA\tG\t0|1\tlow_depth\t1\t1\t0\n''' + input[0] = [[id:'zero_test'], counts_file] + """ + } + } + then { + assert process.success + def rows = path(process.out.results[0][1]).readLines() + assert rows.size() == 2 + def result = [rows[0].split('\t'), rows[1].split('\t')].transpose().collectEntries() + assert result.region == 'low_depth' + assert result.N == '2' + assert Math.abs(result.null_ll.toDouble() - Math.log(0.5)) < 0.02 + } + } + } diff --git a/pipelines/nf-rnaseq/README.md b/pipelines/nf-rnaseq/README.md index 8d7f6478..c431443f 100644 --- a/pipelines/nf-rnaseq/README.md +++ b/pipelines/nf-rnaseq/README.md @@ -117,7 +117,7 @@ nextflow run pipelines/nf-rnaseq -profile docker \ --vcf het_snps.vcf.gz \ --star_index /ref/star_index \ --min_count 20 \ - --pseudocount 0.5 + --pseudocount 0 ``` ### Resume Failed Run diff --git a/pipelines/nf-rnaseq/docs/output.md b/pipelines/nf-rnaseq/docs/output.md index 78d3c321..bcec4419 100644 --- a/pipelines/nf-rnaseq/docs/output.md +++ b/pipelines/nf-rnaseq/docs/output.md @@ -80,27 +80,25 @@ Statistical test results for allelic imbalance. |--------|-------------| | `region` | Gene/region identifier | | `snp_count` | Number of heterozygous SNPs in region | -| `ref_sum` | Total reference allele counts | -| `alt_sum` | Total alternate allele counts | -| `mu` | Estimated mean allelic ratio | +| `ref_count` | Total reference allele counts | +| `alt_count` | Total alternate allele counts | +| `N` | Total reference + alternate reads | +| `mu` | Fitted allelic proportion anchored to the first SNP | | `null_ll` | Null hypothesis log-likelihood | | `alt_ll` | Alternative hypothesis log-likelihood | -| `LRT` | Likelihood ratio test statistic | -| `pvalue` | P-value from chi-squared test | -| `fdr` | Benjamini-Hochberg FDR-adjusted p-value | +| `lrt` | Likelihood ratio test statistic | +| `pval` | P-value from chi-squared test | +| `fdr_pval` | Benjamini-Hochberg FDR-adjusted p-value | **Example:** ```tsv -region snp_count ref_sum alt_sum mu null_ll alt_ll LRT pvalue fdr -ENSG00000001 5 125 180 0.41 -450.2 -445.8 8.8 0.003 0.015 -ENSG00000002 3 45 48 0.48 -120.5 -120.4 0.2 0.65 0.85 -ENSG00000003 8 210 95 0.69 -380.1 -360.2 39.8 2.7e-10 5.4e-09 +region ref_count alt_count N snp_count null_ll alt_ll mu lrt pval fdr_pval +ENSG00000001 125 180 305 5 -450.2 -445.8 0.41 8.8 0.003 0.015 +ENSG00000002 45 48 93 3 -120.5 -120.4 0.48 0.2 0.65 0.85 ``` **Interpreting Results:** -- **mu < 0.5**: Reference allele underexpressed -- **mu > 0.5**: Reference allele overexpressed -- **mu ≈ 0.5**: Balanced expression +The `mu` orientation is anchored to the first SNP. A value below or above 0.5 does not by itself establish reference-allele underexpression or overexpression across a gene. Values near 0.5 indicate balance in the fitted model. - **FDR < 0.05**: Significant allelic imbalance ### `pipeline_info/` @@ -208,8 +206,8 @@ library(tidyverse) results <- read_tsv("results/analysis/SAMPLE1_ai_results.tsv") # Volcano plot -ggplot(results, aes(x = log2(mu/(1-mu)), y = -log10(pvalue))) + - geom_point(aes(color = fdr < 0.05)) + +ggplot(results, aes(x = log2(mu/(1-mu)), y = -log10(pval))) + + geom_point(aes(color = fdr_pval < 0.05)) + geom_hline(yintercept = -log10(0.05), linetype = "dashed") + labs(x = "log2(Allelic Ratio)", y = "-log10(P-value)") ``` diff --git a/pipelines/nf-rnaseq/main.nf b/pipelines/nf-rnaseq/main.nf index 2ed85406..50f7c797 100644 --- a/pipelines/nf-rnaseq/main.nf +++ b/pipelines/nf-rnaseq/main.nf @@ -227,14 +227,12 @@ workflow RNASEQ_ASE { // // STEP 5: Count alleles at heterozygous SNPs - // Note: GTF not passed to count step because WASP2 v1.4.0 find-imbalance - // Rust backend does not support gene-annotated count formats. - // Gene annotation can be added post-hoc by intersecting with GTF. + // Preserve gene membership for region-level allelic imbalance testing. // WASP2_COUNT_ALLELES( WASP2_FILTER_REMAPPED.out.bam, ch_vcf.first(), - [] + ch_gtf ) ch_versions = ch_versions.mix(WASP2_COUNT_ALLELES.out.versions) diff --git a/pipelines/nf-rnaseq/modules/local/wasp2_analyze_imbalance/main.nf b/pipelines/nf-rnaseq/modules/local/wasp2_analyze_imbalance/main.nf index 18dfeb9c..3384673b 100644 --- a/pipelines/nf-rnaseq/modules/local/wasp2_analyze_imbalance/main.nf +++ b/pipelines/nf-rnaseq/modules/local/wasp2_analyze_imbalance/main.nf @@ -20,8 +20,8 @@ process WASP2_ANALYZE_IMBALANCE { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def min_count = params.min_count ?: 10 - def pseudocount = params.pseudocount ?: 1 + def min_count = params.min_count != null ? params.min_count : 10 + def pseudocount = params.pseudocount != null ? params.pseudocount : 1 """ wasp2-analyze find-imbalance \\ ${counts} \\ @@ -39,7 +39,7 @@ process WASP2_ANALYZE_IMBALANCE { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - echo -e "region\\tsnp_count\\tref_sum\\talt_sum\\tmu\\tnull_ll\\talt_ll\\tLRT\\tpvalue\\tfdr" > ${prefix}_ai_results.tsv + echo -e "region\\tref_count\\talt_count\\tN\\tsnp_count\\tnull_ll\\talt_ll\\tmu\\tlrt\\tpval\\tfdr_pval" > ${prefix}_ai_results.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/pipelines/nf-rnaseq/nextflow_schema.json b/pipelines/nf-rnaseq/nextflow_schema.json index 40ab0195..02e61b6e 100644 --- a/pipelines/nf-rnaseq/nextflow_schema.json +++ b/pipelines/nf-rnaseq/nextflow_schema.json @@ -105,7 +105,7 @@ "min_count": { "type": "integer", "default": 10, - "minimum": 1, + "minimum": 0, "description": "Minimum allele count for ASE analysis.", "help_text": "Variants with fewer than this many reads for either allele will be excluded.", "fa_icon": "fas fa-sort-numeric-up" diff --git a/pipelines/nf-rnaseq/tests/main.nf.test b/pipelines/nf-rnaseq/tests/main.nf.test index 04dedf20..77ecb516 100644 --- a/pipelines/nf-rnaseq/tests/main.nf.test +++ b/pipelines/nf-rnaseq/tests/main.nf.test @@ -55,8 +55,8 @@ nextflow_pipeline { // Header validation (first line must match exactly) { assert path("${outputDir}/results/counts/SAMPLE1_counts.tsv").readLines()[0] == "chrom\tpos\tref\talt\tregion\tref_count\talt_count\tother_count\tN" }, { assert path("${outputDir}/results/counts/SAMPLE2_counts.tsv").readLines()[0] == "chrom\tpos\tref\talt\tregion\tref_count\talt_count\tother_count\tN" }, - { assert path("${outputDir}/results/analysis/SAMPLE1_ai_results.tsv").readLines()[0] == "region\tsnp_count\tref_sum\talt_sum\tmu\tnull_ll\talt_ll\tLRT\tpvalue\tfdr" }, - { assert path("${outputDir}/results/analysis/SAMPLE2_ai_results.tsv").readLines()[0] == "region\tsnp_count\tref_sum\talt_sum\tmu\tnull_ll\talt_ll\tLRT\tpvalue\tfdr" } + { assert path("${outputDir}/results/analysis/SAMPLE1_ai_results.tsv").readLines()[0] == "region\tref_count\talt_count\tN\tsnp_count\tnull_ll\talt_ll\tmu\tlrt\tpval\tfdr_pval" }, + { assert path("${outputDir}/results/analysis/SAMPLE2_ai_results.tsv").readLines()[0] == "region\tref_count\talt_count\tN\tsnp_count\tnull_ll\talt_ll\tmu\tlrt\tpval\tfdr_pval" } ) } } @@ -219,7 +219,7 @@ nextflow_pipeline { // Header validation { assert path("${outputDir}/results_single/counts/SAMPLE1_counts.tsv").readLines()[0] == "chrom\tpos\tref\talt\tregion\tref_count\talt_count\tother_count\tN" }, - { assert path("${outputDir}/results_single/analysis/SAMPLE1_ai_results.tsv").readLines()[0] == "region\tsnp_count\tref_sum\talt_sum\tmu\tnull_ll\talt_ll\tLRT\tpvalue\tfdr" } + { assert path("${outputDir}/results_single/analysis/SAMPLE1_ai_results.tsv").readLines()[0] == "region\tref_count\talt_count\tN\tsnp_count\tnull_ll\talt_ll\tmu\tlrt\tpval\tfdr_pval" } ) } } @@ -255,7 +255,7 @@ nextflow_pipeline { // Header validation { assert path("${outputDir}/results_singleend/counts/SAMPLE_SE_counts.tsv").readLines()[0] == "chrom\tpos\tref\talt\tregion\tref_count\talt_count\tother_count\tN" }, - { assert path("${outputDir}/results_singleend/analysis/SAMPLE_SE_ai_results.tsv").readLines()[0] == "region\tsnp_count\tref_sum\talt_sum\tmu\tnull_ll\talt_ll\tLRT\tpvalue\tfdr" } + { assert path("${outputDir}/results_singleend/analysis/SAMPLE_SE_ai_results.tsv").readLines()[0] == "region\tref_count\talt_count\tN\tsnp_count\tnull_ll\talt_ll\tmu\tlrt\tpval\tfdr_pval" } ) } } @@ -293,8 +293,8 @@ nextflow_pipeline { // Header validation { assert path("${outputDir}/results_no_gtf/counts/SAMPLE1_counts.tsv").readLines()[0] == "chrom\tpos\tref\talt\tregion\tref_count\talt_count\tother_count\tN" }, { assert path("${outputDir}/results_no_gtf/counts/SAMPLE2_counts.tsv").readLines()[0] == "chrom\tpos\tref\talt\tregion\tref_count\talt_count\tother_count\tN" }, - { assert path("${outputDir}/results_no_gtf/analysis/SAMPLE1_ai_results.tsv").readLines()[0] == "region\tsnp_count\tref_sum\talt_sum\tmu\tnull_ll\talt_ll\tLRT\tpvalue\tfdr" }, - { assert path("${outputDir}/results_no_gtf/analysis/SAMPLE2_ai_results.tsv").readLines()[0] == "region\tsnp_count\tref_sum\talt_sum\tmu\tnull_ll\talt_ll\tLRT\tpvalue\tfdr" } + { assert path("${outputDir}/results_no_gtf/analysis/SAMPLE1_ai_results.tsv").readLines()[0] == "region\tref_count\talt_count\tN\tsnp_count\tnull_ll\talt_ll\tmu\tlrt\tpval\tfdr_pval" }, + { assert path("${outputDir}/results_no_gtf/analysis/SAMPLE2_ai_results.tsv").readLines()[0] == "region\tref_count\talt_count\tN\tsnp_count\tnull_ll\talt_ll\tmu\tlrt\tpval\tfdr_pval" } ) } } diff --git a/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test b/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test index 788127bc..9cef269f 100644 --- a/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test +++ b/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test @@ -56,4 +56,30 @@ nextflow_process { assert snapshot(process.out).match() } } + test("Preserves zero minimum count and zero pseudocount") { + tag "correctness_regression" + when { + params { + min_count = 0 + pseudocount = 0 + } + process { + """ + def counts_file = file("${workDir}/zero_counts.tsv") + counts_file.text = '''chrom\tpos\tref\talt\tGT\tregion\tref_count\talt_count\tother_count\nchr1\t100\tA\tG\t0|1\tlow_depth\t1\t1\t0\n''' + input[0] = [[id:'zero_test'], counts_file] + """ + } + } + then { + assert process.success + def rows = path(process.out.results[0][1]).readLines() + assert rows.size() == 2 + def result = [rows[0].split('\t'), rows[1].split('\t')].transpose().collectEntries() + assert result.region == 'low_depth' + assert result.N == '2' + assert Math.abs(result.null_ll.toDouble() - Math.log(0.5)) < 0.02 + } + } + } diff --git a/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test.snap b/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test.snap index 0f460b0d..d17698df 100644 --- a/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test.snap +++ b/pipelines/nf-rnaseq/tests/modules/local/wasp2_analyze_imbalance.nf.test.snap @@ -8,7 +8,7 @@ "id": "test_sample", "single_end": false }, - "test_sample_ai_results.tsv:md5,c4148ccd0d9ff199b9161514a575a2cf" + "test_sample_ai_results.tsv:md5,351e14240535962ec58c2cb8424dd122" ] ], "1": [ @@ -20,7 +20,7 @@ "id": "test_sample", "single_end": false }, - "test_sample_ai_results.tsv:md5,c4148ccd0d9ff199b9161514a575a2cf" + "test_sample_ai_results.tsv:md5,351e14240535962ec58c2cb8424dd122" ] ], "versions": [ diff --git a/pipelines/nf-scatac/README.md b/pipelines/nf-scatac/README.md index 28b2f539..2f0b3aba 100644 --- a/pipelines/nf-scatac/README.md +++ b/pipelines/nf-scatac/README.md @@ -1,17 +1,19 @@ # nf-scatac -Single-Cell ATAC-seq Allelic Imbalance Pipeline +Single-cell ATAC-seq counting and coverage workflow + +Fragment files contain coordinates, not observed alleles. Their coverage cannot support allelic imbalance testing. This workflow does not run statistical inference: BAM-to-inference aggregation is also not yet implemented. Use the WASP2 single-cell analysis CLI on validated BAM-derived allele counts for downstream testing. ## Features - **10x Genomics scATAC fragment support** - Direct input from CellRanger ATAC output -- **Allelic imbalance analysis** - At heterozygous SNPs using WASP2 +- **BAM allele counting** - Reference and alternate counts at heterozygous SNPs - **Cell barcode filtering** - Optional whitelist for quality-filtered cells - **Peak region filtering** - Restrict analysis to accessible regions - **AnnData/H5AD output** - For scverse ecosystem (Scanpy, ArchR, Signac) - **Zarr output** - For GenVarLoader integration -- **Pseudo-bulk aggregation** - Sample-level aggregation for statistical power -- **nf-core compliant** - Subworkflow architecture (Issue #57) +- **Coverage aggregation** - Sample-level fragment overlap totals, without allele inference +- **Nextflow DSL2** - Reusable modules and subworkflows ## Architecture @@ -31,7 +33,7 @@ nf-scatac/ ├── modules/ │ └── local/ │ ├── scatac_add_haplotype_layers/ # Hap1/hap2 layer creation from phased VCF -│ ├── scatac_count_alleles/ # Per-cell allele counting (fragment-based) +│ ├── scatac_count_alleles/ # Per-cell fragment overlap coverage │ ├── scatac_create_anndata/ # AnnData H5AD output │ └── scatac_pseudobulk/ # Pseudo-bulk aggregation ├── conf/ @@ -90,13 +92,13 @@ NA12878_bam,,,/path/to/possorted_bam.bam,CB,10x-atac-v2,/path/to/barcodes.txt, ### Processing Options - `--min_fragments_per_cell` - Minimum fragments per cell to include [default: 1000] - `--min_cells_per_snp` - Minimum cells per SNP for pseudo-bulk [default: 3] -- `--min_count` - Minimum count for imbalance testing [default: 10] +- `--min_count` - Reserved bulk-analysis threshold; not used by this workflow ### Output Options - `--outdir` - Output directory [default: ./results] - `--create_zarr` - Also output Zarr format for GenVarLoader [default: false] - `--skip_anndata` - Skip AnnData H5AD creation [default: false] -- `--skip_pseudobulk` - Skip pseudo-bulk aggregation and analysis [default: false] +- `--skip_pseudobulk` - Skip fragment coverage aggregation [default: false] ## Single-Cell Meta Map @@ -116,23 +118,23 @@ The pipeline propagates scATAC-specific metadata through all stages: ### WASP_ALLELIC_SC -Single-cell WASP2 allelic imbalance analysis: +Single-cell counting and coverage aggregation: ```groovy include { WASP_ALLELIC_SC } from './subworkflows/local/wasp_allelic_sc/main' WASP_ALLELIC_SC ( - ch_fragments, // [ val(meta), path(fragments), path(tbi), path(barcodes), path(peaks) ] + ch_input, // [ meta, fragments, tbi, barcodes, peaks, bam, bai ] ch_vcf // [ val(meta), path(vcf), path(tbi) ] ) // Outputs: -// - cell_counts: Per-cell allele counts at SNPs +// - cell_counts: Per-cell fragment overlap coverage at SNPs // - anndata: AnnData H5AD files // - zarr: Zarr directories (if enabled) // - cell_qc: Cell QC metrics -// - pseudobulk: Aggregated counts -// - imbalance: Allelic imbalance results +// - pseudobulk: Aggregated fragment coverage +// - imbalance: Empty (inference is not implemented) ``` ### GENERATE_FRAGMENTS @@ -188,21 +190,21 @@ or through environment variables consumed by `conf/test_real.config` and ``` results/ -├── allele_counts/ # Per-cell allele counts at het SNPs -│ └── {sample}_allele_counts.tsv +├── fragment_coverage/ # Per-cell fragment overlap coverage +│ └── {sample}_fragment_coverage.tsv ├── count_stats/ # Counting statistics │ └── {sample}_count_stats.tsv ├── anndata/ # AnnData H5AD files for scverse -│ └── {sample}_allelic.h5ad +│ ├── {sample}_coverage.h5ad # Fragment input +│ └── {sample}_with_haplotypes.h5ad # BAM input ├── zarr/ # Zarr directories (if --create_zarr) -│ └── {sample}_allelic.zarr/ +│ ├── {sample}_coverage.zarr/ # Fragment input +│ └── {sample}_with_haplotypes.zarr/ # BAM input ├── cell_qc/ # Cell QC metrics │ └── {sample}_cell_qc.tsv -├── pseudobulk/ # Pseudo-bulk aggregated counts -│ ├── {sample}_pseudobulk_counts.tsv +├── pseudobulk/ # Aggregated fragment coverage +│ ├── {sample}_pseudobulk_coverage.tsv │ └── {sample}_aggregation_stats.tsv -├── imbalance/ # Allelic imbalance analysis -│ └── {sample}_ai_results.tsv ├── variants/ # Processed variant BED │ └── variants.variants.bed └── pipeline_info/ # Execution reports @@ -244,7 +246,7 @@ The H5AD file contains different layers depending on input type: - **uns**: Unstructured metadata - `sample_id`: Sample identifier - `pipeline`: 'nf-scatac' - - `data_type`: 'scATAC_allelic_counts' or 'scATAC_allelic_counts_phased' + - `data_type`: 'scATAC_fragment_coverage' for fragment input; 'scATAC_allelic_counts_phased' for phased BAM input - `phased_snps`: Number of phased SNPs (BAM input only) - `phasing_rate`: Fraction of SNPs with phasing info (BAM input only) diff --git a/pipelines/nf-scatac/conf/modules.config b/pipelines/nf-scatac/conf/modules.config index 2aff0a4b..08b2fc2f 100644 --- a/pipelines/nf-scatac/conf/modules.config +++ b/pipelines/nf-scatac/conf/modules.config @@ -15,9 +15,9 @@ process { withName: 'SCATAC_COUNT_ALLELES' { publishDir = [ [ - path: { "${params.outdir}/allele_counts" }, + path: { "${params.outdir}/fragment_coverage" }, mode: params.publish_dir_mode, - pattern: "*_allele_counts.tsv" + pattern: "*_fragment_coverage.tsv" ], [ path: { "${params.outdir}/count_stats" }, @@ -52,7 +52,7 @@ process { [ path: { "${params.outdir}/pseudobulk" }, mode: params.publish_dir_mode, - pattern: "*_pseudobulk_counts.tsv" + pattern: "*_pseudobulk_coverage.tsv" ], [ path: { "${params.outdir}/pseudobulk" }, @@ -70,14 +70,6 @@ process { ] } - withName: 'WASP2_ANALYZE_IMBALANCE' { - publishDir = [ - path: { "${params.outdir}/imbalance" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: 'SINTO_FRAGMENTS' { publishDir = [ path: { "${params.outdir}/fragments" }, diff --git a/pipelines/nf-scatac/docs/output.md b/pipelines/nf-scatac/docs/output.md index ab487c16..4c4cf987 100644 --- a/pipelines/nf-scatac/docs/output.md +++ b/pipelines/nf-scatac/docs/output.md @@ -1,217 +1,39 @@ -# nf-scatac: Output +# nf-scatac output -## Introduction +Fragment inputs provide overlap coverage. BAM inputs provide observed reference/alternate allele counts and optional haplotype layers. **This workflow produces no allelic imbalance results:** fragment coverage contains no observed alleles, and BAM-to-inference aggregation is not yet implemented. The `imbalance` output channel is empty. -This document describes the output files and directory structure produced by the nf-scatac pipeline for single-cell ATAC-seq allelic imbalance analysis. +## Fragment coverage -## Pipeline Output +`fragment_coverage/{sample}_fragment_coverage.tsv` contains: -The pipeline outputs are organized in the following directory structure: +| Column | Meaning | +|--------|---------| +| `barcode` | Cell barcode | +| `chrom`, `pos` | SNP chromosome and 1-based position | +| `ref`, `alt` | Variant annotation from the VCF; these do not identify an observed allele | +| `overlap_count` | Fragment multiplicity summed over overlaps at this SNP and cell | -``` -results/ -├── allele_counts/ # Per-cell allele counts at het SNPs -│ └── {sample}_allele_counts.tsv -├── imbalance/ # Allelic imbalance analysis -│ └── {sample}_imbalance.tsv -├── variants/ # VCF-derived files -│ └── {sample}_het_snps.bed -└── pipeline_info/ # Execution reports - ├── timeline.html - ├── report.html - └── trace.txt -``` - -## Output Files - -### Allele Counts - -**Directory**: `allele_counts/` - -**File**: `{sample}_allele_counts.tsv` - -Per-cell allele counts at heterozygous SNPs. Each row represents a cell-SNP combination: - -| Column | Description | -|--------|-------------| -| cell_barcode | Cell barcode (from BAM CB tag) | -| chrom | Chromosome | -| pos | Position (1-based) | -| ref | Reference allele | -| alt | Alternate allele | -| ref_count | Reference allele read count for this cell | -| alt_count | Alternate allele read count for this cell | -| total_count | Total reads (ref + alt) for this cell | - -### Allelic Imbalance Results - -**Directory**: `imbalance/` - -**File**: `{sample}_imbalance.tsv` - -Statistical analysis of allelic imbalance at each SNP, aggregated across cells: - -| Column | Description | -|--------|-------------| -| chrom | Chromosome | -| pos | Position (1-based) | -| ref | Reference allele | -| alt | Alternate allele | -| n_cells | Number of cells with coverage at this SNP | -| total_ref | Total reference reads across all cells | -| total_alt | Total alternate reads across all cells | -| total_count | Total reads (ref + alt) | -| pval | Beta-binomial p-value | -| fdr_pval | FDR-corrected p-value (Benjamini-Hochberg) | -| log2_ratio | log2(ref/alt) ratio | -| dispersion | Estimated overdispersion parameter | - -### Variant Files - -**Directory**: `variants/` - -**File**: `{sample}_het_snps.bed` - -BED file of heterozygous SNP positions extracted from the VCF: - -| Column | Description | -|--------|-------------| -| chrom | Chromosome | -| start | Start position (0-based) | -| end | End position | -| name | SNP ID (chr:pos:ref>alt) | - -### Pipeline Info - -**Directory**: `pipeline_info/` - -- `execution_report_*.html`: Nextflow execution report with process statistics -- `execution_timeline_*.html`: Timeline visualization of process execution -- `execution_trace_*.txt`: Detailed trace file for each process -- `pipeline_dag_*.html`: Pipeline DAG visualization - -## Interpreting Results - -### Allelic Imbalance Significance - -Variants with significant allelic imbalance (AI) have: -- `fdr_pval < 0.05`: Statistically significant after FDR correction -- `|log2_ratio| > 0.5`: At least 1.4-fold difference between alleles -- `n_cells >= 10`: Sufficient cell coverage (recommended) - -### Cell-Level vs Pseudo-bulk - -The pipeline provides: -1. **Cell-level counts** (`allele_counts/`): For exploring cell-to-cell heterogeneity -2. **Pseudo-bulk analysis** (`imbalance/`): Aggregated statistical testing - -### Quality Metrics +`pseudobulk/{sample}_pseudobulk_coverage.tsv` aggregates the above rows across cells and contains `chrom`, `pos`, `ref`, `alt`, `overlap_count`, and `n_cells`. SNPs must meet `--min_cells_per_snp`. These files are **not** allele-count input for `find-imbalance`. -Check the execution report for: -- Number of cells processed -- Total variants analyzed -- Processing time per sample +`count_stats/{sample}_count_stats.tsv` and `pseudobulk/{sample}_aggregation_stats.tsv` contain counting and filtering summaries. -## Downstream Analysis +## AnnData -### Loading Results in R +Fragment inputs produce `{sample}_coverage.h5ad` (and optional `_coverage.zarr`). `X` contains overlap coverage with cells in `obs` and SNPs in `var`; allele-specific layers are absent. Cell QC is published under `cell_qc/`. -```r -library(readr) -library(dplyr) +BAM inputs produce allele-count AnnData with `ref` and `alt` layers. The haplotype-layer module uses VCF phase information for `hap1`/`hap2`. These counts can be passed separately to the WASP2 single-cell analysis CLI after supplying validated cell-group metadata. -# Load allelic imbalance results -ai_results <- read_tsv("results/imbalance/GM12878_imbalance.tsv") +## Other outputs -# Filter significant variants -sig_ai <- ai_results %>% - filter(fdr_pval < 0.05, abs(log2_ratio) > 0.5, n_cells >= 10) - -# Load cell-level counts -cell_counts <- read_tsv("results/allele_counts/GM12878_allele_counts.tsv") - -# Analyze per-cell heterogeneity at a specific variant -variant_cells <- cell_counts %>% - filter(chrom == "chr1", pos == 12345) -``` - -### Loading Results in Python - -```python -import pandas as pd - -# Load counts -cell_counts = pd.read_csv( - "results/allele_counts/GM12878_allele_counts.tsv", - sep="\t" -) - -# Load AI results -ai_results = pd.read_csv( - "results/imbalance/GM12878_imbalance.tsv", - sep="\t" -) - -# Filter significant -sig_ai = ai_results[ - (ai_results['fdr_pval'] < 0.05) & - (abs(ai_results['log2_ratio']) > 0.5) & - (ai_results['n_cells'] >= 10) -] - -# Cell-level analysis -pivot_counts = cell_counts.pivot_table( - index='cell_barcode', - columns=['chrom', 'pos'], - values='ref_count', - fill_value=0 -) -``` - -### Integration with Scanpy/AnnData +`variants/` contains the VCF-derived SNP BED. `pipeline_info/` contains execution reports and traces. Consult the trace for the processes actually run for each input path. ```python -import scanpy as sc import pandas as pd -# Load your scATAC AnnData -adata = sc.read_h5ad("scatac_data.h5ad") - -# Load WASP2 cell counts -cell_counts = pd.read_csv( - "results/allele_counts/GM12878_allele_counts.tsv", - sep="\t" +coverage = pd.read_csv("results/fragment_coverage/sample_fragment_coverage.tsv", sep="\t") +cell_by_snp = coverage.pivot_table( + index="barcode", columns=["chrom", "pos"], values="overlap_count", fill_value=0 ) - -# Add as additional layer or obsm -# ... depends on your analysis goals ``` -## Troubleshooting - -### Empty Counts File - -- Check that your VCF contains heterozygous variants for the sample -- Ensure fragments file has the correct index (.tbi) -- Verify cell barcodes in fragments match expected format -- Check that the barcode_tag parameter matches your BAM format - -### Low Cell Coverage - -- Increase sequencing depth -- Check that cell barcodes are being parsed correctly -- Verify the chemistry parameter matches your library prep - -### No Significant AI Results - -- Increase sequencing depth per cell -- Pool related samples for more power -- Consider adjusting `--wasp_min_count` threshold -- Check that variants are truly heterozygous in your sample - -### Memory Issues - -For large datasets (>10,000 cells): -- Increase `--max_memory` parameter -- Consider running on a cluster with more resources -- Split input files by chromosome if necessary +An empty coverage table can indicate barcode mismatch, chromosome naming differences, no SNP overlaps, or filtering thresholds that remove all observations. Coverage alone must not be interpreted as reference-allele support. diff --git a/pipelines/nf-scatac/docs/usage.md b/pipelines/nf-scatac/docs/usage.md index de7d8ee9..5f2a110e 100644 --- a/pipelines/nf-scatac/docs/usage.md +++ b/pipelines/nf-scatac/docs/usage.md @@ -2,15 +2,14 @@ ## Introduction -**nf-scatac** is a Nextflow DSL2 pipeline for single-cell ATAC-seq allelic imbalance (AI) analysis. It processes 10x Genomics scATAC-seq data (fragments files or CellRanger ATAC output) and uses WASP2 for per-cell allele counting and statistical testing to identify regions with significant allelic imbalance. +**nf-scatac** is a Nextflow DSL2 workflow for single-cell ATAC-seq counting and coverage. BAM inputs support allele counting; fragment inputs support overlap coverage only. Fragment coordinates cannot identify observed alleles. Statistical inference is not run for either path; BAM-to-inference aggregation remains unimplemented. See the [output documentation](output.md) and use the WASP2 single-cell CLI separately for inference on validated BAM-derived counts. ## Pipeline Summary 1. Input validation (samplesheet, VCF) 2. VCF to BED conversion for SNP positions -3. Per-cell allele counting at heterozygous SNPs -4. Allelic imbalance statistical analysis -5. Results aggregation +3. BAM allele counting or fragment overlap coverage at heterozygous SNPs +4. AnnData conversion and optional fragment coverage aggregation ## Quick Start @@ -60,13 +59,17 @@ GM12878_cellranger,,/data/cellranger_atac/GM12878,CB,10x-atac-v2 ## Optional Parameters -### WASP2 Options +### Coverage options | Parameter | Default | Description | |-----------|---------|-------------| -| `--wasp_min_count` | 10 | Minimum allele count for AI analysis | -| `--wasp_pseudocount` | 1 | Pseudocount for beta-binomial model | -| `--wasp_phased` | false | Use phased haplotype model | +| `--min_fragments_per_cell` | 1000 | Minimum overlap total per cell for the fragment path | +| `--min_cells_per_snp` | 3 | Minimum covered cells per SNP in aggregated fragment coverage | +| `--skip_pseudobulk` | false | Skip fragment coverage aggregation | +| `--skip_anndata` | false | Skip AnnData output | +| `--create_zarr` | false | Also write Zarr output | + +The legacy `--min_count` and `--pseudocount` parameters do not run inference in this workflow. ### Single-Cell Options @@ -114,27 +117,17 @@ nextflow run nf-scatac -profile test_real,singularity ## Example Commands -### Full Single-Cell Analysis +### Cell-level counting and coverage ```bash -nextflow run nf-scatac \ +nextflow run pipelines/nf-scatac \ --input samplesheet.csv \ --vcf phased_variants.vcf.gz \ - --wasp_min_count 10 \ --outdir results \ -profile docker ``` -### Using Phased Haplotype Model - -```bash -nextflow run nf-scatac \ - --input samplesheet.csv \ - --vcf phased_variants.vcf.gz \ - --wasp_phased true \ - --outdir results \ - -profile singularity -``` +Use `--skip_pseudobulk` to omit fragment coverage aggregation. Phase information is used for BAM haplotype layers; no phased or unphased inference is performed by this workflow. ## Supported Chemistries @@ -144,15 +137,9 @@ nextflow run nf-scatac \ | `10x-atac-v2` | 10x Genomics Single Cell ATAC v2 (default) | | `custom` | Custom scATAC-seq library prep | -## Resource Requirements - -Typical resource usage per sample (5000 cells): +## Resources -| Process | CPUs | Memory | Time | -|---------|------|--------|------| -| VCF to BED conversion | 2 | 4 GB | 2-5 min | -| Per-cell allele counting | 4 | 8 GB | 15-30 min | -| Imbalance analysis | 2 | 4 GB | 5-10 min | +Configure `--max_cpus`, `--max_memory`, and `--max_time` for the input size and execution environment. Check the execution report for observed resource use. ## Troubleshooting diff --git a/pipelines/nf-scatac/main.nf b/pipelines/nf-scatac/main.nf index 53869aa2..57a19c76 100644 --- a/pipelines/nf-scatac/main.nf +++ b/pipelines/nf-scatac/main.nf @@ -1,9 +1,9 @@ #!/usr/bin/env nextflow /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - nf-scatac: Single-Cell ATAC-seq Allelic Imbalance Pipeline + nf-scatac: Single-cell ATAC-seq counting and coverage workflow ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Allelic imbalance analysis for single-cell ATAC-seq data using WASP2. + Counting and coverage outputs for single-cell ATAC-seq data using WASP2. Input modes: - BAM input: True allele-specific counting with ref/alt/hap1/hap2 layers diff --git a/pipelines/nf-scatac/modules/local/scatac_count_alleles/main.nf b/pipelines/nf-scatac/modules/local/scatac_count_alleles/main.nf index 2e3efcf9..3d8b86a2 100644 --- a/pipelines/nf-scatac/modules/local/scatac_count_alleles/main.nf +++ b/pipelines/nf-scatac/modules/local/scatac_count_alleles/main.nf @@ -6,7 +6,7 @@ * * Note: Fragment files contain only coordinates, not sequences, so we count total * overlaps per barcode/SNP. The overlap_count represents coverage at SNP sites; - * allele-specific counts require downstream phasing or sequence-level analysis. + * allele-specific counts require sequence-level observations from BAM. */ process SCATAC_COUNT_ALLELES { @@ -24,7 +24,7 @@ process SCATAC_COUNT_ALLELES { path(peaks) // Optional: BED file with peak regions to restrict analysis output: - tuple val(meta), path("*_allele_counts.tsv"), emit: counts + tuple val(meta), path("*_fragment_coverage.tsv"), emit: counts tuple val(meta), path("*_count_stats.tsv") , emit: stats path "versions.yml" , emit: versions @@ -33,7 +33,7 @@ process SCATAC_COUNT_ALLELES { script: def prefix = task.ext.prefix ?: "${meta.id}" - def min_frags = params.min_fragments_per_cell ?: 1000 + def min_frags = params.min_fragments_per_cell != null ? params.min_fragments_per_cell : 1000 def filter_barcodes = !barcodes.name.startsWith('NO_FILE') ? "true" : "false" def filter_peaks = !peaks.name.startsWith('NO_FILE') ? "true" : "false" """ @@ -68,7 +68,7 @@ process SCATAC_COUNT_ALLELES { END { print "barcode", "chrom", "pos", "ref", "alt", "overlap_count" for (k in counts) { split(k,p,OFS); if (bc_total[p[1]] >= min_frags) print k, counts[k] } - }' > ${prefix}_allele_counts.tsv + }' > ${prefix}_fragment_coverage.tsv # Generate counting statistics awk -v OFS='\\t' 'BEGIN { print "metric", "value" } @@ -78,24 +78,24 @@ process SCATAC_COUNT_ALLELES { print "total_snps", length(snp) print "total_fragment_overlaps", tot print "mean_snps_per_cell", length(bc) > 0 ? length(snp)/length(bc) : 0 - }' ${prefix}_allele_counts.tsv > ${prefix}_count_stats.tsv + }' ${prefix}_fragment_coverage.tsv > ${prefix}_count_stats.tsv - [ \$(wc -l < ${prefix}_allele_counts.tsv) -lt 2 ] && echo "WARNING: No overlaps found" >&2 || true + [ \$(wc -l < ${prefix}_fragment_coverage.tsv) -lt 2 ] && echo "WARNING: No overlaps found" >&2 || true cat <<-END_VERSIONS > versions.yml "${task.process}": bedtools: \$(bedtools --version | sed 's/bedtools v//') - awk: \$(awk --version | head -1 | sed 's/GNU Awk //' | cut -d',' -f1) + awk: \$( (awk --version 2>/dev/null || awk -version) | sed -n '1p') END_VERSIONS """ stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - echo -e "barcode\\tchrom\\tpos\\tref\\talt\\toverlap_count" > ${prefix}_allele_counts.tsv - echo -e "AAACGAACAGTCAGTT-1\\tchr1\\t100000\\tA\\tG\\t8" >> ${prefix}_allele_counts.tsv - echo -e "AAACGAACAGTCAGTT-1\\tchr1\\t200000\\tC\\tT\\t5" >> ${prefix}_allele_counts.tsv - echo -e "AAACGAATCTGCGGCA-1\\tchr1\\t100000\\tA\\tG\\t12" >> ${prefix}_allele_counts.tsv + echo -e "barcode\\tchrom\\tpos\\tref\\talt\\toverlap_count" > ${prefix}_fragment_coverage.tsv + echo -e "AAACGAACAGTCAGTT-1\\tchr1\\t100000\\tA\\tG\\t8" >> ${prefix}_fragment_coverage.tsv + echo -e "AAACGAACAGTCAGTT-1\\tchr1\\t200000\\tC\\tT\\t5" >> ${prefix}_fragment_coverage.tsv + echo -e "AAACGAATCTGCGGCA-1\\tchr1\\t100000\\tA\\tG\\t12" >> ${prefix}_fragment_coverage.tsv echo -e "metric\\tvalue" > ${prefix}_count_stats.tsv echo -e "total_barcodes\\t2" >> ${prefix}_count_stats.tsv diff --git a/pipelines/nf-scatac/modules/local/scatac_count_alleles/meta.yml b/pipelines/nf-scatac/modules/local/scatac_count_alleles/meta.yml index ff9cbd29..6f276bf0 100644 --- a/pipelines/nf-scatac/modules/local/scatac_count_alleles/meta.yml +++ b/pipelines/nf-scatac/modules/local/scatac_count_alleles/meta.yml @@ -55,7 +55,7 @@ output: - counts: type: file description: Per-cell fragment overlap counts at SNP positions (barcode, chrom, pos, ref, alt, overlap_count) - pattern: "*_allele_counts.tsv" + pattern: "*_fragment_coverage.tsv" - stats: type: file description: Counting statistics summary (total_barcodes, total_snps, total_fragment_overlaps) diff --git a/pipelines/nf-scatac/modules/local/scatac_create_anndata/main.nf b/pipelines/nf-scatac/modules/local/scatac_create_anndata/main.nf index d18751b7..45f77f31 100644 --- a/pipelines/nf-scatac/modules/local/scatac_create_anndata/main.nf +++ b/pipelines/nf-scatac/modules/local/scatac_create_anndata/main.nf @@ -1,7 +1,7 @@ /* - * SCATAC_CREATE_ANNDATA - Convert allele counts to AnnData H5AD format + * SCATAC_CREATE_ANNDATA - Convert fragment overlap coverage to AnnData H5AD format * - * Creates AnnData object with per-cell allele counts at heterozygous SNPs. + * Creates AnnData object with per-cell fragment overlap coverage at heterozygous SNPs. * Outputs: * - H5AD file with sparse matrix (X=total overlap counts per cell/SNP) * - Cell metadata (obs) with QC metrics @@ -45,7 +45,7 @@ import pandas as pd from scipy import sparse import anndata as ad -# Read allele counts TSV +# Read fragment overlap coverage TSV df = pd.read_csv("${counts}", sep='\\t') # Validate required columns @@ -58,7 +58,7 @@ if missing: # Handle empty input (early return pattern) if df.empty: - ad.AnnData(X=sparse.csr_matrix((0, 0))).write_h5ad("${prefix}_allelic.h5ad") + ad.AnnData(X=sparse.csr_matrix((0, 0))).write_h5ad("${prefix}_coverage.h5ad") pd.DataFrame(columns=['barcode', 'n_snps', 'total_counts']).to_csv( "${prefix}_cell_qc.tsv", sep='\\t', index=False) print("Created empty AnnData (no data)") @@ -89,12 +89,12 @@ else: # Create and write AnnData adata = ad.AnnData(X=X, obs=cell_stats, var=var, uns={'sample_id': '${meta.id}', 'pipeline': 'nf-scatac', - 'data_type': 'scATAC_allelic_counts'}) - adata.write_h5ad("${prefix}_allelic.h5ad") + 'data_type': 'scATAC_fragment_coverage'}) + adata.write_h5ad("${prefix}_coverage.h5ad") cell_stats.reset_index().to_csv("${prefix}_cell_qc.tsv", sep='\\t', index=False) if ${zarr_flag}: - adata.write_zarr("${prefix}_allelic.zarr") + adata.write_zarr("${prefix}_coverage.zarr") print(f"Created AnnData: {len(barcodes)} cells x {len(snp_ids)} SNPs") PYEOF @@ -121,7 +121,7 @@ adata = ad.AnnData( obs=pd.DataFrame({'n_snps': [2, 2], 'total_counts': [8, 10]}, index=['AAACGAAC-1', 'AAACGAAT-1']), var=pd.DataFrame({'chrom': ['chr1', 'chr1'], 'pos': [100, 200]}, index=['chr1:100:A>G', 'chr1:200:C>T']) ) -adata.write_h5ad("${prefix}_allelic.h5ad") +adata.write_h5ad("${prefix}_coverage.h5ad") pd.DataFrame({ 'barcode': ['AAACGAAC-1', 'AAACGAAT-1'], @@ -131,8 +131,8 @@ pd.DataFrame({ PYEOF if [ "${create_zarr}" == "true" ]; then - mkdir -p ${prefix}_allelic.zarr - touch ${prefix}_allelic.zarr/.zgroup + mkdir -p ${prefix}_coverage.zarr + touch ${prefix}_coverage.zarr/.zgroup fi cat <<-END_VERSIONS > versions.yml diff --git a/pipelines/nf-scatac/modules/local/scatac_create_anndata/meta.yml b/pipelines/nf-scatac/modules/local/scatac_create_anndata/meta.yml index 19b8c74d..4d29fd68 100644 --- a/pipelines/nf-scatac/modules/local/scatac_create_anndata/meta.yml +++ b/pipelines/nf-scatac/modules/local/scatac_create_anndata/meta.yml @@ -1,10 +1,10 @@ name: "scatac_create_anndata" -description: Convert per-cell allele counts to AnnData H5AD format for scverse ecosystem +description: Convert per-cell fragment overlap coverage to AnnData H5AD format keywords: - scatac - anndata - single-cell - - allelic-imbalance + - fragment-coverage - h5ad - zarr tools: @@ -21,15 +21,15 @@ input: e.g. [ id:'test', chemistry:'10x-atac-v2' ] - counts: type: file - description: TSV file with per-cell allele counts - pattern: "*_allele_counts.tsv" + description: TSV file with per-cell fragment overlap coverage + pattern: "*_fragment_coverage.tsv" - create_zarr: type: boolean description: Whether to also create Zarr output for GenVarLoader output: - anndata: type: file - description: AnnData H5AD file with sparse allele count matrices + description: AnnData H5AD file with X containing coverage (cells by SNPs), without allele-specific layers pattern: "*.h5ad" - zarr: type: directory diff --git a/pipelines/nf-scatac/modules/local/scatac_pseudobulk/main.nf b/pipelines/nf-scatac/modules/local/scatac_pseudobulk/main.nf index 9d778f5e..56562362 100644 --- a/pipelines/nf-scatac/modules/local/scatac_pseudobulk/main.nf +++ b/pipelines/nf-scatac/modules/local/scatac_pseudobulk/main.nf @@ -1,14 +1,7 @@ /* - * SCATAC_PSEUDOBULK - Aggregate per-cell counts to pseudo-bulk for statistical analysis - * - * Aggregates per-cell allele counts to sample-level pseudo-bulk counts. - * This increases statistical power for allelic imbalance testing by combining - * sparse per-cell data into denser aggregate counts. - * - * Output format has columns matching WASP2_COUNT_ALLELES (ref_count, alt_count). - * Note: Since fragment files lack sequence data, total overlaps are placed in - * ref_count while alt_count is zero. This is a limitation of fragment-based - * scATAC analysis - true allele-specific counting requires BAM-level data. + * Aggregate fragment overlap coverage across cells at each SNP position. + * Fragment coordinates do not identify observed alleles. Outputs are coverage + * summaries and must not be submitted to allelic imbalance inference. */ process SCATAC_PSEUDOBULK { @@ -24,7 +17,7 @@ process SCATAC_PSEUDOBULK { tuple val(meta), path(cell_counts) output: - tuple val(meta), path("*_pseudobulk_counts.tsv"), emit: counts + tuple val(meta), path("*_pseudobulk_coverage.tsv"), emit: counts tuple val(meta), path("*_aggregation_stats.tsv"), emit: stats path "versions.yml" , emit: versions @@ -33,14 +26,14 @@ process SCATAC_PSEUDOBULK { script: def prefix = task.ext.prefix ?: "${meta.id}" - def min_cells = params.min_cells_per_snp ?: 3 + def min_cells = params.min_cells_per_snp != null ? params.min_cells_per_snp : 3 """ set -euo pipefail # Aggregate per-cell counts to pseudo-bulk and generate stats in one pass awk -v OFS='\\t' -v min_cells="${min_cells}" -v prefix="${prefix}" ' BEGIN { - print "chrom", "pos", "ref", "alt", "ref_count", "alt_count" > prefix "_pseudobulk_counts.tsv" + print "chrom", "pos", "ref", "alt", "overlap_count", "n_cells" > (prefix "_pseudobulk_coverage.tsv") } NR > 1 { key = \$2 OFS \$3 OFS \$4 OFS \$5 @@ -54,30 +47,30 @@ process SCATAC_PSEUDOBULK { filtered_count = 0 for (key in total) { if (cells_per_snp[key] >= min_cells) { - print key, total[key], 0 >> prefix "_pseudobulk_counts.tsv" + print key, total[key], cells_per_snp[key] >> (prefix "_pseudobulk_coverage.tsv") filtered_count++ } } # Write aggregation stats - print "metric", "value" > prefix "_aggregation_stats.tsv" - print "total_cells_input", length(input_cells) >> prefix "_aggregation_stats.tsv" - print "total_snps_input", length(input_snps) >> prefix "_aggregation_stats.tsv" - print "snps_after_filtering", filtered_count >> prefix "_aggregation_stats.tsv" + print "metric", "value" > (prefix "_aggregation_stats.tsv") + print "total_cells_input", length(input_cells) >> (prefix "_aggregation_stats.tsv") + print "total_snps_input", length(input_snps) >> (prefix "_aggregation_stats.tsv") + print "snps_after_filtering", filtered_count >> (prefix "_aggregation_stats.tsv") }' ${cell_counts} cat <<-END_VERSIONS > versions.yml "${task.process}": - awk: \$(awk --version | head -1 | sed 's/GNU Awk //' | cut -d',' -f1) + awk: \$( (awk --version 2>/dev/null || awk -version) | sed -n '1p') END_VERSIONS """ stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - echo -e "chrom\\tpos\\tref\\talt\\tref_count\\talt_count" > ${prefix}_pseudobulk_counts.tsv - echo -e "chr1\\t100000\\tA\\tG\\t45\\t0" >> ${prefix}_pseudobulk_counts.tsv - echo -e "chr1\\t200000\\tC\\tT\\t32\\t0" >> ${prefix}_pseudobulk_counts.tsv + echo -e "chrom\\tpos\\tref\\talt\\toverlap_count\\tn_cells" > ${prefix}_pseudobulk_coverage.tsv + echo -e "chr1\\t100000\\tA\\tG\\t45\\t3" >> ${prefix}_pseudobulk_coverage.tsv + echo -e "chr1\\t200000\\tC\\tT\\t32\\t3" >> ${prefix}_pseudobulk_coverage.tsv echo -e "metric\\tvalue" > ${prefix}_aggregation_stats.tsv echo -e "total_cells_input\\t10" >> ${prefix}_aggregation_stats.tsv diff --git a/pipelines/nf-scatac/modules/local/scatac_pseudobulk/meta.yml b/pipelines/nf-scatac/modules/local/scatac_pseudobulk/meta.yml index 8f3d209c..137f8c54 100644 --- a/pipelines/nf-scatac/modules/local/scatac_pseudobulk/meta.yml +++ b/pipelines/nf-scatac/modules/local/scatac_pseudobulk/meta.yml @@ -1,5 +1,5 @@ name: scatac_pseudobulk -description: Aggregate per-cell allele counts to pseudo-bulk for statistical analysis +description: Aggregate per-cell fragment overlap coverage keywords: - scatac - single-cell @@ -20,8 +20,8 @@ input: e.g. [ id:'test', single_end:false ] - cell_counts: type: file - description: Per-cell allele counts TSV from SCATAC_COUNT_ALLELES - pattern: "*_allele_counts.tsv" + description: Per-cell fragment overlap coverage TSV from SCATAC_COUNT_ALLELES + pattern: "*_fragment_coverage.tsv" output: - meta: @@ -29,8 +29,8 @@ output: description: Groovy Map containing sample information - counts: type: file - description: Pseudo-bulk aggregated counts compatible with WASP2_ANALYZE_IMBALANCE - pattern: "*_pseudobulk_counts.tsv" + description: Aggregated fragment coverage; not valid input to allelic imbalance inference + pattern: "*_pseudobulk_coverage.tsv" - stats: type: file description: Aggregation statistics (cells, SNPs, total counts) diff --git a/pipelines/nf-scatac/nextflow.config b/pipelines/nf-scatac/nextflow.config index ea240c91..dbd6add1 100644 --- a/pipelines/nf-scatac/nextflow.config +++ b/pipelines/nf-scatac/nextflow.config @@ -7,7 +7,7 @@ manifest { name = 'wasp2/nf-scatac' author = 'WASP2 Team' - description = 'Single-Cell ATAC-seq Allelic Imbalance Pipeline' + description = 'Single-cell ATAC-seq counting and coverage workflow' mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' version = '1.0.0' @@ -22,13 +22,13 @@ params { // Processing parameters min_fragments_per_cell = 1000 // Minimum fragments per cell to include min_cells_per_snp = 3 // Minimum cells per SNP for pseudo-bulk - min_count = 10 // Minimum count for imbalance testing - pseudocount = 1 // Pseudocount for imbalance calculation + min_count = 10 // Reserved; this workflow does not run inference + pseudocount = 1 // Reserved; this workflow does not run inference // Output options create_zarr = false // Also output Zarr format for GenVarLoader skip_anndata = false // Skip AnnData H5AD creation - skip_pseudobulk = false // Skip pseudo-bulk aggregation and analysis + skip_pseudobulk = false // Skip fragment coverage aggregation // Publishing publish_dir_mode = 'copy' diff --git a/pipelines/nf-scatac/nextflow_schema.json b/pipelines/nf-scatac/nextflow_schema.json index e8e734d0..36420578 100644 --- a/pipelines/nf-scatac/nextflow_schema.json +++ b/pipelines/nf-scatac/nextflow_schema.json @@ -81,7 +81,7 @@ "min_count": { "type": "integer", "default": 10, - "minimum": 1, + "minimum": 0, "description": "Minimum allele count for imbalance testing.", "help_text": "Variants with fewer than this many total reads will be excluded from statistical testing.", "fa_icon": "fas fa-sort-numeric-up" diff --git a/pipelines/nf-scatac/subworkflows/local/utils_nfscatac_pipeline.nf b/pipelines/nf-scatac/subworkflows/local/utils_nfscatac_pipeline.nf index 8c660311..3189be03 100644 --- a/pipelines/nf-scatac/subworkflows/local/utils_nfscatac_pipeline.nf +++ b/pipelines/nf-scatac/subworkflows/local/utils_nfscatac_pipeline.nf @@ -110,10 +110,10 @@ workflow PIPELINE_COMPLETION { Output: ${outdir} Results: - - allele_counts/: Per-cell allele counts at het SNPs + - fragment_coverage/: Per-cell fragment overlap coverage - anndata/: AnnData H5AD files for scverse ecosystem - cell_qc/: Cell QC metrics - - imbalance/: Allelic imbalance analysis results (pseudo-bulk) + - pseudobulk/: Aggregated fragment coverage (not allele inference) """.stripIndent() } @@ -155,6 +155,7 @@ def helpMessage() { Input priority: At least one of fragments, cellranger_dir, or bam is required. When BAM is provided, true allele-specific counting is performed with ref/alt/hap layers. When only fragments are provided, overlap counting is performed (total counts only). + No allelic imbalance inference is run; BAM-to-inference aggregation is not implemented. Profiles: -profile docker Run with Docker @@ -165,12 +166,11 @@ def helpMessage() { Output: results/ - ├── allele_counts/ # Per-cell allele counts + ├── fragment_coverage/ # Per-cell fragment overlap coverage ├── anndata/ # AnnData H5AD files (with ref/alt/hap layers if BAM provided) ├── zarr/ # Zarr files (if --create_zarr) ├── cell_qc/ # Cell QC metrics - ├── pseudobulk/ # Pseudo-bulk aggregated counts - ├── imbalance/ # Allelic imbalance results + ├── pseudobulk/ # Aggregated fragment coverage └── pipeline_info/ # Execution reports """.stripIndent() } diff --git a/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/main.nf b/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/main.nf index bd98850a..5871a7a7 100644 --- a/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/main.nf +++ b/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/main.nf @@ -8,7 +8,8 @@ 1. BAM-based (allele-specific): True ref/alt counting with hap1/hap2 layers 2. Fragment-based (overlap counting): Total coverage at SNP positions - Includes per-cell counting, AnnData output, and optional pseudo-bulk analysis. + Includes per-cell counting, AnnData output, and optional coverage aggregation. + This workflow does not run allelic imbalance inference for either input path. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -18,7 +19,6 @@ include { SCATAC_COUNT_ALLELES } from '../../../modules/local/scatac_co include { SCATAC_CREATE_ANNDATA } from '../../../modules/local/scatac_create_anndata/main' include { SCATAC_ADD_HAPLOTYPE_LAYERS } from '../../../modules/local/scatac_add_haplotype_layers/main' include { SCATAC_PSEUDOBULK } from '../../../modules/local/scatac_pseudobulk/main' -include { WASP2_ANALYZE_IMBALANCE } from '../../../../nf-modules/modules/wasp2/analyze_imbalance/main' workflow WASP_ALLELIC_SC { take: @@ -65,7 +65,7 @@ workflow WASP_ALLELIC_SC { ch_bam_inputs.barcodes, ch_bam_inputs.peaks ) - ch_versions = ch_versions.mix(WASP2_COUNT_SC.out.versions.first().ifEmpty([])) + ch_versions = ch_versions.mix(WASP2_COUNT_SC.out.versions) // Add haplotype layers using phased VCF if (!params.skip_anndata) { @@ -74,14 +74,14 @@ workflow WASP_ALLELIC_SC { ch_vcf, params.create_zarr ?: false ) - ch_versions = ch_versions.mix(SCATAC_ADD_HAPLOTYPE_LAYERS.out.versions.first().ifEmpty([])) + ch_versions = ch_versions.mix(SCATAC_ADD_HAPLOTYPE_LAYERS.out.versions) ch_anndata = ch_anndata.mix(SCATAC_ADD_HAPLOTYPE_LAYERS.out.anndata) ch_cell_qc = ch_cell_qc.mix(SCATAC_ADD_HAPLOTYPE_LAYERS.out.cell_qc) ch_zarr = ch_zarr.mix(SCATAC_ADD_HAPLOTYPE_LAYERS.out.zarr) } // For BAM-based, use stats from WASP2_COUNT_SC - ch_count_stats = ch_count_stats.mix(WASP2_COUNT_SC.out.stats.ifEmpty([])) + ch_count_stats = ch_count_stats.mix(WASP2_COUNT_SC.out.stats) //========================================================================== // PATH 2: Fragment-based overlap counting (total counts only) @@ -101,7 +101,7 @@ workflow WASP_ALLELIC_SC { ch_frag_count_input.barcodes, ch_frag_count_input.peaks ) - ch_versions = ch_versions.mix(SCATAC_COUNT_ALLELES.out.versions.first().ifEmpty([])) + ch_versions = ch_versions.mix(SCATAC_COUNT_ALLELES.out.versions) ch_cell_counts = ch_cell_counts.mix(SCATAC_COUNT_ALLELES.out.counts) ch_count_stats = ch_count_stats.mix(SCATAC_COUNT_ALLELES.out.stats) @@ -111,26 +111,24 @@ workflow WASP_ALLELIC_SC { SCATAC_COUNT_ALLELES.out.counts, params.create_zarr ?: false ) - ch_versions = ch_versions.mix(SCATAC_CREATE_ANNDATA.out.versions.first().ifEmpty([])) + ch_versions = ch_versions.mix(SCATAC_CREATE_ANNDATA.out.versions) ch_anndata = ch_anndata.mix(SCATAC_CREATE_ANNDATA.out.anndata) ch_cell_qc = ch_cell_qc.mix(SCATAC_CREATE_ANNDATA.out.cell_qc) ch_zarr = ch_zarr.mix(SCATAC_CREATE_ANNDATA.out.zarr) } //========================================================================== - // SHARED: Pseudo-bulk aggregation and statistical analysis + // Fragment coverage aggregation (no allele-specific inference) //========================================================================== if (!params.skip_pseudobulk) { // For fragment-based: use TSV counts SCATAC_PSEUDOBULK ( SCATAC_COUNT_ALLELES.out.counts ) - ch_versions = ch_versions.mix(SCATAC_PSEUDOBULK.out.versions.first().ifEmpty([])) + ch_versions = ch_versions.mix(SCATAC_PSEUDOBULK.out.versions) ch_pseudobulk = ch_pseudobulk.mix(SCATAC_PSEUDOBULK.out.counts) - WASP2_ANALYZE_IMBALANCE ( SCATAC_PSEUDOBULK.out.counts ) - ch_versions = ch_versions.mix(WASP2_ANALYZE_IMBALANCE.out.versions.first().ifEmpty([])) - ch_imbalance = ch_imbalance.mix(WASP2_ANALYZE_IMBALANCE.out.results) + // Fragment coordinates contain no observed alleles and cannot support + // allelic imbalance testing. BAM-to-inference aggregation is not yet wired. - // TODO: Add pseudo-bulk analysis for BAM-based path (aggregate from H5AD) } emit: @@ -139,7 +137,7 @@ workflow WASP_ALLELIC_SC { anndata = ch_anndata // channel: [ val(meta), path(*.h5ad) ] zarr = ch_zarr // channel: [ val(meta), path(*.zarr) ] cell_qc = ch_cell_qc // channel: [ val(meta), path(cell_qc.tsv) ] - pseudobulk = ch_pseudobulk // channel: [ val(meta), path(pseudobulk.tsv) ] + pseudobulk = ch_pseudobulk // channel: [ val(meta), path(pseudobulk_coverage.tsv) ] imbalance = ch_imbalance // channel: [ val(meta), path(results.tsv) ] versions = ch_versions // channel: [ path(versions.yml) ] } diff --git a/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/meta.yml b/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/meta.yml index be4c8b16..95d17a08 100644 --- a/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/meta.yml +++ b/pipelines/nf-scatac/subworkflows/local/wasp_allelic_sc/meta.yml @@ -1,8 +1,8 @@ name: wasp_allelic_sc description: | - WASP2 single-cell allelic imbalance analysis for scATAC-seq data. - Converts VCF to BED, counts alleles per cell from fragments, - aggregates to pseudo-bulk, and analyzes allelic imbalance. + Single-cell BAM allele counting or fragment overlap coverage. + Fragment coverage is aggregated without allele inference; downstream + BAM-to-inference aggregation is not implemented. keywords: - wasp2 - scatac @@ -13,7 +13,6 @@ keywords: components: - wasp2/vcf_to_bed - local/scatac_count_alleles - - wasp2/analyze_imbalance input: - ch_fragments: @@ -33,7 +32,7 @@ output: Structure: [ val(meta), path(counts.tsv) ] - imbalance: description: | - Allelic imbalance analysis results (uses pseudo-bulk aggregation internally). + Empty channel; this workflow does not run allelic imbalance inference. Structure: [ val(meta), path(results.tsv) ] - versions: description: | diff --git a/pipelines/nf-scatac/tests/main.nf.test b/pipelines/nf-scatac/tests/main.nf.test index 1d48ef37..6e0363a1 100644 --- a/pipelines/nf-scatac/tests/main.nf.test +++ b/pipelines/nf-scatac/tests/main.nf.test @@ -1,62 +1,3 @@ -nextflow_pipeline { - - name "Test Pipeline nf-scatac" - script "../main.nf" - - tag "pipeline" - tag "scatac" - - test("nf-scatac - stub run") { - - tag "ci_stub" - - options "-stub -profile test_stub" - - when { - params { - outdir = "$outputDir" - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.out.anndata != null }, - { assert snapshot( - workflow.trace.tasks().size(), - workflow.trace.tasks()*.name.sort() - ).match("stub_workflow_tasks") } - ) - } - } - - test("nf-scatac - integration run - real data") { - - tag "integration" - - when { - params { - input = "${projectDir}/tests/data/samplesheet_test.csv" - vcf = "${projectDir}/tests/data/variants.vcf.gz" - outdir = "${outputDir}/results_integration" - min_fragments_per_cell = 1 - min_cells_per_snp = 1 - min_count = 1 - skip_pseudobulk = true - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.out.anndata != null }, - { assert path("${outputDir}/results_integration").exists() }, - { assert snapshot(workflow.trace.tasks().size()).match("integration_task_count") } - ) - } - } -} - nextflow_workflow { name "Test Workflow SCATAC" @@ -102,8 +43,8 @@ nextflow_workflow { { assert workflow.success }, { assert workflow.out.cell_counts.size() == 1 }, { assert workflow.out.anndata.size() == 1 }, - { assert workflow.out.imbalance.size() == 1 }, - { assert snapshot(workflow.out.versions).match("scatac_versions_fragments") } + { assert workflow.out.imbalance.size() == 0 }, + { assert path(workflow.out.pseudobulk[0][1]).getFileName().toString().endsWith("_pseudobulk_coverage.tsv") } ) } } @@ -499,7 +440,7 @@ AAACGAAC-1\\tchr1\\t200\\tC\\tT\\t3 { assert process.success }, { assert process.out.counts.size() == 1 }, { assert process.out.stats.size() == 1 }, - { assert file(process.out.counts[0][1]).text.contains("ref_count") } + { assert file(process.out.counts[0][1]).text.contains("overlap_count") } ) } } diff --git a/pipelines/nf-scatac/tests/modules/local/scatac_count_alleles.nf.test.snap b/pipelines/nf-scatac/tests/modules/local/scatac_count_alleles.nf.test.snap index b54e8dc2..d7ef73dc 100644 --- a/pipelines/nf-scatac/tests/modules/local/scatac_count_alleles.nf.test.snap +++ b/pipelines/nf-scatac/tests/modules/local/scatac_count_alleles.nf.test.snap @@ -7,7 +7,7 @@ { "id": "test_sample" }, - "test_sample_allele_counts.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" + "test_sample_fragment_coverage.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" ] ], "1": [ @@ -26,7 +26,7 @@ { "id": "test_sample" }, - "test_sample_allele_counts.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" + "test_sample_fragment_coverage.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" ] ], "stats": [ @@ -56,7 +56,7 @@ { "id": "test_sample_filtered" }, - "test_sample_filtered_allele_counts.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" + "test_sample_filtered_fragment_coverage.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" ] ], "1": [ @@ -75,7 +75,7 @@ { "id": "test_sample_filtered" }, - "test_sample_filtered_allele_counts.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" + "test_sample_filtered_fragment_coverage.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" ] ], "stats": [ @@ -105,7 +105,7 @@ { "id": "test_sample_peaks" }, - "test_sample_peaks_allele_counts.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" + "test_sample_peaks_fragment_coverage.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" ] ], "1": [ @@ -124,7 +124,7 @@ { "id": "test_sample_peaks" }, - "test_sample_peaks_allele_counts.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" + "test_sample_peaks_fragment_coverage.tsv:md5,8ecc233071c9dfc9a44e06b76cb4b0a4" ] ], "stats": [ diff --git a/pipelines/nf-scatac/tests/modules/local/scatac_create_anndata.nf.test.snap b/pipelines/nf-scatac/tests/modules/local/scatac_create_anndata.nf.test.snap index 067945c1..958a9ba7 100644 --- a/pipelines/nf-scatac/tests/modules/local/scatac_create_anndata.nf.test.snap +++ b/pipelines/nf-scatac/tests/modules/local/scatac_create_anndata.nf.test.snap @@ -7,7 +7,7 @@ { "id": "test_sample_zarr" }, - "test_sample_zarr_allelic.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" + "test_sample_zarr_coverage.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" ] ], "1": [ @@ -36,7 +36,7 @@ { "id": "test_sample_zarr" }, - "test_sample_zarr_allelic.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" + "test_sample_zarr_coverage.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" ] ], "cell_qc": [ @@ -76,7 +76,7 @@ { "id": "test_sample" }, - "test_sample_allelic.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" + "test_sample_coverage.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" ] ], "1": [ @@ -98,7 +98,7 @@ { "id": "test_sample" }, - "test_sample_allelic.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" + "test_sample_coverage.h5ad:md5,8b66deaf104c8e17f66cd4632c588a1d" ] ], "cell_qc": [ diff --git a/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test b/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test index 1efafa27..d7158d80 100644 --- a/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test +++ b/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test @@ -31,4 +31,41 @@ nextflow_process { assert snapshot(process.out).match() } } + test("SCATAC_PSEUDOBULK - aggregates coverage without fabricating alleles") { + tag "correctness_regression" + + when { + params { + min_cells_per_snp = 1 + } + process { + """ + def counts_content = '''barcode\\tchrom\\tpos\\tref\\talt\\toverlap_count +AAACGAAC-1\\tchr1\\t100\\tA\\tG\\t5 +AAACGAAT-1\\tchr1\\t100\\tA\\tG\\t8 +AAACGAAC-1\\tchr1\\t200\\tC\\tT\\t3 +''' + def counts_file = file("${workDir}/test_allele_counts.tsv") + counts_file.text = counts_content + + input[0] = [ + [id: 'test_sample'], + counts_file + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.counts.size() == 1 }, + { assert process.out.stats.size() == 1 }, + { assert file(process.out.counts[0][1]).readLines()[0] == "chrom\tpos\tref\talt\toverlap_count\tn_cells" }, + { assert file(process.out.counts[0][1]).readLines().drop(1).toSet() == ["chr1\t100\tA\tG\t13\t2", "chr1\t200\tC\tT\t3\t1"].toSet() }, + { assert !file(process.out.counts[0][1]).text.contains("ref_count") } + ) + } + } + } diff --git a/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test.snap b/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test.snap index c9a65fce..c27be26c 100644 --- a/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test.snap +++ b/pipelines/nf-scatac/tests/modules/local/scatac_pseudobulk.nf.test.snap @@ -7,7 +7,7 @@ { "id": "test_sample" }, - "test_sample_pseudobulk_counts.tsv:md5,8dd6eddd02ac9b332e7461893c1f6452" + "test_sample_pseudobulk_coverage.tsv:md5,7aba3b2f98e9feac05e9c2a7c73d199b" ] ], "1": [ @@ -26,7 +26,7 @@ { "id": "test_sample" }, - "test_sample_pseudobulk_counts.tsv:md5,8dd6eddd02ac9b332e7461893c1f6452" + "test_sample_pseudobulk_coverage.tsv:md5,7aba3b2f98e9feac05e9c2a7c73d199b" ] ], "stats": [ diff --git a/pipelines/nf-scatac/tests/pipeline.nf.test b/pipelines/nf-scatac/tests/pipeline.nf.test new file mode 100644 index 00000000..66fc878b --- /dev/null +++ b/pipelines/nf-scatac/tests/pipeline.nf.test @@ -0,0 +1,65 @@ +nextflow_pipeline { + + name "Test Pipeline nf-scatac" + script "../main.nf" + + tag "pipeline" + tag "scatac" + + test("nf-scatac - stub run") { + + tag "ci_stub" + + options "-stub" + + when { + params { + // nf-test launches outside the checkout. Resolve the repository + // fixture paths in a temporary samplesheet for that launch. + def samplesheet = path("${outputDir}/samplesheet.csv").toFile() + samplesheet.parentFile.mkdirs() + samplesheet.text = path("${projectDir}/tests/stub/samplesheet.csv").toFile().text.replace( + 'tests/stub/', "${projectDir}/tests/stub/" + ) + input = samplesheet.toString() + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert path("${outputDir}/fragment_coverage/test_sample_fragment_coverage.tsv").exists() }, + { assert path("${outputDir}/anndata/test_sample_coverage.h5ad").exists() }, + { assert path("${outputDir}/pseudobulk/test_sample_pseudobulk_coverage.tsv").exists() }, + { assert !path("${outputDir}/imbalance").exists() } + ) + } + } + + test("nf-scatac - integration run - real data") { + + tag "integration" + + when { + params { + input = "${projectDir}/tests/data/samplesheet_test.csv" + vcf = "${projectDir}/tests/data/variants.vcf.gz" + outdir = "${outputDir}/results_integration" + min_fragments_per_cell = 1 + min_cells_per_snp = 1 + min_count = 1 + skip_pseudobulk = true + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.out.anndata != null }, + { assert path("${outputDir}/results_integration").exists() }, + { assert snapshot(workflow.trace.tasks().size()).match("integration_task_count") } + ) + } + } +} diff --git a/pipelines/nf-scatac/tests/subworkflows/wasp_allelic_sc.nf.test b/pipelines/nf-scatac/tests/subworkflows/wasp_allelic_sc.nf.test new file mode 100644 index 00000000..3760884f --- /dev/null +++ b/pipelines/nf-scatac/tests/subworkflows/wasp_allelic_sc.nf.test @@ -0,0 +1,36 @@ +nextflow_workflow { + name "Test single-cell coverage inference boundary" + script "../../subworkflows/local/wasp_allelic_sc/main.nf" + workflow "WASP_ALLELIC_SC" + + tag "correctness_regression" + + test("Fragment coverage never invokes allelic imbalance inference") { + options "-stub" + when { + params { + skip_anndata = true + skip_pseudobulk = false + } + workflow { + """ + def inputs = ['fragments.tsv.gz', 'fragments.tsv.gz.tbi', 'NO_FILE_BARCODES', 'NO_FILE_PEAKS', 'NO_FILE_BAM', 'NO_FILE_BAI', 'variants.vcf.gz', 'variants.vcf.gz.tbi'].collect { name -> + def item = file("${workDir}/" + name) + item.text = '' + item + } + input[0] = Channel.of([[id:'coverage_only'], *inputs[0..5]]) + input[1] = Channel.of([[id:'variants'], *inputs[6..7]]) + """ + } + } + then { + assert workflow.success + assert workflow.out.imbalance.size() == 0 + assert workflow.out.pseudobulk.size() == 1 + assert path(workflow.out.pseudobulk[0][1]).getFileName().toString().endsWith('_pseudobulk_coverage.tsv') + assert path(workflow.out.pseudobulk[0][1]).readLines()[0] == 'chrom\tpos\tref\talt\toverlap_count\tn_cells' + assert !workflow.trace.tasks()*.name.any { it.contains('ANALYZE_IMBALANCE') } + } + } +} diff --git a/pipelines/nf-scatac/workflows/scatac.nf b/pipelines/nf-scatac/workflows/scatac.nf index 3659c63f..87063b88 100644 --- a/pipelines/nf-scatac/workflows/scatac.nf +++ b/pipelines/nf-scatac/workflows/scatac.nf @@ -2,7 +2,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SCATAC WORKFLOW - Single-Cell ATAC-seq Allelic Imbalance ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Allelic imbalance analysis on scATAC-seq data using WASP2. + BAM allele counting or fragment coverage on scATAC-seq data using WASP2. Supports two input modes: 1. BAM input: True allele-specific counting with ref/alt/hap layers @@ -49,7 +49,7 @@ workflow SCATAC { [ [id: 'variants'], vcf, idx ] } - // Run WASP2 single-cell allelic imbalance analysis + // Run WASP2 single-cell counting and coverage aggregation // Includes: counting, AnnData creation, pseudo-bulk aggregation, statistical analysis WASP_ALLELIC_SC ( samplesheet, ch_vcf ) ch_versions = ch_versions.mix(WASP_ALLELIC_SC.out.versions) diff --git a/pixi.toml b/pixi.toml index a14060af..b846d5f6 100644 --- a/pixi.toml +++ b/pixi.toml @@ -41,11 +41,15 @@ typer = ">=0.12" rich = ">=13.0" # Rust build toolchain -rust = ">=1.70" +rust = ">=1.88" +cmake = ">=3.14" libclang = "*" clang = "*" pkg-config = "*" +[target.osx-64.dependencies] +numba = ">=0.59,<0.63" + [pypi-dependencies] wasp2 = { path = ".", editable = true } maturin = ">=1.6,<2.0" diff --git a/pyproject.toml b/pyproject.toml index ef6a9b3b..bd2ee00f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=1.6,<2.0"] +requires = ["maturin>=1.6,<2.0", "cmake>=3.14"] build-backend = "maturin" [project] diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0b98dac3..b7a2c179 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -26,12 +26,6 @@ dependencies = [ "cc", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "anes" version = "0.1.6" @@ -389,12 +383,6 @@ dependencies = [ "syn", ] -[[package]] -name = "doc-comment" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" - [[package]] name = "either" version = "1.15.0" @@ -459,12 +447,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -573,7 +555,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash 0.1.5", + "foldhash", ] [[package]] @@ -581,11 +563,6 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] [[package]] name = "heck" @@ -756,15 +733,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.17" @@ -791,16 +759,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lambert_w" -version = "1.2.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f0846ee4f0299ca4c5b9ca06ff55cf88b3430a763bf591474cc734479c9b24" -dependencies = [ - "num-complex", - "num-traits", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -897,15 +855,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "lru" -version = "0.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" -dependencies = [ - "hashbrown 0.16.1", -] - [[package]] name = "matrixmultiply" version = "0.3.10" @@ -949,8 +898,8 @@ dependencies = [ "num-complex", "num-rational", "num-traits", - "rand 0.8.7", - "rand_distr 0.4.3", + "rand", + "rand_distr", "simba", "typenum", ] @@ -1070,20 +1019,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -1112,17 +1047,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-rational" version = "0.4.2" @@ -1350,18 +1274,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", + "rand_chacha", + "rand_core", ] [[package]] @@ -1371,17 +1285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "rand_core", ] [[package]] @@ -1393,15 +1297,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - [[package]] name = "rand_distr" version = "0.4.3" @@ -1409,26 +1304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.7", -] - -[[package]] -name = "rand_distr" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" -dependencies = [ - "num-traits", - "rand 0.9.2", -] - -[[package]] -name = "rand_xoshiro" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" -dependencies = [ - "rand_core 0.9.5", + "rand", ] [[package]] @@ -1548,24 +1424,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "rv" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb89285b0862665a769f9e34fc308ed627be1ff149ea6b16ba245921782adcf6" -dependencies = [ - "doc-comment", - "itertools 0.14.0", - "lru", - "num", - "num-traits", - "paste", - "rand 0.9.2", - "rand_distr 0.5.1", - "rand_xoshiro", - "special", -] - [[package]] name = "safe_arch" version = "0.7.4" @@ -1676,16 +1534,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "special" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2037227570e0bedf82a7f866a3e7cebe218ec9cd0d5399151942ee7358f90bb6" -dependencies = [ - "lambert_w", - "libm", -] - [[package]] name = "spin" version = "0.9.8" @@ -1710,7 +1558,7 @@ dependencies = [ "approx", "nalgebra", "num-traits", - "rand 0.8.7", + "rand", ] [[package]] @@ -2002,7 +1850,6 @@ dependencies = [ "rayon", "rust-htslib", "rustc-hash 2.1.1", - "rv", "smallvec", "statrs", "tempfile", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index cf426b30..c5f49459 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -2,6 +2,11 @@ name = "wasp2" version = "1.5.0" edition = "2021" +rust-version = "1.88" +description = "Allele-specific analysis of next-generation sequencing data" +license = "MIT" +repository = "https://github.com/mcvickerlab/WASP2" +documentation = "https://mcvickerlab.github.io/WASP2/" [lib] name = "wasp2_rust" @@ -9,13 +14,12 @@ crate-type = ["cdylib", "rlib"] [dependencies] pyo3 = { version = "0.29", features = ["extension-module"] } -rust-htslib = { version = "1.0", default-features = false } # Keep stable version (0.47+ has NFS build issues) +rust-htslib = { version = "1.0", default-features = false } rayon = "1.8" anyhow = "1.0" rustc-hash = "2.1" statrs = "0.18" -rv = "0.19" -coitrees = "0.4" # Fast interval tree for BAM-BED intersection (15-30x faster than pybedtools) +coitrees = "0.4" # Interval tree for BAM-BED intersection crossbeam-channel = "0.5" # Fast MPMC channels for parallel FASTQ writing gzp = { version = "2.0", default-features = false, features = ["deflate_default"] } # Parallel gzip compression itoa = "1.0" # Fast integer-to-ascii for FASTQ/sidecar writing @@ -23,10 +27,10 @@ smallvec = "1.13" # Reduce heap allocs for small overlap/span vectors # VCF/BCF parsing (noodles - pure Rust, no C dependencies) # Note: noodles-bcf depends on noodles-vcf, so we use compatible versions -noodles-vcf = "0.84" # Updated from 0.72 -noodles-bcf = "0.82" # Updated to match noodles-vcf -noodles-core = "0.19" # Core types remain stable -noodles-bgzf = "0.35" # Updated for compatibility +noodles-vcf = "0.84" +noodles-bcf = "0.82" +noodles-core = "0.19" +noodles-bgzf = "0.35" flate2 = "1.1" # For gzip decompression [dev-dependencies] @@ -36,10 +40,5 @@ tempfile = "3.25" [build-dependencies] pyo3-build-config = "0.29" -# Benchmarks removed for clean release (benchmark files in paper branch only) -# [[bench]] -# name = "mapping_filter_bench" -# harness = false - [profile.release] debug = true # Enable debug symbols for profiling diff --git a/rust/src/analysis.rs b/rust/src/analysis.rs index 872323bb..8b6817c5 100644 --- a/rust/src/analysis.rs +++ b/rust/src/analysis.rs @@ -233,7 +233,7 @@ fn clamp_rho(rho: f64) -> f64 { /// /// Python keeps columns: chrom, pos, ref_count, alt_count, N, region (+ GT if phased) /// and removes exact duplicate rows. This ensures parity with Python's behavior. -fn deduplicate_variants(variants: Vec, phased: bool) -> Vec { +fn deduplicate_variants(variants: Vec, phased: bool) -> Result> { let mut seen: HashSet = HashSet::new(); let mut result = Vec::with_capacity(variants.len()); @@ -244,14 +244,14 @@ fn deduplicate_variants(variants: Vec, phased: bool) -> Vec Result { @@ -885,7 +885,7 @@ fn single_model_with_dispersion( let DispersionParameters::Single { rho: disp } = parameters else { unreachable!("single fit returned linear parameters") }; - eprintln!(" Dispersion: {:.6}", disp); + eprintln!(" Dispersion: {disp:.6}"); let results = test_fixed_dispersion(variants, phased, parameters, false, 0)?; Ok((results, disp)) } @@ -914,7 +914,7 @@ pub fn linear_model_with_dispersion( let DispersionParameters::Linear { d1, d2, .. } = parameters else { unreachable!("linear fit returned scalar parameters") }; - eprintln!(" d1 = {:.6}, d2 = {:.6}", d1, d2); + eprintln!(" d1 = {d1:.6}, d2 = {d2:.6}"); let results = test_fixed_dispersion(variants, phased, parameters, false, 0)?; Ok((results, d1, d2)) } @@ -934,10 +934,7 @@ fn fit_dispersion_parameters( } let ref_counts: Vec = variants.iter().map(|variant| variant.ref_count).collect(); - let n_array: Vec = variants - .iter() - .map(|variant| variant.ref_count + variant.alt_count) - .collect(); + let n_array: Vec = variants.iter().map(checked_depth).collect::>()?; match method { AnalysisMethod::Single => Ok(DispersionParameters::Single { rho: optimize_dispersion(&ref_counts, &n_array)?, @@ -959,6 +956,26 @@ fn fit_dispersion_parameters( } } +fn checked_depth(variant: &VariantCounts) -> Result { + variant + .ref_count + .checked_add(variant.alt_count) + .with_context(|| { + format!( + "read depth exceeds UInt32 at {}:{} (region {})", + variant.chrom, variant.pos, variant.region + ) + }) +} + +fn checked_region_sum(mut values: impl Iterator, region: &str) -> Result { + values.try_fold(0_u32, |total, count| { + total + .checked_add(count) + .with_context(|| format!("aggregated read counts exceed UInt32 for region {region}")) + }) +} + fn raw_depth(variant: &VariantCounts, pseudocount: u32) -> Result { let raw_ref = variant.ref_count.checked_sub(pseudocount).ok_or_else(|| { anyhow::anyhow!("reference count is smaller than the configured pseudocount") @@ -986,10 +1003,7 @@ fn test_fixed_dispersion( } let ref_counts: Vec = variants.iter().map(|variant| variant.ref_count).collect(); - let n_array: Vec = variants - .iter() - .map(|variant| variant.ref_count + variant.alt_count) - .collect(); + let n_array: Vec = variants.iter().map(checked_depth).collect::>()?; let raw_depths: Vec = variants .iter() .map(|variant| raw_depth(variant, pseudocount)) @@ -1007,6 +1021,10 @@ fn test_fixed_dispersion( .push(i); } + for (region, indices) in ®ion_map { + checked_region_sum(indices.iter().map(|&i| n_array[i]), region)?; + } + eprintln!( "Optimizing imbalance likelihood for {} regions...", region_map.len() @@ -1064,9 +1082,10 @@ fn test_fixed_dispersion( let chi2 = ChiSquared::new(1.0).context("Failed to create chi-squared distribution")?; let pval = chi2.sf(lrt); - let total_ref: u32 = region_ref.iter().sum(); - let total_alt: u32 = indices.iter().map(|&i| variants[i].alt_count).sum(); - let total_n = total_ref + total_alt; + let total_ref = checked_region_sum(region_ref.iter().copied(), region)?; + let total_alt = + checked_region_sum(indices.iter().map(|&i| variants[i].alt_count), region)?; + let total_n = checked_region_sum([total_ref, total_alt].into_iter(), region)?; Ok(ImbalanceResult { region: region.clone(), @@ -1106,18 +1125,31 @@ fn prepare_analysis( variants: Vec, config: &AnalysisConfig, ) -> Result { - let filtered: Vec = variants - .into_iter() - .map(|mut v| { - v.ref_count += config.pseudocount; - v.alt_count += config.pseudocount; - v - }) - .filter(|v| { - let n = v.ref_count + v.alt_count; - n >= config.min_count + (2 * config.pseudocount) - }) - .collect(); + let threshold = config + .pseudocount + .checked_mul(2) + .and_then(|offset| config.min_count.checked_add(offset)) + .context("minimum count plus pseudocounts exceeds UInt32")?; + let mut filtered = Vec::with_capacity(variants.len()); + for mut variant in variants { + let context = || { + format!( + "pseudocounted allele count exceeds UInt32 at {}:{} (region {})", + variant.chrom, variant.pos, variant.region + ) + }; + variant.ref_count = variant + .ref_count + .checked_add(config.pseudocount) + .with_context(context)?; + variant.alt_count = variant + .alt_count + .checked_add(config.pseudocount) + .with_context(context)?; + if checked_depth(&variant)? >= threshold { + filtered.push(variant); + } + } eprintln!("Processing {} variants after filtering", filtered.len()); @@ -1126,7 +1158,7 @@ fn prepare_analysis( // Deduplicate variants for Python parity. // Python's get_imbalance() does: df[keep_cols].drop_duplicates() let before = filtered.len(); - let deduped = deduplicate_variants(filtered, phased); + let deduped = deduplicate_variants(filtered, phased)?; if deduped.len() < before { eprintln!( "Deduplicated {} → {} variants ({} duplicates removed)", @@ -1141,22 +1173,34 @@ fn prepare_analysis( }) } -fn remove_result_pseudocounts(results: &mut [ImbalanceResult], pseudocount: u32) { +fn remove_result_pseudocounts(results: &mut [ImbalanceResult], pseudocount: u32) -> Result<()> { for result in results { - let region_pseudocount = pseudocount.saturating_mul(result.snp_count as u32); - if result.ref_count < region_pseudocount - || result.alt_count < region_pseudocount - || result.n < 2 * region_pseudocount - { - eprintln!( - "[WARN] Counts smaller than pseudocount for region {}: ref={}, alt={}, n={}, pc={}", - result.region, result.ref_count, result.alt_count, result.n, region_pseudocount - ); - } - result.ref_count = result.ref_count.saturating_sub(region_pseudocount); - result.alt_count = result.alt_count.saturating_sub(region_pseudocount); - result.n = result.n.saturating_sub(2 * region_pseudocount); - } + let region_pseudocount = u32::try_from(result.snp_count) + .ok() + .and_then(|count| pseudocount.checked_mul(count)) + .with_context(|| { + format!( + "regional pseudocount exceeds UInt32 for region {}", + result.region + ) + })?; + let total_pseudocount = region_pseudocount + .checked_mul(2) + .context("regional depth pseudocount exceeds UInt32")?; + result.ref_count = result + .ref_count + .checked_sub(region_pseudocount) + .context("regional reference count is smaller than its pseudocount")?; + result.alt_count = result + .alt_count + .checked_sub(region_pseudocount) + .context("regional alternate count is smaller than its pseudocount")?; + result.n = result + .n + .checked_sub(total_pseudocount) + .context("regional depth is smaller than its pseudocount")?; + } + Ok(()) } fn analyze_prepared( @@ -1173,7 +1217,7 @@ fn analyze_prepared( exact_snv_pvalues, config.pseudocount, )?; - remove_result_pseudocounts(&mut results, config.pseudocount); + remove_result_pseudocounts(&mut results, config.pseudocount)?; Ok(AnalysisOutput { results, @@ -1207,7 +1251,7 @@ pub fn fit_imbalance_dispersion( let parameters = fit_dispersion_parameters(&prepared.variants, config.method, config.pseudocount)?; match parameters { - DispersionParameters::Single { rho } => eprintln!(" Dispersion: {:.6}", rho), + DispersionParameters::Single { rho } => eprintln!(" Dispersion: {rho:.6}"), DispersionParameters::Linear { d1, d2, @@ -1215,8 +1259,7 @@ pub fn fit_imbalance_dispersion( depth_scale, } => { eprintln!( - " d1 = {:.6}, d2 = {:.6}, depth center = {:.6}, depth scale = {:.6}", - d1, d2, depth_center, depth_scale + " d1 = {d1:.6}, d2 = {d2:.6}, depth center = {depth_center:.6}, depth scale = {depth_scale:.6}" ) } } @@ -1263,7 +1306,7 @@ pub fn analyze_imbalance_detailed( let parameters = fit_dispersion_parameters(&prepared.variants, config.method, config.pseudocount)?; match parameters { - DispersionParameters::Single { rho } => eprintln!(" Dispersion: {:.6}", rho), + DispersionParameters::Single { rho } => eprintln!(" Dispersion: {rho:.6}"), DispersionParameters::Linear { d1, d2, @@ -1271,8 +1314,7 @@ pub fn analyze_imbalance_detailed( depth_scale, } => { eprintln!( - " d1 = {:.6}, d2 = {:.6}, depth center = {:.6}, depth scale = {:.6}", - d1, d2, depth_center, depth_scale + " d1 = {d1:.6}, d2 = {d2:.6}, depth center = {depth_center:.6}, depth scale = {depth_scale:.6}" ) } } @@ -1326,11 +1368,7 @@ mod tests { // Test values very close to boundaries for rho in [1e-15, 1e-12, 1e-10, 0.999999999, 0.9999999999999] { let result = opt_prob(0.5, rho, 10, 20).unwrap(); - assert!( - result.is_finite(), - "rho={} should produce finite result", - rho - ); + assert!(result.is_finite(), "rho={rho} should produce finite result"); } } @@ -1524,9 +1562,7 @@ mod tests { let expected = opt_prob(0.5, 0.1, 10, 20).unwrap(); assert!( (result - expected).abs() < 1e-10, - "Single position: result={}, expected={}", - result, - expected + "Single position: result={result}, expected={expected}" ); } @@ -1583,6 +1619,53 @@ mod tests { } } + #[test] + fn test_prepare_analysis_rejects_overflow_without_wrapping() { + for (reference, alternate, pseudocount, minimum) in [ + (u32::MAX, 0, 1, 0), + (u32::MAX, 1, 0, 0), + (1, 1, u32::MAX, 0), + (1, 1, 1, u32::MAX), + ] { + let config = AnalysisConfig { + min_count: minimum, + pseudocount, + ..AnalysisConfig::default() + }; + let error = prepare_analysis(vec![vc_peak(1, reference, alternate, None)], &config) + .err() + .unwrap(); + assert!(error.to_string().contains("UInt32"), "{error}"); + } + let config = AnalysisConfig { + min_count: 0, + pseudocount: 0, + ..AnalysisConfig::default() + }; + let prepared = prepare_analysis(vec![vc_peak(1, u32::MAX - 1, 1, None)], &config).unwrap(); + assert_eq!(checked_depth(&prepared.variants[0]).unwrap(), u32::MAX); + } + + #[test] + fn test_region_depth_overflow_is_rejected_before_inference() { + let error = test_fixed_dispersion( + vec![ + vc_peak(1, u32::MAX / 2, 0, None), + vc_peak(2, u32::MAX / 2, 2, None), + ], + false, + DispersionParameters::Single { rho: 0.1 }, + false, + 0, + ) + .unwrap_err(); + assert!(error.to_string().contains("region peak1"), "{error}"); + assert_eq!( + checked_region_sum([u32::MAX - 1, 1].into_iter(), "peak1").unwrap(), + u32::MAX + ); + } + fn assert_result_close(a: &ImbalanceResult, b: &ImbalanceResult) { assert_eq!(a.region, b.region); assert_eq!(a.ref_count, b.ref_count); diff --git a/rust/src/bam_counter.rs b/rust/src/bam_counter.rs index 39796c74..6ac6594e 100644 --- a/rust/src/bam_counter.rs +++ b/rust/src/bam_counter.rs @@ -36,7 +36,7 @@ impl BamCounter { // Verify BAM file exists if !Path::new(&bam_path).exists() { return Err(PyErr::new::( - format!("BAM file not found: {}", bam_path), + format!("BAM file not found: {bam_path}"), )); } @@ -72,14 +72,12 @@ impl BamCounter { // Validate alleles are non-empty if ref_allele.is_empty() { return Err(PyErr::new::(format!( - "Empty ref_allele for variant at {}:{}", - chrom, pos + "Empty ref_allele for variant at {chrom}:{pos}" ))); } if alt_allele.is_empty() { return Err(PyErr::new::(format!( - "Empty alt_allele for variant at {}:{}", - chrom, pos + "Empty alt_allele for variant at {chrom}:{pos}" ))); } @@ -118,8 +116,7 @@ impl BamCounter { .build() .map_err(|e| { PyErr::new::(format!( - "Failed to create thread pool: {}", - e + "Failed to create thread pool: {e}" )) })? .install(|| { @@ -176,7 +173,7 @@ impl BamCounter { return self.process_chromosome_snps(chrom, regions, min_qual, debug_sites); } let mut bam = bam::IndexedReader::from_path(&self.bam_path).map_err(|e| { - PyErr::new::(format!("Failed to open BAM: {}", e)) + PyErr::new::(format!("Failed to open BAM: {e}")) })?; let mut seen_reads: FxHashSet> = FxHashSet::default(); @@ -236,7 +233,7 @@ impl BamCounter { Err(e) => { skipped_records += 1; if skipped_records <= MAX_SKIP_WARNINGS { - eprintln!("[WARN] Skipped corrupted BAM record on {}: {}", chrom, e); + eprintln!("[WARN] Skipped corrupted BAM record on {chrom}: {e}"); } continue; } diff --git a/rust/src/bam_counter_sc.rs b/rust/src/bam_counter_sc.rs index 1d8f0d76..fcaf9c1b 100644 --- a/rust/src/bam_counter_sc.rs +++ b/rust/src/bam_counter_sc.rs @@ -54,7 +54,7 @@ impl BamCounterSC { fn new(bam_path: String) -> PyResult { if !Path::new(&bam_path).exists() { return Err(PyErr::new::( - format!("BAM file not found: {}", bam_path), + format!("BAM file not found: {bam_path}"), )); } @@ -92,14 +92,12 @@ impl BamCounterSC { if ref_allele.is_empty() { return Err(PyErr::new::(format!( - "Empty ref_allele for variant at {}:{}", - chrom, pos + "Empty ref_allele for variant at {chrom}:{pos}" ))); } if alt_allele.is_empty() { return Err(PyErr::new::(format!( - "Empty alt_allele for variant at {}:{}", - chrom, pos + "Empty alt_allele for variant at {chrom}:{pos}" ))); } @@ -144,8 +142,7 @@ impl BamCounterSC { .build() .map_err(|e| { PyErr::new::(format!( - "Failed to create thread pool: {}", - e + "Failed to create thread pool: {e}" )) })? .install(|| { @@ -225,7 +222,7 @@ impl BamCounterSC { min_qual: u8, ) -> PyResult> { let mut bam = bam::IndexedReader::from_path(&self.bam_path).map_err(|e| { - PyErr::new::(format!("Failed to open BAM: {}", e)) + PyErr::new::(format!("Failed to open BAM: {e}")) })?; // Multithreaded BGZF decode. Defaults to 2 background decode threads, which @@ -265,10 +262,7 @@ impl BamCounterSC { let start = (region.pos.saturating_sub(1)) as i64; let end = region.pos as i64; if let Err(e) = bam.fetch((chrom, start, end)) { - eprintln!( - "[WARN] Failed to fetch {}:{}-{}: {}. Skipping variant.", - chrom, start, end, e - ); + eprintln!("[WARN] Failed to fetch {chrom}:{start}-{end}: {e}. Skipping variant."); continue; } @@ -281,7 +275,7 @@ impl BamCounterSC { Err(e) => { skipped_records += 1; if skipped_records <= MAX_SKIP_WARNINGS { - eprintln!("[WARN] Skipped corrupted BAM record on {}: {}", chrom, e); + eprintln!("[WARN] Skipped corrupted BAM record on {chrom}: {e}"); } continue; } diff --git a/rust/src/bam_filter.rs b/rust/src/bam_filter.rs index 6d424f62..19b9cb59 100644 --- a/rust/src/bam_filter.rs +++ b/rust/src/bam_filter.rs @@ -270,7 +270,7 @@ pub fn filter_bam_by_variants( // Phase 1: Build variant store (reuse from bam_intersect) let t0 = Instant::now(); - eprintln!("Phase 1: Building variant store from {}...", bed_path); + eprintln!("Phase 1: Building variant store from {bed_path}..."); let store = build_variant_store(bed_path)?; stats.phase1_ms = t0.elapsed().as_millis() as u64; eprintln!( diff --git a/rust/src/bam_intersect.rs b/rust/src/bam_intersect.rs index 8e675bab..1b20c454 100644 --- a/rust/src/bam_intersect.rs +++ b/rust/src/bam_intersect.rs @@ -151,7 +151,7 @@ pub fn build_variant_store(bed_path: &str) -> Result { .map(|(chrom, intervals)| { let interval_count = intervals.len(); let tree = COITree::new(&intervals); - eprintln!(" {}: {} variants", chrom, interval_count); + eprintln!(" {chrom}: {interval_count} variants"); (chrom, tree) }) .collect(); @@ -296,8 +296,7 @@ pub fn intersect_bam_with_store( writer.flush()?; eprintln!( - " Processed {} reads, {} with overlaps, {} total intersections", - read_count, reads_with_overlaps, intersection_count + " Processed {read_count} reads, {reads_with_overlaps} with overlaps, {intersection_count} total intersections" ); Ok(intersection_count) @@ -319,7 +318,7 @@ pub fn intersect_bam_with_variants( bed_path: &str, out_path: &str, ) -> Result { - eprintln!("Building variant store from {}...", bed_path); + eprintln!("Building variant store from {bed_path}..."); let store = build_variant_store(bed_path)?; eprintln!( " {} chromosomes, {} total variants", @@ -329,7 +328,7 @@ pub fn intersect_bam_with_variants( eprintln!("Intersecting reads with variants..."); let count = intersect_bam_with_store(bam_path, &store, out_path)?; - eprintln!(" {} intersections found", count); + eprintln!(" {count} intersections found"); Ok(count) } @@ -532,7 +531,7 @@ pub fn intersect_bam_with_store_multi( // Write all sample genotypes for gt in &info.sample_genotypes { - write!(writer, "\t{}", gt).ok(); + write!(writer, "\t{gt}").ok(); } writeln!(writer).ok(); @@ -557,10 +556,7 @@ pub fn intersect_bam_with_variants_multi( out_path: &str, num_samples: usize, ) -> Result { - eprintln!( - "Building multi-sample variant store from {} ({} samples)...", - bed_path, num_samples - ); + eprintln!("Building multi-sample variant store from {bed_path} ({num_samples} samples)..."); let store = build_variant_store_multi(bed_path, num_samples)?; eprintln!( " {} chromosomes, {} total variants", @@ -570,7 +566,7 @@ pub fn intersect_bam_with_variants_multi( eprintln!("Intersecting reads with variants (multi-sample)..."); let count = intersect_bam_with_store_multi(bam_path, &store, out_path)?; - eprintln!(" {} intersections found", count); + eprintln!(" {count} intersections found"); Ok(count) } diff --git a/rust/src/bam_remapper.rs b/rust/src/bam_remapper.rs index 527ffaaf..cfcfd9e9 100644 --- a/rust/src/bam_remapper.rs +++ b/rust/src/bam_remapper.rs @@ -472,7 +472,7 @@ pub fn swap_alleles_for_chrom( let header = bam.header().clone(); let tid = header .tid(chrom.as_bytes()) - .ok_or_else(|| anyhow::anyhow!("Chromosome {} not found in BAM", chrom))?; + .ok_or_else(|| anyhow::anyhow!("Chromosome {chrom} not found in BAM"))?; bam.fetch(tid as i32) .context("Failed to fetch chromosome")?; @@ -1957,11 +1957,7 @@ fn generate_wasp_name( total_seqs: usize, ) -> Vec { let name_str = std::str::from_utf8(original_name).unwrap_or("unknown"); - format!( - "{}_WASP_{}_{}_{}_{}", - name_str, pos1, pos2, seq_num, total_seqs - ) - .into_bytes() + format!("{name_str}_WASP_{pos1}_{pos2}_{seq_num}_{total_seqs}").into_bytes() } // ============================================================================ @@ -2053,30 +2049,6 @@ mod tests { .any(|s| s.vcf_start == 87401 && s.vcf_stop == 87402)); } - #[test] - #[ignore] - fn test_generate_haplotype_seqs() { - // TODO: Create mock BAM record - // TODO: Create test variants - // TODO: Generate haplotypes - // TODO: Verify sequences are correct - } - - #[test] - #[ignore] - fn test_build_alignment_map() { - // TODO: Create read with known alignment - // TODO: Build map - // TODO: Verify positions are correct - } - - #[test] - #[ignore] - fn test_generate_wasp_name() { - // TODO: Generate name with test inputs - // TODO: Verify format matches Python implementation - } - // ============================================================================ // INDEL Trim Combination Tests // ============================================================================ diff --git a/rust/src/cigar_utils.rs b/rust/src/cigar_utils.rs index 0ff98a66..fba37ef5 100644 --- a/rust/src/cigar_utils.rs +++ b/rust/src/cigar_utils.rs @@ -267,7 +267,7 @@ pub fn apply_cigar_aware_substitution( let query_start = ref2query_left .get(&ref_start) .copied() - .ok_or_else(|| anyhow::anyhow!("Ref position {} not in left map", ref_start))?; + .ok_or_else(|| anyhow::anyhow!("Ref position {ref_start} not in left map"))?; // For end position, we want the position AFTER the last ref base // ref_end is exclusive, so we look up ref_end - 1 and add 1 diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 81e0e068..5329478f 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -66,7 +66,7 @@ fn parse_intersect_bed(py: Python, intersect_bed: &str) -> PyResult> { // Call Rust parser let variants = bam_remapper::parse_intersect_bed(intersect_bed) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse BED: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse BED: {e}")))?; // Convert to Python dict let py_dict = PyDict::new(py); @@ -136,16 +136,16 @@ fn remap_chromosome( // Parse intersection file let variants = bam_remapper::parse_intersect_bed(intersect_bed) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse BED: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse BED: {e}")))?; // Process chromosome let (haplotypes, stats) = bam_remapper::swap_alleles_for_chrom(bam_path, &variants, chrom, &config) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to swap alleles: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to swap alleles: {e}")))?; // Write FASTQ files let (_r1_count, _r2_count) = bam_remapper::write_fastq_pair(&haplotypes, out_r1, out_r2) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to write FASTQ: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to write FASTQ: {e}")))?; Ok((stats.pairs_processed, stats.haplotypes_generated)) } @@ -185,15 +185,12 @@ fn remap_all_chromosomes( // Parse intersect file ONCE, grouped by chromosome // This is the key optimization: 22x fewer parse operations for RNA-seq let variants_by_chrom = bam_remapper::parse_intersect_bed_by_chrom(intersect_bed) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse intersect BED: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse intersect BED: {e}")))?; // Report chromosome count let num_chroms = variants_by_chrom.len(); let total_reads: usize = variants_by_chrom.values().map(|v| v.len()).sum(); - eprintln!( - "Parsed {} chromosomes with {} reads from intersect file", - num_chroms, total_reads - ); + eprintln!("Parsed {num_chroms} chromosomes with {total_reads} reads from intersect file"); let stats = if parallel { // Use streaming parallel version with crossbeam channels @@ -203,8 +200,7 @@ fn remap_all_chromosomes( rayon::current_num_threads() }; eprintln!( - "Processing {} chromosomes in parallel ({} threads) with streaming writes...", - num_chroms, effective_threads + "Processing {num_chroms} chromosomes in parallel ({effective_threads} threads) with streaming writes..." ); bam_remapper::process_and_write_parallel( @@ -215,18 +211,18 @@ fn remap_all_chromosomes( out_r2, num_threads, ) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to process chromosomes: {}", e)))? + .map_err(|e| PyRuntimeError::new_err(format!("Failed to process chromosomes: {e}")))? } else { - eprintln!("Processing {} chromosomes sequentially...", num_chroms); + eprintln!("Processing {num_chroms} chromosomes sequentially..."); let (haplotypes, stats) = bam_remapper::process_all_chromosomes_sequential(bam_path, &variants_by_chrom, &config) .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to process chromosomes: {}", e)) + PyRuntimeError::new_err(format!("Failed to process chromosomes: {e}")) })?; // Write FASTQ output files bam_remapper::write_fastq_pair(&haplotypes, out_r1, out_r2) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to write FASTQ: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to write FASTQ: {e}")))?; stats }; @@ -676,7 +672,7 @@ fn analyze_imbalance_run( #[pyfunction] fn intersect_bam_bed(bam_path: &str, bed_path: &str, out_path: &str) -> PyResult { bam_intersect::intersect_bam_with_variants(bam_path, bed_path, out_path) - .map_err(|e| PyRuntimeError::new_err(format!("Intersect failed: {}", e))) + .map_err(|e| PyRuntimeError::new_err(format!("Intersect failed: {e}"))) } /// Intersect BAM reads with multi-sample variant BED file @@ -697,7 +693,7 @@ fn intersect_bam_bed_multi( num_samples: usize, ) -> PyResult { bam_intersect::intersect_bam_with_variants_multi(bam_path, bed_path, out_path, num_samples) - .map_err(|e| PyRuntimeError::new_err(format!("Multi-sample intersect failed: {}", e))) + .map_err(|e| PyRuntimeError::new_err(format!("Multi-sample intersect failed: {e}"))) } // ============================================================================ @@ -756,7 +752,7 @@ fn filter_bam_by_variants_py( is_paired, threads, ) - .map_err(|e| PyRuntimeError::new_err(format!("BAM filter failed: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("BAM filter failed: {e}")))?; Ok(( stats.remap_reads, @@ -871,7 +867,7 @@ fn unified_make_reads_py( }; let stats = unified_pipeline::unified_make_reads(bam_path, bed_path, out_r1, out_r2, &config) - .map_err(|e| PyRuntimeError::new_err(format!("Unified pipeline failed: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Unified pipeline failed: {e}")))?; stats_to_pydict(py, &stats) } @@ -955,13 +951,13 @@ fn unified_make_reads_parallel_py( .num_threads(threads) .build() .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to build Rayon thread pool: {}", e)) + PyRuntimeError::new_err(format!("Failed to build Rayon thread pool: {e}")) })?; pool.install(run) } else { run() } - .map_err(|e| PyRuntimeError::new_err(format!("Parallel unified pipeline failed: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Parallel unified pipeline failed: {e}")))?; stats_to_pydict(py, &stats) } @@ -1020,7 +1016,7 @@ fn vcf_to_bed_py( }; vcf_to_bed::vcf_to_bed(vcf_path, bed_path, &config) - .map_err(|e| PyRuntimeError::new_err(format!("VCF to BED failed: {}", e))) + .map_err(|e| PyRuntimeError::new_err(format!("VCF to BED failed: {e}"))) } // ============================================================================ @@ -1053,7 +1049,7 @@ fn parse_intersect_bed_multi( use pyo3::types::{PyDict, PyList}; let variants = multi_sample::parse_intersect_bed_multi(intersect_bed, num_samples) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse multi-sample BED: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse multi-sample BED: {e}")))?; // Convert to Python dict let py_dict = PyDict::new(py); @@ -1133,13 +1129,13 @@ fn remap_chromosome_multi( ) -> PyResult<(usize, usize)> { // Parse multi-sample intersection file let variants = multi_sample::parse_intersect_bed_multi(intersect_bed, num_samples) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse multi-sample BED: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to parse multi-sample BED: {e}")))?; // Process chromosome let stats = multi_sample::swap_alleles_for_chrom_multi( bam_path, &variants, chrom, out_r1, out_r2, max_seqs, ) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to swap alleles: {}", e)))?; + .map_err(|e| PyRuntimeError::new_err(format!("Failed to swap alleles: {e}")))?; Ok((stats.pairs_processed, stats.haplotypes_generated)) } @@ -1261,8 +1257,7 @@ mod tests { assert_eq!( parse_numeric_phased_gt(gt), None, - "{} should not be treated as phased GT", - gt + "{gt} should not be treated as phased GT" ); } } diff --git a/rust/src/mapping_filter.rs b/rust/src/mapping_filter.rs index 32ea7d7e..9fac60da 100644 --- a/rust/src/mapping_filter.rs +++ b/rust/src/mapping_filter.rs @@ -102,10 +102,7 @@ fn parse_wasp_name(qname: &[u8]) -> Option> { /// Check if remapped positions match expected positions (mate-order agnostic) fn positions_match(rec_pos: i64, mate_pos: i64, exp_pos1: i64, exp_pos2: i64, slop: i64) -> bool { if slop < 0 { - eprintln!( - "[WARN] positions_match: negative slop ({}), clamping to 0", - slop - ); + eprintln!("[WARN] positions_match: negative slop ({slop}), clamping to 0"); } let slop = slop.max(0); if slop == 0 { @@ -151,8 +148,7 @@ pub fn filter_bam_wasp( if let Some(sidecar_path) = expected_sidecar.as_ref() { let file = std::fs::File::open(sidecar_path).map_err(|e| { PyErr::new::(format!( - "Failed to open sidecar {}: {}", - sidecar_path, e + "Failed to open sidecar {sidecar_path}: {e}" )) })?; let mut reader = BufReader::new(file); @@ -163,8 +159,7 @@ pub fn filter_bam_wasp( buf.clear(); let n = reader.read_until(b'\n', &mut buf).map_err(|e| { PyErr::new::(format!( - "Failed to read sidecar {}: {}", - sidecar_path, e + "Failed to read sidecar {sidecar_path}: {e}" )) })?; if n == 0 { @@ -212,7 +207,7 @@ pub fn filter_bam_wasp( let mut read_buffer: FxHashMap, BufferedRead> = FxHashMap::default(); let mut remapped_reader = bam::Reader::from_path(&remapped_bam).map_err(|e| { - PyErr::new::(format!("Failed to open remapped BAM: {}", e)) + PyErr::new::(format!("Failed to open remapped BAM: {e}")) })?; if threads > 1 { let _ = remapped_reader.set_threads(threads); @@ -321,8 +316,7 @@ pub fn filter_bam_wasp( if let Some(path) = keep_read_file.as_ref() { let mut file = std::fs::File::create(path).map_err(|e| { PyErr::new::(format!( - "Failed to create keep_read_file: {}", - e + "Failed to create keep_read_file: {e}" )) })?; for name in keep_set.iter() { @@ -331,8 +325,7 @@ pub fn filter_bam_wasp( .and_then(|_| file.write_all(b"\n")) .map_err(|e| { PyErr::new::(format!( - "Failed to write keep_read_file: {}", - e + "Failed to write keep_read_file: {e}" )) })?; } @@ -340,7 +333,7 @@ pub fn filter_bam_wasp( // Write filtered BAM from original to_remap input let mut to_reader = bam::Reader::from_path(&to_remap_bam).map_err(|e| { - PyErr::new::(format!("Failed to open to_remap BAM: {}", e)) + PyErr::new::(format!("Failed to open to_remap BAM: {e}")) })?; if threads > 1 { let _ = to_reader.set_threads(threads); @@ -349,8 +342,7 @@ pub fn filter_bam_wasp( let mut writer = Writer::from_path(&remap_keep_bam, &header, bam::Format::Bam).map_err(|e| { PyErr::new::(format!( - "Failed to create remap_keep_bam: {}", - e + "Failed to create remap_keep_bam: {e}" )) })?; if threads > 1 { @@ -365,7 +357,7 @@ pub fn filter_bam_wasp( }; if keep_set.contains(rec.qname()) { writer.write(&rec).map_err(|e| { - PyErr::new::(format!("Write failed: {}", e)) + PyErr::new::(format!("Write failed: {e}")) })?; kept_written += 1; } diff --git a/rust/src/multi_sample.rs b/rust/src/multi_sample.rs index 111bbfa5..557637f3 100644 --- a/rust/src/multi_sample.rs +++ b/rust/src/multi_sample.rs @@ -330,8 +330,7 @@ pub fn apply_allele_substitutions_cigar_aware( // Get query positions using CIGAR-aware mapping let query_start = ref2query_left.get(&ref_start).copied().ok_or_else(|| { anyhow::anyhow!( - "Variant at ref {} not in left map (read may not cover variant)", - ref_start + "Variant at ref {ref_start} not in left map (read may not cover variant)" ) })?; @@ -501,7 +500,7 @@ pub fn generate_multi_sample_sequences_from_record( Ok((new_seq, new_qual)) => results.push((new_seq, new_qual)), Err(e) => { // Log error but continue - variant may not overlap read properly - eprintln!("Warning: failed to apply substitution: {}", e); + eprintln!("Warning: failed to apply substitution: {e}"); continue; } } @@ -596,7 +595,7 @@ pub fn swap_alleles_for_chrom_multi( let tid = match header.tid(chrom.as_bytes()) { Some(t) => t, None => { - eprintln!(" Chromosome {} not found in BAM, skipping", chrom); + eprintln!(" Chromosome {chrom} not found in BAM, skipping"); return Ok(stats); } }; @@ -772,7 +771,7 @@ fn write_fastq_record( seq: &[u8], qual: &[u8], ) -> Result<()> { - writeln!(writer, "@{}", name)?; + writeln!(writer, "@{name}")?; writer.write_all(seq)?; writeln!(writer)?; writeln!(writer, "+")?; diff --git a/rust/src/read_pairer.rs b/rust/src/read_pairer.rs index 3ea6b828..0468a726 100644 --- a/rust/src/read_pairer.rs +++ b/rust/src/read_pairer.rs @@ -221,37 +221,3 @@ pub fn pair_reads_from_chromosome( // ============================================================================ // Tests // ============================================================================ - -#[cfg(test)] -mod tests { - #[test] - #[ignore] // Remove when implemented - fn test_read_pairer_basic() { - // TODO: Create test BAM file - // TODO: Pair reads - // TODO: Verify pairs are correct - } - - #[test] - #[ignore] - fn test_read_pairer_stats() { - // TODO: Create test BAM with various read types - // TODO: Pair with statistics enabled - // TODO: Verify stats are accurate - } - - #[test] - #[ignore] - fn test_filters() { - // TODO: Test each filter individually - // TODO: Verify discarded reads are counted correctly - } - - #[test] - #[ignore] - fn test_chromosome_specific() { - // TODO: Create BAM with multiple chromosomes - // TODO: Pair only chr10 - // TODO: Verify only chr10 pairs returned - } -} diff --git a/rust/src/unified_pipeline.rs b/rust/src/unified_pipeline.rs index 877fc311..35677d4c 100644 --- a/rust/src/unified_pipeline.rs +++ b/rust/src/unified_pipeline.rs @@ -657,8 +657,7 @@ fn process_pair( // Invariant: total_seqs > 0 guarantees at least one output. // If violated, log and fall back rather than propagate empty NeedsRemap. eprintln!( - "[WARN] process_pair: outputs empty despite total_seqs={}; treating as KeepAsIs", - total_seqs + "[WARN] process_pair: outputs empty despite total_seqs={total_seqs}; treating as KeepAsIs" ); return ProcessPairResult::KeepAsIs; } @@ -1083,7 +1082,7 @@ pub fn unified_make_reads( // Phase 1: Build variant store let t0 = Instant::now(); - eprintln!("Building variant store from {}...", bed_path); + eprintln!("Building variant store from {bed_path}..."); let store = build_variant_store(bed_path)?; stats.tree_build_ms = t0.elapsed().as_millis() as u64; eprintln!( @@ -1104,7 +1103,7 @@ pub fn unified_make_reads( // Spawn writer thread (with optional compression) let r1_owned = r1_path.to_string(); let r2_owned = r2_path.to_string(); - let sidecar_owned = format!("{}.expected_positions.tsv", r1_owned); + let sidecar_owned = format!("{r1_owned}.expected_positions.tsv"); let compression_threads = config.compression_threads; let compress = config.compress_output; let writer_handle = thread::spawn(move || { @@ -1684,12 +1683,9 @@ pub fn unified_make_reads_parallel( let enable_timing = std::env::var_os("WASP2_TIMING").is_some(); // Check BAM index exists - fall back to sequential if not - let bai_path = format!("{}.bai", bam_path); + let bai_path = format!("{bam_path}.bai"); if !std::path::Path::new(&bai_path).exists() { - eprintln!( - "BAM index not found ({}), falling back to sequential processing", - bai_path - ); + eprintln!("BAM index not found ({bai_path}), falling back to sequential processing"); return unified_make_reads(bam_path, bed_path, r1_path, r2_path, config); } @@ -1702,7 +1698,7 @@ pub fn unified_make_reads_parallel( // Phase 1: Build variant store (shared, read-only) let t0 = Instant::now(); - eprintln!("Building variant store from {}...", bed_path); + eprintln!("Building variant store from {bed_path}..."); let store = Arc::new(build_variant_store(bed_path)?); let tree_build_ms = t0.elapsed().as_millis() as u64; eprintln!( @@ -1735,7 +1731,7 @@ pub fn unified_make_reads_parallel( let r1_owned = r1_path.to_string(); let r2_owned = r2_path.to_string(); - let sidecar_owned = format!("{}.expected_positions.tsv", r1_owned); + let sidecar_owned = format!("{r1_owned}.expected_positions.tsv"); let compression_threads = config.compression_threads; let compress = config.compress_output; let writer_handle = thread::spawn(move || { @@ -1784,7 +1780,7 @@ pub fn unified_make_reads_parallel( final_stats = final_stats.merge(stats); } Err(e) => { - eprintln!("Warning: Chromosome processing failed: {}", e); + eprintln!("Warning: Chromosome processing failed: {e}"); } } } diff --git a/rust/src/vcf_to_bed.rs b/rust/src/vcf_to_bed.rs index 5476ddb8..af06a504 100644 --- a/rust/src/vcf_to_bed.rs +++ b/rust/src/vcf_to_bed.rs @@ -192,10 +192,7 @@ fn vcf_to_bed_from_reader( } writer.flush()?; - eprintln!( - " Processed {} records, wrote {} variants to BED", - total_records, variant_count - ); + eprintln!(" Processed {total_records} records, wrote {variant_count} variants to BED"); Ok(variant_count) } @@ -269,11 +266,7 @@ fn process_vcf_record( } } - writeln!( - writer, - "{}\t{}\t{}\t{}\t{}", - chrom, pos0, end, ref_allele, alt_allele - )?; + writeln!(writer, "{chrom}\t{pos0}\t{end}\t{ref_allele}\t{alt_allele}")?; written += 1; } return Ok(Some(written)); @@ -342,15 +335,10 @@ fn process_vcf_record( if config.include_genotypes { writeln!( writer, - "{}\t{}\t{}\t{}\t{}\t{}", - chrom, pos0, end, ref_allele, alt_allele, gt_string + "{chrom}\t{pos0}\t{end}\t{ref_allele}\t{alt_allele}\t{gt_string}" )?; } else { - writeln!( - writer, - "{}\t{}\t{}\t{}\t{}", - chrom, pos0, end, ref_allele, alt_allele - )?; + writeln!(writer, "{chrom}\t{pos0}\t{end}\t{ref_allele}\t{alt_allele}")?; } written += 1; @@ -383,7 +371,7 @@ fn get_genotype_indices( // Convert value to string via Debug format and extract genotype // Note: noodles Value doesn't implement Display; Debug gives e.g. String("0|1") - let gt_string = format!("{:?}", gt_value); + let gt_string = format!("{gt_value:?}"); let gt_clean = extract_genotype_string(>_string); // Check for missing genotype diff --git a/scripts/compare_archive_analysis.py b/scripts/compare_archive_analysis.py index 0bafbbee..080aa2ae 100644 --- a/scripts/compare_archive_analysis.py +++ b/scripts/compare_archive_analysis.py @@ -9,6 +9,7 @@ from __future__ import annotations +import ast import contextlib import io import subprocess @@ -27,11 +28,30 @@ ROOT = Path(__file__).resolve().parents[1] ARCHIVE_REF = "41354fc" STAT_COLUMNS = ["null_ll", "alt_ll", "mu", "lrt", "pval", "fdr_pval"] -COUNT_COLUMNS = ["ref_count", "alt_count", "N", "snp_count", "num_snps"] - - -def load_archive(ref: str = ARCHIVE_REF) -> dict[str, types.ModuleType]: - """Load unmodified historical modules, restoring their legacy import names.""" +COUNT_COLUMNS = [ + "ref_count", + "alt_count", + "N", + "snp_count", + "num_snps", + "num_snps_group1", + "num_snps_group2", +] + + +def load_archive( + ref: str = ARCHIVE_REF, + *, + widen_single_cell_sums: bool = False, + widen_bulk_csv_counts: bool = False, +) -> dict[str, types.ModuleType]: + """Load historical modules with separately selectable count-width adapters. + + The explicit dtype repair prevents count overflow and SciPy float32 dispatch. + Single-cell changes only sum accumulator keywords; bulk CSV changes only + the three count-column reader dtypes. Neither changes statistical algorithms. + The default executes the unmodified archive. + """ names = ["as_analysis", "as_analysis_sc", "compare_ai"] saved = {name: sys.modules.get(name) for name in names} modules = {} @@ -43,7 +63,60 @@ def load_archive(ref: str = ARCHIVE_REF) -> dict[str, types.ModuleType]: module = types.ModuleType(f"archive_{name}") modules[name] = module sys.modules[name] = module - exec(compile(source, f"{ref}:src/analysis/{name}.py", "exec"), module.__dict__) + tree = ast.parse(source) + if widen_bulk_csv_counts and name == "as_analysis": + changed = set() + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "get_imbalance" + ) + for node in ast.walk(function): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "pd" + and node.func.attr == "read_csv" + ): + continue + for keyword in node.keywords: + if keyword.arg != "dtype" or not isinstance(keyword.value, ast.Dict): + continue + for key, value in zip( + keyword.value.keys, keyword.value.values, strict=True + ): + if ( + isinstance(key, ast.Constant) + and key.value in {"ref_count", "alt_count", "other_count"} + and isinstance(value, ast.Attribute) + and isinstance(value.value, ast.Name) + and value.value.id == "np" + and value.attr == "uint16" + ): + value.attr = "uint64" + changed.add(key.value) + if changed != {"ref_count", "alt_count", "other_count"}: + raise ValueError("Cannot locate the three archived bulk CSV count dtypes") + if widen_single_cell_sums and name in {"as_analysis_sc", "compare_ai"}: + for node in ast.walk(tree): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "sum" + ): + continue + for keyword in node.keywords: + value = keyword.value + if ( + keyword.arg == "dtype" + and isinstance(value, ast.Attribute) + and isinstance(value.value, ast.Name) + and value.value.id == "np" + and value.attr == "uint16" + ): + value.attr = "uint64" + exec(compile(tree, f"{ref}:src/analysis/{name}.py", "exec"), module.__dict__) finally: for name, module in saved.items(): if module is None: @@ -219,12 +292,24 @@ def compare_frames(expected: pd.DataFrame, observed: pd.DataFrame, key: str = "r if column not in observed: failures.append(f"missing {column}") continue - left = expected[column].to_numpy(dtype=float) - right = observed[column].to_numpy(dtype=float) - deltas[column] = float(np.max(np.abs(left - right))) if len(left) else 0.0 if column in COUNT_COLUMNS: - equal = np.array_equal(left, right) + # Python integers retain UInt64 values and signed/unsigned differences. + # Casting first to float64 hides one-count changes above 2**53. + left_counts = expected[column].tolist() + right_counts = observed[column].tolist() + valid = all( + pd.notna(a) and pd.notna(b) for a, b in zip(left_counts, right_counts, strict=True) + ) + equal = valid and left_counts == right_counts + deltas[column] = ( + max((abs(a - b) for a, b in zip(left_counts, right_counts, strict=True)), default=0) + if valid + else float("nan") + ) else: + left = expected[column].to_numpy(dtype=float) + right = observed[column].to_numpy(dtype=float) + deltas[column] = float(np.max(np.abs(left - right))) if len(left) else 0.0 equal = ( np.all(np.isfinite(left)) and np.all(np.isfinite(right)) @@ -242,6 +327,81 @@ def compare_frames(expected: pd.DataFrame, observed: pd.DataFrame, key: str = "r } +def classify_reference_differences( + legacy: pd.DataFrame, widened: pd.DataFrame, observed: pd.DataFrame, key: str = "region" +) -> dict: + """Retain strict failures and localize changes caused by a count-width adapter. + + A complementary estimate is only a candidate: equivalence requires a separate + check of the actual anchored objective. This helper never alters an output. + """ + strict = compare_frames(legacy, observed, key) + corrected = compare_frames(widened, observed, key) + frames = [ + frame.set_index(key, drop=False).rename_axis(None) for frame in (legacy, widened, observed) + ] + if any(not frame.index.is_unique for frame in frames): + raise ValueError("Reference classification requires unique result identities") + rows = [] + for identity in sorted(set().union(*(frame.index for frame in frames))): + present = [identity in frame.index for frame in frames] + if not all(present): + rows.append( + { + "identity": identity, + "stage": "tested_identity", + "present": present, + "reference_width_explains": present[1:] == [True, True] + and corrected["status"] == "PASS", + } + ) + continue + parts = [frame.loc[[identity]] for frame in frames] + comparison = compare_frames(parts[0], parts[2], key) + if comparison["status"] == "PASS": + continue + fields = comparison.get("failures", []) + stages = [] + if any(field in COUNT_COLUMNS for field in fields): + stages.append("count_precision") + if any(field in fields for field in ("null_ll", "alt_ll", "lrt")): + stages.append("likelihood_precision") + effects = [field for field in ("mu", "mu1", "mu2", "combined_mu") if field in fields] + if effects: + stages.append("effect_estimate_precision") + if "pval" in fields: + stages.append("p_value_precision") + if "fdr_pval" in fields or any(field.startswith("significance at") for field in fields): + stages.append("multiple_testing") + candidates = [ + field + for field in effects + if np.isclose( + float(parts[0][field].iloc[0]) + float(parts[2][field].iloc[0]), + 1.0, + rtol=5e-5, + atol=5e-5, + ) + ] + rows.append( + { + "identity": identity, + "stages": stages, + "fields": fields, + "complementary_estimate_candidates": candidates, + "reference_width_explains": compare_frames(parts[1], parts[2], key)["status"] + == "PASS", + "max_absolute_differences": comparison["max_absolute_differences"], + } + ) + return { + "legacy_strict_comparison": strict, + "widened_comparison": corrected, + "mismatched_rows": rows, + "unexplained_rows": sum(not row["reference_width_explains"] for row in rows), + } + + def prepare_bulk_reference_counts( counts: pd.DataFrame, *, diff --git a/scripts/release_version.py b/scripts/release_version.py index 4db6c4d1..0f67dce8 100755 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -71,11 +71,15 @@ def set_version(version: str, release_date: str) -> None: ) replace_one("pixi.toml", rf'^(version\s*=\s*)"{SEMVER}"', rf'\1"{version}"') replace_one("CITATION.cff", rf'^(version:\s*)"{SEMVER}"', rf'\1"{version}"') - replace_one( - "CITATION.cff", - r'^(date-released:\s*)"[0-9]{4}-[0-9]{2}-[0-9]{2}"', - rf'\1"{release_date}"', - ) + citation = read("CITATION.cff") + date_line = f'date-released: "{release_date}"' + if re.search(r"^date-released:.*$", citation, flags=re.MULTILINE): + citation = re.sub(r"^date-released:.*$", date_line, citation, flags=re.MULTILINE) + else: + citation = re.sub( + r"^(version:.*)$", rf"\1\n{date_line}", citation, count=1, flags=re.MULTILINE + ) + write("CITATION.cff", citation) replace_one("Dockerfile", rf"^(ARG VERSION=){SEMVER}", rf"\g<1>{version}") replace_one( "Singularity.def", @@ -110,6 +114,12 @@ def check_version() -> list[str]: expected = cargo_version() errors: list[str] = [] + lock_match = re.search( + r'^name = "wasp2"\nversion = "([^\"]+)"', read("rust/Cargo.lock"), flags=re.MULTILINE + ) + if lock_match is None or lock_match.group(1) != expected: + errors.append("rust/Cargo.lock root package version differs from rust/Cargo.toml") + pixi_version = toml_section_version("pixi.toml", "project") if pixi_version != expected: errors.append(f"pixi.toml: {pixi_version} != {expected}") diff --git a/src/analysis/__main__.py b/src/analysis/__main__.py index a34b230c..3bbeaef0 100644 --- a/src/analysis/__main__.py +++ b/src/analysis/__main__.py @@ -149,7 +149,7 @@ def find_imbalance( "--parent", help=( "Report allelic imbalance by parent group instead of feature level in RNA-seq counts. " - "Name of parent column. Not valid if no parent column or if using ATAC-seq peaks. " + "Name of parent column; groups SNPs in Rust before deduplication and fitting. " "(Default: Report by feature level instead of parent level)" ), ), @@ -306,9 +306,7 @@ def find_imbalance_sc( try: from .run_analysis_sc import run_ai_analysis_sc except (ImportError, ModuleNotFoundError) as error: - raise typer.BadParameter( - "Single-cell analysis requires: pip install 'wasp2[single-cell]'" - ) from error + raise typer.BadParameter("Single-cell analysis requires: pip install wasp2") from error run_ai_analysis_sc( count_file=counts, @@ -404,9 +402,7 @@ def compare_imbalance( try: from .run_compare_ai import run_ai_comparison except (ImportError, ModuleNotFoundError) as error: - raise typer.BadParameter( - "Single-cell comparison requires: pip install 'wasp2[single-cell]'" - ) from error + raise typer.BadParameter("Single-cell comparison requires: pip install wasp2") from error run_ai_comparison( count_file=counts, diff --git a/src/analysis/as_analysis.py b/src/analysis/as_analysis.py index fcd5a4a1..ae7b1bcd 100644 --- a/src/analysis/as_analysis.py +++ b/src/analysis/as_analysis.py @@ -173,6 +173,19 @@ def opt_unphased_dp( # Get likelihood of first pos first_ll = opt_prob(prob, first_disp, first_ref[0], first_n[0]) + # Preserve the archived sequential arithmetic while products remain normal. + # Equivalent log-space operations can select another symmetric optimum. + forward = np.atleast_1d(opt_prob(prob, phase_disp, phase_ref, phase_n, log=False)) + reverse = np.atleast_1d(opt_prob(1 - prob, phase_disp, phase_ref, phase_n, log=False)) + previous = 1.0 + for p1, p2 in zip(forward, reverse, strict=True): + phase1, phase2 = previous * p1, previous * p2 + previous = 0.5 * phase1 + 0.5 * phase2 + if previous < np.finfo(float).tiny: + break + else: + return float(first_ll - np.log(previous)) + # Get LOG likelihood with regard to phasing of first pos. # opt_prob(..., log=True) returns the NEGATIVE log-pmf, so negate to get the # log-pmf itself for stable log-space accumulation. diff --git a/src/analysis/as_analysis_sc.py b/src/analysis/as_analysis_sc.py index e42ede4b..fcfaee40 100644 --- a/src/analysis/as_analysis_sc.py +++ b/src/analysis/as_analysis_sc.py @@ -3,7 +3,7 @@ Provides functions for analyzing allelic imbalance in single-cell data stored in AnnData format with SNP counts in layers. -Orientation contract (scverse convention, see ISSUE2_ORIENTATION_DECISION §2b): +Orientation contract (scverse convention): obs = cells (barcodes); var = SNPs (variants). @@ -126,7 +126,7 @@ def get_imbalance_sc( n_counts = ref_counts + alt_counts # Calculate dispersion across dataset - def opt_disp(rho: float, ref_data: NDArray[np.uint16], n_data: NDArray[np.uint16]) -> float: + def opt_disp(rho: float, ref_data: NDArray[np.uint64], n_data: NDArray[np.uint64]) -> float: rho_safe = float(clamp_rho(rho)) # Prevent division by zero return float( -np.sum( diff --git a/src/analysis/compare_ai.py b/src/analysis/compare_ai.py index abc7b0a0..4c67c696 100644 --- a/src/analysis/compare_ai.py +++ b/src/analysis/compare_ai.py @@ -114,7 +114,8 @@ def get_compared_imbalance( raise ValueError("Please provide 2 or more groups to compare.") # Process initial minimums for whole data dispersion - snp_cutoff: int = min_count + (2 * pseudocount) + # SNP eligibility removes zero-count observations; min_count applies per region. + snp_cutoff: int = 2 * pseudocount ref_counts: NDArray[np.uint64] = ( adata.layers["ref"].sum(axis=0, dtype=np.uint64).T.A1 + pseudocount diff --git a/src/analysis/run_analysis.py b/src/analysis/run_analysis.py index fea4542a..761a2e8c 100644 --- a/src/analysis/run_analysis.py +++ b/src/analysis/run_analysis.py @@ -82,9 +82,9 @@ def __init__( ) -> None: # Per-variant (SNV-solo) mode cannot also name a region column: per-variant tests each # SNV independently, whereas region_col groups variants by that column. - if per_variant and region_col is not None: + if per_variant and (region_col is not None or groupby is not None): raise ValueError( - "per_variant cannot be combined with region_col: per-variant tests each SNV " + "per_variant cannot be combined with region_col or groupby: per-variant tests each SNV " "independently, while region_col groups by that column. Choose one." ) self.per_variant: bool = per_variant @@ -171,7 +171,7 @@ def run_ai_analysis( region_col : str | None, optional Column name for grouping variants. groupby : str | None, optional - Additional grouping column (not supported by the Rust backend; raises if set). + Parent column to use instead of the feature column for grouping in Rust. per_variant : bool, optional Test each SNV independently (per-variant) instead of grouping by a region column. Forces per-variant even when a region column is present. Default False. @@ -226,16 +226,6 @@ def run_ai_analysis( if expected_manifest_sha256 is not None: raise ValueError("--expected-manifest-sha256 requires --scope per-donor") - # Fail closed: --groupby is not supported by the Rust backend. It only re-keys the grouping - # column (region_col = groupby), so the identical result is obtained with --region_col . - # Erroring prevents silently returning feature-level results when parent-level was requested. - if groupby is not None: - raise RuntimeError( - "--groupby (parent-level grouping) is not supported by the Rust analysis backend. " - "Since groupby only re-keys the grouping column, group by the parent column directly " - "with --region_col instead." - ) - # Store analysis data and params ai_files = WaspAnalysisData( count_file, @@ -262,7 +252,7 @@ def run_ai_analysis( pseudocount=ai_files.pseudocount, method=ai_files.model, phased=ai_files.phased, - region_col=ai_files.region_col, + region_col=ai_files.groupby or ai_files.region_col, ) ai_df = pd.DataFrame(results) diff --git a/src/analysis/run_analysis_sc.py b/src/analysis/run_analysis_sc.py index 5a242127..af341108 100644 --- a/src/analysis/run_analysis_sc.py +++ b/src/analysis/run_analysis_sc.py @@ -123,7 +123,7 @@ def process_adata_inputs( else: sample = sample_list[0] - else: + if sample is not None: sample_list = adata.uns["samples"] if sample not in sample_list: @@ -153,7 +153,7 @@ def process_adata_inputs( # Check phasing if True or None if phased is not False: - if {"0|1", "1|0"}.issubset(set(adata.var[sample].unique())): + if {"0|1", "1|0"} == set(adata.var[sample].unique()): phased = True else: phased = False diff --git a/src/counting/__main__.py b/src/counting/__main__.py index b3475c1a..1eb04629 100644 --- a/src/counting/__main__.py +++ b/src/counting/__main__.py @@ -133,7 +133,7 @@ def count_variants( "--feat", help=( "Feature type in gtf/gff3 for counting intersecting SNPs. " - "Defaults to 'gene' when present." + "Defaults to exon, then transcript, then gene when present." ), ), ] = None, diff --git a/src/counting/count_alleles_sc.py b/src/counting/count_alleles_sc.py index eb0e0ae5..2980d9ad 100644 --- a/src/counting/count_alleles_sc.py +++ b/src/counting/count_alleles_sc.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -# Try to import the Rust single-cell counter (optional; Python fallback always available) +# The normal counting path requires Rust; Python is an explicit reference path. try: from wasp2_rust import BamCounterSC as _RustSC @@ -34,11 +34,11 @@ def _sparse_from_counts( shape: tuple[int, int], ) -> csr_matrix: if not counts: - return csr_matrix(shape, dtype=np.uint16) + return csr_matrix(shape, dtype=np.uint64) return csr_matrix( (list(counts.values()), list(zip(*counts.keys()))), shape=shape, - dtype=np.uint16, + dtype=np.uint64, ) @@ -112,14 +112,23 @@ def make_count_matrix( include_features : list[str] | None, optional Feature columns to include, by default None. use_rust : bool, optional - Use the Rust per-cell counter when available, by default True. Falls back - to the pure-Python ``count_bc_snp_alleles`` if the extension is missing. + Use the Rust per-cell counter, by default True. Raises ImportError when + the native extension is unavailable. Set False explicitly to use the + Python reference counter. Returns ------- ad.AnnData AnnData object with count matrices in layers (ref, alt, other). """ + if use_rust and not RUST_SC_AVAILABLE: + raise ImportError( + "Single-cell counting requires the wasp2_rust native extension. " + "Reinstall WASP2 using a compatible wheel or build the extension with " + "`maturin develop --release` from the source checkout. " + "The Python reference counter requires explicit use_rust=False." + ) + chrom_list = df.get_column("chrom").unique(maintain_order=True) # Add genotypes annotations @@ -136,9 +145,7 @@ def make_count_matrix( sc_counts = CountStatsSC() # Class that holds total count data - rust_path = use_rust and RUST_SC_AVAILABLE - - if rust_path: + if use_rust: start = timeit.default_timer() count_bc_snp_alleles_rust( bam_file=bam_file, @@ -189,7 +196,7 @@ def make_count_matrix( layers={"ref": sparse_ref, "alt": sparse_alt, "other": sparse_other}, ) - adata.obs = snp_df.to_pandas() + adata.obs = pd.DataFrame(snp_df.to_dict(as_series=False)) adata.obs["ref_count"] = adata.layers["ref"].sum(axis=1).T.A1 adata.obs["alt_count"] = adata.layers["alt"].sum(axis=1).T.A1 @@ -212,7 +219,7 @@ def make_count_matrix( ) if "region" not in feature_df.columns: feature_df = feature_df.with_columns(pl.col(feature_cols[0]).alias("region")) - adata.uns["feature"] = feature_df.to_pandas() + adata.uns["feature"] = pd.DataFrame(feature_df.to_dict(as_series=False)) # region_snp_dict = dict( # df.join(snp_df, on=["chrom", "pos", "ref", "alt"], how="left" diff --git a/src/counting/parse_gene_data.py b/src/counting/parse_gene_data.py index 3df9ac8d..6d438080 100644 --- a/src/counting/parse_gene_data.py +++ b/src/counting/parse_gene_data.py @@ -2,14 +2,12 @@ from __future__ import annotations -import logging +import re from pathlib import Path from typing import NamedTuple import polars as pl -logger = logging.getLogger(__name__) - class ParsedGeneData(NamedTuple): """Parsed gene data from GTF/GFF3 file.""" @@ -60,6 +58,12 @@ def update_data(self, data: ParsedGeneData) -> None: setattr(self, key, getattr(data, key)) +def _attribute_values(attribute: str) -> pl.Expr: + """Extract an exact GTF/GFF3 attribute key, including the final attribute.""" + pattern = rf"(?:^|;)\s*{re.escape(attribute)}(?:=|\s+)\s*([^;]*)" + return pl.col("attribute").str.extract(pattern).str.strip_chars(" \t\"'") + + def parse_gene_file( gene_file: str | Path, feature: str | None = None, @@ -73,7 +77,7 @@ def parse_gene_file( gene_file : str | Path Path to GTF/GFF3 file. feature : str | None, optional - Feature type to extract (auto-detected if None). + Feature type to extract (exon, then transcript, then gene if None). attribute : str | None, optional Attribute for region names (auto-detected if None). parent_attribute : str | None, optional @@ -102,61 +106,59 @@ def parse_gene_file( gene_file, separator="\t", comment_prefix="#", has_header=False, new_columns=gtf_cols ) - # Check if we need to preparse feature if feature is None: - feature_list = df.select(pl.col("feature").unique()).to_series() - - if "gene" in feature_list: - feature = "gene" - elif "transcript" in feature_list: - feature = "transcript" - elif "exon" in feature_list: - feature = "exon" - else: - logger.warning("exon, gene or transcript not found in feature list: %s", feature_list) - - assert feature is not None, "Could not determine feature type from gene file" + features = df.get_column("feature") + feature = next((name for name in ("exon", "transcript", "gene") if name in features), None) + if feature is None: + raise ValueError("Could not determine feature type: expected exon, transcript or gene") - # feature filter df = df.filter(pl.col("feature") == feature) + if df.is_empty(): + raise ValueError(f"No {feature!r} features found in {gene_file}") + + def has_attribute(name: str, *, every_row: bool = True) -> bool: + present = _attribute_values(name).is_not_null() + return bool(df.select(present.all() if every_row else present.any()).item()) - # Parse attributes if attribute is None: - if df.get_column("attribute").str.contains(f"{feature}_id").any(): - attribute = f"{feature}_id" - elif df.get_column("attribute").str.contains("ID").any(): - attribute = "ID" - elif df.get_column("attribute").str.contains("Name").any(): - attribute = "Name" - else: - logger.warning("No 'ID', '%s_id' or 'Name' attribute found. Please include ID", feature) - - # Parse parent attributes - if parent_attribute is None: - if df.get_column("attribute").str.contains("Parent").any(): - parent_attribute = "Parent" - elif df.get_column("attribute").str.contains("transcript_id").any(): - parent_attribute = "transcript_id" - elif df.get_column("attribute").str.contains("gene_id").any(): - parent_attribute = "gene_id" - else: - parent_attribute = attribute - - assert attribute is not None, "Could not determine attribute from gene file" - assert parent_attribute is not None, "Could not determine parent_attribute from gene file" - if parent_attribute == attribute: - parent_col = f"groupby_{attribute}" - else: - parent_col = parent_attribute + candidates = (f"{feature}_id", "ID", "Name") + attribute = next((name for name in candidates if has_attribute(name)), None) + if attribute is None: + # If no complete key exists, report which records lack the preferred ID. + attribute = next( + (name for name in candidates if has_attribute(name, every_row=False)), None + ) + if attribute is None: + raise ValueError( + f"No feature identifier found for {feature!r} in {gene_file}; " + f"expected {feature}_id, ID or Name. Specify --gene_attribute." + ) - # Extract relevant attributes - attr_regex = rf"{attribute}[=\s]\"?\'?(.*?)\"?\'?;" - parent_regex = rf"{parent_attribute}[=\s]\"?\'?(.*?)\"?\'?;" + if parent_attribute is None: + parent_attribute = next( + (name for name in ("Parent", "transcript_id", "gene_id") if has_attribute(name)), + attribute, + ) + parent_col = f"groupby_{attribute}" if parent_attribute == attribute else parent_attribute + + # Validate before writing a BED so missing IDs cannot reach interval intersection. + for name in dict.fromkeys((attribute, parent_attribute)): + invalid = df.filter(_attribute_values(name).is_null() | (_attribute_values(name) == "")) + if not invalid.is_empty(): + examples = ", ".join( + f"{row['seqname']}:{row['start']}-{row['end']}" + for row in invalid.head(3).iter_rows(named=True) + ) + raise ValueError( + f"Missing or empty annotation attribute {name!r} on {invalid.height} " + f"{feature!r} record(s) in {gene_file}: {examples}. " + "Provide complete identifiers or select --gene_attribute/--gene_parent." + ) df = df.with_columns( pl.col("start").sub(1), - pl.col("attribute").str.extract(attr_regex).alias(attribute), - pl.col("attribute").str.extract(parent_regex).alias(parent_col), + _attribute_values(attribute).alias(attribute), + _attribute_values(parent_attribute).alias(parent_col), ).select(["seqname", "start", "end", attribute, parent_col]) return ParsedGeneData(df, feature, attribute, parent_col) diff --git a/src/counting/run_counting_sc.py b/src/counting/run_counting_sc.py index cb19b2b0..6de1ff9a 100644 --- a/src/counting/run_counting_sc.py +++ b/src/counting/run_counting_sc.py @@ -51,19 +51,19 @@ def __init__( self.feature_file = feature_file self.use_region_names = use_region_names - # Make sure samples turned into str list - # Ideally single sample for single cell - normalized_samples: list[str] | None - if isinstance(samples, str): - # Check if sample file or comma delim string - if Path(samples).is_file(): - with open(samples) as sample_file: - normalized_samples = [l.strip() for l in sample_file] - else: - normalized_samples = [s.strip() for s in samples.split(",")] - else: - normalized_samples = samples - self.samples: list[str] | None = normalized_samples + # Typer supplies repeated options as a list; normalize each value equally. + normalized_samples: list[str] | None = None + if samples is not None: + normalized_samples = [] + for value in [samples] if isinstance(samples, str) else samples: + entries = Path(value).read_text().splitlines() if Path(value).is_file() else [value] + normalized_samples.extend( + sample.strip() + for entry in entries + for sample in entry.split(",") + if sample.strip() + ) + self.samples = normalized_samples # parse output? self.out_file: str = ( diff --git a/tests/analysis/fixtures/public_atac/README.md b/tests/analysis/fixtures/public_atac/README.md index 0a413be4..9feb48a9 100644 --- a/tests/analysis/fixtures/public_atac/README.md +++ b/tests/analysis/fixtures/public_atac/README.md @@ -12,7 +12,13 @@ SNP rows and 556 peak-overlapping rows; the complete counting CLI reproduced all 556 peak rows exactly. `metadata.json` records source and table checksums. Tests execute archived Python commit `41354fc` on these inputs rather than storing -Rust outputs as expected results. Phased comparisons select truly phased SNPs. +Rust outputs as expected results. The original 32-run matrix supplies wide-integer +DataFrames to the archive, bypassing its UInt16 CSV reader. It therefore establishes +wide-input compatibility, not exact reproduction of legacy file-input rounding. +The bulk tests additionally execute untouched CSV input and a separately labeled +CSV adapter changing only the three count-column dtypes to UInt64. Per-row strict +legacy failures and widened-reference comparisons are retained in test properties; +a legacy failure is not relabeled as a strict parity pass. Phased comparisons select truly phased SNPs. Linear-reference shape repairs and forwarding of its ignored phased flag are explicit in the test. Strict statistical tolerances remain `rtol=atol=5e-5`; counts must match exactly. Sixteen legacy runs plus sixteen donor feature runs diff --git a/tests/analysis/test_archive_parity.py b/tests/analysis/test_archive_parity.py index 0c5bcd55..2a1151c9 100644 --- a/tests/analysis/test_archive_parity.py +++ b/tests/analysis/test_archive_parity.py @@ -1,6 +1,7 @@ """Production-code parity with outputs frozen from unmodified Python 41354fc. -The fixture is independent of Rust and does not need archived git objects in CI. +Frozen fixtures are independent of Rust. Live reference tests also require the +pinned archived git objects. These tests cover statistical compatibility on synthetic counts, not calibration. """ @@ -10,7 +11,12 @@ import numpy as np import pandas as pd import pytest -from scripts.compare_archive_analysis import bulk_counts, compare_frames, single_cell_counts +from scripts.compare_archive_analysis import ( + bulk_counts, + compare_frames, + load_archive, + single_cell_counts, +) from typer.testing import CliRunner from analysis.as_analysis_sc import get_imbalance_sc @@ -23,6 +29,157 @@ ) +@pytest.mark.parametrize("scenario", ["base", "overlap", "zeros", "unequal_depth"]) +@pytest.mark.parametrize("mode", ["single", "shared", "different"]) +@pytest.mark.parametrize("phased", [False, True]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("min_count", [0, 10, 20]) +def test_live_single_cell_archive_matrix(scenario, mode, phased, pc, min_count): + from scipy.sparse import csr_matrix + + historical = single_cell_counts() + if scenario == "overlap": + historical.uns["feature"] = pd.concat( + [historical.uns["feature"], pd.DataFrame({"index": [0, 1], "region": ["overlap"] * 2})], + ignore_index=True, + ) + elif scenario in ["zeros", "unequal_depth"]: + for layer in ["ref", "alt"]: + values = historical.layers[layer].toarray() + if scenario == "zeros": + values[0, :] = 0 + else: + values[:, 2:] *= 4 + historical.layers[layer] = csr_matrix(values) + # Explicit reference repair: current sums are uint64 to prevent overflow. + # The unchanged archive remains covered by the frozen tests below. + archive = load_archive(widen_single_cell_sums=True) + reference = { + "single": archive["as_analysis_sc"].get_imbalance_sc, + "shared": archive["compare_ai"].get_compared_imbalance, + "different": archive["compare_ai"].get_compared_imbalance_diff_snps, + }[mode] + current = { + "single": get_imbalance_sc, + "shared": get_compared_imbalance, + "different": get_compared_imbalance_diff_snps, + }[mode] + options = { + "sample": "sample1", + "groups": ["A", "B"], + "phased": phased, + "pseudocount": pc, + "min_count": min_count, + } + expected = reference(historical.copy(), **options) + actual = current(historical.T.copy(), **options) + assert expected.keys() == actual.keys() + for key in expected: + comparison = compare_frames(expected[key], actual[key]) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.parametrize("phased", [False, True, None]) +def test_automatic_sample_selection_filters_like_explicit_selection(phased): + current = single_cell_counts().T.copy() + current.var.loc[current.var.index[0], "sample1"] = "0/0" + automatic = process_adata_inputs(current.copy(), phased=phased) + explicit = process_adata_inputs(current.copy(), sample="sample1", phased=phased) + assert automatic.sample == explicit.sample == "sample1" + assert automatic.phased == explicit.phased + assert automatic.adata.n_vars == explicit.adata.n_vars == 5 + pd.testing.assert_frame_equal(automatic.adata.var, explicit.adata.var) + pd.testing.assert_frame_equal(automatic.adata.uns["feature"], explicit.adata.uns["feature"]) + np.testing.assert_array_equal( + automatic.adata.layers["ref"].toarray(), explicit.adata.layers["ref"].toarray() + ) + + +@pytest.mark.parametrize("sample", [None, "sample1"]) +@pytest.mark.parametrize("requested_phase", [None, True, False]) +@pytest.mark.parametrize( + "genotypes,expected_phase", + [ + (["0|1", "1|0"] * 3, True), + (["0|1", "1|0", "0/1", "1/0", "0|1", "1|0"], False), + (["0|1"] * 6, False), + (["0/1", "1/0"] * 3, False), + ], +) +def test_single_cell_phasing_requires_exact_archived_genotype_set( + sample, requested_phase, genotypes, expected_phase +): + current = single_cell_counts().T.copy() + current.var["sample1"] = genotypes + if requested_phase is not False and not expected_phase: + with pytest.warns(UserWarning, match="Switching to unphased model"): + prepared = process_adata_inputs(current, sample=sample, phased=requested_phase) + else: + prepared = process_adata_inputs(current, sample=sample, phased=requested_phase) + assert prepared.phased is (expected_phase and requested_phase is not False) + assert prepared.adata.n_vars == len(genotypes) + + +@pytest.mark.parametrize("sample", [None, "sample1"]) +@pytest.mark.parametrize("mode", ["single", "shared", "different"]) +def test_mixed_phasing_matches_unphased_archived_inference(sample, mode): + historical = single_cell_counts() + historical.obs["sample1"] = ["0|1", "1|0", "0/1", "1/0", "0|1", "1|0"] + with pytest.warns(UserWarning, match="Switching to unphased model"): + prepared = process_adata_inputs(historical.T.copy(), sample=sample, phased=True) + archive = load_archive(widen_single_cell_sums=True) + reference = { + "single": archive["as_analysis_sc"].get_imbalance_sc, + "shared": archive["compare_ai"].get_compared_imbalance, + "different": archive["compare_ai"].get_compared_imbalance_diff_snps, + }[mode] + function = { + "single": get_imbalance_sc, + "shared": get_compared_imbalance, + "different": get_compared_imbalance_diff_snps, + }[mode] + options = {"sample": "sample1", "groups": prepared.groups, "min_count": 10} + expected = reference(historical, phased=False, **options) + actual = function(prepared.adata, phased=prepared.phased, **options) + assert expected.keys() == actual.keys() + for key in expected: + comparison = compare_frames(expected[key], actual[key]) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.parametrize("sample", [None, "sample1"]) +def test_sample_selection_rejects_no_heterozygotes(sample): + current = single_cell_counts().T.copy() + current.var["sample1"] = "0/0" + with pytest.raises(ValueError, match="Heterozygous genotypes not found"): + process_adata_inputs(current, sample=sample) + + +@pytest.mark.parametrize("cutoff", [0.5, 1, 3]) +def test_outlier_filtering_matches_archived_feature_reindexing(cutoff): + from analysis.as_analysis_sc import adata_count_qc + + historical = single_cell_counts() + historical.obs.loc[historical.obs.index[0], "ref_count"] = 10000 + expected = load_archive()["as_analysis_sc"].adata_count_qc(historical.copy(), z_cutoff=cutoff) + actual = adata_count_qc(historical.T.copy(), z_cutoff=cutoff) + np.testing.assert_array_equal( + actual.layers["ref"].toarray(), expected.layers["ref"].toarray().T + ) + pd.testing.assert_frame_equal(actual.uns["feature"], expected.uns["feature"]) + + +@pytest.mark.parametrize("phased", [False, True, None]) +def test_missing_genotypes_force_unphased_and_missing_groups_fail(phased): + current = single_cell_counts().T.copy() + del current.uns["samples"] + prepared = process_adata_inputs(current.copy(), phased=phased) + assert prepared.phased is False + assert prepared.sample is None + with pytest.raises(KeyError, match="not found"): + process_adata_inputs(current, groups=["missing"], phased=phased) + + @pytest.mark.rust @pytest.mark.parametrize("unit", ["snv", "atac", "rna"]) @pytest.mark.parametrize("phased", [False, True]) @@ -171,3 +328,26 @@ def test_single_cell_cli_reads_current_count_layout(tmp_path, command): files = sorted(tmp_path.glob("ai_*.tsv")) assert len(files) == (2 if command == "find-imbalance-sc" else 1) assert all(not pd.read_csv(file, sep="\t").empty for file in files) + + +@pytest.mark.parametrize("command", ["find-imbalance-sc", "compare-imbalance"]) +@pytest.mark.parametrize("explicit_sample", [False, True]) +def test_single_cell_cli_mixed_phasing_selects_unphased_inference( + tmp_path, command, explicit_sample +): + from analysis.__main__ import app + + current = single_cell_counts().T.copy() + current.var["sample1"] = ["0|1", "1|0", "0/1", "1/0", "0|1", "1|0"] + counts, groups = tmp_path / "counts.h5ad", tmp_path / "groups.tsv" + current.write_h5ad(counts) + groups.write_text("cell0\tA\ncell1\tA\ncell2\tB\ncell3\tB\n") + arguments = [command, str(counts), str(groups), "--phased", "-o", str(tmp_path / "ai.tsv")] + if explicit_sample: + arguments.extend(["--sample", "sample1"]) + with pytest.warns(UserWarning, match="Switching to unphased model"): + result = CliRunner().invoke(app, arguments) + assert result.exit_code == 0, result.output + repr(result.exception) + files = sorted(tmp_path.glob("ai_*.tsv")) + assert len(files) == (2 if command == "find-imbalance-sc" else 1) + assert all(not pd.read_csv(file, sep="\t").empty for file in files) diff --git a/tests/analysis/test_public_atac_matrix.py b/tests/analysis/test_public_atac_matrix.py index 62d0e597..eaaa2a9a 100644 --- a/tests/analysis/test_public_atac_matrix.py +++ b/tests/analysis/test_public_atac_matrix.py @@ -1,7 +1,8 @@ """Repeat the full 32-run real ATAC matrix against executed archived Python. Count tables are frozen measurements from public reads, not Rust-generated -expectations. Linux Python 3.11 CI pins the original numerical environment. +expectations. DataFrame, untouched CSV and widened CSV references are distinct; +legacy precision differences are recorded separately from wide-input parity. Linux Python 3.11 CI pins the original numerical environment. """ import contextlib @@ -18,7 +19,12 @@ import pytest import scipy from scipy.special import expit -from scripts.compare_archive_analysis import adapt_linear_reference, compare_frames, load_archive +from scripts.compare_archive_analysis import ( + adapt_linear_reference, + classify_reference_differences, + compare_frames, + load_archive, +) from analysis.run_analysis_per_donor import run_per_donor_analysis @@ -35,7 +41,7 @@ @pytest.mark.parametrize("model", ["single", "linear"]) @pytest.mark.parametrize("phased", [False, True]) @pytest.mark.parametrize("pc", [0, 1]) -def test_full_public_atac_matrix(tmp_path, unit, model, phased, pc): +def test_full_public_atac_matrix(tmp_path, unit, model, phased, pc, record_property): import wasp2_rust source = METADATA["counts"][unit] @@ -77,6 +83,35 @@ def capture_fit(*args, **kwargs): output = wasp2_rust.analyze_imbalance_run( str(path), method=model, phased=phased, region_col=region, pseudocount=pc ) + # DataFrame input uses inferred wide counts. Also execute the original CSV + # reader and a separately labeled adapter changing only its count dtypes. + csv_results = {} + for wide in (False, True): + csv_modules = load_archive(METADATA["archive_commit"], widen_bulk_csv_counts=wide) + if model == "linear": + adapt_linear_reference(csv_modules) + if phased: + original_parse_csv = csv_modules["as_analysis"].parse_opt + csv_modules["as_analysis"].parse_opt = ( + lambda df, disp=None, phased=False, _parse=original_parse_csv: _parse( + df, disp=disp, phased=True + ) + ) + with contextlib.redirect_stdout(io.StringIO()): + csv_results[wide] = ( + csv_modules["as_analysis"] + .get_imbalance( + str(path), method=model, phased=phased, region_col=region, pseudocount=pc + ) + .rename(columns={"variant": "region"}) + ) + classification = classify_reference_differences( + csv_results[False], csv_results[True], pd.DataFrame(output["results"]) + ) + record_property("csv_reference_comparisons", json.dumps(classification)) + assert classification["widened_comparison"]["status"] == "PASS", classification + assert classification["unexplained_rows"] == 0, classification + # A legacy FAIL remains a FAIL in the diagnostic; it is not reported as parity. comparison = compare_frames(expected, pd.DataFrame(output["results"])) assert comparison["status"] == "PASS", json.dumps( { diff --git a/tests/analysis/test_reference_adapters.py b/tests/analysis/test_reference_adapters.py index d4465eab..81973444 100644 --- a/tests/analysis/test_reference_adapters.py +++ b/tests/analysis/test_reference_adapters.py @@ -38,3 +38,136 @@ def test_vector_adapter_only_slices_dispersion(mu): mu, dispersions, first_ref, first_n, rest_ref, rest_n ) assert actual == expected + + +@pytest.mark.parametrize("mu", [0.06, 0.25, 0.38196601125, 0.7, 0.94]) +def test_python_unphased_likelihood_preserves_archive_recurrence(mu): + from analysis.as_analysis import opt_unphased_dp + + archive = load_archive()["as_analysis"] + ref, depth = np.array([10, 3, 18, 0]), np.array([20, 20, 24, 10]) + expected = archive.opt_unphased_dp(mu, 0.03, ref[:1], depth[:1], ref[1:], depth[1:]) + actual = opt_unphased_dp(mu, 0.03, ref[:1], depth[:1], ref[1:], depth[1:]) + assert actual == float(np.asarray(expected).item()) + + +def test_python_unphased_underflow_fallback_remains_finite(): + from scipy.stats import betabinom + + from analysis.as_analysis import opt_unphased_dp + + mu, rho = 0.2, 0.03 + ref, depth = np.arange(2000, 2200), np.full(200, 100_000) + shape = (1 - rho) / rho + forward = betabinom.logpmf(ref, depth, mu * shape, (1 - mu) * shape) + reverse = betabinom.logpmf(ref, depth, (1 - mu) * shape, mu * shape) + expected = -forward[0] - np.sum(np.logaddexp(forward[1:], reverse[1:]) - np.log(2)) + actual = opt_unphased_dp(mu, rho, ref[:1], depth[:1], ref[1:], depth[1:]) + assert np.isfinite(actual) + assert np.isclose(actual, expected, rtol=5e-5, atol=5e-5) + + +def test_wide_archive_accumulators_prevent_overflow(monkeypatch): + from scipy.sparse import csr_matrix + from scripts.compare_archive_analysis import single_cell_counts + + archive = load_archive(widen_single_cell_sums=True)["as_analysis_sc"] + historical = single_cell_counts() + historical.layers["ref"] = csr_matrix(np.full(historical.shape, 40000, dtype=np.uint16)) + + def inspect_fit(function, args, **kwargs): + ref, depth = args + np.testing.assert_array_equal(ref, np.full(historical.n_obs, 160001)) + assert ref.dtype == np.uint64 + assert np.all(depth >= ref) + raise RuntimeError("inspected reference inputs") + + monkeypatch.setattr(archive, "minimize_scalar", inspect_fit) + with pytest.raises(RuntimeError, match="inspected reference inputs"): + archive.get_imbalance_sc(historical, sample="sample1", groups=["A", "B"], pseudocount=1) + + +@pytest.mark.parametrize("column", ["num_snps_group1", "num_snps_group2"]) +def test_differential_snp_membership_counts_require_exact_agreement(column): + import pandas as pd + from scripts.compare_archive_analysis import compare_frames + + expected = pd.DataFrame({"region": ["feature"], column: [2], "fdr_pval": [0.5]}) + observed = expected.copy() + observed[column] = 3 + comparison = compare_frames(expected, observed) + assert comparison["status"] == "FAIL" + assert column in comparison["failures"] + + +@pytest.mark.parametrize("column", ["ref_count", "N", "num_snps_group1", "num_snps_group2"]) +@pytest.mark.parametrize("value", [2**53, 2**63, 2**64 - 2]) +def test_count_verifier_preserves_uint64_precision(column, value): + import pandas as pd + from scripts.compare_archive_analysis import compare_frames + + expected = pd.DataFrame( + {"region": ["feature"], column: pd.Series([value], dtype="uint64"), "fdr_pval": [0.5]} + ) + observed = expected.copy() + assert compare_frames(expected, observed)["status"] == "PASS" + observed[column] = pd.Series([value + 1], dtype="uint64") + result = compare_frames(expected, observed) + assert result["status"] == "FAIL" + assert result["max_absolute_differences"][column] == 1 + + +@pytest.mark.parametrize("wide", [False, True]) +def test_bulk_csv_adapter_only_changes_count_reader_dtypes(tmp_path, monkeypatch, wide): + import pandas as pd + + path = tmp_path / "counts.tsv" + pd.DataFrame( + { + "chrom": ["chr1"], + "pos": [101], + "ref": ["A"], + "alt": ["G"], + "ref_count": [40000], + "alt_count": [40000], + "other_count": [0], + } + ).to_csv(path, sep="\t", index=False) + module = load_archive(widen_bulk_csv_counts=wide)["as_analysis"] + + def inspect_fit(frame, *args, **kwargs): + expected_type = np.dtype("uint64" if wide else "uint16") + assert frame.ref_count.dtype == frame.alt_count.dtype == frame.N.dtype == expected_type + assert int(frame.ref_count.iloc[0]) == 40001 + assert int(frame.N.iloc[0]) == (80002 if wide else 14466) + raise RuntimeError("inspected bulk CSV fitting inputs") + + monkeypatch.setattr(module, "single_model", inspect_fit) + with pytest.raises(RuntimeError, match="inspected bulk CSV"): + module.get_imbalance(str(path), pseudocount=1) + # Other modules are untouched by this independently selected adapter. + other = load_archive(widen_bulk_csv_counts=wide)["as_analysis_sc"] + from scripts.compare_archive_analysis import single_cell_counts + + def inspect_sc(function, args, **kwargs): + assert all(value.dtype == np.uint16 for value in args) + raise RuntimeError("single-cell sums untouched") + + monkeypatch.setattr(other, "minimize_scalar", inspect_sc) + with pytest.raises(RuntimeError, match="single-cell sums untouched"): + other.get_imbalance_sc(single_cell_counts(), sample="sample1", groups=["A", "B"]) + + +def test_reference_classification_keeps_strict_failures(): + import pandas as pd + from scripts.compare_archive_analysis import classify_reference_differences + + legacy = pd.DataFrame({"region": ["feature"], "mu": [0.2], "pval": [0.8], "fdr_pval": [0.8]}) + wide = legacy.assign(mu=0.8) + result = classify_reference_differences(legacy, wide, wide.copy()) + assert result["legacy_strict_comparison"]["status"] == "FAIL" + assert result["widened_comparison"]["status"] == "PASS" + assert result["unexplained_rows"] == 0 + assert result["mismatched_rows"][0]["complementary_estimate_candidates"] == ["mu"] + damaged = wide.assign(pval=0.3) + assert classify_reference_differences(legacy, wide, damaged)["unexplained_rows"] == 1 diff --git a/tests/analysis/test_region_col_modes.py b/tests/analysis/test_region_col_modes.py index 92213202..2534a643 100644 --- a/tests/analysis/test_region_col_modes.py +++ b/tests/analysis/test_region_col_modes.py @@ -8,8 +8,7 @@ labels regions ``{chrom}_{pos}`` (must match Python's key for parity); * ``--per-variant`` forces per-variant even when a region column is present; * ``--per-variant`` + ``--region_col`` is rejected; -* ``--groupby`` fails closed on the Rust backend (it only re-keys the grouping column, - so ``--region_col `` is the supported equivalent); +* ``--groupby`` selects the parent grouping column in Rust; * ``--phased`` / ``--region_col`` are forwarded to the Rust backend. """ @@ -156,18 +155,26 @@ def test_cli_forwards_phased_and_region_col(tmp_path, spy_rust): @pytest.mark.unit -def test_cli_groupby_fails_closed(tmp_path, spy_rust): +@pytest.mark.parametrize( + "groupby,expected", [("parent", "parent"), ("region", "region"), ("absent", "region")] +) +def test_cli_groupby_resolves_parent(tmp_path, spy_rust, groupby, expected): from analysis.run_analysis import run_ai_analysis tsv = _write_counts(tmp_path / "counts.tsv") - with pytest.raises(RuntimeError, match="groupby"): - run_ai_analysis( - str(tsv), - min_count=0, - pseudocount=0, - groupby="parent", - out_file=str(tmp_path / "out.tsv"), - ) + text = tsv.read_text().splitlines() + tsv.write_text("\n".join([text[0] + "\tparent", *(row + "\tgeneA" for row in text[1:])]) + "\n") + run_ai_analysis(str(tsv), groupby=groupby, out_file=str(tmp_path / "out.tsv")) + assert spy_rust["kwargs"]["region_col"] == expected + + +@pytest.mark.unit +def test_per_variant_groupby_conflict(tmp_path, spy_rust): + from analysis.run_analysis import run_ai_analysis + + with pytest.raises(ValueError, match="per_variant"): + run_ai_analysis(_write_counts(tmp_path / "counts.tsv"), groupby="parent", per_variant=True) + assert not spy_rust @pytest.mark.unit diff --git a/tests/analysis/test_scipy_numerical_kernel.py b/tests/analysis/test_scipy_numerical_kernel.py index 871612ad..e6bc32cf 100644 --- a/tests/analysis/test_scipy_numerical_kernel.py +++ b/tests/analysis/test_scipy_numerical_kernel.py @@ -135,3 +135,152 @@ def test_many_high_depth_unphased_observations_remain_finite(tmp_path): for p1, p2 in zip(np.exp(forward[1:]), np.exp(reverse[1:]), strict=True): product = 0.5 * (product * p1) + 0.5 * (product * p2) assert product == 0.0 + + +def decimal_beta_binomial_logpmf(k, n, alpha, beta): + """Independent rising-factorial PMF; no SciPy or gamma/beta function reuse.""" + from decimal import Decimal, localcontext + from math import comb + + with localcontext() as context: + context.prec = 70 + a, b = Decimal(str(alpha)), Decimal(str(beta)) + value = Decimal(comb(n, k)) + for i in range(k): + value *= a + i + for i in range(n - k): + value *= b + i + for i in range(n): + value /= a + b + i + return float(value.ln()) + + +@pytest.mark.rust +@pytest.mark.parametrize("ref_count,total", [(0, 0), (0, 14), (4, 8), (7, 14), (14, 14)]) +def test_likelihood_precision_against_independent_decimal(tmp_path, ref_count, total): + import wasp2_rust + from analysis.as_analysis import opt_prob + + rho, alpha = 0.1, 4.5 + expected = decimal_beta_binomial_logpmf(ref_count, total, alpha, alpha) + vector = opt_prob( + 0.5, rho, np.array([ref_count], dtype=np.uint64), np.array([total], dtype=np.uint64) + ) + scalar = opt_prob(0.5, rho, np.uint64(ref_count), np.uint64(total)) + assert float(np.asarray(vector).item()) == scalar + assert scalar == pytest.approx(-expected, abs=1e-12, rel=1e-12) + path = tmp_path / "counts.tsv" + pd.DataFrame( + { + "chrom": ["chr1"], + "pos": [101], + "ref": ["A"], + "alt": ["G"], + "region": ["peak"], + "ref_count": [ref_count], + "alt_count": [total - ref_count], + } + ).to_csv(path, sep="\t", index=False) + result = wasp2_rust.analyze_imbalance_run( + str(path), method="single", min_count=0, pseudocount=0, region_col="region", rho=rho + )["results"][0] + assert result["null_ll"] == pytest.approx(expected, abs=1e-12, rel=1e-12) + if 2 * ref_count == total: + assert result["alt_ll"] == pytest.approx(result["null_ll"], abs=1e-12, rel=1e-12) + assert result["pval"] == pytest.approx(1.0, abs=1e-12) + + +@pytest.mark.parametrize("first_ref,symmetric", [(3, True), (4, False)]) +def test_unphased_anchor_symmetry_is_conditional(first_ref, symmetric): + from analysis.as_analysis import opt_unphased_dp + + rho = 0.1 + ref = np.array([first_ref, 1, 0, 0, 0], dtype=np.uint64) + depth = np.array([6, 20, 0, 0, 1], dtype=np.uint64) + values = [] + for mu in (0.2, 0.8): + expected_first = decimal_beta_binomial_logpmf( + int(ref[0]), int(depth[0]), mu * 9, (1 - mu) * 9 + ) + rest = [ + np.logaddexp( + decimal_beta_binomial_logpmf(int(k), int(n), mu * 9, (1 - mu) * 9), + decimal_beta_binomial_logpmf(int(k), int(n), (1 - mu) * 9, mu * 9), + ) + - np.log(2) + for k, n in zip(ref[1:], depth[1:], strict=True) + ] + expected = -expected_first - sum(rest) + actual = opt_unphased_dp(mu, rho, ref[:1], depth[:1], ref[1:], depth[1:]) + assert actual == pytest.approx(expected, abs=1e-12, rel=1e-12) + values.append(actual) + if symmetric: + assert values[0] == pytest.approx(values[1], abs=1e-12, rel=1e-12) + else: + assert abs(values[0] - values[1]) > 1.0 + + +@pytest.mark.skipif( + (np.__version__, scipy.__version__) != ("1.26.4", "1.13.1"), + reason="Historical dtype dispatch reproduction uses pinned NumPy/SciPy", +) +def test_untouched_archive_balanced_precision_defect_is_reported(tmp_path): + from scripts.compare_archive_analysis import load_archive + + path = tmp_path / "balanced.tsv" + pd.DataFrame( + { + "chrom": ["chr1"], + "pos": [101], + "ref": ["A"], + "alt": ["G"], + "region": ["peak"], + "ref_count": [7], + "alt_count": [7], + "other_count": [0], + } + ).to_csv(path, sep="\t", index=False) + narrow = load_archive()["as_analysis"].get_imbalance( + str(path), region_col="region", min_count=0, pseudocount=0 + ) + wide = load_archive(widen_bulk_csv_counts=True)["as_analysis"].get_imbalance( + str(path), region_col="region", min_count=0, pseudocount=0 + ) + assert float(narrow.pval.iloc[0]) < 0.9999 + assert float(wide.pval.iloc[0]) == 1.0 + assert compare_frames(narrow, wide)["status"] == "FAIL" + + +@pytest.mark.rust +@pytest.mark.parametrize("ref_count,alt_count,pc", [(40000, 40000, 1), (65536, 65537, 0)]) +def test_wide_bulk_counts_preserve_raw_outputs_and_fitting_depth( + tmp_path, ref_count, alt_count, pc +): + import wasp2_rust + + path = tmp_path / "counts.tsv" + pd.DataFrame( + { + "chrom": ["chr1"], + "pos": [101], + "ref": ["A"], + "alt": ["G"], + "region": ["peak"], + "ref_count": [ref_count], + "alt_count": [alt_count], + } + ).to_csv(path, sep="\t", index=False) + rho = 0.1 + result = wasp2_rust.analyze_imbalance_run( + str(path), method="single", min_count=0, pseudocount=pc, region_col="region", rho=rho + )["results"][0] + assert (result["ref_count"], result["alt_count"], result["N"]) == ( + ref_count, + alt_count, + ref_count + alt_count, + ) + shape = 0.5 * (1 - rho) / rho + expected_null = betabinom.logpmf( + np.uint64(ref_count + pc), np.uint64(ref_count + alt_count + 2 * pc), shape, shape + ) + assert result["null_ll"] == pytest.approx(expected_null, abs=1e-9, rel=1e-9) diff --git a/tests/counting/test_gene_data.py b/tests/counting/test_gene_data.py new file mode 100644 index 00000000..ad2244a4 --- /dev/null +++ b/tests/counting/test_gene_data.py @@ -0,0 +1,371 @@ +"""RNA annotation and Rust-backed grouping compatibility with Python 41354fc.""" + +import gzip +import subprocess +import types +from pathlib import Path + +import pandas as pd +import polars as pl +import pysam +import pytest +from scripts.compare_archive_analysis import adapt_linear_reference, compare_frames, load_archive +from typer.testing import CliRunner + +from analysis.__main__ import app as analysis_app +from counting.count_alleles import make_count_df +from counting.parse_gene_data import make_gene_data, parse_gene_file + +ROOT = Path(__file__).resolve().parents[2] + + +def _archive_annotations(): + source = subprocess.check_output( + ["git", "show", "41354fc:src/counting/parse_gene_data.py"], cwd=ROOT, text=True + ) + module = types.ModuleType("archived_gene_data") + exec(compile(source, "archived_gene_data", "exec"), module.__dict__) + return module + + +def _annotation(path, rows): + text = "".join( + f"chr1\ttest\t{feature}\t{start}\t{end}\t.\t+\t.\t{attrs}\n" + for feature, start, end, attrs in rows + ) + if str(path).endswith(".gz"): + with gzip.open(path, "wt") as stream: + stream.write(text) + else: + path.write_text(text) + return path + + +@pytest.mark.parametrize("suffix", ["gtf", "gtf.gz", "gff3"]) +@pytest.mark.parametrize("feature", [None, "exon", "transcript", "gene"]) +def test_annotation_selection_matches_archive(tmp_path, suffix, feature): + rows = [] + for kind in ("gene", "transcript", "exon"): + attrs = ( + ('gene_id "g1"; transcript_id "t1"; ' + ('exon_id "e1";' if kind == "exon" else "")) + if suffix.startswith("gtf") + else f"ID={kind}1;Parent=g1;" + ) + rows.append((kind, 91, 220, attrs)) + path = _annotation(tmp_path / f"annotation.{suffix}", rows) + expected = _archive_annotations().parse_gene_file(path, feature=feature) + actual = parse_gene_file(path, feature=feature) + assert actual[1:] == expected[1:] + assert actual.gene_df.to_dicts() == expected.gene_df.to_dicts() + if feature is None: + assert actual.feature == "exon" + + +@pytest.mark.parametrize( + "features,expected", [(["gene", "transcript"], "transcript"), (["gene"], "gene")] +) +def test_default_feature_priority_without_exons(tmp_path, features, expected): + path = _annotation( + tmp_path / "annotation.gff3", [(name, 1, 20, f"ID={name}1") for name in features] + ) + assert parse_gene_file(path).feature == expected + + +@pytest.mark.parametrize( + "attrs,options,identifier,parent", + [ + ("ID=e1;Parent=t1", {}, "e1", "t1"), + ("ID=e1;Parent=t1;", {}, "e1", "t1"), + ('exon_id "e1"; gene_id "g1"', {}, "e1", "g1"), + ("OtherID=wrong;ID=e1;OtherParent=wrong;Parent=t1", {}, "e1", "t1"), + ("OtherID=wrong;Name=e1", {}, "e1", "e1"), + ("ID=e1;Parent=t1,t2", {}, "e1", "t1,t2"), + ( + "custom.id=e1;custom.parent=g1", + {"attribute": "custom.id", "parent_attribute": "custom.parent"}, + "e1", + "g1", + ), + ("ID=e1", {}, "e1", "e1"), + ], +) +def test_final_attributes_and_exact_keys(tmp_path, attrs, options, identifier, parent): + path = _annotation(tmp_path / "annotation.gff3", [("exon", 91, 220, attrs)]) + result = parse_gene_file(path, **options) + assert result.gene_df.row(0) == ("chr1", 90, 220, identifier, parent) + + +@pytest.mark.parametrize("missing", ["exon_id", "gene_id"]) +@pytest.mark.parametrize("value", [None, "", " "]) +def test_missing_identifiers_fail_before_bed_write(tmp_path, missing, value): + attrs = {"exon_id": "e2", "gene_id": "g1"} + if value is None: + del attrs[missing] + else: + attrs[missing] = value + path = _annotation( + tmp_path / "annotation.gtf", + [ + ("exon", 1, 20, 'exon_id "e1"; gene_id "g1";'), + ("exon", 31, 50, " ".join(f'{key} "{val}";' for key, val in attrs.items())), + ], + ) + bed = tmp_path / "features.bed" + with pytest.raises(ValueError, match=rf"{missing}.*1.*chr1:31-50"): + make_gene_data(path, bed, attribute="exon_id", parent_attribute="gene_id") + assert not bed.exists() + + +@pytest.mark.parametrize( + "attrs,options,message", + [ + ("OtherID=e1", {}, "No feature identifier"), + ("ID=e1", {"feature": "gene"}, "No 'gene' features"), + ("ID=e1", {"attribute": "absent"}, "absent"), + ], +) +def test_invalid_annotation_errors(tmp_path, attrs, options, message): + path = _annotation(tmp_path / "annotation.gff3", [("exon", 1, 20, attrs)]) + with pytest.raises(ValueError, match=message): + parse_gene_file(path, **options) + + +@pytest.fixture +def rna_counts(tmp_path): + """Spliced reads and paired mates overlap shared exons in two genes.""" + bam = tmp_path / "reads.bam" + header = {"HD": {"VN": "1.6", "SO": "coordinate"}, "SQ": [{"SN": "chr1", "LN": 1000}]} + with pysam.AlignmentFile(str(bam), "wb", header=header) as stream: + + def write(name, start, sequence, cigar="1M", flag=0, mate=-1): + read = pysam.AlignedSegment(stream.header) + read.query_name, read.query_sequence = name, sequence + read.flag, read.reference_id, read.reference_start = flag, 0, start + read.cigarstring, read.mapping_quality = cigar, 60 + if mate >= 0: + read.next_reference_id, read.next_reference_start = 0, mate + stream.write(read) + + for base in ("A", "G"): + write(f"spliced_{base}", 90, base * 40, "20M90N20M") + positions = [101, 151, 201, 251, 301, 401] + for pos, (ref, alt) in zip( + positions, [(16, 4), (5, 15), (12, 8), (8, 12), (18, 2), (7, 13)], strict=True + ): + if pos in (151, 301): + write( + "pair", + pos - 1, + "A" if pos == 151 else "G", + flag=65 if pos == 151 else 129, + mate=300 if pos == 151 else 150, + ) + for base, count in (("A", ref), ("G", alt)): + for index in range(count): + write(f"{pos}_{base}_{index}", pos - 1, base) + pysam.index(str(bam)) + # These are the SNP/exon intersection rows: positions 151 and 201 overlap two exons. + frame = pl.DataFrame( + { + "chrom": ["chr1"] * 8, + "pos": [101, 151, 151, 201, 201, 251, 301, 401], + "ref": ["A"] * 8, + "alt": ["G"] * 8, + "GT": ["0|1", "1|0", "1|0", "0|1", "0|1", "1|0", "0|1", "1|0"], + "exon_id": ["e1", "e1", "e2", "e1", "e2", "e2", "e3", "e4"], + "gene_id": ["g1"] * 7 + ["g2"], + } + ).with_columns(pl.col("chrom").cast(pl.Categorical)) + return bam, frame + + +@pytest.mark.rust +@pytest.mark.parametrize("with_gt", [False, True]) +@pytest.mark.parametrize("order", ["forward", "reverse", "gap_first"]) +@pytest.mark.parametrize("threads", [1, 4]) +def test_rna_counting_matches_archive(rna_counts, monkeypatch, with_gt, order, threads): + from tests.test_rust_python_counting_parity import python_count_snp_alleles + + bam, frame = rna_counts + if not with_gt: + frame = frame.drop("GT") + if order == "reverse": + frame = frame.reverse() + elif order == "gap_first": + frame = pl.concat([frame.filter(pl.col("pos") == 151), frame.filter(pl.col("pos") != 151)]) + variants = list(frame.select("pos", "ref", "alt").unique(maintain_order=True).iter_rows()) + expected = { + row[1]: tuple(row[2:]) for row in python_count_snp_alleles(str(bam), "chr1", variants) + } + monkeypatch.setenv("WASP2_RUST_THREADS", str(threads)) + actual = make_count_df(str(bam), frame) + assert actual.height == frame.height + for row in actual.iter_rows(named=True): + assert (row["ref_count"], row["alt_count"], row["other_count"]) == expected[row["pos"]] + + +@pytest.mark.rust +@pytest.mark.parametrize("phased", [False, True]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("model", ["single", "linear"]) +@pytest.mark.parametrize("grouping", ["exon_id", "gene_id"]) +def test_rna_grouping_cli_matches_archive(tmp_path, rna_counts, phased, pc, model, grouping): + bam, frame = rna_counts + measured = make_count_df(str(bam), frame) + counts = tmp_path / "counts.tsv" + measured.write_csv(counts, separator="\t") + modules = load_archive() + if model == "linear": + # Explicit reference-only repairs: array/scalar shapes and the archive's + # ignored phased argument. Likelihoods and optimizers stay unchanged. + adapt_linear_reference(modules) + if phased: + original = modules["as_analysis"].parse_opt + + def parse_phased(df, disp=None, phased=False): + return original(df, disp=disp, phased=True) + + modules["as_analysis"].parse_opt = parse_phased + expected = ( + modules["as_analysis"] + .get_imbalance( + pd.DataFrame(measured.to_dict(as_series=False)), + region_col="exon_id", + groupby="gene_id" if grouping == "gene_id" else None, + method=model, + phased=phased, + pseudocount=pc, + ) + .rename(columns={grouping: "region"}) + ) + aliases = ( + [None] + if grouping == "exon_id" + else [None, "--groupby", "--group", "--parent_col", "--parent"] + ) + for alias in aliases: + output = tmp_path / "results.tsv" + options = ( + ["--region_col", grouping] + if alias is None + else ["--region_col", "exon_id", alias, "gene_id"] + ) + result = CliRunner().invoke( + analysis_app, + [ + "find-imbalance", + str(counts), + *options, + "--model", + model, + "--pseudocount", + str(pc), + *(["--phased"] if phased else []), + "-o", + str(output), + ], + ) + assert result.exit_code == 0, result.exception + comparison = compare_frames(expected, pd.read_csv(output, sep="\t")) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.rust +@pytest.mark.parametrize("suffix", ["gtf", "gtf.gz", "gff3"]) +def test_rna_annotation_counting_cli(tmp_path, rna_counts, suffix): + import shutil + + from counting.__main__ import app as count_app + + if shutil.which("bedtools") is None: + pytest.skip("bedtools is required for annotation intersection") + bam, frame = rna_counts + features = [ + ("gene", 81, 330, "g1", "g1"), + ("gene", 381, 430, "g2", "g2"), + ("exon", 91, 210, "e1", "g1"), + ("exon", 141, 260, "e2", "g1"), + ("exon", 291, 320, "e3", "g1"), + ("exon", 391, 420, "e4", "g2"), + ] + annotation = _annotation( + tmp_path / f"annotation.{suffix}", + [ + ( + kind, + start, + end, + f'gene_id "{gene}"; exon_id "{identifier}";' + if suffix.startswith("gtf") + else f"ID={identifier};Parent={gene}", + ) + for kind, start, end, identifier, gene in features + ], + ) + vcf = tmp_path / "variants.vcf" + records = frame.select("pos", "GT").unique(maintain_order=True) + vcf.write_text( + "##fileformat=VCFv4.2\n##contig=\n" + '##FORMAT=\n' + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n" + + "".join( + f"chr1\t{pos}\t.\tA\tG\t60\tPASS\t.\tGT\t{gt}\n" for pos, gt in records.iter_rows() + ) + ) + output = tmp_path / "counts.tsv" + result = CliRunner().invoke( + count_app, + [ + "count-variants", + str(bam), + str(vcf), + "--regions", + str(annotation), + "--sample", + "sample1", + "-o", + str(output), + ], + ) + assert result.exit_code == 0, result.exception + actual = pl.read_csv(output, separator="\t") + if suffix == "gff3": + actual = actual.rename({"ID": "exon_id", "Parent": "gene_id"}) + expected = make_count_df(str(bam), frame) + columns = ["pos", "exon_id", "gene_id", "ref_count", "alt_count", "other_count"] + assert ( + actual.select(columns).sort(columns).to_dicts() + == expected.select(columns).sort(columns).to_dicts() + ) + + +@pytest.mark.parametrize("partial", ["identifier", "parent"]) +def test_auto_attributes_require_complete_keys_like_archive(tmp_path, partial): + attributes = ( + ["ID=e1;Name=first;", "Name=second;"] + if partial == "identifier" + else ['exon_id "e1"; transcript_id "t1"; gene_id "g1";', 'exon_id "e2"; gene_id "g1";'] + ) + path = _annotation( + tmp_path / "annotation.gtf", + [ + ("exon", start, start + 10, attrs) + for start, attrs in zip([1, 31], attributes, strict=True) + ], + ) + expected = _archive_annotations().parse_gene_file(path) + actual = parse_gene_file(path) + assert actual[1:] == expected[1:] + assert actual.gene_df.to_dicts() == expected.gene_df.to_dicts() + + +def test_partial_auto_identifier_reports_affected_records(tmp_path): + path = _annotation( + tmp_path / "annotation.gtf", + [ + ("exon", 1, 20, 'exon_id "e1"; gene_id "g1";'), + ("exon", 31, 50, 'gene_id "g1";'), + ], + ) + with pytest.raises(ValueError, match="exon_id.*1.*chr1:31-50"): + parse_gene_file(path) diff --git a/tests/counting/test_single_cell_pipeline.py b/tests/counting/test_single_cell_pipeline.py new file mode 100644 index 00000000..06bb9faf --- /dev/null +++ b/tests/counting/test_single_cell_pipeline.py @@ -0,0 +1,301 @@ +"""Single-cell counting and command execution against independent Python inputs.""" + +import ast +import contextlib +import io +import subprocess +from bisect import bisect_left +from collections import defaultdict +from pathlib import Path + +import anndata as ad +import numpy as np +import pandas as pd +import polars as pl +import pysam +import pytest +from scripts.compare_archive_analysis import compare_frames, load_archive, single_cell_counts +from typer.testing import CliRunner + +import counting.count_alleles_sc as sc_counter +from analysis.__main__ import app as analysis_app +from counting.__main__ import app as count_app +from counting.count_alleles_sc import CountStatsSC, _sparse_from_counts, count_bc_snp_alleles +from counting.run_counting_sc import WaspCountSC + +ROOT = Path(__file__).resolve().parents[2] +ARCHIVE = "41354fc087404ba1fa99542803ee9cb9b7e0e6bf" + + +def archive_counter(): + namespace = {"bisect_left": bisect_left} + for path, name in [ + ("src/counting/count_alleles.py", "find_read_aln_pos"), + ("src/counting/count_alleles_sc.py", "count_bc_snp_alleles"), + ]: + source = subprocess.check_output(["git", "show", f"{ARCHIVE}:{path}"], cwd=ROOT) + tree = ast.parse(source) + function = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == name) + exec(compile(ast.Module(body=[function], type_ignores=[]), path, "exec"), namespace) + return namespace["count_bc_snp_alleles"] + + +@pytest.mark.rust +@pytest.mark.parametrize( + "scenario", ["overlap", "paired", "deletion", "skip", "barcodes", "missing_sequence"] +) +@pytest.mark.parametrize("reverse", [False, True]) +@pytest.mark.parametrize("threads", [1, 4]) +def test_three_way_single_cell_counting(tmp_path, scenario, reverse, threads): + import wasp2_rust + + bam_path = tmp_path / "reads.bam" + header = { + "HD": {"VN": "1.6", "SO": "coordinate"}, + "SQ": [{"SN": c, "LN": 1000} for c in ["chr1", "chr2"]], + } + with pysam.AlignmentFile(str(bam_path), "wb", header=header) as out: + for chrom in [0, 1]: + for i in range(6): + read = pysam.AlignedSegment(out.header) + read.query_name = "pair" if scenario == "paired" and i < 2 else f"read{i}" + read.reference_id, read.reference_start = chrom, 90 + i + read.mapping_quality, read.flag = 60, 0 + if scenario == "paired" and i < 2: + read.flag = 65 if i == 0 else 129 + read.next_reference_id = chrom + read.next_reference_start = 91 if i == 0 else 90 + read.cigarstring = {"deletion": "10M1D10M", "skip": "10M1N10M"}.get(scenario, "30M") + length = 20 if scenario in ["deletion", "skip"] else 30 + read.query_sequence = ("A" if i % 2 == 0 else "G") * length + if scenario == "missing_sequence" and i == 0: + read.query_sequence = None + if scenario != "barcodes" or i != 0: + read.set_tag( + "CB", "unknown" if scenario == "barcodes" and i == 1 else ["A", "B"][i % 2] + ) + out.write(read) + pysam.index(str(bam_path)) + snps = [(0, 101, "A", "G"), (1, 111, "A", "G")] + if reverse: + snps.reverse() + barcodes = {"A": 0, "B": 1} + expected = [] + for counter in [archive_counter(), count_bc_snp_alleles]: + counts = CountStatsSC() + with pysam.AlignmentFile(str(bam_path), "rb") as bam: + for chrom_index, chrom in enumerate(["chr1", "chr2"]): + chromosome_snps = [ + (idx + 2 * chrom_index, pos, ref, alt) for idx, pos, ref, alt in snps + ] + counter(bam, barcodes, chrom, iter(chromosome_snps), counts) + expected.append([dict(counts.ref_count), dict(counts.alt_count), dict(counts.other_count)]) + actual = [{}, {}, {}] + regions = [ + (idx + 2 * chrom_index, chrom, pos, ref, alt) + for chrom_index, chrom in enumerate(["chr1", "chr2"]) + for idx, pos, ref, alt in snps + ] + for snp, bc, ref, alt, other in wasp2_rust.BamCounterSC(str(bam_path)).count_bc_alleles( + regions, barcodes, 0, threads + ): + for layer, value in zip(actual, [ref, alt, other], strict=True): + if value: + layer[snp, bc] = value + assert expected[0] == expected[1] == actual + + +@pytest.mark.parametrize("value", [0, 65535, 65536, 70000, 2**32]) +def test_sparse_counts_preserve_large_values_through_h5ad(tmp_path, value): + counts = defaultdict(int, {(0, 0): value}) if value else defaultdict(int) + matrix = _sparse_from_counts(counts, (1, 1)) + adata = ad.AnnData(X=matrix + matrix, layers={"ref": matrix, "alt": matrix}) + path = tmp_path / "counts.h5ad" + adata.write_h5ad(path) + restored = ad.read_h5ad(path) + assert restored.layers["ref"][0, 0] == value + assert restored.X[0, 0] == 2 * value + + +@pytest.mark.parametrize("form", ["string", "list", "mixed", "file", "file_list"]) +def test_sample_arguments_normalize_consistently(tmp_path, form): + path = tmp_path / "samples.txt" + path.write_text("sample1\n sample2 \n\n") + samples = { + "string": "sample1, sample2", + "list": ["sample1", "sample2"], + "mixed": ["sample1,sample2"], + "file": str(path), + "file_list": [str(path)], + }[form] + options = WaspCountSC("reads.bam", "variants.vcf", "barcodes.tsv", None, samples=samples) + assert options.samples == ["sample1", "sample2"] + + +@pytest.fixture +def synthetic_count_bundle(tmp_path): + historical = single_cell_counts() + positions = [100 + 100 * i for i in range(6)] + bam = tmp_path / "reads.bam" + with pysam.AlignmentFile( + str(bam), + "wb", + header={"HD": {"VN": "1.6", "SO": "coordinate"}, "SQ": [{"SN": "chr1", "LN": 1000}]}, + ) as out: + for snp, pos in enumerate(positions): + for cell in range(4): + for allele, layer in [("A", "ref"), ("G", "alt")]: + for k in range(int(historical.layers[layer][snp, cell])): + read = pysam.AlignedSegment(out.header) + read.query_name = f"{snp}_{cell}_{allele}_{k}" + read.query_sequence, read.flag = allele, 0 + read.reference_id, read.reference_start = 0, pos - 1 + read.mapping_quality, read.cigarstring = 60, "1M" + read.set_tag("CB", f"cell{cell}") + out.write(read) + pysam.index(str(bam)) + vcf = tmp_path / "variants.vcf" + vcf.write_text( + '##fileformat=VCFv4.2\n##contig=\n##FORMAT=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n' + + "".join( + f"chr1\t{pos}\t.\tA\tG\t60\tPASS\t.\tGT\t" + ["0|1", "1|0"][i % 2] + "\n" + for i, pos in enumerate(positions) + ) + ) + barcodes, groups, features = [ + tmp_path / name for name in ["barcodes.tsv", "groups.tsv", "features.bed"] + ] + barcodes.write_text("".join(f"cell{i}\n" for i in range(4))) + groups.write_text("cell0\tA\ncell1\tA\ncell2\tB\ncell3\tB\n") + features.write_text("chr1\t90\t210\tp0\nchr1\t290\t310\tp1\nchr1\t390\t610\tp2\n") + return bam, vcf, barcodes, groups, features, historical + + +@pytest.mark.rust +@pytest.mark.parametrize("command", ["find-imbalance-sc", "compare-imbalance"]) +@pytest.mark.parametrize("phased", [False, True]) +@pytest.mark.parametrize("pc", [0, 1]) +def test_bam_to_h5ad_to_analysis_matches_archive( + tmp_path, synthetic_count_bundle, monkeypatch, command, phased, pc +): + import shutil + + if not shutil.which("bedtools"): + pytest.skip("bedtools required for feature intersection") + bam, vcf, barcodes, groups, features, historical = synthetic_count_bundle + assert sc_counter.RUST_SC_AVAILABLE, "Native counting must be available for CLI parity" + monkeypatch.setattr( + sc_counter, + "count_bc_snp_alleles", + lambda *a, **k: pytest.fail("CLI silently used the Python reference counter"), + ) + # Metadata conversion must work without the undeclared optional PyArrow package. + monkeypatch.setattr( + pl.DataFrame, "to_pandas", lambda *a, **k: pytest.fail("PyArrow-dependent conversion") + ) + counts = tmp_path / "counts.h5ad" + result = CliRunner().invoke( + count_app, + [ + "count-variants-sc", + str(bam), + str(vcf), + str(barcodes), + "--samples", + "sample1", + "--feature", + str(features), + "-o", + str(counts), + ], + ) + assert result.exit_code == 0, (result.output, result.exception) + measured = ad.read_h5ad(counts) + assert measured.shape == (4, 6) + assert measured.obs_names.tolist() == [f"cell{i}" for i in range(4)] + for layer in ["ref", "alt"]: + np.testing.assert_array_equal( + measured.layers[layer].toarray(), historical.layers[layer].toarray().T + ) + output = tmp_path / "results.tsv" + result = CliRunner().invoke( + analysis_app, + [ + command, + str(counts), + str(groups), + "--sample", + "sample1", + "--phased" if phased else "--unphased", + "--pseudocount", + str(pc), + "-o", + str(output), + ], + ) + assert result.exit_code == 0, (result.output, result.exception) + archive = load_archive() + function = ( + archive["as_analysis_sc"].get_imbalance_sc + if command == "find-imbalance-sc" + else archive["compare_ai"].get_compared_imbalance + ) + reference = measured.T.copy() + reference.var["group"] = ["A", "A", "B", "B"] + with contextlib.redirect_stdout(io.StringIO()): + expected = function( + reference, + sample="sample1", + groups=["A", "B"], + phased=phased, + pseudocount=pc, + min_count=10, + ) + assert expected + for key, frame in expected.items(): + suffix = key if isinstance(key, str) else "_".join(key) + observed = pd.read_csv(tmp_path / f"results_{suffix}.tsv", sep="\t") + comparison = compare_frames(frame, observed) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.parametrize("explicit_default", [False, True]) +def test_native_counter_unavailable_fails_before_reading_input(monkeypatch, explicit_default): + monkeypatch.setattr(sc_counter, "RUST_SC_AVAILABLE", False) + options = {"use_rust": True} if explicit_default else {} + with pytest.raises(ImportError, match="requires the wasp2_rust native extension"): + sc_counter.make_count_matrix("missing.bam", pl.DataFrame(), {}, **options) + + +def test_python_reference_counting_is_explicitly_available(synthetic_count_bundle, monkeypatch): + bam, _vcf, _barcodes, _groups, _features, historical = synthetic_count_bundle + monkeypatch.setattr(sc_counter, "RUST_SC_AVAILABLE", False) + variants = pl.DataFrame( + { + "chrom": ["chr1"] * 6, + "pos": [100 + 100 * i for i in range(6)], + "ref": ["A"] * 6, + "alt": ["G"] * 6, + } + ) + observed = sc_counter.make_count_matrix( + str(bam), variants, {f"cell{i}": i for i in range(4)}, use_rust=False + ) + for layer in ["ref", "alt"]: + np.testing.assert_array_equal( + observed.layers[layer].toarray(), historical.layers[layer].toarray().T + ) + + +def test_counting_cli_requires_native_counter(synthetic_count_bundle, monkeypatch, tmp_path): + bam, vcf, barcodes, _groups, _features, _historical = synthetic_count_bundle + monkeypatch.setattr(sc_counter, "RUST_SC_AVAILABLE", False) + output = tmp_path / "counts.h5ad" + result = CliRunner().invoke( + count_app, + ["count-variants-sc", str(bam), str(vcf), str(barcodes), "-o", str(output)], + ) + assert result.exit_code != 0 + assert isinstance(result.exception, ImportError) + assert "requires the wasp2_rust native extension" in str(result.exception) + assert not output.exists() diff --git a/tests/test_extension_version.py b/tests/test_extension_version.py index 68f73afd..aa237f9c 100644 --- a/tests/test_extension_version.py +++ b/tests/test_extension_version.py @@ -1,7 +1,76 @@ import importlib.metadata +import pytest + import wasp2_rust def test_rust_extension_version_matches_distribution() -> None: assert wasp2_rust.__version__ == importlib.metadata.version("wasp2") + + +@pytest.fixture +def release_root(tmp_path, monkeypatch): + """Exercise release synchronization without changing checkout metadata.""" + from scripts import release_version + + paths = [ + "rust/Cargo.toml", + "rust/Cargo.lock", + "pixi.toml", + "CITATION.cff", + "Dockerfile", + "Singularity.def", + "README.md", + "docs/CONTAINER_USAGE.md", + "docs/source/installation.rst", + "docs/source/conf.py", + "src/wasp2/__init__.py", + "galaxy/tools/wasp2/macros.xml", + ] + for relative in paths: + destination = tmp_path / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(release_version.read(relative), encoding="utf-8") + monkeypatch.setattr(release_version, "ROOT", tmp_path) + return release_version + + +@pytest.mark.parametrize("root_version", ["current", "missing", "9.8.7"]) +def test_release_version_checks_root_lock_package(release_root, root_version): + import re + + version = release_root.cargo_version() + lock = release_root.read("rust/Cargo.lock") + # Leave dependency versions intact: only the root package should determine + # the lockfile's release-version compatibility. + pattern = rf'(?m)^name = "wasp2"\nversion = "{re.escape(version)}"' + replacement = ( + 'name = "different-package"\nversion = "9.8.7"' + if root_version == "missing" + else f'name = "wasp2"\nversion = "{version if root_version == "current" else root_version}"' + ) + updated, count = re.subn(pattern, lambda _match: replacement, lock) + assert count == 1 + release_root.write("rust/Cargo.lock", updated) + errors = release_root.check_version() + if root_version == "current": + assert errors == [] + else: + assert errors == ["rust/Cargo.lock root package version differs from rust/Cargo.toml"] + + +@pytest.mark.parametrize("existing_date", [None, "2020-01-01"]) +def test_setting_release_version_inserts_or_updates_citation_date(release_root, existing_date): + import re + + citation = re.sub(r"(?m)^date-released:.*\n?", "", release_root.read("CITATION.cff")) + if existing_date is not None: + citation += f'\ndate-released: "{existing_date}"\n' + release_root.write("CITATION.cff", citation) + release_root.set_version("9.8.7", "2027-02-03") + updated = release_root.read("CITATION.cff") + assert re.findall(r"(?m)^date-released:.*$", updated) == ['date-released: "2027-02-03"'] + assert 'version: "9.8.7"' in updated + assert release_root.cargo_version() == "9.8.7" + assert release_root.check_version() == [] From 061e7e7905be5921b089d1a398236db67363059a Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:59:27 -0700 Subject: [PATCH 2/2] Install H5AD dependencies for the active single-cell pipeline check --- .github/workflows/ecosystem.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml index 67287ed1..9edb4763 100644 --- a/.github/workflows/ecosystem.yml +++ b/.github/workflows/ecosystem.yml @@ -73,6 +73,14 @@ jobs: - uses: nf-core/setup-nf-test@f2db585128ae681918440f6c074dededeceadce1 # v1 with: version: "0.9.2" + - name: Set up Python for single-cell H5AD stubs + if: matrix.pipeline == 'nf-scatac' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.11" + - name: Install single-cell stub dependencies + if: matrix.pipeline == 'nf-scatac' + run: python -m pip install "pandas>=1.5,<3.0" "scipy>=1.10" "anndata>=0.8,<0.12" - name: Run pipeline compatibility test run: | test_file=tests/main.nf.test