diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0852ff41..1e54ae20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,8 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + # Runtime reference tests read the pinned pre-Rust archive commit. + fetch-depth: 0 - name: Install system dependencies run: | @@ -114,6 +116,14 @@ jobs: source .venv/bin/activate python -c "import wasp2_rust; print('wasp2_rust loaded:', wasp2_rust.__file__)" + - name: Pin historical numerical reference for the full real ATAC matrix + if: matrix.python-version == '3.11' + run: uv pip install --python .venv/bin/python numpy==1.26.4 scipy==1.13.1 + + - name: Diagnose platform numerical arithmetic + if: matrix.python-version == '3.11' + run: .venv/bin/python scripts/diagnose_bulk_numerics.py + - name: Run tests run: | source .venv/bin/activate diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fa442975..6674dd5f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,18 +19,28 @@ permissions: jobs: build: name: Container ${{ matrix.platform }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: fail-fast: false matrix: - platform: [linux/amd64, linux/arm64] + include: + - {runner: ubuntu-24.04, platform: linux/amd64} + - {runner: ubuntu-24.04-arm, platform: linux/arm64} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Read package version + id: version + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import tomllib + with open("rust/Cargo.toml", "rb") as handle: + print("value=" + tomllib.load(handle)["package"]["version"]) + PY - name: Build local test image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: @@ -38,13 +48,20 @@ jobs: load: true platforms: ${{ matrix.platform }} tags: wasp2:ci - build-args: VERSION=0+ci.${{ github.sha }} + build-args: | + VERSION=${{ steps.version.outputs.value }} + REVISION=${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max - name: Smoke-test image env: PLATFORM: ${{ matrix.platform }} - run: docker run --rm --platform "$PLATFORM" wasp2:ci /opt/wasp2/scripts/container_smoke_test.sh + EXPECTED_VERSION: ${{ steps.version.outputs.value }} + EXPECTED_REVISION: ${{ github.sha }} + run: | + test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' wasp2:ci)" = "$EXPECTED_VERSION" + test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' wasp2:ci)" = "$EXPECTED_REVISION" + docker run --rm --platform "$PLATFORM" -e EXPECTED_WASP2_VERSION="$EXPECTED_VERSION" wasp2:ci /opt/wasp2/scripts/container_smoke_test.sh required: name: Required / Container diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ce91548..12150483 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -281,14 +281,21 @@ jobs: load: true platforms: ${{ matrix.platform }} tags: wasp2:release-smoke-${{ matrix.tag }} - build-args: VERSION=${{ needs.validate.outputs.version }} + build-args: | + VERSION=${{ needs.validate.outputs.version }} + REVISION=${{ github.sha }} cache-from: type=gha,scope=release-${{ matrix.tag }} cache-to: type=gha,mode=max,scope=release-${{ matrix.tag }} - name: Smoke-test local release image env: IMAGE: wasp2:release-smoke-${{ matrix.tag }} PLATFORM: ${{ matrix.platform }} - run: docker run --rm --platform "$PLATFORM" "$IMAGE" /opt/wasp2/scripts/container_smoke_test.sh + EXPECTED_VERSION: ${{ needs.validate.outputs.version }} + EXPECTED_REVISION: ${{ github.sha }} + run: | + test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' "$IMAGE")" = "$EXPECTED_VERSION" + test "$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE")" = "$EXPECTED_REVISION" + docker run --rm --platform "$PLATFORM" -e EXPECTED_WASP2_VERSION="$EXPECTED_VERSION" "$IMAGE" /opt/wasp2/scripts/container_smoke_test.sh singularity-smoke: name: Singularity smoke @@ -311,7 +318,9 @@ jobs: context: . platforms: linux/amd64 tags: wasp2:release-singularity - build-args: VERSION=${{ needs.validate.outputs.version }} + build-args: | + VERSION=${{ needs.validate.outputs.version }} + REVISION=${{ github.sha }} outputs: type=docker,dest=${{ runner.temp }}/wasp2-docker.tar cache-from: type=gha,scope=release-amd64 cache-to: type=gha,mode=max,scope=release-amd64 @@ -373,7 +382,9 @@ jobs: tags: | ${{ env.IMAGE_NAME }}:${{ needs.validate.outputs.version }} ${{ env.IMAGE_NAME }}:latest - build-args: VERSION=${{ needs.validate.outputs.version }} + build-args: | + VERSION=${{ needs.validate.outputs.version }} + REVISION=${{ github.sha }} provenance: mode=max sbom: true - name: Smoke-test published architectures diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c3bfd1d..7aa426b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,26 +5,68 @@ All notable changes to WASP2 will be documented in this file. ## [Unreleased] ### Fixed +- Donor feature deduplication uses the parsed first GT allele, as Python does. + Original GT tokens remain available for the backend's pre-dedup phasing + decision; fitting weights, raw counts, QC and provenance reflect retained rows. +- Cephes arithmetic follows the measured reference compiler policy: separate + multiplication/addition for Linux x86_64 and fused operations for the macOS + arm64 reference. Platform probes distinguish this from statistical-model changes. +- Bulk phasing retains the original genotype tokens for archived table-wide + checks, parses numeric first alleles consistently, and rejects malformed mixed + phased input instead of silently changing a region's likelihood. Interior + dispersion values are no longer clipped to the endpoint regularization range. +- Full 32-run public ATAC comparisons execute archived Python in pinned Linux + CI, including CLI inference and donor routes. Heterogeneous donor checks cover + both dispersion models, phasing modes and pseudocounts with independent fits. +- Container checks exercise reordered ATAC headers, region grouping, phased + likelihoods and raw counts. Images carry package-version and source-revision + labels; smoke tests require a matching Python/Rust distribution version. +- CLI help now reports the actual donor-local pseudocount default of 1. +- Bulk beta-binomial calculations use a SciPy 1.13.1-compatible Cephes kernel, + reference FMA arithmetic, and NumPy float64 reductions. Unphased inference + follows archived probability accumulation when safe and uses log-space fallback + when products lose range. Linear fits preserve adjusted-depth coefficients in + the existing four-field representation to avoid conversion-rounding changes. + Strict real ATAC comparisons include raw allelic proportions. +- Bulk archive validation separately verifies symmetric unphased optima using + retained anchor counts and independent Python objective evaluations, while + preserving strict comparisons, raw proportions, and numerical tolerances. +- Allelic-proportion optimization uses archived Python bounded Brent defaults + for single-SNP, phased, and unphased bulk tests, including empty-allele inputs + with pseudocount 0. Balanced first-SNP unphased objectives can still select + mirrored proportions across numerical libraries; no tie-breaking rule is added. +- Linear dispersion follows archived Python single-start Nelder-Mead fitting + on pseudocounted depth, retains coefficients with a pseudocount depth-origin + shift for exact reusable prediction. Equal depths use a unit scale; testing no longer applies + the fitting-only predictor clip. +- Retain distinct repeated SNP count observations in donor analysis and apply + Python phase-aware deduplication after depth filtering. SNV outputs aggregate + retained observations per position; fitting preserves their phase identities. +- Donor feature analysis follows archived Python phasing fallback for missing, + identical, or nonnumeric GT after depth filtering, without converting allele + strings. Provenance records requested and effective phase per included donor. +- Donor dispersion fits retain overlapping SNP-feature observations as archived + Python does, with separate donor-feature identities in global fits. Unique-SNP + QC and donor eligibility counts retain their existing meanings. - Bulk SNP counting preserves archived Python input-order priority across paired mates, with read names consumed before checking aligned bases at each SNP. -- Bulk region outputs remove one pseudocount per retained SNP and allele, restoring - the archived Python raw-count aggregation without changing the fitted statistics. -- Single-cell analysis input preparation and differential comparisons now follow - the cells-by-variants AnnData layout throughout, including barcode groups and genotypes. -- Differential pseudobulk sums widen to UInt64 rather than wrapping at 65,535. +- Restore archived chi-square LRT p-values and first-SNP-anchored unphased + feature likelihoods in donor-local and legacy bulk analysis. +- Default donor-local pseudocounts to 1, matching archived Python. Per-donor + nuisance fitting and independent donor tests/BH correction remain available. +- Integrate the raw-count and single-cell layout compatibility fixes from #135. + Exact-SNV requests are rejected rather than switching the p-value method. -### Tests -- Frozen outputs from archived Python commit `41354fc` cover bulk, single-cell, - and differential analysis. Linear-reference array-shape adaptations are explicit; - the archive's ignored phased-linear flag is reported as a reference limitation. - -## [1.5.0] - 2026-07-14 +## [1.5.0] - 2026-07-21 ### Added - `wasp2-count count-cohort` for one-final-BAM-per-donor bulk counting with explicit `donor_id`/`vcf_sample` mapping, atomic locked outputs, and SHA-256 provenance manifests. - Linear-dispersion and donor-specific-dispersion beta-binomial implementations in the Rust analysis API, including phased feature-level inference. +- Donor-local bulk imbalance orchestration through `wasp2-analyze find-imbalance` with explicit + SNV or feature units, global or per-donor nuisance fitting, donor-local inference and BH + correction, manifest pinning, and non-overwriting result, dispersion, QC, and provenance files. - `--per-variant` / `--snv-solo` for independent SNV tests and explicit `--region_col` support for feature- or peak-level aggregation. - A Rust single-cell ATAC allele counter with chromosome-parallel execution and Python fallback. @@ -32,10 +74,12 @@ All notable changes to WASP2 will be documented in this file. ### Changed - VCF conversion now selects biallelic SNVs by default; multi-allelic records require the explicit `--include-multiallelic` option. -- Phased inference accepts only numeric VCF heterozygote orientation (`0|1` and `1|0`) for parity - with the archived Python implementation; allele-string genotypes do not imply cross-SNV phase. +- Phased inference follows archived numeric first-allele parsing after table-wide + genotype checks; allele-string genotypes do not imply cross-SNV phase. - `--phased` and `--region_col` are forwarded to the Rust backend; unsupported `--groupby` requests fail closed and direct users to `--region_col `. +- SNV analysis is unphased by contract, while feature analysis supports phased and unphased + likelihoods without pooling effect inference across donors. - Single-cell AnnData output now follows the scverse `obs=cells`, `var=variants` orientation. - Packaging derives the Python version from installed metadata and synchronizes Cargo, pixi, containers, Galaxy, Nextflow, documentation, and citation metadata from one release command. @@ -48,6 +92,10 @@ All notable changes to WASP2 will be documented in this file. - Unphased multi-SNV likelihoods now use log-space dynamic programming to prevent underflow. - Linear inference uses per-SNV dispersion throughout the alternative optimization, phased donor-specific inference uses each observation's donor dispersion, and LRTs are non-negative. +- Linear nuisance fits now standardize raw depth and persist the fitted center and scale with both + coefficients so fixed global fits are reused exactly in every donor-local run. +- Ambiguous multi-allelic count rows are rejected instead of being silently coerced into + biallelic SNV identities. - Bulk counts use `UInt32`, and single-cell pseudobulk/observation sums widen safely instead of overflowing at 65,535; single-cell counting also works without an optional feature file. - Count-analysis TSV parsing is header-driven, so `pos0`, `GT`, sample, and donor metadata cannot diff --git a/CITATION.cff b/CITATION.cff index 2dd28b90..6d023466 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -23,7 +23,7 @@ authors: affiliation: "Salk Institute for Biological Studies" version: "1.5.0" -date-released: "2026-07-14" +date-released: "2026-07-21" license: MIT repository-code: "https://github.com/mcvickerlab/WASP2" diff --git a/Dockerfile b/Dockerfile index 726f34a5..e86fa6ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,6 +70,7 @@ FROM python:3.11-slim-bookworm # Version: keep in sync with rust/Cargo.toml (single source of truth) # Run scripts/check-version-consistency.sh to verify ARG VERSION=1.5.0 +ARG REVISION=unknown LABEL org.opencontainers.image.source="https://github.com/mcvickerlab/WASP2" LABEL org.opencontainers.image.description="WASP2: Allele-specific analysis of NGS data with Rust acceleration" @@ -77,6 +78,7 @@ LABEL org.opencontainers.image.licenses="MIT" LABEL org.opencontainers.image.vendor="mcvickerlab" LABEL org.opencontainers.image.title="WASP2" LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${REVISION}" LABEL maintainer="Jeff Jaureguy " # Install runtime deps + temporary build deps for pybedtools C++ extension diff --git a/README.md b/README.md index eac466bf..326e04d1 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,77 @@ to retain peak identifiers for feature-level analysis. wasp2-analyze find-imbalance counts.tsv -o ai_results.tsv ``` +### Donor-local cohort analysis + +Analyze the locked count directory (or its `count_manifest.json` or +`counts.tsv.gz`) with an explicit unit. SNV analysis rejects `--phased` and tests +each variant independently within each donor: + +```bash +wasp2-analyze find-imbalance cohort_counts \ + --scope per-donor --unit snv --min-donor-observations 50 \ + -o atac_snv_results.tsv +``` + +`--dispersion-scope per-donor` is the default. `--dispersion-scope global` +fits one nuisance model over retained eligible donor-feature-SNV observations and reuses that exact +fit for every included donor; testing and BH correction remain donor-local. Both +`single` and `linear` dispersion models are supported. Linear fits record and +reuse `linear_d1`, `linear_d2`, `linear_depth_center`, and +`linear_depth_scale` together. +New fits preserve the original adjusted-depth coefficients: `linear_depth_center` +is `-2 * pseudocount` and `linear_depth_scale` is 1. This avoids rounding from +converting the predictor to mean/std coordinates. Existing fixed parameter sets +continue to use their supplied center and scale. + +Allelic-proportion fitting follows archived Python bounded Brent defaults for +SNP, phased, and unphased bulk tests. An unphased region whose first retained SNP +has equal reference and alternate counts has a symmetric likelihood: `mu` and +`1 - mu` are equally likely. Numerical libraries may select opposite optima; +the reported direction is not identifiable in that case. + +Archive validation reports strict numerical matching separately from statistical +equivalence. A mirrored unphased proportion qualifies only with a balanced first +retained SNP, complementary proportions, independently verified equal archived +likelihoods, and passing counts, dispersion and all other statistical checks. +Reports retain the raw proportion differences and strict failures. + +Feature analysis preserves one SNV in every overlapping feature. `peak` remains +an alias for `feature`: + +```bash +wasp2-count count-cohort donors.tsv variants.vcf.gz feature_counts \ + --unit feature --regions peaks.bed + +wasp2-analyze find-imbalance feature_counts \ + --scope per-donor --unit feature --model linear --phased \ + -o atac_feature_results.tsv +``` + +The donor-local route defaults to pseudocount 1. It writes +`atac_feature_results.tsv`, `atac_feature_results.dispersion.tsv`, +`atac_feature_results.qc.tsv`, and `atac_feature_results.provenance.json` and +refuses to overwrite any of them. For standalone legacy count tables only, +`sample` is accepted and normalized to `donor_id`; providing both columns is an +error. Use `--expected-manifest-sha256` to require an externally trusted locked +bundle manifest. An input `snv_id` is preserved exactly after validating its +donor and allele identity; coordinate-based IDs are synthesized only when the +column is absent. Dispersion and provenance metadata report optimizer status +only when Rust provides it; `parameters_returned` does not assert convergence. + +Phasing checks run after depth filtering. Missing GT, identical GT tokens, or +absence of numeric `0|1`/`1|0` genotypes selects unphased inference. Otherwise, +every retained GT must have a numeric first allele: mixed `0|0`/`1|1` tokens +follow archived Python's first-allele behavior, while malformed mixed tokens +(such as `0/1` or `A|G`) raise an error. For phased inference, prepare a table of +truly phased heterozygous SNPs; otherwise omit `--phased`. + +Valid interior dispersion values are retained without clipping. Exact rho +endpoints and probability-product underflow use documented finite extensions +because archived Python is undefined there. The full public ATAC comparison +and its reference adaptations are described in +[the validation report](docs/BULK_PARITY_VALIDATION.md). + ## Single-Cell Example ```bash diff --git a/docs/BULK_PARITY_VALIDATION.md b/docs/BULK_PARITY_VALIDATION.md new file mode 100644 index 00000000..4aecb8c6 --- /dev/null +++ b/docs/BULK_PARITY_VALIDATION.md @@ -0,0 +1,118 @@ +# Bulk Python/Rust compatibility validation + +Validation date: 2026-10-14. Changes are proposed in PR #133, which includes +the compatibility changes from PR #135. Neither PR is merged or released. + +## Reference and data + +The reference is archived **Python**, commit +`41354fc087404ba1fa99542803ee9cb9b7e0e6bf`, with NumPy 1.26.4 and SciPy 1.13.1. +Tests execute its functions; Rust outputs are not used as expected results. + +Public real ATAC inputs are GM12878 experiment ENCSR637XSC, replicate-2 +BAM ENCFF962FMH, peaks ENCFF452ZBX, and matched HG001 GIAB NISTv3.3.1 +GRCh38 genotypes. Source MD5 checks passed. Chromosome 21 contains 1,760,836 +alignment records, 27,182 eligible SNPs and 556 peak-overlapping SNP rows. +Counting matches the unmodified archived function at thread settings 1 and 4. +The full counting CLI independently reproduced the 556 peak rows and memberships. +Source links, compressed count tables and checksums are in +[the public-data fixture](../tests/analysis/fixtures/public_atac/README.md). + +## Results and reproduction + +- The 32-run matrix passes locally on macOS arm64: 16 legacy runs across + SNP/peak, single/linear, phased/unphased and pseudocount 0/1, plus 16 donor + feature runs across global/per-donor dispersion. Reported raw mu is compared. +- Counts require exact agreement. Statistical tolerances remain + `rtol=atol=5e-5`; no proportion flipping or region-specific exceptions occur. +- Every legacy matrix case also executes CLI inference and checks its output. + Mixed-genotype tests compare the actual archive's successful and error paths. +- Heterogeneous donor tests independently fit dispersion with Python and compare + donor effects, chi-square LRT p-values and donor-local BH correction. They cover + overlapping features, both dispersion models, phase modes and pseudocounts. +- Local suite: 474 Python tests passed, 82 skipped, 1 deselected; 132 Rust tests + passed, 7 ignored. Ruff, formatting and diff checks pass. Rust 1.88 Clippy + passes with the pre-existing `uninlined_format_args` lint disabled. + +In a full clone with the built extension and pinned numerical dependencies: + +```bash +python -m pytest tests/analysis/test_public_atac_matrix.py \ + tests/analysis/test_bulk_edge_parity.py \ + tests/analysis/test_donor_archive_parity.py -q +``` + +Python 3.11 Linux CI pins the numerical dependencies and runs the complete real +matrix. Other Python versions run the remaining compatibility regressions. +Final-head Linux and container outcomes must be reviewed separately in GitHub +Actions; local results do not establish those outcomes. + +The first full Linux run at `cb7b834` exposed six dynamic-reference failures: +three public unphased peak cases differed in raw mu, and three heterogeneous +per-donor linear cases differed in fitted statistics. Earlier frozen regressions +passed, so they did not establish full Linux compatibility. Both native amd64 +and arm64 container builds, ATAC smoke tests and label checks passed on that head. +These strict failures remain integration gates while platform arithmetic is +investigated. `scripts/diagnose_bulk_numerics.py` compares fused/unfused kernels +and NumPy scalar/vector arithmetic without modifying production source. +The probe confirms that the pinned Linux x86_64 SciPy build uses separate +multiply/add arithmetic at the tested Cephes components, while the macOS arm64 +build uses fused arithmetic. Rust now follows that measured compiler policy; +the full strict matrix must verify the resulting inference on each platform. +At `9d3006d`, all 32 executed-reference Linux matrix runs and heterogeneous donor +comparisons pass. At `625252d`, Linux Python 3.11 passes 464 tests, 76 skipped, +1 deselected, including all 16 native conditional checks; Python 3.10 and 3.12 +also pass. Independently executed archived Linux Python returns +`mu=0.6928487541708976` for the fixed symmetric linear case where the Mac +reference returns `0.3071512458291038`. The Rust trace test records each measured +platform's raw outcome, without accepting complementary outputs. +Conditional tests execute the native archived oracle at identical supplied +dispersion parameters; the original Mac snapshot is unchanged and also checked +in its pinned environment. Matching version numbers alone does not establish +identical compiler/library arithmetic across platforms. + +## Deliberate extensions and reference repairs + +Global/per-donor dispersion is the intended extension. Shared dispersion does +not pool effect tests or multiple-testing correction across donors. Archived +linear array shapes require explicit reference adapters; phased linear tests +explicitly forward the phase flag that the archived linear wrapper ignored. +The shape adapter preserves the original sequential probability recurrence; +exact scalar/vector recurrence tests guard against replacing it with a +mathematically equivalent but differently rounded product reduction. + +Interior rho values are preserved, including tested values from `1e-12` to +`1-1e-12`. Exact endpoints retain a finite regularization because archived +beta-binomial shapes are undefined there. Long probability products use +log-space fallback after losing normal range; this extends the archive's +underflowing arithmetic rather than reproducing its failure. + +Archived phasing checks use distinct original GT tokens and the presence of +`0|1`/`1|0`, then parse every retained first allele. Numeric homozygous tokens +follow that behavior. Malformed mixed tokens raise an error instead of silently +changing a region's likelihood. Numeric-GT duplicates are removed using the parsed first allele, while the +original tokens remain available for the backend's pre-dedup phase decision. +Sixteen extra regressions cover both models, both dispersion scopes, both +orientations and pseudocounts, including fitting weights, QC and phase provenance. +One public peak SNP has `0/1`: the unfiltered +phased command is rejected by both implementations. Phased comparisons explicitly +select truly phased heterozygous SNPs; unphased comparisons retain all SNPs. + +## Integration and release gates + +Review compatibility PR #135 first, then refresh #133 against the resulting dev +branch and rerun required checks. No merge or tag is included in this validation. +All active version metadata currently uses 1.5.0. CI/release image builds pass +the package version and source SHA to OCI labels; container smoke tests verify +those labels and Python/Rust package versions and exercise ATAC grouping/phasing. + +Review the exact final head's tests and containers before integration. A release +dry run remains necessary after the reviewed code reaches the permitted release +branch, including the separate Singularity preflight. Existing published images +and packages will remain old until a release is actually published. + +These comparisons cover one real donor/chromosome and controlled heterogeneous +donors. They do not establish biological calibration, a new WASP remapping pass, +or universal bitwise agreement across numerical environments. A symmetric +unphased likelihood has no identifiable biological direction even when its +reported mu agrees numerically with the pinned reference. diff --git a/docs/source/api/analysis.rst b/docs/source/api/analysis.rst index a4e09d3c..c7b3f814 100644 --- a/docs/source/api/analysis.rst +++ b/docs/source/api/analysis.rst @@ -44,6 +44,14 @@ run_analysis :undoc-members: :show-inheritance: +run_analysis_per_donor +~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: analysis.run_analysis_per_donor + :members: + :undoc-members: + :show-inheritance: + run_analysis_sc ~~~~~~~~~~~~~~~ diff --git a/docs/source/user_guide/analysis.rst b/docs/source/user_guide/analysis.rst index d9edc971..1b11b7e0 100644 --- a/docs/source/user_guide/analysis.rst +++ b/docs/source/user_guide/analysis.rst @@ -33,6 +33,83 @@ Useful options: * ``--region_col``: explicit region column name if auto-detection is not desired * ``--groupby``: group on an alternate annotation column, such as a parent gene column +Donor-Local Bulk Analysis +------------------------- + +Locked cohort count bundles use ``donor_id`` as the donor identity. Analyze the +bundle directory, its ``count_manifest.json``, or its locked ``counts.tsv.gz``: + +.. code-block:: bash + + wasp2-analyze find-imbalance cohort_counts \ + --scope per-donor \ + --unit snv \ + --model single \ + --dispersion-scope per-donor \ + --min-donor-observations 50 \ + --output donor_snv.tsv + +The supported units are ``snv`` and ``feature``; ``peak`` is an alias for +``feature``. SNV analysis is unphased. Feature analysis may use ``--phased`` and +uses the locked ``region`` column, or an explicit ``--region-col`` for a +standalone count table. Overlapping feature memberships are retained for feature +tests and dispersion fitting, matching archived Python's retained SNP-feature +observations. Deduplication follows Python's chromosome, position, counts, and +feature key, including GT only when phasing is effective. Different count +observations at the same SNP are retained. SNV output aggregates those retained +observations into one result per position, as archived Python does. Allele and +SNV-ID consistency checks remain enforced. + +``--dispersion-scope per-donor`` fits ``single`` or ``linear`` nuisance +parameters independently for each included donor. ``--dispersion-scope global`` +fits the combined eligible donor-SNP-feature observations and reuses the returned fit ID and +parameters unchanged in each donor run. In both modes, effect inference and +Benjamini-Hochberg correction are independent within each donor. + +A feature-level ``--phased`` request follows archived Python's fallback checks +within each donor after depth filtering. Missing GT, a single distinct GT, or +no numeric ``0|1``/``1|0`` genotype selects unphased inference. Allele-string GT +such as ``A|G`` is retained rather than converted to numeric phase. Provenance +``donors.phasing`` records each included donor's ``requested_phased``, +``effective_phased``, and ``fallback_reason`` before phase-aware deduplication; +``analysis.phased`` remains the +original request. A shared dispersion fit does not share phasing decisions. + +QC ``eligible_snv_observations`` and dispersion-sidecar +``donor_eligible_snv_observations`` count unique eligible SNPs per donor, so +multiple memberships do not inflate the donor eligibility threshold. +``fit_observations`` counts actual retained fitting rows: all included donors +for a global fit, or the individual donor for a per-donor fit. + +The ``linear`` model carries four inseparable nuisance values: +``linear_d1``, ``linear_d2``, ``linear_depth_center``, and +``linear_depth_scale``. + +Linear fitting follows archived Python's balanced-null objective on +pseudocounted depth, with Nelder-Mead starting at ``(0, 0)`` and SciPy's default +simplex, tolerances, and evaluation limits. The fitted coefficients are then +converted to the standardized raw-depth representation above. Equal-depth +inputs use a unit scale, preserving their fitted predictions. The ``[-10, 10]`` +predictor guard applies during fitting only; testing uses the unclipped sigmoid +prediction with the existing numerical rho boundary guard. + +The fit ID includes all four, and fixed-nuisance donor +runs must echo all four exactly. The dispersion TSV and provenance JSON retain +these values. They retain optimizer or fit status fields when Rust supplies +them; otherwise ``fit_status=parameters_returned`` records only that a usable +parameter payload was returned and does not claim optimizer convergence. + +The donor-local route defaults to pseudocount 1 and creates result, +``.dispersion.tsv``, ``.qc.tsv``, and ``.provenance.json`` artifacts without +overwriting existing files. ``--expected-manifest-sha256`` can pin the locked +bundle manifest to an external digest. Standalone legacy tables may use +``sample`` instead of ``donor_id``; a locked bundle may not, and a table with +both columns is rejected. If the count table has ``snv_id``, its exact text is +retained after enforcing a one-to-one mapping with donor, chromosome, position, +reference, and alternate allele. Only tables without the column receive a +synthesized ``chrom:pos:ref:alt`` identifier, so canonical zero-based input IDs +are not rewritten. + Single-Cell Analysis -------------------- @@ -81,16 +158,24 @@ Notes Defaults and contracts ---------------------- -- ``pseudocount`` defaults to **1** (added to both ``ref_count`` and +Both routes retain the archived chi-square LRT p-values (one degree of freedom). +Unphased feature likelihoods anchor the first SNP to its reference allele and +average the two orientations only for subsequent SNPs. Input order therefore +sets the reference orientation; the first SNP is not symmetrized. + +- ``pseudocount`` defaults to **1** on both the legacy route and with + ``--scope per-donor``. It is added to both ``ref_count`` and ``alt_count`` — Laplace-style shrinkage toward :math:`\mu = 0.5`, - conservative under :math:`H_0`). + before likelihood fitting; raw output counts exclude these pseudocounts. - ``min_count`` defaults to **10**; regions with :math:`N < 10` are dropped. - Beta-binomial LRT: :math:`\rho` is held at its **null-model MLE** while maximizing the alternative likelihood over :math:`\mu` (profile likelihood, df = 1). :math:`\rho` is not jointly re-estimated under :math:`H_1`. -- Dispersion optimizer bounds :math:`\rho \in (10^{-6},\, 1-10^{-6})`; the - linear-dispersion model clips the logit at :math:`\pm 10` for numerical - stability on extreme :math:`N`. +- Scalar dispersion is fit on :math:`(0,1)` with boundary values clamped for + numerical stability. The linear predictor retains the :math:`\pm 10` guard. The + linear model uses raw-depth standardization, + :math:`\rho(N)=\operatorname{expit}(d_1 + d_2(N-c)/s)`, and persists the fitted + center :math:`c` and positive scale :math:`s` with :math:`d_1` and :math:`d_2`. - FDR correction uses :func:`scipy.stats.false_discovery_control` with ``method='bh'`` (Benjamini–Hochberg). ``scipy`` raises on NaN p-values, but a hand-written BH loop using ``np.minimum.accumulate`` silently diff --git a/pyproject.toml b/pyproject.toml index 1a520d6e..ef6a9b3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,7 @@ module-name = "wasp2_rust" python-packages = ["counting", "mapping", "analysis", "wasp2"] bindings = "pyo3" strip = false -include = ["LICENSE", "README.md"] +include = ["LICENSE", "README.md", "rust/licenses/SCIPY-LICENSE.txt", "rust/licenses/NUMPY-LICENSE.txt"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/rust/licenses/NUMPY-LICENSE.txt b/rust/licenses/NUMPY-LICENSE.txt new file mode 100644 index 00000000..014d51c9 --- /dev/null +++ b/rust/licenses/NUMPY-LICENSE.txt @@ -0,0 +1,30 @@ +Copyright (c) 2005-2023, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rust/licenses/SCIPY-LICENSE.txt b/rust/licenses/SCIPY-LICENSE.txt new file mode 100644 index 00000000..11711761 --- /dev/null +++ b/rust/licenses/SCIPY-LICENSE.txt @@ -0,0 +1,30 @@ +Copyright (c) 2001-2002 Enthought, Inc. 2003-2024, SciPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/rust/src/analysis.rs b/rust/src/analysis.rs index 822ba516..872323bb 100644 --- a/rust/src/analysis.rs +++ b/rust/src/analysis.rs @@ -1,3 +1,4 @@ +use crate::scipy_compat::{betabinom_logpmf, numpy_sum}; /// WASP2 Analysis Module - Beta-binomial Allelic Imbalance Detection /// /// Rust implementation of the Python analysis stage (src/analysis/as_analysis.py) @@ -6,8 +7,6 @@ /// Performance target: 3-5x speedup over Python (2.7s → 0.5-0.9s) use anyhow::{Context, Result}; use rayon::prelude::*; -use rv::dist::BetaBinomial; -use rv::traits::HasDensity; use statrs::distribution::{ChiSquared, ContinuousCDF}; use std::collections::{HashMap, HashSet}; @@ -24,11 +23,10 @@ pub struct VariantCounts { pub ref_count: u32, pub alt_count: u32, pub region: String, - /// Donor/sample identifier; required only by the per_donor model. - pub sample: Option, - /// Genotype phase relative to the reference allele: 0 = ref|alt, 1 = alt|ref. - /// `None` when genotypes are absent or unphased. Required by phased analysis. + /// First numeric allele, parsed as in archived Python after phasing checks. pub gt: Option, + /// Original GT token for Python's table-wide diversity/heterozygosity checks. + pub raw_gt: Option, } /// Deduplication key for matching Python's `df[keep_cols].drop_duplicates()`. @@ -60,6 +58,18 @@ pub struct ImbalanceResult { pub fdr_pval: f64, // FDR-corrected p-value } +/// Results and fitted nuisance parameters from one independent analysis table. +#[derive(Debug, Clone)] +pub struct AnalysisOutput { + pub results: Vec, + pub rho: Option, + pub linear_params: Option<(f64, f64)>, + pub linear_depth_center: Option, + pub linear_depth_scale: Option, + pub n_observations: usize, + pub effective_phased: bool, +} + /// Configuration for analysis #[derive(Debug, Clone)] pub struct AnalysisConfig { @@ -71,9 +81,111 @@ pub struct AnalysisConfig { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AnalysisMethod { - Single, // Single global dispersion parameter - Linear, // Linear dispersion model: rho = expit(d1 + N*d2) - PerDonor, // Per-donor dispersion: rho fit separately per sample + Single, // Single global dispersion parameter + Linear, // Linear dispersion model: rho = expit(d1 + d2*((N-center)/scale)) +} + +/// Nuisance parameters fitted under the balanced beta-binomial null. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum DispersionParameters { + Single { + rho: f64, + }, + Linear { + d1: f64, + d2: f64, + depth_center: f64, + depth_scale: f64, + }, +} + +impl DispersionParameters { + fn validate_for(self, method: AnalysisMethod) -> Result { + match (method, self) { + (AnalysisMethod::Single, Self::Single { rho }) => { + if !rho.is_finite() || !(0.0..=1.0).contains(&rho) { + anyhow::bail!("fixed scalar rho must be finite and in [0, 1]"); + } + } + ( + AnalysisMethod::Linear, + Self::Linear { + d1, + d2, + depth_center, + depth_scale, + }, + ) => { + if !d1.is_finite() || !d2.is_finite() { + anyhow::bail!("fixed linear dispersion parameters must be finite"); + } + if !depth_center.is_finite() { + anyhow::bail!("fixed linear depth center must be finite"); + } + if !depth_scale.is_finite() || depth_scale <= 0.0 { + anyhow::bail!("fixed linear depth scale must be finite and positive"); + } + } + (AnalysisMethod::Single, Self::Linear { .. }) => { + anyhow::bail!("method 'single' requires a fixed scalar rho"); + } + (AnalysisMethod::Linear, Self::Single { .. }) => { + anyhow::bail!( + "method 'linear' requires fixed linear_d1, linear_d2, \ + linear_depth_center, and linear_depth_scale" + ); + } + } + Ok(self) + } + + #[inline] + fn rho_at_depth(self, n: u32) -> f64 { + match self { + Self::Single { rho } => clamp_rho(rho), + Self::Linear { + d1, + d2, + depth_center, + depth_scale, + } => { + let standardized_depth = (n as f64 - depth_center) / depth_scale; + clamp_rho(expit(d1 + standardized_depth * d2)) + } + } + } + + pub fn rho(self) -> Option { + match self { + Self::Single { rho } => Some(rho), + Self::Linear { .. } => None, + } + } + + pub fn linear_params(self) -> Option<(f64, f64)> { + match self { + Self::Single { .. } => None, + Self::Linear { d1, d2, .. } => Some((d1, d2)), + } + } + + pub fn linear_depth_standardization(self) -> Option<(f64, f64)> { + match self { + Self::Single { .. } => None, + Self::Linear { + depth_center, + depth_scale, + .. + } => Some((depth_center, depth_scale)), + } + } +} + +/// Result of nuisance-only fitting on one independent table. +#[derive(Debug, Clone, Copy)] +pub struct DispersionFit { + pub parameters: DispersionParameters, + pub n_observations: usize, } impl Default for AnalysisConfig { @@ -91,11 +203,11 @@ impl Default for AnalysisConfig { // Rho Parameter Bounds (Issue #228) // ============================================================================ -/// Epsilon for clamping rho parameter to avoid division by zero. -/// Matches Python's RHO_EPSILON in as_analysis.py. +/// Regularization for exact endpoints, where archived beta-binomial shapes +/// are undefined. Interior values retain the archived parameter unchanged. const RHO_EPSILON: f64 = 1e-10; -/// Clamp rho to safe range (epsilon, 1-epsilon) to prevent division by zero. +/// Regularize exact endpoints without clipping valid interior dispersion. /// /// The beta-binomial parameterization uses alpha = mu * (1-rho) / rho, which /// causes division by zero when rho=0 and produces zero alpha/beta when rho=1. @@ -108,7 +220,13 @@ const RHO_EPSILON: f64 = 1e-10; /// implementation (`as_analysis.py`) supports optional warning via `warn=True`. #[inline] fn clamp_rho(rho: f64) -> f64 { - rho.clamp(RHO_EPSILON, 1.0 - RHO_EPSILON) + if rho <= 0.0 { + RHO_EPSILON + } else if rho >= 1.0 { + 1.0 - RHO_EPSILON + } else { + rho + } } /// Deduplicate variants to match Python's `df[keep_cols].drop_duplicates()`. @@ -136,27 +254,30 @@ fn deduplicate_variants(variants: Vec, phased: bool) -> Vec bool { +fn effective_phased(variants: &[VariantCounts], requested_phased: bool) -> Result { if !requested_phased { - return false; - } - - let mut saw_ref_alt = false; - let mut saw_alt_ref = false; - for variant in variants { - match variant.gt { - Some(0) => saw_ref_alt = true, - Some(1) => saw_alt_ref = true, - _ => {} - } + return Ok(false); } - - if saw_ref_alt && saw_alt_ref { - true - } else { + let unique: HashSet<_> = variants.iter().map(|v| v.raw_gt.as_deref()).collect(); + let informative = variants + .iter() + .any(|v| matches!(v.raw_gt.as_deref(), Some("0|1" | "1|0"))); + if unique.len() <= 1 || !informative { eprintln!("No informative numeric phased GT found: switching to unphased model"); - false + return Ok(false); + } + // Once Python enables phasing it parses every retained GT, and fails on + // malformed mixed input. Do not silently change likelihoods per region. + for variant in variants { + anyhow::ensure!( + matches!(variant.gt, Some(0 | 1)), + "Invalid GT {:?} at {}:{} for phased analysis; provide numeric phased genotypes or run unphased", + variant.raw_gt, + variant.chrom, + variant.pos + ); } + Ok(true) } /// Sigmoid function (inverse logit): expit(x) = 1 / (1 + e^-x) @@ -164,12 +285,9 @@ fn effective_phased(variants: &[VariantCounts], requested_phased: bool) -> bool /// Numerically stable implementation that avoids overflow for large |x|. #[inline] fn expit(x: f64) -> f64 { - if x >= 0.0 { - 1.0 / (1.0 + (-x).exp()) - } else { - let e = x.exp(); - e / (1.0 + e) - } + // Preserve SciPy's expression, including its rounding for negative inputs. + // IEEE exp overflow gives infinity and the quotient remains the valid 0. + 1.0 / (1.0 + (-x).exp()) } // ============================================================================ @@ -196,13 +314,7 @@ pub fn opt_prob(prob: f64, rho: f64, k: u32, n: u32) -> Result { let alpha = prob * (1.0 - rho) / rho; let beta = (1.0 - prob) * (1.0 - rho) / rho; - // Create beta-binomial distribution (rv uses: n as u32, alpha, beta) - let bb = - BetaBinomial::new(n, alpha, beta).context("Failed to create beta-binomial distribution")?; - - // Return negative log-likelihood (rv uses reference for ln_f, k as u64) - let log_pmf = bb.ln_f(&(k as u64)); - Ok(-log_pmf) + Ok(-betabinom_logpmf(k, n, alpha, beta)?) } /// Calculate beta-binomial log-likelihood for array of counts @@ -214,15 +326,12 @@ pub fn betabinom_logpmf_sum( alpha: f64, beta: f64, ) -> Result { - let mut sum = 0.0; - - for (k, n) in ref_counts.iter().zip(n_array.iter()) { - let bb = BetaBinomial::new(*n, alpha, beta) - .context("Failed to create beta-binomial distribution")?; - sum += bb.ln_f(&(*k as u64)); - } - - Ok(sum) + let values: Vec<_> = ref_counts + .iter() + .zip(n_array) + .map(|(&k, &n)| betabinom_logpmf(k, n, alpha, beta)) + .collect::>()?; + Ok(numpy_sum(&values)) } // ============================================================================ @@ -246,65 +355,66 @@ fn optimize_dispersion(ref_counts: &[u32], n_array: &[u32]) -> Result { } }; - // Use golden section search (simple but effective) - let result = golden_section_search(objective, 0.001, 0.999, 1e-6)?; - Ok(result) + // Match scipy.optimize.minimize_scalar(method="bounded") used by AHO. + // In sparse donors the MLE can be far below 0.001, so imposing a floor + // changes the null likelihood and every downstream p-value. + scipy_bounded_minimize(objective, 0.0, 1.0, 1e-5, 500) } -/// Optimize linear dispersion parameters using Nelder-Mead -/// -/// Python equivalent: `minimize(opt_linear, x0=(0, 0), method="Nelder-Mead")` -/// Fits ρ = expit(d1 + N × d2) to the data. -fn optimize_dispersion_linear(ref_counts: &[u32], n_array: &[u32]) -> Result<(f64, f64)> { +/// Fit the archived pseudocounted-depth predictor without converting its +/// coefficients. The reusable raw-depth origin is shifted by the pseudocount. +fn optimize_dispersion_linear( + ref_counts: &[u32], + n_array: &[u32], + raw_depths: &[u32], +) -> Result<(f64, f64, f64, f64)> { + if ref_counts.len() != n_array.len() || ref_counts.len() != raw_depths.len() { + anyhow::bail!("linear dispersion inputs have inconsistent lengths"); + } + if raw_depths.is_empty() { + anyhow::bail!("cannot fit linear dispersion without observations"); + } + let objective = |params: [f64; 2]| -> f64 { let (d1, d2) = (params[0], params[1]); - let mut neg_ll = 0.0; + if !d1.is_finite() || !d2.is_finite() { + return f64::INFINITY; + } + let mut values = Vec::with_capacity(ref_counts.len()); for (&ref_count, &n) in ref_counts.iter().zip(n_array.iter()) { - // Clamp to prevent overflow in expit - let exp_in = (d1 + n as f64 * d2).clamp(-10.0, 10.0); - let rho = clamp_rho(expit(exp_in)); + let predictor = d1 + f64::from(n) * d2; + if predictor.is_nan() { + return f64::INFINITY; + } + let rho = clamp_rho(expit(predictor.clamp(-10.0, 10.0))); let alpha = 0.5 * (1.0 - rho) / rho; // Beta-binomial log-pmf with α = β (null: p=0.5) - if let Ok(bb) = rv::dist::BetaBinomial::new(n, alpha, alpha) { - use rv::traits::HasDensity; - neg_ll -= bb.ln_f(&(ref_count as u64)); + if let Ok(log_pmf) = betabinom_logpmf(ref_count, n, alpha, alpha) { + values.push(log_pmf); } else { return f64::INFINITY; } } - neg_ll + -numpy_sum(&values) }; - // Multi-start Nelder-Mead. A single start at (0,0) can stall on the flat plateau - // created by the exp_in clamp (when |d1 + N*d2| > 10 for all N, rho is constant, so - // the surface has no descent direction and the simplex collapses at an unconverged, - // worse point). Run NM from a neutral grid of starts spanning the plausible (d1,d2) - // range and keep the best (lowest neg-LL). Deterministic; starts are NOT seeded near - // any expected answer. Fixes the case where the simple simplex returned e.g. (1.5,-2.0). - let starts: [[f64; 2]; 8] = [ - [0.0, 0.0], - [-3.0, 0.0], - [-6.0, 0.0], - [3.0, 0.0], - [0.0, -0.2], - [-3.0, -0.2], - [-6.0, -0.2], - [3.0, -0.2], - ]; - let mut best: Option<(f64, (f64, f64))> = None; - for s in starts { - if let Ok((d1, d2)) = nelder_mead_2d(objective, s, 1e-6, 1000) { - let val = objective([d1, d2]); - if val.is_finite() && best.is_none_or(|(bv, _)| val < bv) { - best = Some((val, (d1, d2))); - } - } + // Match scipy.optimize.minimize(method="Nelder-Mead", x0=(0, 0)): + // default simplex, xatol=fatol=1e-4, maxiter=maxfev=400 for two parameters. + let (intercept, slope) = nelder_mead_2d(objective, [0.0, 0.0], 1e-4, 400)?; + let offset = f64::from(n_array[0]) - f64::from(raw_depths[0]); + if n_array + .iter() + .zip(raw_depths) + .any(|(&n, &raw)| (f64::from(n) - f64::from(raw)) != offset) + { + anyhow::bail!("linear fit requires a consistent pseudocount offset"); } - best.map(|(_, p)| p) - .ok_or_else(|| anyhow::anyhow!("linear dispersion optimization failed from all starts")) + // Nraw - (-offset) evaluates the original adjusted depth exactly. Avoid + // mean/std coefficient conversion, whose roundtrip can change a tied basin. + Ok((intercept, slope, -offset, 1.0)) } /// Optimize probability parameter for alternative model @@ -317,7 +427,7 @@ fn optimize_prob(ref_counts: &[u32], n_array: &[u32], disp: f64) -> Result<(f64, opt_prob(prob, disp, ref_counts[0], n_array[0]).unwrap_or(f64::INFINITY) }; - let mu = golden_section_search(objective, 0.0, 1.0, 1e-6)?; + let mu = scipy_bounded_minimize(objective, 0.0, 1.0, 1e-5, 500)?; let alt_ll = -objective(mu); return Ok((alt_ll, mu)); } @@ -334,7 +444,7 @@ fn optimize_prob(ref_counts: &[u32], n_array: &[u32], disp: f64) -> Result<(f64, sum }; - let mu = golden_section_search(objective, 0.0, 1.0, 1e-6)?; + let mu = scipy_bounded_minimize(objective, 0.0, 1.0, 1e-5, 500)?; let alt_ll = -objective(mu); Ok((alt_ll, mu)) } @@ -395,7 +505,7 @@ fn optimize_prob_phased( sum }; - let mu = golden_section_search(objective, 0.0, 1.0, 1e-6)?; + let mu = scipy_bounded_minimize(objective, 0.0, 1.0, 1e-5, 500)?; let alt_ll = -objective(mu); Ok((alt_ll, mu)) } @@ -404,9 +514,8 @@ fn optimize_prob_phased( /// /// Python equivalent: `opt_unphased_dp()` in as_analysis.py /// -/// This marginalizes over unknown phase using dynamic programming. At each -/// position after the first, we consider both phase possibilities and weight -/// them equally (0.5 each), accumulating likelihood across the region. +/// Preserve archived Python: anchor the first SNP to its reference allele, +/// then marginalize both orientations for each subsequent SNP. /// /// # Arguments /// * `prob` - Probability parameter (0 to 1) @@ -427,6 +536,10 @@ pub fn optimize_prob_unphased_dp( phase_ref: &[u32], phase_n: &[u32], ) -> Result { + if disp.is_empty() { + anyhow::bail!("at least one dispersion value is required"); + } + // Split per-variant dispersion for first vs subsequent positions (linear model) let (first_disp, phase_disp): (f64, &[f64]) = if disp.len() > 1 { (disp[0], &disp[1..]) @@ -435,112 +548,185 @@ pub fn optimize_prob_unphased_dp( (disp[0], disp) }; - // Get likelihood of first position (negative log-likelihood) - let first_ll = opt_prob(prob, first_disp, first_ref, first_n)?; - - // If no subsequent positions, just return first_ll - if phase_ref.is_empty() { - return Ok(first_ll); - } - - // Accumulate likelihoods for subsequent positions under both phase hypotheses - // in LOG-space (mathematically identical to the old linear product, but without - // underflow). For each position: + // Average orientations only after the first SNP, as in archived Python. + // For each SNP: // log(0.5 * p1 + 0.5 * p2) = ln(0.5) + logaddexp(ln(p1), ln(p2)) - // where lp1 = ln-pmf with prob and lp2 = ln-pmf with (1 - prob). - let mut log_prev: f64 = 0.0; + let orientation_log_likelihood = |ref_count: u32, n: u32, rho: f64| -> Result<(f64, f64)> { + let rho = clamp_rho(rho); + let alpha1 = prob * (1.0 - rho) / rho; + let beta1 = (1.0 - prob) * (1.0 - rho) / rho; + let lp1 = betabinom_logpmf(ref_count, n, alpha1, beta1)?; + // Python calls opt_prob(1-prob), recomputing both shape parameters. + // Swapping alpha/beta avoids a subtraction and changes last-bit rounding. + let opposite_prob = 1.0 - prob; + let alpha2 = opposite_prob * (1.0 - rho) / rho; + let beta2 = (1.0 - opposite_prob) * (1.0 - rho) / rho; + let lp2 = betabinom_logpmf(ref_count, n, alpha2, beta2)?; + Ok((lp1, lp2)) + }; + + let first_nll = opt_prob(prob, first_disp, first_ref, first_n)?; + let mut log_likelihood = -first_nll; + let mut previous_probability = 1.0; + let mut product_is_normal = true; for (i, (&ref_count, &n)) in phase_ref.iter().zip(phase_n.iter()).enumerate() { - // Get dispersion for this position let rho = if phase_disp.len() > 1 { - clamp_rho(phase_disp[i.min(phase_disp.len() - 1)]) + phase_disp[i.min(phase_disp.len() - 1)] } else { - clamp_rho(phase_disp[0]) + phase_disp[0] }; - - // Phase 1: use prob - let alpha1 = prob * (1.0 - rho) / rho; - let beta1 = (1.0 - prob) * (1.0 - rho) / rho; - let bb1 = BetaBinomial::new(n, alpha1, beta1) - .context("Failed to create beta-binomial for phase1")?; - let lp1 = bb1.ln_f(&(ref_count as u64)); // log-pmf - - // Phase 2: use (1 - prob) - let alpha2 = (1.0 - prob) * (1.0 - rho) / rho; - let beta2 = prob * (1.0 - rho) / rho; - let bb2 = BetaBinomial::new(n, alpha2, beta2) - .context("Failed to create beta-binomial for phase2")?; - let lp2 = bb2.ln_f(&(ref_count as u64)); // log-pmf - - // DP update in log-space: marginalize over both phases with equal weight. - log_prev += (0.5_f64).ln() + logaddexp(lp1, lp2); + let (lp1, lp2) = orientation_log_likelihood(ref_count, n, rho)?; + log_likelihood += (0.5_f64).ln() + logaddexp(lp1, lp2); + if product_is_normal { + // Preserve the archived sequential probability recurrence when safe. + let phase1 = previous_probability * lp1.exp(); + let phase2 = previous_probability * lp2.exp(); + previous_probability = 0.5 * phase1 + 0.5 * phase2; + product_is_normal = previous_probability.is_normal(); + } } - // Return first_ll + (-log_prev) - if !log_prev.is_finite() { + if product_is_normal { + return Ok(first_nll - previous_probability.ln()); + } + // Preserve stable inference when the archive's product loses range. + if !log_likelihood.is_finite() { return Ok(f64::INFINITY); } - - Ok(first_ll + (-log_prev)) + Ok(-log_likelihood) } /// Numerically stable two-argument log-sum-exp: ln(exp(a) + exp(b)). /// -/// Equivalent to NumPy's `np.logaddexp`. Subtracting the max before exponentiating -/// avoids overflow/underflow. If the max is non-finite (e.g. -inf when both -/// inputs are -inf), the max is returned directly. +/// Follows NumPy's `np.logaddexp` arithmetic with log1p of the smaller +/// exponential, avoiding overflow and retaining small contributions. #[inline] fn logaddexp(a: f64, b: f64) -> f64 { - let m = a.max(b); - if !m.is_finite() { - m + // NumPy uses log1p for the smaller exponential; log(1 + exp(...)) + // rounds differently and can alter a symmetric optimizer's branch. + if a == b { + a + std::f64::consts::LN_2 + } else if a > b { + a + (b - a).exp().ln_1p() } else { - m + ((a - m).exp() + (b - m).exp()).ln() + b + (a - b).exp().ln_1p() } } -/// Golden section search for 1D optimization -/// -/// Simple but robust method for bounded scalar optimization. -/// Equivalent to scipy's minimize_scalar with method='bounded' -#[allow(unused_assignments)] -fn golden_section_search(f: F, a: f64, mut b: f64, tol: f64) -> Result +/// Bounded Brent minimizer matching SciPy's default scalar-bounded routine. +fn scipy_bounded_minimize(f: F, x1: f64, x2: f64, xatol: f64, maxfun: usize) -> Result +where + F: Fn(f64) -> f64, +{ + scipy_bounded_minimize_traced(f, x1, x2, xatol, maxfun, |_, _, _, _, _| {}) +} + +/// Diagnostic hook records candidate, objective, updated interval and acceptance. +/// The regular wrapper supplies an empty hook; no production output is changed. +fn scipy_bounded_minimize_traced( + f: F, + x1: f64, + x2: f64, + xatol: f64, + maxfun: usize, + mut trace: T, +) -> Result where F: Fn(f64) -> f64, + T: FnMut(f64, f64, f64, f64, bool), { - let phi = (1.0 + 5.0_f64.sqrt()) / 2.0; - let inv_phi = 1.0 / phi; - let inv_phi2 = 1.0 / (phi * phi); - - let mut a = a; - let mut h = b - a; - - // Initial points - let mut c = a + inv_phi2 * h; - let mut d = a + inv_phi * h; - let mut fc = f(c); - let mut fd = f(d); - - // Iterate until convergence - while h.abs() > tol { - if fc < fd { - b = d; - d = c; - fd = fc; - h *= inv_phi; - c = a + inv_phi2 * h; - fc = f(c); + if !x1.is_finite() || !x2.is_finite() || x1 > x2 { + anyhow::bail!("invalid bounded optimization interval"); + } + let sqrt_eps = (2.2e-16_f64).sqrt(); + let golden_mean = 0.5 * (3.0 - 5.0_f64.sqrt()); + let (mut a, mut b) = (x1, x2); + let mut fulc = a + golden_mean * (b - a); + let (mut nfc, mut xf) = (fulc, fulc); + let (mut rat, mut e) = (0.0_f64, 0.0_f64); + let mut fx = f(xf); + let (mut ffulc, mut fnfc) = (fx, fx); + trace(xf, fx, a, b, true); + let mut fu = f64::INFINITY; + let mut calls = 1_usize; + let mut xm = 0.5 * (a + b); + let mut tol1 = sqrt_eps * xf.abs() + xatol / 3.0; + let mut tol2 = 2.0 * tol1; + + while (xf - xm).abs() > tol2 - 0.5 * (b - a) { + let mut golden = true; + if e.abs() > tol1 { + let r = (xf - nfc) * (fx - ffulc); + let q0 = (xf - fulc) * (fx - fnfc); + let mut p = (xf - fulc) * q0 - (xf - nfc) * r; + let mut q = 2.0 * (q0 - r); + if q > 0.0 { + p = -p; + } + q = q.abs(); + let previous_e = e; + e = rat; + if p.abs() < (0.5 * q * previous_e).abs() && p > q * (a - xf) && p < q * (b - xf) { + rat = p / q; + let candidate = xf + rat; + if candidate - a < tol2 || b - candidate < tol2 { + rat = tol1 * if xm - xf >= 0.0 { 1.0 } else { -1.0 }; + } + golden = false; + } + } + if golden { + e = if xf >= xm { a - xf } else { b - xf }; + rat = golden_mean * e; + } + + let direction = if rat >= 0.0 { 1.0 } else { -1.0 }; + let x = xf + direction * rat.abs().max(tol1); + fu = f(x); + calls += 1; + let accepted = fu <= fx; + if accepted { + if x >= xf { + a = xf; + } else { + b = xf; + } + fulc = nfc; + ffulc = fnfc; + nfc = xf; + fnfc = fx; + xf = x; + fx = fu; } else { - a = c; - c = d; - fc = fd; - h *= inv_phi; - d = a + inv_phi * h; - fd = f(d); + if x < xf { + a = x; + } else { + b = x; + } + if fu <= fnfc || nfc == xf { + fulc = nfc; + ffulc = fnfc; + nfc = x; + fnfc = fu; + } else if fu <= ffulc || fulc == xf || fulc == nfc { + fulc = x; + ffulc = fu; + } } - } - Ok(if fc < fd { c } else { d }) + trace(x, fu, a, b, accepted); + xm = 0.5 * (a + b); + tol1 = sqrt_eps * xf.abs() + xatol / 3.0; + tol2 = 2.0 * tol1; + if calls >= maxfun { + anyhow::bail!("bounded scalar optimization exceeded {maxfun} evaluations"); + } + } + if !xf.is_finite() || !fx.is_finite() || fu.is_nan() { + anyhow::bail!("bounded scalar optimization produced a non-finite result"); + } + Ok(xf) } /// Nelder-Mead simplex optimization for 2D functions @@ -551,103 +737,98 @@ fn nelder_mead_2d(f: F, x0: [f64; 2], tol: f64, max_iter: usize) -> Result<(f where F: Fn([f64; 2]) -> f64, { - // Standard Nelder-Mead coefficients - let alpha = 1.0; // reflection - let gamma = 2.0; // expansion - let rho = 0.5; // contraction - let sigma = 0.5; // shrinkage - - // Initialize simplex: equilateral triangle around x0 - let step = 1.0; + let step = |x: f64| if x == 0.0 { 0.00025 } else { 1.05 * x }; let mut simplex = [ (x0, f(x0)), - ([x0[0] + step, x0[1]], f([x0[0] + step, x0[1]])), - ([x0[0], x0[1] + step], f([x0[0], x0[1] + step])), + ([step(x0[0]), x0[1]], f([step(x0[0]), x0[1]])), + ([x0[0], step(x0[1])], f([x0[0], step(x0[1])])), ]; - - for _iter in 0..max_iter { - // Sort by function value (ascending) - simplex.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap()); - - let (x_best, f_best) = simplex[0]; - let (x_worst, f_worst) = simplex[2]; - let (x_second, f_second) = simplex[1]; - - // Check convergence: simplex size - let size = ((x_worst[0] - x_best[0]).powi(2) + (x_worst[1] - x_best[1]).powi(2)).sqrt(); - if size < tol { - return Ok((x_best[0], x_best[1])); + let mut evaluations = 3; + let mut evaluate = |point| { + if evaluations >= max_iter { + None + } else { + evaluations += 1; + Some(f(point)) } - - // Centroid of all points except worst - let centroid = [ - (x_best[0] + x_second[0]) / 2.0, - (x_best[1] + x_second[1]) / 2.0, - ]; - - // Reflection - let x_r = [ - centroid[0] + alpha * (centroid[0] - x_worst[0]), - centroid[1] + alpha * (centroid[1] - x_worst[1]), - ]; - let f_r = f(x_r); - - if f_r < f_second && f_r >= f_best { - // Accept reflection - simplex[2] = (x_r, f_r); - continue; + }; + 'iterations: for _ in 1..max_iter { + simplex.sort_by(|a, b| a.1.total_cmp(&b.1)); + let (best, best_value) = simplex[0]; + let (second, second_value) = simplex[1]; + let (worst, worst_value) = simplex[2]; + let coordinate_spread = simplex[1..] + .iter() + .flat_map(|(x, _)| [(x[0] - best[0]).abs(), (x[1] - best[1]).abs()]) + .fold(0.0, f64::max); + let objective_spread = simplex[1..] + .iter() + .map(|(_, value)| (value - best_value).abs()) + .fold(0.0, f64::max); + if coordinate_spread <= tol && objective_spread <= tol { + break; } - - if f_r < f_best { - // Try expansion - let x_e = [ - centroid[0] + gamma * (x_r[0] - centroid[0]), - centroid[1] + gamma * (x_r[1] - centroid[1]), + let center = [0.5 * (best[0] + second[0]), 0.5 * (best[1] + second[1])]; + let reflected = [2.0 * center[0] - worst[0], 2.0 * center[1] - worst[1]]; + let Some(reflected_value) = evaluate(reflected) else { + break; + }; + if reflected_value < best_value { + let expanded = [ + 3.0 * center[0] - 2.0 * worst[0], + 3.0 * center[1] - 2.0 * worst[1], ]; - let f_e = f(x_e); - - if f_e < f_r { - simplex[2] = (x_e, f_e); + let Some(expanded_value) = evaluate(expanded) else { + break; + }; + simplex[2] = if expanded_value < reflected_value { + (expanded, expanded_value) } else { - simplex[2] = (x_r, f_r); - } - continue; - } - - // f_r >= f_second: try contraction - let x_c = if f_r < f_worst { - // Outside contraction - [ - centroid[0] + rho * (x_r[0] - centroid[0]), - centroid[1] + rho * (x_r[1] - centroid[1]), - ] + (reflected, reflected_value) + }; + } else if reflected_value < second_value { + simplex[2] = (reflected, reflected_value); } else { - // Inside contraction - [ - centroid[0] + rho * (x_worst[0] - centroid[0]), - centroid[1] + rho * (x_worst[1] - centroid[1]), - ] - }; - let f_c = f(x_c); - - if f_c < f_worst { - simplex[2] = (x_c, f_c); - continue; - } - - // Shrink toward best point - for point in simplex.iter_mut().skip(1) { - let (x_i, _) = *point; - let x_new = [ - x_best[0] + sigma * (x_i[0] - x_best[0]), - x_best[1] + sigma * (x_i[1] - x_best[1]), - ]; - *point = (x_new, f(x_new)); + let outside = reflected_value < worst_value; + let contracted = if outside { + [ + 1.5 * center[0] - 0.5 * worst[0], + 1.5 * center[1] - 0.5 * worst[1], + ] + } else { + [ + 0.5 * center[0] + 0.5 * worst[0], + 0.5 * center[1] + 0.5 * worst[1], + ] + }; + let Some(contracted_value) = evaluate(contracted) else { + break; + }; + let accept = if outside { + contracted_value <= reflected_value + } else { + contracted_value < worst_value + }; + if accept { + simplex[2] = (contracted, contracted_value); + } else { + for point in simplex.iter_mut().skip(1) { + point.0 = [ + best[0] + 0.5 * (point.0[0] - best[0]), + best[1] + 0.5 * (point.0[1] - best[1]), + ]; + let Some(value) = evaluate(point.0) else { + break 'iterations; + }; + point.1 = value; + } + } } } - - // Return best point after max iterations - simplex.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap()); + simplex.sort_by(|a, b| a.1.total_cmp(&b.1)); + if !simplex[0].1.is_finite() { + anyhow::bail!("linear dispersion optimization produced no finite objective"); + } Ok((simplex[0].0[0], simplex[0].0[1])) } @@ -690,160 +871,134 @@ pub fn fdr_correction(pvals: &[f64]) -> Vec { /// Single dispersion model analysis /// /// Python equivalent: `single_model()` in as_analysis.py -pub fn single_model(variants: Vec, phased: bool) -> Result> { +#[allow(dead_code)] +fn single_model_with_dispersion( + variants: Vec, + phased: bool, +) -> Result<(Vec, f64)> { if variants.is_empty() { - return Ok(vec![]); + return Ok((vec![], f64::NAN)); } - // Extract ref_counts and N for all variants - let ref_counts: Vec = variants.iter().map(|v| v.ref_count).collect(); - let n_array: Vec = variants.iter().map(|v| v.ref_count + v.alt_count).collect(); - - // Step 1: Optimize global dispersion parameter eprintln!("Optimizing dispersion parameter..."); - let disp = optimize_dispersion(&ref_counts, &n_array)?; + let parameters = fit_dispersion_parameters(&variants, AnalysisMethod::Single, 0)?; + let DispersionParameters::Single { rho: disp } = parameters else { + unreachable!("single fit returned linear parameters") + }; eprintln!(" Dispersion: {:.6}", disp); + let results = test_fixed_dispersion(variants, phased, parameters, false, 0)?; + Ok((results, disp)) +} - // Step 2: Group by region - let mut region_map: HashMap> = HashMap::new(); - for (i, variant) in variants.iter().enumerate() { - region_map - .entry(variant.region.clone()) - .or_default() - .push(i); - } - - eprintln!( - "Optimizing imbalance likelihood for {} regions...", - region_map.len() - ); - - // Step 3: Calculate null and alternative likelihoods per region (parallel) - // Clamp disp before calculating null_param (Issue #228) - let disp = clamp_rho(disp); - let null_param = 0.5 * (1.0 - disp) / disp; - - let results: Result> = region_map - .par_iter() - .map(|(region, indices)| -> Result { - // Extract counts for this region - let region_ref: Vec = indices.iter().map(|&i| ref_counts[i]).collect(); - let region_n: Vec = indices.iter().map(|&i| n_array[i]).collect(); - - // Null model: prob = 0.5 (no imbalance) - let null_ll = betabinom_logpmf_sum(®ion_ref, ®ion_n, null_param, null_param)?; +#[allow(dead_code)] +pub fn single_model(variants: Vec, phased: bool) -> Result> { + Ok(single_model_with_dispersion(variants, phased)?.0) +} - // Alternative model: optimize prob (phase-aware routing) - let all_gt = indices.iter().all(|&i| variants[i].gt.is_some()); - let (alt_ll, mu) = if phased && indices.len() > 1 && all_gt { - // Phased multi-SNP region: use known genotype phase. - let region_gt: Vec = indices - .iter() - .map(|&i| variants[i].gt.expect("gt present (checked by all_gt)")) - .collect(); - // single_model uses one global scalar dispersion; pass it as a - // 1-element slice so the broadcast fallback keeps results identical. - optimize_prob_phased(®ion_ref, ®ion_n, ®ion_gt, &[disp])? - } else if indices.len() > 1 { - // Unphased multi-SNP region: marginalize over phase via DP, - // minimizing opt_unphased_dp over prob in (0, 1) - // (first = index 0, phase = remaining), matching Python's - // parse_opt() unphased branch (minimize_scalar(opt_unphased_dp,...)). - let disp_slice = [disp]; - let first_ref = region_ref[0]; - let first_n = region_n[0]; - let phase_ref = ®ion_ref[1..]; - let phase_n = ®ion_n[1..]; - let objective = |prob: f64| -> f64 { - optimize_prob_unphased_dp( - prob, - &disp_slice, - first_ref, - first_n, - phase_ref, - phase_n, - ) - .unwrap_or(f64::INFINITY) - }; - let mu = golden_section_search(objective, 0.0, 1.0, 1e-6)?; - let alt_ll = -objective(mu); - (alt_ll, mu) - } else { - optimize_prob(®ion_ref, ®ion_n, disp)? - }; +/// Linear dispersion model analysis +/// +/// Python equivalent: `linear_model()` in as_analysis.py +/// Fits the archived pseudocounted-depth predictor and returns equivalent +/// standardized raw-depth parameters for reuse. +#[allow(dead_code)] +pub fn linear_model_with_dispersion( + variants: Vec, + phased: bool, +) -> Result<(Vec, f64, f64)> { + if variants.is_empty() { + return Ok((vec![], f64::NAN, f64::NAN)); + } - // Likelihood ratio test - let lrt = -2.0 * (null_ll - alt_ll); + eprintln!("Optimizing linear dispersion parameters..."); + let parameters = fit_dispersion_parameters(&variants, AnalysisMethod::Linear, 0)?; + let DispersionParameters::Linear { d1, d2, .. } = parameters else { + unreachable!("linear fit returned scalar parameters") + }; + eprintln!(" d1 = {:.6}, d2 = {:.6}", d1, d2); + let results = test_fixed_dispersion(variants, phased, parameters, false, 0)?; + Ok((results, d1, d2)) +} - // P-value from chi-squared distribution (df=1) - let chi2 = ChiSquared::new(1.0).context("Failed to create chi-squared distribution")?; - let pval = 1.0 - chi2.cdf(lrt); +#[allow(dead_code)] +pub fn linear_model(variants: Vec, phased: bool) -> Result> { + Ok(linear_model_with_dispersion(variants, phased)?.0) +} - // Sum counts for this region - let total_ref: u32 = region_ref.iter().sum(); - let total_alt: u32 = indices.iter().map(|&i| variants[i].alt_count).sum(); - let total_n = total_ref + total_alt; +fn fit_dispersion_parameters( + variants: &[VariantCounts], + method: AnalysisMethod, + pseudocount: u32, +) -> Result { + if variants.is_empty() { + anyhow::bail!("cannot fit dispersion without observations"); + } - Ok(ImbalanceResult { - region: region.clone(), - ref_count: total_ref, - alt_count: total_alt, - n: total_n, - snp_count: indices.len(), - null_ll, - alt_ll, - mu, - lrt, - pval, - fdr_pval: 0.0, // Will be filled later - }) - }) + let ref_counts: Vec = variants.iter().map(|variant| variant.ref_count).collect(); + let n_array: Vec = variants + .iter() + .map(|variant| variant.ref_count + variant.alt_count) .collect(); - - let mut results = results?; - - // Sort by region name for deterministic output across runs - results.sort_by(|a, b| a.region.cmp(&b.region)); - - // Step 4: FDR correction - let pvals: Vec = results.iter().map(|r| r.pval).collect(); - let fdr_pvals = fdr_correction(&pvals); - - for (result, fdr_pval) in results.iter_mut().zip(fdr_pvals.iter()) { - result.fdr_pval = *fdr_pval; + match method { + AnalysisMethod::Single => Ok(DispersionParameters::Single { + rho: optimize_dispersion(&ref_counts, &n_array)?, + }), + AnalysisMethod::Linear => { + let raw_depths: Vec = variants + .iter() + .map(|variant| raw_depth(variant, pseudocount)) + .collect::>()?; + let (d1, d2, depth_center, depth_scale) = + optimize_dispersion_linear(&ref_counts, &n_array, &raw_depths)?; + Ok(DispersionParameters::Linear { + d1, + d2, + depth_center, + depth_scale, + }) + } } +} - Ok(results) +fn raw_depth(variant: &VariantCounts, pseudocount: u32) -> Result { + let raw_ref = variant.ref_count.checked_sub(pseudocount).ok_or_else(|| { + anyhow::anyhow!("reference count is smaller than the configured pseudocount") + })?; + let raw_alt = variant.alt_count.checked_sub(pseudocount).ok_or_else(|| { + anyhow::anyhow!("alternate count is smaller than the configured pseudocount") + })?; + raw_ref + .checked_add(raw_alt) + .ok_or_else(|| anyhow::anyhow!("raw read depth exceeds the supported range")) } -/// Linear dispersion model analysis -/// -/// Python equivalent: `linear_model()` in as_analysis.py -/// Uses ρ = expit(d1 + N × d2) where d1, d2 are globally optimized parameters. -pub fn linear_model(variants: Vec, phased: bool) -> Result> { +fn test_fixed_dispersion( + variants: Vec, + phased: bool, + parameters: DispersionParameters, + exact_snv_pvalues: bool, + pseudocount: u32, +) -> Result> { + if exact_snv_pvalues { + anyhow::bail!("Exact SNV p-values are not supported; use the archived chi-square LRT"); + } if variants.is_empty() { return Ok(vec![]); } - // Extract ref_counts and N for all variants - let ref_counts: Vec = variants.iter().map(|v| v.ref_count).collect(); - let n_array: Vec = variants.iter().map(|v| v.ref_count + v.alt_count).collect(); - - // Step 1: Optimize linear dispersion parameters (d1, d2) - eprintln!("Optimizing linear dispersion parameters..."); - let (d1, d2) = optimize_dispersion_linear(&ref_counts, &n_array)?; - eprintln!(" d1 = {:.6}, d2 = {:.6}", d1, d2); - - // Step 2: Compute per-variant rho array - let rho_array: Vec = n_array + let ref_counts: Vec = variants.iter().map(|variant| variant.ref_count).collect(); + let n_array: Vec = variants .iter() - .map(|&n| { - let exp_in = (d1 + n as f64 * d2).clamp(-10.0, 10.0); - clamp_rho(expit(exp_in)) - }) + .map(|variant| variant.ref_count + variant.alt_count) + .collect(); + let raw_depths: Vec = variants + .iter() + .map(|variant| raw_depth(variant, pseudocount)) + .collect::>()?; + let rho_array: Vec = raw_depths + .iter() + .map(|&depth| parameters.rho_at_depth(depth)) .collect(); - // Step 3: Group by region let mut region_map: HashMap> = HashMap::new(); for (i, variant) in variants.iter().enumerate() { region_map @@ -857,16 +1012,13 @@ pub fn linear_model(variants: Vec, phased: bool) -> Result> = region_map .par_iter() .map(|(region, indices)| -> Result { - // Extract counts and rho for this region let region_ref: Vec = indices.iter().map(|&i| ref_counts[i]).collect(); let region_n: Vec = indices.iter().map(|&i| n_array[i]).collect(); let region_rho: Vec = indices.iter().map(|&i| rho_array[i]).collect(); - // Null model: prob = 0.5, using per-variant rho let mut null_ll = 0.0; for ((&ref_count, &n), &rho) in region_ref .iter() @@ -874,32 +1026,17 @@ pub fn linear_model(variants: Vec, phased: bool) -> Result 1 && all_gt { - // Phased multi-SNP region: use known genotype phase. let region_gt: Vec = indices .iter() .map(|&i| variants[i].gt.expect("gt present (checked by all_gt)")) .collect(); optimize_prob_phased(®ion_ref, ®ion_n, ®ion_gt, ®ion_rho)? } else if indices.len() > 1 { - // Unphased multi-SNP region: marginalize over phase via DP, - // minimizing opt_unphased_dp over prob in (0, 1) - // (first = index 0, phase = remaining), matching Python's - // parse_opt() unphased branch (minimize_scalar(opt_unphased_dp,...)). - // region_rho is in the region's index order ([0] = first, - // [1..] = phase), matching opt_unphased_dp's internal split. let first_ref = region_ref[0]; let first_n = region_n[0]; let phase_ref = ®ion_ref[1..]; @@ -915,22 +1052,18 @@ pub fn linear_model(variants: Vec, phased: bool) -> Result, phased: bool) -> Result = results.iter().map(|r| r.pval).collect(); let fdr_pvals = fdr_correction(&pvals); - for (result, fdr_pval) in results.iter_mut().zip(fdr_pvals.iter()) { result.fdr_pval = *fdr_pval; } - - Ok(results) -} - -/// Fit per-donor dispersion (rho) via bounded MLE, one value per `sample`. -/// -/// Python equivalent: `compute_per_donor_rho()` — each donor's rho is the -/// symmetric beta-binomial MLE over all that donor's observations, bounded to -/// (1e-6, 1-1e-6). No extra floor (matches locked donor_rho.tsv: min ~7e-6). -fn compute_donor_rho(variants: &[VariantCounts]) -> Result> { - // Group observation indices by donor - let mut donor_map: HashMap> = HashMap::new(); - for (i, v) in variants.iter().enumerate() { - let donor = v.sample.clone().ok_or_else(|| { - anyhow::anyhow!("per_donor model requires a 'sample' column in the counts TSV") - })?; - donor_map.entry(donor).or_default().push(i); - } - - // Fit rho per donor in parallel (golden-section on symmetric beta-binomial) - donor_map - .par_iter() - .map(|(donor, idx)| -> Result<(String, f64)> { - let dref: Vec = idx.iter().map(|&i| variants[i].ref_count).collect(); - let dn: Vec = idx - .iter() - .map(|&i| variants[i].ref_count + variants[i].alt_count) - .collect(); - let objective = |rho: f64| -> f64 { - let rho = clamp_rho(rho); - let a = 0.5 * (1.0 - rho) / rho; - match betabinom_logpmf_sum(&dref, &dn, a, a) { - Ok(ll) => -ll, - Err(_) => f64::INFINITY, - } - }; - // Bounds (1e-6, 1-1e-6) match scipy minimize_scalar(method="bounded") - let rho = golden_section_search(objective, 1e-6, 1.0 - 1e-6, 1e-6)?; - Ok((donor.clone(), rho)) - }) - .collect() -} - -/// Per-donor dispersion model analysis. -/// -/// Python equivalent: WASP2 `per_donor` extension — fit rho per `sample`, then -/// run the standard per-region beta-binomial LRT using each observation's -/// donor-specific rho (per-observation rho in BOTH null and alternative). -pub fn per_donor_model(variants: Vec, phased: bool) -> Result> { - if variants.is_empty() { - return Ok(vec![]); - } - - // Step 1: fit per-donor dispersion - eprintln!("Fitting per-donor dispersion..."); - let donor_rho = compute_donor_rho(&variants)?; - eprintln!(" Fit rho for {} donors", donor_rho.len()); - // Emit fitted rho (for validation against locked donor_rho.tsv) - let mut donors_sorted: Vec<(&String, &f64)> = donor_rho.iter().collect(); - donors_sorted.sort_by(|a, b| a.0.cmp(b.0)); - for (donor, rho) in donors_sorted { - eprintln!("DONOR_RHO\t{}\t{:.8}", donor, rho); - } - - // Step 2: per-observation rho = each obs's donor rho - let ref_counts: Vec = variants.iter().map(|v| v.ref_count).collect(); - let n_array: Vec = variants.iter().map(|v| v.ref_count + v.alt_count).collect(); - let rho_array: Vec = variants - .iter() - .map(|v| { - let donor = v - .sample - .as_ref() - .expect("sample present (checked in compute_donor_rho)"); - clamp_rho( - *donor_rho - .get(donor) - .expect("donor rho fit for every sample"), - ) - }) - .collect(); - - // Step 3: group by region - let mut region_map: HashMap> = HashMap::new(); - for (i, variant) in variants.iter().enumerate() { - region_map - .entry(variant.region.clone()) - .or_default() - .push(i); - } - eprintln!( - "Optimizing imbalance likelihood for {} regions...", - region_map.len() - ); - - // Step 4: per-region null/alt LL + LRT, using per-observation rho - let results: Result> = region_map - .par_iter() - .map(|(region, indices)| -> Result { - let region_ref: Vec = indices.iter().map(|&i| ref_counts[i]).collect(); - let region_n: Vec = indices.iter().map(|&i| n_array[i]).collect(); - let region_rho: Vec = indices.iter().map(|&i| rho_array[i]).collect(); - - use rv::traits::HasDensity; - - // Null model: prob = 0.5, per-observation rho - let mut null_ll = 0.0; - for ((&ref_count, &n), &rho) in region_ref - .iter() - .zip(region_n.iter()) - .zip(region_rho.iter()) - { - let alpha = 0.5 * (1.0 - rho) / rho; - let bb = rv::dist::BetaBinomial::new(n, alpha, alpha) - .context("Failed to create beta-binomial for null")?; - null_ll += bb.ln_f(&(ref_count as u64)); - } - - // Alternative model: optimize one pi, per-observation rho. - // Phase-aware routing (mirrors single_model): for a phased multi-SNP - // region with known GT, use the haplotype-phased likelihood, feeding - // the per-observation (per-donor) rho slice into optimize_prob_phased's - // &[f64] disp argument. Otherwise (unphased, or single-SNP, or missing - // GT) use the unphased shared-pi product UNCHANGED — so phased=false is - // byte-identical to the pre-phasing per_donor model. - let all_gt = indices.iter().all(|&i| variants[i].gt.is_some()); - let (alt_ll, mu) = if phased && indices.len() > 1 && all_gt { - let region_gt: Vec = indices - .iter() - .map(|&i| variants[i].gt.expect("gt present (checked by all_gt)")) - .collect(); - // Per-observation donor rho flows straight into the &[f64] slice. - optimize_prob_phased(®ion_ref, ®ion_n, ®ion_gt, ®ion_rho)? - } else { - // Unphased: optimize one pi over the per-observation beta-binomial - // product (matches Python _alt_ll: a=pi(1-rho)/rho, b=(1-pi)(1-rho)/rho) - let alt_obj = |pi: f64| -> f64 { - let pi = pi.clamp(1e-6, 1.0 - 1e-6); - let mut nll = 0.0; - for ((&ref_count, &n), &rho) in region_ref - .iter() - .zip(region_n.iter()) - .zip(region_rho.iter()) - { - let a = pi * (1.0 - rho) / rho; - let b = (1.0 - pi) * (1.0 - rho) / rho; - match rv::dist::BetaBinomial::new(n, a, b) { - Ok(bb) => nll -= bb.ln_f(&(ref_count as u64)), - Err(_) => return f64::INFINITY, - } - } - nll - }; - let mu = golden_section_search(alt_obj, 1e-6, 1.0 - 1e-6, 1e-6)?; - let alt_ll = -alt_obj(mu); - (alt_ll, mu) - }; - - // Likelihood ratio test (clamped at 0 like Python) - let lrt = (-2.0 * (null_ll - alt_ll)).max(0.0); - let chi2 = ChiSquared::new(1.0).context("Failed to create chi-squared distribution")?; - let pval = 1.0 - chi2.cdf(lrt); - - let total_ref: u32 = region_ref.iter().sum(); - let total_alt: u32 = indices.iter().map(|&i| variants[i].alt_count).sum(); - let total_n = total_ref + total_alt; - - Ok(ImbalanceResult { - region: region.clone(), - ref_count: total_ref, - alt_count: total_alt, - n: total_n, - snp_count: indices.len(), - null_ll, - alt_ll, - mu, - lrt, - pval, - fdr_pval: 0.0, - }) - }) - .collect(); - - let mut results = results?; - - // Deterministic output order - results.sort_by(|a, b| a.region.cmp(&b.region)); - - // FDR correction (BH) - let pvals: Vec = results.iter().map(|r| r.pval).collect(); - let fdr_pvals = fdr_correction(&pvals); - for (result, fdr_pval) in results.iter_mut().zip(fdr_pvals.iter()) { - result.fdr_pval = *fdr_pval; - } - Ok(results) } /// Main entry point for allelic imbalance analysis /// /// Python equivalent: `get_imbalance()` in as_analysis.py -pub fn analyze_imbalance( +struct PreparedAnalysis { + variants: Vec, + effective_phased: bool, +} + +fn prepare_analysis( variants: Vec, config: &AnalysisConfig, -) -> Result> { - // Apply filters and pseudocounts +) -> Result { let filtered: Vec = variants .into_iter() .map(|mut v| { @@ -1181,38 +1121,29 @@ pub fn analyze_imbalance( eprintln!("Processing {} variants after filtering", filtered.len()); - let phased = effective_phased(&filtered, config.phased); + let phased = effective_phased(&filtered, config.phased)?; - // Deduplicate variants for Python parity (single and linear models only). + // Deduplicate variants for Python parity. // Python's get_imbalance() does: df[keep_cols].drop_duplicates() - // Per-donor model skips dedup as it's a Rust-only feature. - let deduped = if config.method != AnalysisMethod::PerDonor { - let before = filtered.len(); - let after = deduplicate_variants(filtered, phased); - if after.len() < before { - eprintln!( - "Deduplicated {} → {} variants ({} duplicates removed)", - before, - after.len(), - before - after.len() - ); - } - after - } else { - filtered - }; - - // Run analysis based on method - let mut results = match config.method { - AnalysisMethod::Single => single_model(deduped, phased)?, - AnalysisMethod::Linear => linear_model(deduped, phased)?, - AnalysisMethod::PerDonor => per_donor_model(deduped, phased)?, - }; + let before = filtered.len(); + let deduped = deduplicate_variants(filtered, phased); + if deduped.len() < before { + eprintln!( + "Deduplicated {} → {} variants ({} duplicates removed)", + before, + deduped.len(), + before - deduped.len() + ); + } + Ok(PreparedAnalysis { + variants: deduped, + effective_phased: phased, + }) +} - // Remove pseudocounts from results - for result in results.iter_mut() { - // Each retained SNP contributes one pseudocount per allele. - let region_pseudocount = config.pseudocount.saturating_mul(result.snp_count as u32); +fn remove_result_pseudocounts(results: &mut [ImbalanceResult], pseudocount: u32) { + for result in results { + let region_pseudocount = pseudocount.saturating_mul(result.snp_count as u32); if result.ref_count < region_pseudocount || result.alt_count < region_pseudocount || result.n < 2 * region_pseudocount @@ -1226,47 +1157,139 @@ pub fn analyze_imbalance( result.alt_count = result.alt_count.saturating_sub(region_pseudocount); result.n = result.n.saturating_sub(2 * region_pseudocount); } +} - Ok(results) +fn analyze_prepared( + prepared: PreparedAnalysis, + config: &AnalysisConfig, + parameters: DispersionParameters, + exact_snv_pvalues: bool, +) -> Result { + let n_observations = prepared.variants.len(); + let mut results = test_fixed_dispersion( + prepared.variants, + prepared.effective_phased, + parameters, + exact_snv_pvalues, + config.pseudocount, + )?; + remove_result_pseudocounts(&mut results, config.pseudocount); + + Ok(AnalysisOutput { + results, + rho: parameters.rho(), + linear_params: parameters.linear_params(), + linear_depth_center: parameters + .linear_depth_standardization() + .map(|standardization| standardization.0), + linear_depth_scale: parameters + .linear_depth_standardization() + .map(|standardization| standardization.1), + n_observations, + effective_phased: prepared.effective_phased, + }) } -#[cfg(test)] -mod tests { - use super::*; +/// Fit only the balanced-null nuisance parameters for one independent table. +pub fn fit_imbalance_dispersion( + variants: Vec, + config: &AnalysisConfig, +) -> Result { + let prepared = prepare_analysis(variants, config)?; + if prepared.variants.is_empty() { + anyhow::bail!("no variants passed the count threshold for dispersion fitting"); + } - #[test] - fn test_region_counts_remove_pseudocounts_per_retained_snp() { - let variant = |pos, ref_count, alt_count| VariantCounts { - chrom: "chr1".to_string(), - pos, - ref_count, - alt_count, - region: "peak".to_string(), - sample: None, - gt: None, - }; - for pseudocount in [0, 1, 2] { - // The duplicate must not contribute counts or extra pseudocounts. - let variants = vec![ - variant(101, 18, 2), - variant(201, 3, 17), - variant(101, 18, 2), - ]; - let config = AnalysisConfig { - min_count: 10, - pseudocount, - method: AnalysisMethod::Single, - phased: false, - }; - let results = analyze_imbalance(variants, &config).unwrap(); - assert_eq!(results.len(), 1); - let result = &results[0]; - assert_eq!(result.snp_count, 2); - assert_eq!(result.ref_count, 21); - assert_eq!(result.alt_count, 19); - assert_eq!(result.n, 40); + match config.method { + AnalysisMethod::Single => eprintln!("Optimizing dispersion parameter..."), + AnalysisMethod::Linear => eprintln!("Optimizing linear dispersion parameters..."), + } + let parameters = + fit_dispersion_parameters(&prepared.variants, config.method, config.pseudocount)?; + match parameters { + DispersionParameters::Single { rho } => eprintln!(" Dispersion: {:.6}", rho), + DispersionParameters::Linear { + d1, + d2, + depth_center, + depth_scale, + } => { + eprintln!( + " d1 = {:.6}, d2 = {:.6}, depth center = {:.6}, depth scale = {:.6}", + d1, d2, depth_center, depth_scale + ) + } + } + Ok(DispersionFit { + parameters, + n_observations: prepared.variants.len(), + }) +} + +/// Test effects with nuisance parameters supplied by a separate fitting run. +pub fn analyze_imbalance_with_fixed_dispersion( + variants: Vec, + config: &AnalysisConfig, + parameters: DispersionParameters, + exact_snv_pvalues: bool, +) -> Result { + let parameters = parameters.validate_for(config.method)?; + let prepared = prepare_analysis(variants, config)?; + analyze_prepared(prepared, config, parameters, exact_snv_pvalues) +} + +pub fn analyze_imbalance_detailed( + variants: Vec, + config: &AnalysisConfig, +) -> Result { + let prepared = prepare_analysis(variants, config)?; + let n_observations = prepared.variants.len(); + if n_observations == 0 { + return Ok(AnalysisOutput { + results: vec![], + rho: None, + linear_params: None, + linear_depth_center: None, + linear_depth_scale: None, + n_observations, + effective_phased: prepared.effective_phased, + }); + } + + match config.method { + AnalysisMethod::Single => eprintln!("Optimizing dispersion parameter..."), + AnalysisMethod::Linear => eprintln!("Optimizing linear dispersion parameters..."), + } + let parameters = + fit_dispersion_parameters(&prepared.variants, config.method, config.pseudocount)?; + match parameters { + DispersionParameters::Single { rho } => eprintln!(" Dispersion: {:.6}", rho), + DispersionParameters::Linear { + d1, + d2, + depth_center, + depth_scale, + } => { + eprintln!( + " d1 = {:.6}, d2 = {:.6}, depth center = {:.6}, depth scale = {:.6}", + d1, d2, depth_center, depth_scale + ) } } + analyze_prepared(prepared, config, parameters, false) +} + +#[allow(dead_code)] +pub fn analyze_imbalance( + variants: Vec, + config: &AnalysisConfig, +) -> Result> { + Ok(analyze_imbalance_detailed(variants, config)?.results) +} + +#[cfg(test)] +mod tests { + use super::*; #[test] fn test_opt_prob() { @@ -1319,6 +1342,51 @@ mod tests { assert!((clamp_rho(0.5) - 0.5).abs() < 1e-15); assert!((clamp_rho(-1.0) - RHO_EPSILON).abs() < 1e-15); assert!((clamp_rho(2.0) - (1.0 - RHO_EPSILON)).abs() < 1e-15); + assert_eq!(clamp_rho(1e-12), 1e-12); + assert_eq!(clamp_rho(1.0 - 1e-12), 1.0 - 1e-12); + } + + #[test] + fn test_standardized_linear_predictor_is_algebraically_equivalent() { + let raw_intercept = -4.0; + let raw_slope = 0.02; + let depth_center = 20.0; + let depth_scale = 5.0; + let parameters = DispersionParameters::Linear { + d1: raw_intercept + raw_slope * depth_center, + d2: raw_slope * depth_scale, + depth_center, + depth_scale, + }; + + for depth in [0, 10, 20, 40, 100] { + let expected = clamp_rho(expit(raw_intercept + raw_slope * depth as f64)); + assert!((parameters.rho_at_depth(depth) - expected).abs() < 1e-15); + } + } + + #[test] + fn test_expit_matches_scipy_expression() { + for (x, expected) in [ + (-0.7, 0.3318122278318339), + (-2.3, 0.09112296101485616), + (-2.5, 0.07585818002124355), + ] { + assert_eq!(expit(x), expected); + } + assert_eq!(expit(-1000.0), 0.0); + assert_eq!(expit(1000.0), 1.0); + } + + #[test] + fn test_linear_prediction_is_not_clipped_at_fit_guard() { + let parameters = DispersionParameters::Linear { + d1: 0.0, + d2: 12.0, + depth_center: 0.0, + depth_scale: 1.0, + }; + assert!((parameters.rho_at_depth(1) - expit(12.0)).abs() < 1e-15); } #[test] @@ -1333,13 +1401,120 @@ mod tests { } #[test] - fn test_golden_section() { + fn test_linear_real_optimizer_trace() { + let counts = [6, 10, 3, 11, 1, 8]; + let depths = [12, 27, 13, 14, 15, 12]; + let parameters = DispersionParameters::Linear { + d1: -2.5581565149445686, + d2: -0.011773137230599792, + depth_center: 0.0, + depth_scale: 1.0, + }; + let rho: Vec<_> = depths + .iter() + .map(|n| parameters.rho_at_depth(n - 2)) + .collect(); + for mu in [0.3819660112501051, 0.6180339887498948] { + let forward: Vec<_> = counts + .iter() + .zip(&depths) + .zip(&rho) + .map(|((&k, &n), &r)| -opt_prob(mu, r, k, n).unwrap()) + .collect(); + let reverse: Vec<_> = counts + .iter() + .zip(&depths) + .zip(&rho) + .map(|((&k, &n), &r)| -opt_prob(1.0 - mu, r, k, n).unwrap()) + .collect(); + eprintln!("mu={mu:.17}, rho={rho:?}, forward={forward:?}, reverse={reverse:?}"); + } + let objective = |mu| { + optimize_prob_unphased_dp(mu, &rho, counts[0], depths[0], &counts[1..], &depths[1..]) + .unwrap() + }; + let mut evaluations = Vec::new(); + let mu = scipy_bounded_minimize_traced( + objective, + 0.0, + 1.0, + 1e-5, + 500, + |x, fx, a, b, accepted| evaluations.push((x, fx, a, b, accepted)), + ) + .unwrap(); + eprintln!("linear fixed-rho mu={mu:.17}, trace={evaluations:?}"); + // Independently executed archived Python at these exact fixed parameters: + // Linux x86_64/SciPy 1.13.1 and macOS arm64/SciPy 1.13.1 respectively. + // Preserve both raw reference outcomes, rather than accepting a flip. + let expected_mu = if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + 0.6928487541708976 + } else { + 0.3071512458291038 + }; + assert!((mu - expected_mu).abs() < 5e-5); + } + + #[test] + fn test_real_unphased_optimizer_trace_matches_python_basin() { + // Archived SciPy 1.13.1 outcomes, fixed nuisance to isolate arithmetic. + let examples = [ + (vec![7, 5], vec![14, 18], 0.41804483019409894), + ( + vec![5, 9, 2, 10, 0, 7], + vec![10, 25, 11, 12, 13, 10], + 0.2550470457103756, + ), + ]; + for (ref_counts, depths, expected_mu) in examples { + let rho = [0.03295621518540146]; + let objective = |mu| { + optimize_prob_unphased_dp( + mu, + &rho, + ref_counts[0], + depths[0], + &ref_counts[1..], + &depths[1..], + ) + .unwrap() + }; + let mut evaluations = Vec::new(); + let mu = scipy_bounded_minimize_traced( + objective, + 0.0, + 1.0, + 1e-5, + 500, + |x, fx, a, b, accepted| evaluations.push((x, fx, a, b, accepted)), + ) + .unwrap(); + assert!( + (mu - expected_mu).abs() < 5e-5, + "mu={mu}, trace={evaluations:?}" + ); + assert!((evaluations[0].0 - 0.3819660112501051).abs() < 1e-15); + assert!((evaluations[1].0 - 0.6180339887498948).abs() < 1e-15); + eprintln!("fixed-rho optimizer mu={mu:.17}, trace={evaluations:?}"); + } + } + + #[test] + fn test_scipy_bounded_minimize_interior_solution() { // Test optimization on simple quadratic let f = |x: f64| (x - 0.7).powi(2); - let min = golden_section_search(f, 0.0, 1.0, 1e-6).unwrap(); + let min = scipy_bounded_minimize(f, 0.0, 1.0, 1e-5, 500).unwrap(); assert!((min - 0.7).abs() < 1e-5); } + #[test] + fn test_scipy_bounded_minimize_preserves_near_zero_solution() { + let minimum = scipy_bounded_minimize(|rho| rho, 0.0, 1.0, 1e-5, 500).unwrap(); + let scipy_boundary = 5.9608609865491405e-6; + assert!((minimum - scipy_boundary).abs() < 1e-15); + assert!(minimum < 0.001); + } + #[test] fn test_optimize_prob_unphased_dp_single_position() { // Test with only first position (no subsequent positions) @@ -1403,8 +1578,8 @@ mod tests { ref_count: rc, alt_count: ac, region: "peak1".to_string(), - sample: Some("d1".to_string()), gt, + raw_gt: gt.map(|g| if g == 0 { "0|1" } else { "1|0" }.to_string()), } } @@ -1421,19 +1596,22 @@ mod tests { } #[test] - fn test_effective_phased_requires_both_numeric_orientations() { + fn test_effective_phased_follows_python_table_checks() { let both = vec![vc_peak(100, 18, 4, Some(0)), vc_peak(200, 5, 15, Some(1))]; - assert!(effective_phased(&both, true)); - assert!(!effective_phased(&both, false)); + assert!(effective_phased(&both, true).unwrap()); + assert!(!effective_phased(&both, false).unwrap()); let none = vec![vc_peak(100, 18, 4, None), vc_peak(200, 5, 15, None)]; - assert!(!effective_phased(&none, true)); + assert!(!effective_phased(&none, true).unwrap()); let same = vec![vc_peak(100, 18, 4, Some(0)), vc_peak(200, 12, 8, Some(0))]; - assert!(!effective_phased(&same, true)); + assert!(!effective_phased(&same, true).unwrap()); let partial = vec![vc_peak(100, 18, 4, Some(0)), vc_peak(200, 12, 8, None)]; - assert!(!effective_phased(&partial, true)); + assert!(effective_phased(&partial, true).is_err()); + let mut numeric = same; + numeric[1].raw_gt = Some("0|0".to_string()); + assert!(effective_phased(&numeric, true).unwrap()); } #[test] @@ -1467,69 +1645,146 @@ mod tests { assert_result_close(&phased[0], &unphased[0]); } - // Helper: a per_donor VariantCounts for one donor "d1". - fn vc_d1(pos: u32, rc: u32, ac: u32, gt: u8) -> VariantCounts { - VariantCounts { - chrom: "chr1".to_string(), - pos, - ref_count: rc, - alt_count: ac, - region: "peak1".to_string(), - sample: Some("d1".to_string()), - gt: Some(gt), + #[test] + fn test_multi_snp_output_removes_every_pseudocount() { + let variants = vec![vc_peak(100, 10, 5, None), vc_peak(200, 7, 8, None)]; + let config = AnalysisConfig { + min_count: 0, + pseudocount: 1, + method: AnalysisMethod::Single, + phased: false, + }; + let output = analyze_imbalance_detailed(variants, &config).unwrap(); + assert_eq!(output.results.len(), 1); + let result = &output.results[0]; + assert_eq!(result.snp_count, 2); + assert_eq!(result.ref_count, 17); + assert_eq!(result.alt_count, 13); + assert_eq!(result.n, 30); + } + + #[test] + fn test_fixed_nuisance_keeps_archived_chi_square_pvalues() { + let config = AnalysisConfig { + min_count: 0, + pseudocount: 1, + method: AnalysisMethod::Single, + phased: false, + }; + let nuisance = DispersionParameters::Single { rho: 0.04 }; + let output = analyze_imbalance_with_fixed_dispersion( + vec![vc_peak(100, 8, 2, None)], + &config, + nuisance, + false, + ) + .unwrap(); + let result = &output.results[0]; + let expected = ChiSquared::new(1.0).unwrap().sf(result.lrt); + assert!((result.pval - expected).abs() < 1e-12); + assert_eq!(result.ref_count, 8); + assert_eq!(result.alt_count, 2); + assert!(analyze_imbalance_with_fixed_dispersion( + vec![vc_peak(100, 8, 2, None)], + &config, + nuisance, + true + ) + .is_err()); + } + + #[test] + fn test_fixed_linear_nuisance_is_reported_without_refitting() { + let config = AnalysisConfig { + min_count: 0, + pseudocount: 0, + method: AnalysisMethod::Linear, + phased: false, + }; + let nuisance = DispersionParameters::Linear { + d1: -4.25, + d2: 0.031, + depth_center: 10.0, + depth_scale: 2.0, + }; + let output = analyze_imbalance_with_fixed_dispersion( + vec![vc_peak(100, 7, 5, None)], + &config, + nuisance, + false, + ) + .unwrap(); + assert_eq!(output.linear_params, Some((-4.25, 0.031))); + assert_eq!(output.linear_depth_center, Some(10.0)); + assert_eq!(output.linear_depth_scale, Some(2.0)); + assert_eq!(output.rho, None); + assert_eq!(output.n_observations, 1); + } + + #[test] + fn test_fixed_linear_nuisance_rejects_invalid_scale() { + let config = AnalysisConfig { + min_count: 0, + pseudocount: 0, + method: AnalysisMethod::Linear, + phased: false, + }; + for depth_scale in [0.0, -1.0, f64::NAN, f64::INFINITY] { + let error = analyze_imbalance_with_fixed_dispersion( + vec![vc_peak(100, 7, 5, None)], + &config, + DispersionParameters::Linear { + d1: -4.0, + d2: 0.1, + depth_center: 12.0, + depth_scale, + }, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("finite and positive")); } } #[test] - fn test_per_donor_phased_wiring() { - // Phased per_donor on a single-donor multi-SNP region must equal a DIRECT - // optimize_prob_phased call with that donor's fitted rho — proving - // region_rho / region_gt are wired into the phased likelihood correctly. + fn test_linear_fit_preserves_adjusted_depth_predictor() { let variants = vec![ - vc_d1(100, 18, 4, 0), - vc_d1(200, 5, 15, 1), - vc_d1(300, 12, 8, 0), + vc_peak(100, 8, 2, None), + vc_peak(200, 10, 10, None), + vc_peak(300, 18, 12, None), ]; - - let rho_map = compute_donor_rho(&variants).unwrap(); - let rho = clamp_rho(rho_map["d1"]); - let region_ref = vec![18u32, 5, 12]; - let region_n = vec![22u32, 20, 20]; - let region_gt = vec![0u8, 1, 0]; - let region_rho = vec![rho, rho, rho]; - let (exp_alt, exp_mu) = - optimize_prob_phased(®ion_ref, ®ion_n, ®ion_gt, ®ion_rho).unwrap(); - - let res = per_donor_model(variants, true).unwrap(); - assert_eq!(res.len(), 1); - assert!( - (res[0].alt_ll - exp_alt).abs() < 1e-9, - "phased per_donor alt_ll {} != direct optimize_prob_phased {}", - res[0].alt_ll, - exp_alt - ); - assert!( - (res[0].mu - exp_mu).abs() < 1e-9, - "mu {} != {}", - res[0].mu, - exp_mu - ); - assert!(res[0].lrt >= 0.0, "LRT must be clamped >= 0"); + let config = AnalysisConfig { + min_count: 0, + pseudocount: 3, + method: AnalysisMethod::Linear, + phased: false, + }; + let fit = fit_imbalance_dispersion(variants, &config).unwrap(); + let (center, scale) = fit.parameters.linear_depth_standardization().unwrap(); + assert_eq!((center, scale), (-6.0, 1.0)); + let (intercept, slope) = fit.parameters.linear_params().unwrap(); + for depth in [10, 20, 30] { + assert_eq!( + fit.parameters.rho_at_depth(depth), + clamp_rho(expit(intercept + (depth + 6) as f64 * slope)) + ); + } } #[test] - fn test_per_donor_unphased_ignores_gt() { - // phased=false uses the shared-pi product, which does NOT use GT — - // flipping GT must leave the unphased result unchanged. - let a = vec![vc_d1(100, 18, 4, 0), vc_d1(200, 5, 15, 1)]; - let b = vec![vc_d1(100, 18, 4, 1), vc_d1(200, 5, 15, 0)]; // GT flipped - let ra = per_donor_model(a, false).unwrap(); - let rb = per_donor_model(b, false).unwrap(); - assert!( - (ra[0].lrt - rb[0].lrt).abs() < 1e-12, - "unphased per_donor must ignore GT: {} vs {}", - ra[0].lrt, - rb[0].lrt + fn test_linear_fit_accepts_constant_raw_depth() { + let variants = vec![vc_peak(100, 8, 2, None), vc_peak(200, 6, 4, None)]; + let config = AnalysisConfig { + min_count: 0, + pseudocount: 2, + method: AnalysisMethod::Linear, + phased: false, + }; + let fit = fit_imbalance_dispersion(variants, &config).unwrap(); + assert_eq!( + fit.parameters.linear_depth_standardization(), + Some((-4.0, 1.0)) ); + assert!(fit.parameters.rho_at_depth(10).is_finite()); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a1f0e348..81e0e068 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -19,6 +19,7 @@ mod cigar_utils; // Shared CIGAR-aware position mapping utilities mod mapping_filter; mod multi_sample; mod read_pairer; +mod scipy_compat; mod seq_decode; mod unified_pipeline; mod vcf_to_bed; @@ -32,11 +33,10 @@ use bam_counter_sc::BamCounterSC; use mapping_filter::filter_bam_wasp; fn parse_numeric_phased_gt(gt: &str) -> Option { - match gt { - "0|1" => Some(0), - "1|0" => Some(1), - _ => None, - } + // Python parses the first allele after its table-wide phasing checks. + // Retain homozygous numeric GT too; the original string determines whether + // the table is informative, not the number of distinct first alleles. + gt.split('|').next()?.parse::().ok() } // ============================================================================ @@ -268,169 +268,215 @@ fn remap_all_chromosomes( /// for r in results: /// print(f"{r['region']}: pval={r['pval']:.4f}") /// ``` -#[pyfunction] -#[pyo3(signature = (tsv_path, min_count=10, pseudocount=1, method="single", phased=false, region_col=None))] -fn analyze_imbalance( - py: Python, +fn parse_analysis_method(method: &str) -> PyResult { + match method { + "single" => Ok(analysis::AnalysisMethod::Single), + "linear" => Ok(analysis::AnalysisMethod::Linear), + "per_donor" => Err(PyRuntimeError::new_err( + "method='per_donor' has been removed because it pooled effect likelihoods across donors; split the count table by donor, fit each donor with fit_imbalance_dispersion(), and pass those fixed nuisance parameters to analyze_imbalance_run()", + )), + _ => Err(PyRuntimeError::new_err(format!( + "Unknown method '{method}'; expected 'single' or 'linear'" + ))), + } +} + +fn fixed_dispersion_parameters( + method: analysis::AnalysisMethod, + rho: Option, + linear_d1: Option, + linear_d2: Option, + linear_depth_center: Option, + linear_depth_scale: Option, +) -> PyResult> { + match method { + analysis::AnalysisMethod::Single => { + if linear_d1.is_some() + || linear_d2.is_some() + || linear_depth_center.is_some() + || linear_depth_scale.is_some() + { + return Err(PyRuntimeError::new_err( + "method='single' accepts rho, not linear dispersion parameters", + )); + } + Ok(rho.map(|rho| analysis::DispersionParameters::Single { rho })) + } + analysis::AnalysisMethod::Linear => { + if rho.is_some() { + return Err(PyRuntimeError::new_err( + "method='linear' accepts linear_d1/linear_d2, not rho", + )); + } + match ( + linear_d1, + linear_d2, + linear_depth_center, + linear_depth_scale, + ) { + (None, None, None, None) => Ok(None), + (Some(d1), Some(d2), Some(depth_center), Some(depth_scale)) => { + Ok(Some(analysis::DispersionParameters::Linear { + d1, + d2, + depth_center, + depth_scale, + })) + } + _ => Err(PyRuntimeError::new_err( + "fixed method='linear' requires linear_d1, linear_d2, \ + linear_depth_center, and linear_depth_scale", + )), + } + } + } +} + +fn read_analysis_variants( tsv_path: &str, - min_count: u32, - pseudocount: u32, - method: &str, - phased: bool, - region_col: Option, -) -> PyResult> { - use pyo3::types::{PyDict, PyList}; + region_col: Option<&str>, +) -> PyResult> { + use flate2::read::MultiGzDecoder; + use std::collections::HashSet; use std::fs::File; use std::io::{BufRead, BufReader}; - // Parse method - let analysis_method = match method { - "single" => analysis::AnalysisMethod::Single, - "linear" => analysis::AnalysisMethod::Linear, - "per_donor" => analysis::AnalysisMethod::PerDonor, - _ => { + let file = File::open(tsv_path) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to open TSV: {e}")))?; + let mut reader: Box = if tsv_path.ends_with(".gz") { + Box::new(BufReader::new(MultiGzDecoder::new(file))) + } else { + Box::new(BufReader::new(file)) + }; + + let mut header_line = String::new(); + if reader + .read_line(&mut header_line) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to read TSV header: {e}")))? + == 0 + { + return Err(PyRuntimeError::new_err("Count TSV is empty")); + } + let headers: Vec<&str> = header_line + .trim_end_matches(['\r', '\n']) + .split('\t') + .collect(); + let unique: HashSet<&str> = headers.iter().copied().collect(); + if unique.len() != headers.len() { + return Err(PyRuntimeError::new_err( + "Count TSV contains duplicate column names", + )); + } + for name in ["chrom", "pos", "ref", "alt", "ref_count", "alt_count"] { + if !unique.contains(name) { return Err(PyRuntimeError::new_err(format!( - "Unknown method: {}", - method - ))) + "Required column '{name}' is missing from count TSV" + ))); } + } + let index = |name: &str| { + headers + .iter() + .position(|header| *header == name) + .expect("required header checked") }; - - let config = analysis::AnalysisConfig { - min_count, - pseudocount, - method: analysis_method, - phased, + let chrom_idx = index("chrom"); + let pos_idx = index("pos"); + let ref_idx = index("ref"); + let alt_idx = index("alt"); + let ref_count_idx = index("ref_count"); + let alt_count_idx = index("alt_count"); + let gt_idx = headers.iter().position(|header| *header == "GT"); + let region_idx = match region_col { + Some(name) => Some( + headers + .iter() + .position(|header| *header == name) + .ok_or_else(|| { + PyRuntimeError::new_err(format!( + "region_col '{name}' not found in header columns: {headers:?}" + )) + })?, + ), + None => None, }; - // Read TSV file - let file = File::open(tsv_path) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to open TSV: {}", e)))?; - let reader = BufReader::new(file); - let mut variants = Vec::new(); - - // Resolve columns from the header. Count tables can carry pos0, GT, feature, - // parent, sample, or donor metadata, so fixed offsets are not reliable. - let mut chrom_idx: usize = 0; - let mut pos_idx: usize = 1; - let mut ref_count_idx: usize = 4; - let mut alt_count_idx: usize = 5; - let mut gt_idx: Option = None; - // Region column index (use input region names instead of chrom_pos format) - let mut region_idx: Option = None; - // Optional donor/sample column (used by the per_donor model) - let mut sample_idx: Option = None; - let mut header_seen = false; - - for (line_idx, line) in reader.lines().enumerate() { - let line = - line.map_err(|e| PyRuntimeError::new_err(format!("Failed to read line: {}", e)))?; - - if !header_seen { - header_seen = true; - let headers: Vec<&str> = line.split('\t').collect(); - let required_column = |name: &str| { - headers - .iter() - .position(|header| *header == name) - .ok_or_else(|| { - PyRuntimeError::new_err(format!( - "Required column '{}' not found in header columns: {:?}", - name, headers - )) - }) - }; - chrom_idx = required_column("chrom")?; - pos_idx = required_column("pos")?; - ref_count_idx = required_column("ref_count")?; - alt_count_idx = required_column("alt_count")?; - gt_idx = headers.iter().position(|header| *header == "GT"); - // Detect an optional `sample` column by name (per_donor input) - sample_idx = headers.iter().position(|h| *h == "sample"); - // Resolve grouping column from the region_col argument (mirror Python get_imbalance): - // Some(name) => group by that column (error if the column is absent) - // None => per-variant grouping (chrom_pos), ignoring any "region" column - region_idx = match ®ion_col { - Some(name) => match headers.iter().position(|h| *h == name.as_str()) { - Some(ix) => Some(ix), - None => { - return Err(PyRuntimeError::new_err(format!( - "region_col '{}' not found in header columns: {:?}", - name, headers - ))) - } - }, - None => None, - }; - continue; - } - - if line.trim().is_empty() { - continue; + for (offset, line) in reader.lines().enumerate() { + let line_number = offset + 2; + let line = line.map_err(|e| { + PyRuntimeError::new_err(format!("Failed to read line {line_number}: {e}")) + })?; + if line.is_empty() { + return Err(PyRuntimeError::new_err(format!( + "Blank row at line {line_number}" + ))); } let fields: Vec<&str> = line.split('\t').collect(); - let field = |index: usize, name: &str| { - fields.get(index).copied().ok_or_else(|| { - PyRuntimeError::new_err(format!( - "Missing '{}' value on TSV line {}", - name, - line_idx + 1 - )) - }) - }; - - let chrom = field(chrom_idx, "chrom")?.to_string(); - let pos = field(pos_idx, "pos")? - .parse::() - .map_err(|e| PyRuntimeError::new_err(format!("Invalid pos: {}", e)))?; - let ref_count = field(ref_count_idx, "ref_count")? - .parse::() - .map_err(|e| PyRuntimeError::new_err(format!("Invalid ref_count: {}", e)))?; - let alt_count = field(alt_count_idx, "alt_count")? - .parse::() - .map_err(|e| PyRuntimeError::new_err(format!("Invalid alt_count: {}", e)))?; - - // region_col=Some => use that column verbatim; region_col=None => per-variant chrom_pos. - // The chrom_pos label matches Python's df["chrom"] + "_" + df["pos"] for SNV-solo parity. - let region = match region_idx { - Some(ri) => field(ri, "region")?.to_string(), - None => format!("{}_{}", chrom, pos), - }; - - let sample = sample_idx.and_then(|si| fields.get(si).map(|s| s.to_string())); - - // Parse the phased genotype (GT) column when present. - // Only fully-phased hets map to a value; everything else (unphased, - // homozygous, missing) is None. Emits ONLY Some(0)/Some(1)/None so - // downstream `1 - g` on u8 never underflows. - // - // AHO parity: only numeric VCF-style phased heterozygotes are trusted. - // Allele-string GT values such as "A|G" are not interpreted as - // cross-SNV haplotype phase by the archived Python implementation. - let gt: Option = gt_idx.and_then(|gi| { - let gt_str = fields.get(gi)?; - parse_numeric_phased_gt(gt_str) - }); + if fields.len() != headers.len() { + return Err(PyRuntimeError::new_err(format!( + "Line {line_number} has {} fields; expected {}", + fields.len(), + headers.len() + ))); + } + let chrom = fields[chrom_idx].to_string(); + let ref_allele = fields[ref_idx]; + let alt_allele = fields[alt_idx]; + if chrom.is_empty() || ref_allele.is_empty() || alt_allele.is_empty() { + return Err(PyRuntimeError::new_err(format!( + "Empty chromosome, REF, or ALT value at line {line_number}" + ))); + } + if ref_allele == alt_allele || alt_allele.contains(',') { + return Err(PyRuntimeError::new_err(format!( + "Line {line_number} is not a biallelic REF/ALT observation" + ))); + } + let pos = fields[pos_idx].parse::().map_err(|e| { + PyRuntimeError::new_err(format!("Invalid pos at line {line_number}: {e}")) + })?; + if pos == 0 { + return Err(PyRuntimeError::new_err(format!( + "Position must be one-based at line {line_number}" + ))); + } + let ref_count = fields[ref_count_idx].parse::().map_err(|e| { + PyRuntimeError::new_err(format!("Invalid ref_count at line {line_number}: {e}")) + })?; + let alt_count = fields[alt_count_idx].parse::().map_err(|e| { + PyRuntimeError::new_err(format!("Invalid alt_count at line {line_number}: {e}")) + })?; + let region = region_idx + .map(|idx| fields[idx].to_string()) + .unwrap_or_else(|| format!("{chrom}_{pos}")); + if region.is_empty() { + return Err(PyRuntimeError::new_err(format!( + "Empty region at line {line_number}" + ))); + } + let gt = gt_idx.and_then(|idx| parse_numeric_phased_gt(fields[idx])); variants.push(analysis::VariantCounts { chrom, pos, ref_count, alt_count, region, - sample, gt, + raw_gt: gt_idx.map(|idx| fields[idx].to_string()), }); } + Ok(variants) +} - // Run analysis - let results = analysis::analyze_imbalance(variants, &config) - .map_err(|e| PyRuntimeError::new_err(format!("Analysis failed: {}", e)))?; +fn results_to_py<'py>( + py: Python<'py>, + results: Vec, +) -> PyResult> { + use pyo3::types::{PyDict, PyList}; - // Convert to Python list of dicts let py_list = PyList::empty(py); - for result in results { let py_dict = PyDict::new(py); py_dict.set_item("region", result.region)?; @@ -446,8 +492,162 @@ fn analyze_imbalance( py_dict.set_item("fdr_pval", result.fdr_pval)?; py_list.append(py_dict)?; } + Ok(py_list) +} + +#[pyfunction] +#[pyo3(signature = (tsv_path, min_count=10, pseudocount=1, method="single", phased=false, region_col=None))] +fn analyze_imbalance( + py: Python, + tsv_path: &str, + min_count: u32, + pseudocount: u32, + method: &str, + phased: bool, + region_col: Option, +) -> PyResult> { + let config = analysis::AnalysisConfig { + min_count, + pseudocount, + method: parse_analysis_method(method)?, + phased, + }; + let variants = read_analysis_variants(tsv_path, region_col.as_deref())?; + let output = analysis::analyze_imbalance_detailed(variants, &config) + .map_err(|e| PyRuntimeError::new_err(format!("Analysis failed: {e}")))?; + Ok(results_to_py(py, output.results)?.unbind().into_any()) +} + +/// Fit balanced-null dispersion without testing allelic effects. +#[pyfunction] +#[pyo3(signature = (tsv_path, min_count=10, pseudocount=1, method="single", phased=false, region_col=None))] +fn fit_imbalance_dispersion( + py: Python, + tsv_path: &str, + min_count: u32, + pseudocount: u32, + method: &str, + phased: bool, + region_col: Option, +) -> PyResult> { + use pyo3::types::PyDict; + + let analysis_method = parse_analysis_method(method)?; + let config = analysis::AnalysisConfig { + min_count, + pseudocount, + method: analysis_method, + phased, + }; + let variants = read_analysis_variants(tsv_path, region_col.as_deref())?; + let fit = analysis::fit_imbalance_dispersion(variants, &config) + .map_err(|e| PyRuntimeError::new_err(format!("Dispersion fitting failed: {e}")))?; + + let result = PyDict::new(py); + result.set_item("method", method)?; + result.set_item("rho", fit.parameters.rho())?; + result.set_item( + "linear_d1", + fit.parameters.linear_params().map(|params| params.0), + )?; + result.set_item( + "linear_d2", + fit.parameters.linear_params().map(|params| params.1), + )?; + result.set_item( + "linear_depth_center", + fit.parameters + .linear_depth_standardization() + .map(|standardization| standardization.0), + )?; + result.set_item( + "linear_depth_scale", + fit.parameters + .linear_depth_standardization() + .map(|standardization| standardization.1), + )?; + result.set_item("n_observations", fit.n_observations)?; + Ok(result.unbind().into_any()) +} + +/// Analyze one independent count table and return dispersion metadata. +/// +/// Supplying `rho` (single) or all four linear parameters runs a fixed-nuisance +/// test. Omitting them retains the legacy fit-and-test behavior. +#[pyfunction] +#[pyo3(signature = (tsv_path, min_count=10, pseudocount=1, phased=false, region_col=None, method="single", rho=None, linear_d1=None, linear_d2=None, linear_depth_center=None, linear_depth_scale=None, exact_snv=None))] +#[allow(clippy::too_many_arguments)] +fn analyze_imbalance_run( + py: Python, + tsv_path: &str, + min_count: u32, + pseudocount: u32, + phased: bool, + region_col: Option, + method: &str, + rho: Option, + linear_d1: Option, + linear_d2: Option, + linear_depth_center: Option, + linear_depth_scale: Option, + exact_snv: Option, +) -> PyResult> { + use pyo3::types::PyDict; + + let analysis_method = parse_analysis_method(method)?; + let fixed_parameters = fixed_dispersion_parameters( + analysis_method, + rho, + linear_d1, + linear_d2, + linear_depth_center, + linear_depth_scale, + )?; + let fixed_nuisance = fixed_parameters.is_some(); + let exact_snv = exact_snv.unwrap_or(false); + if exact_snv { + return Err(PyRuntimeError::new_err( + "exact_snv=True is not supported; use the archived chi-square LRT", + )); + } + let config = analysis::AnalysisConfig { + min_count, + pseudocount, + method: analysis_method, + phased, + }; + let variants = read_analysis_variants(tsv_path, region_col.as_deref())?; + let output = match fixed_parameters { + Some(parameters) => analysis::analyze_imbalance_with_fixed_dispersion( + variants, &config, parameters, exact_snv, + ), + None => analysis::analyze_imbalance_detailed(variants, &config), + } + .map_err(|e| PyRuntimeError::new_err(format!("Analysis failed: {e}")))?; + if output.results.is_empty() { + return Err(PyRuntimeError::new_err( + "No variants passed the count threshold; no results were produced", + )); + } - Ok(py_list.unbind().into_any()) + let result = PyDict::new(py); + result.set_item("results", results_to_py(py, output.results)?)?; + result.set_item("method", method)?; + result.set_item("rho", output.rho)?; + result.set_item("linear_d1", output.linear_params.map(|params| params.0))?; + result.set_item("linear_d2", output.linear_params.map(|params| params.1))?; + result.set_item("linear_depth_center", output.linear_depth_center)?; + result.set_item("linear_depth_scale", output.linear_depth_scale)?; + result.set_item("n_observations", output.n_observations)?; + result.set_item("requested_phased", phased)?; + result.set_item("effective_phased", output.effective_phased)?; + result.set_item( + "nuisance_source", + if fixed_nuisance { "fixed" } else { "fitted" }, + )?; + result.set_item("exact_snv", exact_snv)?; + result.set_item("pvalue_method", "chi_square_lrt")?; + Ok(result.unbind().into_any()) } // ============================================================================ @@ -972,6 +1172,8 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { /// - analyze_imbalance: Fast beta-binomial analysis for AI detection (IMPLEMENTED) #[pymodule] fn wasp2_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + // Legacy test function m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; @@ -1015,6 +1217,8 @@ fn wasp2_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { // Analysis module (beta-binomial allelic imbalance detection) m.add_function(wrap_pyfunction!(analyze_imbalance, m)?)?; + m.add_function(wrap_pyfunction!(fit_imbalance_dispersion, m)?)?; + m.add_function(wrap_pyfunction!(analyze_imbalance_run, m)?)?; Ok(()) } @@ -1047,11 +1251,13 @@ mod tests { use super::parse_numeric_phased_gt; #[test] - fn test_parse_numeric_phased_gt_accepts_only_aho_numeric_hets() { + fn test_parse_numeric_phased_gt_follows_python_first_allele() { assert_eq!(parse_numeric_phased_gt("0|1"), Some(0)); assert_eq!(parse_numeric_phased_gt("1|0"), Some(1)); - for gt in ["A|G", "G|A", "0/1", "0|0", "1|1", "./.", ".|."] { + assert_eq!(parse_numeric_phased_gt("0|0"), Some(0)); + assert_eq!(parse_numeric_phased_gt("1|1"), Some(1)); + for gt in ["A|G", "G|A", "0/1", "./.", ".|."] { assert_eq!( parse_numeric_phased_gt(gt), None, diff --git a/rust/src/scipy_compat.rs b/rust/src/scipy_compat.rs new file mode 100644 index 00000000..6e181f6e --- /dev/null +++ b/rust/src/scipy_compat.rs @@ -0,0 +1,392 @@ +//! Positive-domain beta-binomial arithmetic from SciPy 1.13.1. +//! +//! Sources: scipy/special/special/cephes/{gamma,beta,polevl}.h and +//! scipy/stats/_discrete_distns.py at tag v1.13.1. Preserve coefficients, +//! branch boundaries, and arithmetic order: algebraic rearrangement can +//! change which of two symmetric optima a bounded optimizer selects. +//! +//! Copyright (c) 2001-2002 Enthought, Inc. 2003-2024, SciPy Developers. +//! Cephes Gamma: Copyright 1984, 1987, 1989, 1992 Stephen L. Moshier. +//! Cephes Beta: Copyright 1984, 1987 Stephen L. Moshier. +//! See the distributed third-party SciPy license for redistribution terms. + +// These literals intentionally preserve the upstream decimal coefficients. +#![allow(clippy::excessive_precision)] + +use anyhow::{bail, Result}; + +const GAMMA_P: [f64; 7] = [ + 1.60119522476751861407E-4, + 1.19135147006586384913E-3, + 1.04213797561761569935E-2, + 4.76367800457137231464E-2, + 2.07448227648435975150E-1, + 4.94214826801497100753E-1, + 9.99999999999999996796E-1, +]; +const GAMMA_Q: [f64; 8] = [ + -2.31581873324120129819E-5, + 5.39605580493303397842E-4, + -4.45641913851797240494E-3, + 1.18139785222060435552E-2, + 3.58236398605498653373E-2, + -2.34591795718243348568E-1, + 7.14304917030273074085E-2, + 1.00000000000000000320E0, +]; +const GAMMA_STIR: [f64; 5] = [ + 7.87311395793093628397E-4, + -2.29549961613378126380E-4, + -2.68132617805781232825E-3, + 3.47222221605458667310E-3, + 8.33333333333482257126E-2, +]; +const GAMMA_A: [f64; 5] = [ + 8.11614167470508450300E-4, + -5.95061904284301438324E-4, + 7.93650340457716943945E-4, + -2.77777777730099687205E-3, + 8.33333333333331927722E-2, +]; +const GAMMA_B: [f64; 6] = [ + -1.37825152569120859100E3, + -3.88016315134637840924E4, + -3.31612992738871184744E5, + -1.16237097492762307383E6, + -1.72173700820839662146E6, + -8.53555664245765465627E5, +]; +const GAMMA_C: [f64; 6] = [ + -3.51815701436523470549E2, + -1.70642106651881159223E4, + -2.20528590553854454839E5, + -1.13933444367982507207E6, + -2.53252307177582951285E6, + -2.01889141433532773231E6, +]; +const MAXGAM: f64 = 171.624376956302725; +// Retain the pinned Cephes literal on Rust versions without EULER_GAMMA. +#[allow(clippy::approx_constant)] +const CEPHES_EULER_GAMMA: f64 = 0.5772156649015329; + +/// Reproduce the measured reference compiler's contraction policy. The pinned +/// manylinux x86_64 SciPy wheel evaluates Cephes operations separately, whereas +/// the macOS arm64 reference contracts these expressions. This is independent +/// of the dataset and preserves the original arithmetic on other targets. +#[inline] +fn cephes_mul_add(a: f64, b: f64, c: f64) -> f64 { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + { + a * b + c + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + { + a.mul_add(b, c) + } +} + +fn polevl(x: f64, coefficients: &[f64]) -> f64 { + let mut result = coefficients[0]; + for &coefficient in &coefficients[1..] { + result = cephes_mul_add(result, x, coefficient); + } + result +} + +fn p1evl(x: f64, coefficients: &[f64]) -> f64 { + let mut result = x + coefficients[0]; + for &coefficient in &coefficients[1..] { + result = cephes_mul_add(result, x, coefficient); + } + result +} + +fn stirf(x: f64) -> f64 { + if x >= MAXGAM { + return f64::INFINITY; + } + let mut w = 1.0 / x; + w = cephes_mul_add(w, polevl(w, &GAMMA_STIR), 1.0); + let mut y = x.exp(); + if x > 143.01608 { + let v = x.powf(0.5 * x - 0.25); + y = v * (v / y); + } else { + y = x.powf(x - 0.5) / y; + } + 2.50662827463100050242 * y * w +} + +fn gamma_positive(mut x: f64) -> f64 { + if !x.is_finite() { + return x; + } + if x > 33.0 { + return stirf(x); + } + let mut z = 1.0; + while x >= 3.0 { + x -= 1.0; + z *= x; + } + while x < 2.0 { + if x < 1e-9 { + return z / (cephes_mul_add(CEPHES_EULER_GAMMA, x, 1.0) * x); + } + z /= x; + x += 1.0; + } + if x == 2.0 { + return z; + } + x -= 2.0; + z * polevl(x, &GAMMA_P) / polevl(x, &GAMMA_Q) +} + +fn lgamma_positive(mut x: f64) -> f64 { + if !x.is_finite() { + return x; + } + if x < 13.0 { + let mut z = 1.0; + let mut p = 0.0; + let mut u = x; + while u >= 3.0 { + p -= 1.0; + u = x + p; + z *= u; + } + while u < 2.0 { + if u == 0.0 { + return f64::INFINITY; + } + z /= u; + p += 1.0; + u = x + p; + } + if u == 2.0 { + return z.ln(); + } + p -= 2.0; + x += p; + p = x * polevl(x, &GAMMA_B) / p1evl(x, &GAMMA_C); + return z.ln() + p; + } + if x > 2.556348e305 { + return f64::INFINITY; + } + let mut q = cephes_mul_add(x - 0.5, x.ln(), -x) + 0.91893853320467274178; + if x > 1.0e8 { + return q; + } + let p = 1.0 / (x * x); + if x >= 1000.0 { + q += ((7.9365079365079365079365e-4 * p - 2.7777777777777777777778e-3) * p + + 0.0833333333333333333333) + / x; + } else { + q += polevl(p, &GAMMA_A) / x; + } + q +} + +// Preserve the upstream operand order rather than rewriting as +=. +#[allow(clippy::assign_op_pattern)] +fn betaln_positive(mut a: f64, mut b: f64) -> f64 { + if a < b { + std::mem::swap(&mut a, &mut b); + } + if a > 1e6 * b && a > 1e6 { + let mut r = lgamma_positive(b); + r -= b * a.ln(); + r += b * (1.0 - b) / (2.0 * a); + r += b * (1.0 - b) * (1.0 - 2.0 * b) / (12.0 * a * a); + r += -b * b * (1.0 - b) * (1.0 - b) / (12.0 * a * a * a); + return r; + } + let mut y = a + b; + if y > MAXGAM || a > MAXGAM || b > MAXGAM { + y = lgamma_positive(y); + y = lgamma_positive(b) - y; + y = lgamma_positive(a) + y; + return y; + } + y = gamma_positive(y); + a = gamma_positive(a); + b = gamma_positive(b); + if y == 0.0 { + return f64::INFINITY; + } + if (a.abs() - y.abs()).abs() > (b.abs() - y.abs()).abs() { + y = b / y; + y *= a; + } else { + y = a / y; + y *= b; + } + y.abs().ln() +} + +/// NumPy 1.26.4 contiguous float64 pairwise reduction (128-value blocks). +/// Source: numpy/core/src/umath/loops_utils.h.src; see NUMPY-LICENSE.txt. +pub(crate) fn numpy_sum(values: &[f64]) -> f64 { + let n = values.len(); + if n < 8 { + return values.iter().fold(-0.0, |sum, &value| sum + value); + } + if n <= 128 { + let mut lanes = [0.0; 8]; + lanes.copy_from_slice(&values[..8]); + let end = n - n % 8; + for block in values[8..end].as_chunks::<8>().0 { + for (lane, &value) in lanes.iter_mut().zip(block) { + *lane += value; + } + } + let mut result = ((lanes[0] + lanes[1]) + (lanes[2] + lanes[3])) + + ((lanes[4] + lanes[5]) + (lanes[6] + lanes[7])); + for &value in &values[end..] { + result += value; + } + return result; + } + let mut middle = n / 2; + middle -= middle % 8; + numpy_sum(&values[..middle]) + numpy_sum(&values[middle..]) +} + +/// SciPy's beta-binomial log PMF, preserving its three betaln evaluations. +pub(crate) fn betabinom_logpmf(k: u32, n: u32, alpha: f64, beta: f64) -> Result { + if !alpha.is_finite() || !beta.is_finite() || alpha <= 0.0 || beta <= 0.0 { + bail!("beta-binomial shape parameters must be finite and positive"); + } + if k > n { + return Ok(f64::NEG_INFINITY); + } + if n == 0 { + return Ok(0.0); + } + let k = f64::from(k); + let n = f64::from(n); + let combiln = -(n + 1.0).ln() - betaln_positive(n - k + 1.0, k + 1.0); + Ok(combiln + betaln_positive(k + alpha, n - k + beta) - betaln_positive(alpha, beta)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn close(actual: f64, expected: f64) { + assert!((actual - expected).abs() <= 5e-14 * expected.abs().max(1.0)); + } + + #[test] + fn scipy_1131_gamma_and_loggamma_branches() { + // Generated with scipy.special.gamma/gammaln in SciPy 1.13.1. + for (x, gamma, lgamma) in [ + (1e-12, 999999999999.4226, 27.63102111592797), + (0.1, 9.513507698668732, 2.252712651734206), + (2.5, 1.329340388179137, 0.2846828704729192), + (13.0, 479001600.0, 19.98721449566189), + (33.01, 2.724528906472356e35, 81.59277378960284), + (143.02, 2.976421070788908e245, 565.2240693793439), + (171.0, 7.257415615308e306, 706.5730622457875), + ] { + close(gamma_positive(x), gamma); + close(lgamma_positive(x), lgamma); + } + close(lgamma_positive(1000.0), 5905.220423209181); + close(lgamma_positive(1e8), 1742068066.1038349); + } + + #[test] + fn scipy_1131_beta_branches() { + for (a, b, expected) in [ + (0.1, 0.2, 2.680978479346915), + (30.0, 40.0, -48.30174909591612), + (80.0, 100.0, -124.629853209889), + (1e8, 0.2, -2.160072325559689), + (1e-8, 10.0, 18.42068071566268), + (500.0, 500.0, -694.988722485713), + ] { + close(betaln_positive(a, b), expected); + close(betaln_positive(b, a), expected); + } + } + + #[test] + fn numpy_1264_reduction_boundaries() { + // Independently generated np.sum(float64) references, including cancellation. + for (n, expected) in [ + (7, 1e16), + (8, 0.0), + (16, 1e16), + (127, 1.0000000000000004e16), + (128, 0.0), + (129, 0.0), + (256, 1e16), + (502, 1.0000000000000004e16), + (513, 0.0), + ] { + let values: Vec<_> = (0..n) + .map(|i| match i % 3 { + 0 => 1e16, + 1 => -1e16, + _ => 1.0, + }) + .collect(); + assert_eq!(numpy_sum(&values), expected); + } + } + + #[test] + fn real_probe_components() { + let a = 8.977092566062371; + let b = 5.5481483259805575; + let x = 8.0 + a; + let y = 4.0 + b; + eprintln!( + "betaln={:?}, {:?}, {:?}; gamma={:?}, {:?}, {:?}; numerator shapes={:?},{:?}", + betaln_positive(5.0, 9.0), + betaln_positive(x, y), + betaln_positive(a, b), + gamma_positive(x), + gamma_positive(y), + gamma_positive(x + y), + x, + y + ); + // Independently measured SciPy 1.13.1 values in both reference builds. + let expected = if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + (19621650968466.156, 132624.78469324368) + } else { + (19621650968466.152, 132624.78469324365) + }; + assert_eq!(gamma_positive(x), expected.0); + assert_eq!(gamma_positive(y), expected.1); + close(betaln_positive(x, y), -17.307163194505247); + } + + #[test] + fn scipy_1131_beta_binomial_expression() { + for (k, n, a, b, expected) in [ + (0, 10, 0.1, 0.2, -0.656035869515514), + (5, 10, 1.0, 1.0, -2.3978952727983707), + (100, 200, 50.0, 50.0, -3.4271728015926755), + (12, 50, 1e7, 1e7, -9.134989546611905), + (3, 12, 0.0001, 0.5, -10.168874588765625), + ] { + close(betabinom_logpmf(k, n, a, b).unwrap(), expected); + } + } + + #[test] + fn beta_binomial_support_and_parameter_validation() { + assert_eq!(betabinom_logpmf(0, 0, 1.0, 2.0).unwrap(), 0.0); + assert_eq!(betabinom_logpmf(2, 1, 1.0, 2.0).unwrap(), f64::NEG_INFINITY); + for shape in [0.0, -1.0, f64::INFINITY, f64::NAN] { + assert!(betabinom_logpmf(0, 1, shape, 1.0).is_err()); + assert!(betabinom_logpmf(0, 1, 1.0, shape).is_err()); + } + } +} diff --git a/scripts/compare_archive_analysis.py b/scripts/compare_archive_analysis.py index 90598529..7cfadec2 100644 --- a/scripts/compare_archive_analysis.py +++ b/scripts/compare_archive_analysis.py @@ -19,12 +19,15 @@ import tempfile import types import warnings +from collections.abc import Callable from pathlib import Path import numpy as np import pandas as pd from anndata import AnnData +from scipy.optimize import OptimizeResult from scipy.sparse import csr_matrix +from scipy.special import expit ROOT = Path(__file__).resolve().parents[1] ARCHIVE_REF = "41354fc" @@ -79,12 +82,77 @@ def aligned_dp(prob, disp, first_ref, first_n, phase_ref, phase_n): first_ll = scalar_prob(prob, first_disp, first_ref[0], first_n[0]) p1 = np.atleast_1d(scalar_prob(prob, phase_disp, phase_ref, phase_n, log=False)) p2 = np.atleast_1d(scalar_prob(1 - prob, phase_disp, phase_ref, phase_n, log=False)) - return float(first_ll - np.log(np.prod(0.5 * p1 + 0.5 * p2))) + # Preserve the archive's arithmetic as well as its likelihood. Reducing + # pre-mixed probabilities with np.prod changes last-bit rounding. + previous = 1 + for forward, reverse in zip(p1, p2, strict=True): + forward_combined = previous * forward + reverse_combined = previous * reverse + previous = 0.5 * forward_combined + 0.5 * reverse_combined + return float(first_ll - np.log(previous)) module.opt_prob = scalar_prob module.opt_unphased_dp = aligned_dp +def fixed_bulk_reference(case: dict, ref: str = ARCHIVE_REF) -> pd.DataFrame: + """Execute archived inference with supplied nuisance parameters, on this platform. + + Only nuisance fitting is replaced. The archived data preparation, effect + optimizer, likelihood recurrence, LRT and BH functions remain the oracle. + Linear predictions evaluate the supplied four-field API directly to avoid + a coefficient-conversion roundtrip. Forwarding its ignored phase flag and + scalar/array shape repairs are the same explicit reference adaptations. + """ + modules = load_archive(ref) + archive = modules["as_analysis"] + fixed = case["fixed_dispersion"] + pc = case["pseudocount"] + if case["method"] == "single": + original = archive.minimize_scalar + + def fixed_scalar_optimizer(fun, *args, **kwargs): + if len(kwargs.get("args", ())) == 2: + return OptimizeResult(x=fixed["rho"]) + return original(fun, *args, **kwargs) + + archive.minimize_scalar = fixed_scalar_optimizer + else: + adapt_linear_reference(modules) + original_model = archive.linear_model + original_parse = archive.parse_opt + + def fixed_linear_model(frame, region_col, phased=False): + # frame already has the archive's pseudocounts, filtering and dedup. + depth = frame["N"].to_numpy() - 2 * pc + prediction = expit( + fixed["linear_d1"] + + ((depth - fixed["linear_depth_center"]) / fixed["linear_depth_scale"]) + * fixed["linear_d2"] + ) + # The original model obtains coefficients from minimize, then calls + # expit once to build this vector. Supply the fixed vector there. + archive.minimize = lambda *args, **kwargs: OptimizeResult( + x=[fixed["linear_d1"], fixed["linear_d2"]] + ) + archive.expit = lambda _: prediction + archive.parse_opt = lambda df, disp=None, phased=False: original_parse( + df, disp=disp, phased=effective_phase + ) + effective_phase = phased + return original_model(frame, region_col, phased=phased) + + archive.linear_model = fixed_linear_model + with contextlib.redirect_stdout(io.StringIO()): + return archive.get_imbalance( + pd.DataFrame(case["counts"]), + method=case["method"], + phased=case["phased"], + pseudocount=pc, + region_col="region", + ) + + def bulk_counts() -> pd.DataFrame: pairs = [ (18, 2), @@ -179,6 +247,158 @@ def compare_frames(expected: pd.DataFrame, observed: pd.DataFrame, key: str = "r } +def prepare_bulk_reference_counts( + counts: pd.DataFrame, + *, + min_count: int, + pseudocount: int, + effective_phased: bool, + region_col: str | None = None, +) -> pd.DataFrame: + """Reproduce archived filtering/deduplication while preserving the anchor order. + + Input counts are raw. Returned counts include pseudocounts and N, as used + by the archived objective. Phase fallback must already have been resolved. + """ + frame = counts.copy() + if region_col is None: + frame["region"] = frame["chrom"].astype(str) + "_" + frame["pos"].astype(str) + elif region_col != "region": + frame["region"] = frame[region_col] + frame[["ref_count", "alt_count"]] = ( + frame[["ref_count", "alt_count"]].astype("int64") + pseudocount + ) + frame["N"] = frame["ref_count"] + frame["alt_count"] + frame = frame.loc[frame["N"] >= min_count + 2 * pseudocount].copy() + columns = ["chrom", "pos", "ref_count", "alt_count", "N", "region"] + if effective_phased: + frame["GT"] = frame["GT"].str.split("|", regex=False).str[0].astype("uint8") + columns.append("GT") + return frame[columns].drop_duplicates() + + +def compare_bulk_equivalence( + expected: pd.DataFrame, + observed: pd.DataFrame, + *, + retained_counts: pd.DataFrame, + effective_phased: bool, + reference_log_likelihood: Callable[[str, float], float], + dispersion_comparison: dict, +) -> dict: + """Keep strict results; independently verify mirrored unphased bulk optima. + + The caller supplies the ordered, filtered/deduplicated objective observations, + a Python reference objective, and the independently checked nuisance fit. + This helper never edits results or applies to single-cell comparisons. + """ + required = {"region", "ref_count", "alt_count", "N", "snp_count", *STAT_COLUMNS} + if ( + expected.empty + or not required.issubset(expected.columns) + or not required.issubset(observed.columns) + ): + return { + "strict_comparison": {"status": "FAIL", "reason": "missing required bulk outputs"}, + "statistical_equivalence": "FAIL", + "failures": ["missing required bulk outputs"], + "regions": [], + } + strict = compare_frames(expected, observed) + failures = [] + evidence = [] + if dispersion_comparison.get("status") != "PASS": + failures.append("dispersion comparison did not pass") + if expected["region"].duplicated().any() or observed["region"].duplicated().any(): + failures.append("duplicate tested identities") + if strict.get("reason") or any(item != "mu" for item in strict.get("failures", [])): + failures.append("strict differences other than mu") + if failures: + return { + "strict_comparison": strict, + "statistical_equivalence": "FAIL", + "failures": failures, + "regions": evidence, + } + actual = observed.set_index("region") + for row in expected.itertuples(index=False): + identity = row.region + python_mu = float(row.mu) + rust_mu = float(actual.loc[identity, "mu"]) + finite = np.isfinite(python_mu) and np.isfinite(rust_mu) + valid = finite and 0 <= python_mu <= 1 and 0 <= rust_mu <= 1 + same = valid and np.isclose(python_mu, rust_mu, rtol=5e-5, atol=5e-5) + columns = {"region", "ref_count", "alt_count", "N"} + rows = ( + retained_counts.loc[retained_counts["region"] == identity] + if columns.issubset(retained_counts.columns) + else pd.DataFrame() + ) + symmetric = bool( + not effective_phased + and len(rows) > 1 + and len(rows) == row.snp_count + and rows.iloc[0]["ref_count"] == rows.iloc[0]["alt_count"] + and np.isfinite(rows[["ref_count", "alt_count", "N"]].to_numpy()).all() + and (rows[["ref_count", "alt_count", "N"]].to_numpy() >= 0).all() + and (rows["N"] == rows["ref_count"] + rows["alt_count"]).all() + ) + item = { + "region": identity, + "python_mu": python_mu if finite else None, + "rust_mu": rust_mu if finite else None, + "symmetric_unphased_anchor": symmetric, + "retained_observations": len(rows), + "strict_mu_match": bool(same), + } + if len(rows) and np.isfinite(rows[["ref_count", "alt_count", "N"]].to_numpy()).all(): + item["first_retained_counts"] = { + "ref_count": int(rows.iloc[0]["ref_count"]), + "alt_count": int(rows.iloc[0]["alt_count"]), + "N": int(rows.iloc[0]["N"]), + } + if same: + item["status"] = "STRICT_MATCH" + elif not valid or not symmetric: + item["status"] = "FAIL" + failures.append(f"{identity}: mu mismatch without a valid symmetric anchor") + elif not np.isclose(rust_mu, 1 - python_mu, rtol=5e-5, atol=5e-5): + item["status"] = "FAIL" + failures.append(f"{identity}: proportions are not complementary") + else: + try: + values = np.array( + [ + reference_log_likelihood(identity, mu) + for mu in [python_mu, rust_mu, 1 - python_mu] + ], + dtype=float, + ) + reported = np.array([row.alt_ll, actual.loc[identity, "alt_ll"]], dtype=float) + equal = ( + np.isfinite(values).all() + and np.allclose(values, values[0], rtol=5e-5, atol=5e-5) + and np.allclose(values[:2], reported, rtol=5e-5, atol=5e-5) + ) + item["reference_log_likelihoods"] = ( + values.tolist() if np.isfinite(values).all() else None + ) + item["reference_evaluation_order"] = ["python_mu", "rust_mu", "1-python_mu"] + item["status"] = "SYMMETRIC_EQUIVALENT" if equal else "FAIL" + if not equal: + failures.append(f"{identity}: reference likelihoods do not agree") + except Exception as exc: + item["status"] = "FAIL" + failures.append(f"{identity}: reference evaluation failed: {exc}") + evidence.append(item) + return { + "strict_comparison": strict, + "statistical_equivalence": "FAIL" if failures else "PASS", + "failures": failures, + "regions": evidence, + } + + def run_comparison(archive: dict[str, types.ModuleType]) -> dict: import wasp2_rust from analysis.as_analysis_sc import get_imbalance_sc diff --git a/scripts/container_smoke_test.sh b/scripts/container_smoke_test.sh index 8981e3d5..7a52d13d 100755 --- a/scripts/container_smoke_test.sh +++ b/scripts/container_smoke_test.sh @@ -53,7 +53,7 @@ echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 1: CLI binaries exist and print version # ───────────────────────────────────────────────────────────────────────────── -echo "[1/4] Version checks..." +echo "[1/5] Version checks..." for cmd in wasp2-count wasp2-map wasp2-analyze; do if $cmd --version > /dev/null 2>&1; then @@ -64,18 +64,32 @@ for cmd in wasp2-count wasp2-map wasp2-analyze; do FAIL=$((FAIL + 1)) fi done +python - <<'PY' +import os +from importlib.metadata import version +import wasp2 +import wasp2_rust + +installed = version("wasp2") +assert wasp2.__version__ == wasp2_rust.__version__ == installed +expected = os.environ.get("EXPECTED_WASP2_VERSION", installed) +assert installed == expected, (installed, expected) +print(f" PASS: Python, Rust and distribution versions agree: {installed}") +PY +PASS=$((PASS + 1)) echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 2: Python imports # ───────────────────────────────────────────────────────────────────────────── -echo "[2/4] Python imports..." +echo "[2/5] Python imports..." if python -c "import wasp2_rust; print('Rust extension OK')" 2>/dev/null; then echo " PASS: wasp2_rust import" PASS=$((PASS + 1)) else - echo " INFO: wasp2_rust not available (Python-only mode)" + echo " FAIL: wasp2_rust import" + FAIL=$((FAIL + 1)) fi if python -c "from counting.run_counting import run_count_variants; print('counting OK')" 2>/dev/null; then @@ -88,9 +102,9 @@ fi echo "" # ───────────────────────────────────────────────────────────────────────────── -# Test 3: Count variants with real data +# Test 3: Count variants with the bundled synthetic BAM/VCF # ───────────────────────────────────────────────────────────────────────────── -echo "[3/4] Count variants with real data..." +echo "[3/5] Count variants with bundled BAM/VCF..." if [[ -f "$DATA_DIR/sample1.bam" && -f "$DATA_DIR/variants.vcf.gz" ]]; then if wasp2-count count-variants \ @@ -120,7 +134,7 @@ echo "" # ───────────────────────────────────────────────────────────────────────────── # Test 4: External tools (samtools, bcftools, bedtools) # ───────────────────────────────────────────────────────────────────────────── -echo "[4/4] External tool availability..." +echo "[4/5] External tool availability..." for tool in samtools bcftools bedtools; do if command -v $tool > /dev/null 2>&1; then @@ -133,6 +147,49 @@ for tool in samtools bcftools bedtools; do done echo "" +# Exercise the original ATAC failure: reordered columns, nonnumeric region IDs, +# and both phase orientations. Compare phased and unphased CLI runs to catch +# silently ignored --phased as well as positional parsing and ignored grouping. +echo "[5/5] ATAC region analysis..." +python - "$TMP_DIR/regions.tsv" <<'PY' +import csv +import sys + +with open(sys.argv[1], "w") as handle: + writer = csv.writer(handle, delimiter="\t") + writer.writerow(["ref", "alt", "region", "chrom", "start", "pos", "GT", "ref_count", "alt_count"]) + for pos, gt, ref, alt in [(101, "0|1", 18, 2), (201, "1|0", 3, 17), + (301, "0|1", 22, 18), (401, "1|0", 25, 15)]: + region = "chr1:100-250" if pos < 300 else "chr1:300-450" + writer.writerow(["A", "G", region, "chr1", pos - 1, pos, gt, ref, alt]) +PY +wasp2-analyze find-imbalance "$TMP_DIR/regions.tsv" --phased \ + --region_col region --out_file "$TMP_DIR/phased.tsv" +wasp2-analyze find-imbalance "$TMP_DIR/regions.tsv" \ + --region_col region --out_file "$TMP_DIR/unphased.tsv" +python - "$TMP_DIR/phased.tsv" "$TMP_DIR/unphased.tsv" <<'PY' +import csv +import math +import sys + +def read(path): + with open(path) as handle: + return {row["region"]: row for row in csv.DictReader(handle, delimiter="\t")} + +phased, unphased = map(read, sys.argv[1:]) +assert set(phased) == set(unphased) == {"chr1:100-250", "chr1:300-450"} +for region, counts in [("chr1:100-250", (21, 19)), ("chr1:300-450", (47, 33))]: + for results in (phased, unphased): + row = results[region] + assert (int(row["ref_count"]), int(row["alt_count"])) == counts + assert int(row["snp_count"]) == 2 + assert all(math.isfinite(float(row[key])) for key in ["mu", "null_ll", "alt_ll", "pval", "fdr_pval"]) +assert any(abs(float(phased[k]["alt_ll"]) - float(unphased[k]["alt_ll"])) > 1e-6 for k in phased) +print(" PASS: region grouping, raw counts and phased likelihood through CLI") +PY +PASS=$((PASS + 1)) +echo "" + # ───────────────────────────────────────────────────────────────────────────── # Summary # ───────────────────────────────────────────────────────────────────────────── diff --git a/scripts/diagnose_bulk_numerics.py b/scripts/diagnose_bulk_numerics.py new file mode 100644 index 00000000..9b3c7c64 --- /dev/null +++ b/scripts/diagnose_bulk_numerics.py @@ -0,0 +1,144 @@ +"""Compare fused/unfused Rust Cephes arithmetic with the installed SciPy. + +Builds two isolated probe modules from the production source in a temporary +crate. The repository's source and numerical policy are never modified. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import math +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +import numpy as np +import scipy +from scipy.special import gamma, gammaln +from scipy.stats import betabinom + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> None: + source = (ROOT / "rust/src/scipy_compat.rs").read_text() + start = source.index("#[inline]\nfn cephes_mul_add") + end = source.index("\n}\n", start) + 3 + implementation = source[start:end] + signature = "#[inline]\nfn cephes_mul_add(a: f64, b: f64, c: f64) -> f64" + fused = source.replace(implementation, signature + " { a.mul_add(b, c) }\n") + plain = source.replace(implementation, signature + " { a * b + c }\n") + wrapper = "\npub(crate) fn gamma_probe(x: f64) -> (f64, f64) { (gamma_positive(x), lgamma_positive(x)) }\n" + rust_main = r""" +mod fused; +mod plain; +fn main() { + for x in [0.1, 9.548148325980558, 16.97709256606237, 1000.0, 1e8, 5e11] { + let f = fused::gamma_probe(x); + let p = plain::gamma_probe(x); + println!("gamma,{x:?},{:?},{:?},{:?},{:?}", f.0, p.0, f.1, p.1); + } + for rho in [1e-12, 1e-9, 0.01, 0.1, 0.99] { + for k in [0, 10, 20] { + let a = 0.5 * (1.0-rho)/rho; + let f = fused::betabinom_logpmf(k, 20, a, a).unwrap(); + let p = plain::betabinom_logpmf(k, 20, a, a).unwrap(); + println!("pmf,{k},20,{a:?},{a:?},{f:?},{p:?}"); + } + } + let (a,b) = (8.977092566062371,5.5481483259805575); + println!("pmf,8,12,{a:?},{b:?},{:?},{:?}", + fused::betabinom_logpmf(8,12,a,b).unwrap(), + plain::betabinom_logpmf(8,12,a,b).unwrap()); + // Keep both reduction functions live when compiling with warning checks. + assert_eq!(fused::numpy_sum(&[1.0,2.0]),plain::numpy_sum(&[1.0,2.0])); +} +""" + with tempfile.TemporaryDirectory(prefix="wasp2-numerical-probe-") as directory: + temp = Path(directory) + (temp / "src").mkdir() + (temp / "Cargo.toml").write_text( + '[package]\nname="wasp2-numerical-probe"\nversion="0.0.0"\nedition="2021"\n' + '[dependencies]\nanyhow="1"\n' + ) + (temp / "src/fused.rs").write_text(fused + wrapper) + (temp / "src/plain.rs").write_text(plain + wrapper) + (temp / "src/main.rs").write_text(rust_main) + env = dict(os.environ) + env["CARGO_TARGET_DIR"] = str(ROOT / "rust/target/numerical-probe") + completed = subprocess.run( + ["cargo", "run", "--release", "--manifest-path", str(temp / "Cargo.toml")], + env=env, + capture_output=True, + text=True, + check=True, + ) + rows = [] + logs = [] + for row in csv.reader(io.StringIO(completed.stdout)): + if row[0] == "gamma": + x, fg, pg, fl, pl = map(float, row[1:]) + rows.append( + { + "kind": "gamma", + "x": x, + "python_gamma": float(gamma(x)), + "fused_gamma": fg, + "plain_gamma": pg, + "python_lgamma": float(gammaln(x)), + "fused_lgamma": fl, + "plain_lgamma": pl, + } + ) + else: + k, n, a, b, fused, plain_value = map(float, row[1:]) + reference = float(betabinom.logpmf(k, n, a, b)) + logs.append(reference) + rows.append( + { + "kind": "logpmf", + "k": k, + "n": n, + "alpha": a, + "beta": b, + "python": reference, + "fused": fused, + "plain": plain_value, + } + ) + values = np.array(logs * 32) + scalar_exp = np.array([math.exp(x) for x in values]) + vector_exp = np.exp(values) + numpy_so = Path(np.core._multiarray_umath.__file__) + report = { + "platform": platform.platform(), + "numpy": np.__version__, + "scipy": scipy.__version__, + "numpy_binary_sha256": hashlib.sha256(numpy_so.read_bytes()).hexdigest(), + "numpy_cpu_features": np.core._multiarray_umath.__cpu_features__, + "scalar_vector_exp_differences": int(np.count_nonzero(scalar_exp != vector_exp)), + "scalar_vector_exp_max_delta": float(np.max(np.abs(scalar_exp - vector_exp))), + "rows": rows, + } + print(json.dumps(report, indent=2)) + sys.path.insert(0, str(ROOT)) + from scripts.compare_archive_analysis import fixed_bulk_reference + + reference = json.loads( + (ROOT / "tests/analysis/fixtures/real_atac_optimizer_reference.json").read_text() + ) + # These are newly executed Python outcomes, not adjusted Rust expectations. + fixed_cases = { + case["name"]: fixed_bulk_reference(case).to_dict("records") for case in reference["cases"] + } + print("ARCHIVED_FIXED_NUISANCE_REFERENCES=" + json.dumps(fixed_cases)) + + +if __name__ == "__main__": + main() diff --git a/src/analysis/__main__.py b/src/analysis/__main__.py index 31c69a75..a34b230c 100644 --- a/src/analysis/__main__.py +++ b/src/analysis/__main__.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import Annotated import typer @@ -5,8 +6,26 @@ from wasp2.cli import create_version_callback, verbosity_callback from .run_analysis import run_ai_analysis -from .run_analysis_sc import run_ai_analysis_sc -from .run_compare_ai import run_ai_comparison + + +class AnalysisScopeChoice(str, Enum): + per_donor = "per-donor" + + +class AnalysisUnitChoice(str, Enum): + snv = "snv" + feature = "feature" + peak = "peak" + + +class AnalysisModelChoice(str, Enum): + single = "single" + linear = "linear" + + +class DispersionScopeChoice(str, Enum): + per_donor = "per-donor" + global_ = "global" def _get_analysis_deps() -> dict[str, str]: @@ -55,7 +74,10 @@ def main( @app.command() def find_imbalance( - counts: Annotated[str, typer.Argument(help="Count File")], + counts: Annotated[ + str, + typer.Argument(help="Count table, locked count bundle, or count_manifest.json."), + ], min: Annotated[ int | None, typer.Option( @@ -71,7 +93,10 @@ def find_imbalance( "--ps", "--pseudo", "--pseudocount", - help="Pseudocount added when measuring allelic imbalance. (Default: 1)", + help=( + "Pseudocount added when measuring allelic imbalance. Defaults to 1 " + "for both donor-local and legacy analysis." + ), ), ] = None, out_file: Annotated[ @@ -97,22 +122,22 @@ def find_imbalance( ), ] = False, model: Annotated[ - str | None, + AnalysisModelChoice | None, typer.Option( "-m", "--model", - help=( - "Model used for measuring optimization parameter when finding imbalance. " - "HIGHLY RECOMMENDED TO LEAVE AS DEFAULT FOR SINGLE DISPERSION MODEL. " - "Choice of 'single' or 'linear'. (Default: 'single')" - ), + help=("Dispersion model used for imbalance inference. Defaults to 'single'."), ), ] = None, region_col: Annotated[ str | None, typer.Option( + "--region-col", "--region_col", - help="Name of region column for current data. 'region' for ATAC-seq. Attribute name for RNA-seq. (Default: Auto-parses if none provided)", + help=( + "Feature identifier column. Locked feature bundles use 'region'; standalone " + "feature tables require this option." + ), ), ] = None, groupby: Annotated[ @@ -142,17 +167,61 @@ def find_imbalance( ), ), ] = False, + scope: Annotated[ + AnalysisScopeChoice | None, + typer.Option( + "--scope", + help=( + "Analyze each donor independently. Requires --unit and canonical donor_id; " + "standalone legacy tables may use sample instead." + ), + ), + ] = None, + unit: Annotated[ + AnalysisUnitChoice | None, + typer.Option( + "--unit", + help="Statistical unit: snv or feature. peak is an alias for feature.", + ), + ] = None, + dispersion_scope: Annotated[ + DispersionScopeChoice, + typer.Option( + "--dispersion-scope", + help="Fit dispersion globally across eligible donor-SNV rows or per donor.", + ), + ] = DispersionScopeChoice.per_donor, + min_donor_observations: Annotated[ + int, + typer.Option( + "--min-donor-observations", + min=1, + help="Exclude donors with fewer eligible unique SNV observations.", + ), + ] = 50, + expected_manifest_sha256: Annotated[ + str | None, + typer.Option( + "--expected-manifest-sha256", + help="Require a locked bundle manifest with this SHA-256 digest.", + ), + ] = None, ) -> None: run_ai_analysis( count_file=counts, min_count=min, - model=model, + model=model.value if model is not None else None, pseudocount=pseudocount, phased=phased, out_file=out_file, region_col=region_col, groupby=groupby, per_variant=per_variant, + scope=scope.value if scope is not None else None, + unit=unit.value if unit is not None else None, + dispersion_scope=dispersion_scope.value, + min_donor_observations=min_donor_observations, + expected_manifest_sha256=expected_manifest_sha256, ) @@ -234,6 +303,13 @@ def find_imbalance_sc( ), ] = None, ) -> None: + try: + from .run_analysis_sc import run_ai_analysis_sc + except (ImportError, ModuleNotFoundError) as error: + raise typer.BadParameter( + "Single-cell analysis requires: pip install 'wasp2[single-cell]'" + ) from error + run_ai_analysis_sc( count_file=counts, bc_map=bc_map, @@ -325,6 +401,13 @@ def compare_imbalance( ), ] = None, ) -> None: + try: + from .run_compare_ai import run_ai_comparison + except (ImportError, ModuleNotFoundError) as error: + raise typer.BadParameter( + "Single-cell comparison requires: pip install 'wasp2[single-cell]'" + ) from error + run_ai_comparison( count_file=counts, bc_map=bc_map, diff --git a/src/analysis/run_analysis.py b/src/analysis/run_analysis.py index 94a33e58..fea4542a 100644 --- a/src/analysis/run_analysis.py +++ b/src/analysis/run_analysis.py @@ -13,6 +13,8 @@ import pandas as pd +from .run_analysis_per_donor import run_per_donor_analysis + # Rust analysis (required; no Python fallback) try: from wasp2_rust import analyze_imbalance as rust_analyze_imbalance @@ -143,7 +145,12 @@ def run_ai_analysis( region_col: str | None = None, groupby: str | None = None, per_variant: bool = False, -) -> None: + scope: str | None = None, + unit: str | None = None, + dispersion_scope: str = "per-donor", + min_donor_observations: int = 50, + expected_manifest_sha256: str | None = None, +) -> dict[str, Path] | None: """Run allelic imbalance analysis pipeline. Parameters @@ -153,7 +160,8 @@ def run_ai_analysis( min_count : int | None, optional Minimum total count threshold, by default 10. pseudocount : int | None, optional - Pseudocount to add, by default 1. + Pseudocount to add. Defaults to 1 for donor-local analysis and for + the legacy route. phased : bool | None, optional Use phased genotype information, by default False. model : str | None, optional @@ -167,12 +175,57 @@ def run_ai_analysis( per_variant : bool, optional Test each SNV independently (per-variant) instead of grouping by a region column. Forces per-variant even when a region column is present. Default False. + scope : str | None, optional + Set to ``"per-donor"`` to use donor-local orchestration. + unit : str | None, optional + Donor-local analysis unit: ``"snv"`` or ``"feature"``; ``"peak"`` is an alias. + dispersion_scope : str, optional + Fit dispersion globally or independently per donor, by default ``"per-donor"``. + min_donor_observations : int, optional + Minimum eligible unique donor-SNV rows required for inference, by default 50. + expected_manifest_sha256 : str | None, optional + External trust anchor for a locked count bundle manifest. Raises ------ RuntimeError If Rust analysis extension is not available. """ + if model not in {None, "single", "linear"}: + raise ValueError("model must be 'single' or 'linear'") + if scope is not None: + if scope != "per-donor": + raise ValueError("scope must be 'per-donor'") + if unit not in {"snv", "feature", "peak"}: + raise ValueError( + "Per-donor analysis requires --unit snv or --unit feature (--unit peak is an alias)" + ) + if groupby is not None or per_variant: + raise ValueError("Per-donor analysis cannot use --groupby or --per-variant") + if dispersion_scope not in {"global", "per-donor"}: + raise ValueError("dispersion_scope must be 'global' or 'per-donor'") + output = out_file if out_file is not None else str(Path.cwd() / "ai_results.tsv") + return run_per_donor_analysis( + count_file, + output, + unit=unit, + model="single" if model is None else model, + dispersion_scope=dispersion_scope, + region_col=region_col, + phased=bool(phased), + min_count=10 if min_count is None else min_count, + pseudocount=1 if pseudocount is None else pseudocount, + min_donor_observations=min_donor_observations, + expected_manifest_sha256=expected_manifest_sha256, + ) + + if unit is not None: + raise ValueError("--unit requires --scope per-donor in the bulk analysis command") + if dispersion_scope != "per-donor": + raise ValueError("--dispersion-scope requires --scope per-donor") + if expected_manifest_sha256 is not None: + raise ValueError("--expected-manifest-sha256 requires --scope per-donor") + # Fail closed: --groupby is not supported by the Rust backend. It only re-keys the grouping # column (region_col = groupby), so the identical result is obtained with --region_col . # Erroring prevents silently returning feature-level results when parent-level was requested. @@ -218,3 +271,4 @@ def run_ai_analysis( # Write results ai_df.to_csv(ai_files.out_file, sep="\t", header=True, index=False) + return None diff --git a/src/analysis/run_analysis_per_donor.py b/src/analysis/run_analysis_per_donor.py new file mode 100644 index 00000000..6fba1ba7 --- /dev/null +++ b/src/analysis/run_analysis_per_donor.py @@ -0,0 +1,1290 @@ +"""Donor-local allelic imbalance orchestration.""" + +from __future__ import annotations + +import csv +import gzip +import hashlib +import inspect +import json +import math +import os +import shutil +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, Literal, cast + +import pandas as pd + +try: + from wasp2_rust import analyze_imbalance_run as rust_analyze_imbalance_run + from wasp2_rust import fit_imbalance_dispersion as rust_fit_imbalance_dispersion +except ImportError: + rust_analyze_imbalance_run = None + rust_fit_imbalance_dispersion = None + +AnalysisUnit = Literal["snv", "feature"] +AnalysisUnitInput = Literal["snv", "feature", "peak"] +DispersionScope = Literal["global", "per-donor"] +Model = Literal["single", "linear"] +FitMetadataValue = str | int | float | bool | None + +UTC = timezone.utc +SCHEMA_VERSION = "wasp2.donor-local-analysis.v2" +COMMON_COLUMNS = { + "donor_id", + "chrom", + "pos", + "ref", + "alt", + "ref_count", + "alt_count", +} +SNV_COLUMNS = ["donor_id", "chrom", "pos", "ref", "alt"] +RUST_RESULT_COLUMNS = [ + "region", + "ref_count", + "alt_count", + "N", + "snp_count", + "null_ll", + "alt_ll", + "mu", + "lrt", + "pval", + "fdr_pval", +] +STATISTIC_COLUMNS = [ + "ref_count", + "alt_count", + "N", + "snp_count", + "null_ll", + "alt_ll", + "mu", + "lrt", + "pval", + "fdr_pval", +] + + +@dataclass(frozen=True) +class CountInput: + """A verified count table and its optional locked-bundle manifest.""" + + counts: Path + counts_record: dict[str, str | int] + counts_signature: tuple[int, int, int, int] + manifest: Path | None + manifest_record: dict[str, str | int] | None + manifest_signature: tuple[int, int, int, int] | None + manifest_data: dict[str, Any] | None + + @property + def bundled(self) -> bool: + return self.manifest is not None + + +@dataclass(frozen=True) +class DispersionFit: + """Validated nuisance-parameter fit returned by Rust.""" + + payload: Mapping[str, Any] + fit_id: str + method: Model + rho: float | None + linear_d1: float | None + linear_d2: float | None + linear_depth_center: float | None + linear_depth_scale: float | None + n_observations: int + metadata: Mapping[str, FitMetadataValue] + extended_contract: bool + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _signature(path: Path) -> tuple[int, int, int, int]: + stat = path.stat() + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns + + +def _file_record(path: Path) -> tuple[dict[str, str | int], tuple[int, int, int, int]]: + resolved = path.resolve(strict=True) + before = _signature(resolved) + digest = _sha256(resolved) + if _signature(resolved) != before: + raise RuntimeError(f"Input changed while it was being hashed: {resolved}") + stat = resolved.stat() + return ( + { + "path": str(resolved), + "sha256": digest, + "size_bytes": stat.st_size, + "mtime_utc": datetime.fromtimestamp(stat.st_mtime, UTC).isoformat(), + }, + before, + ) + + +def _package_version() -> str: + try: + return version("wasp2") + except PackageNotFoundError: + return "unknown" + + +def _normalize_unit(unit: AnalysisUnitInput) -> AnalysisUnit: + normalized = "feature" if unit == "peak" else unit + if normalized not in {"snv", "feature"}: + raise ValueError("unit must be 'snv' or 'feature' ('peak' is an alias)") + return cast(AnalysisUnit, normalized) + + +def _validate_sha256(value: str, label: str) -> str: + normalized = value.lower() + if len(normalized) != 64 or any( + character not in "0123456789abcdef" for character in normalized + ): + raise ValueError(f"{label} must be a 64-character hexadecimal SHA-256 digest") + return normalized + + +def _locate_manifest(source: Path) -> Path | None: + if source.is_dir(): + manifest = source / "count_manifest.json" + if not manifest.is_file(): + raise ValueError(f"Count bundle directory has no count_manifest.json: {source}") + return manifest + if source.name == "count_manifest.json": + return source + adjacent = source.parent / "count_manifest.json" + return adjacent if adjacent.is_file() else None + + +def _resolve_count_input( + count_file: str | Path, + *, + unit: AnalysisUnit, + expected_manifest_sha256: str | None, +) -> CountInput: + source = Path(count_file).expanduser() + if not source.exists(): + raise FileNotFoundError(f"Count input does not exist: {source}") + source = source.resolve(strict=True) + manifest_path = _locate_manifest(source) + + if manifest_path is None: + if source.is_dir(): + raise ValueError(f"Count input is not a regular file: {source}") + if expected_manifest_sha256 is not None: + raise ValueError( + "--expected-manifest-sha256 requires a locked count bundle; " + "standalone count tables have no manifest trust anchor" + ) + counts_record, counts_signature = _file_record(source) + return CountInput( + counts=source, + counts_record=counts_record, + counts_signature=counts_signature, + manifest=None, + manifest_record=None, + manifest_signature=None, + manifest_data=None, + ) + + from counting.run_counting_cohort import verify_count_bundle + + expected = ( + _validate_sha256(expected_manifest_sha256, "expected manifest SHA-256") + if expected_manifest_sha256 is not None + else None + ) + manifest_data = verify_count_bundle( + manifest_path, + expected_manifest_sha256=expected, + ) + bundle_unit = manifest_data.get("analysis_unit") + if bundle_unit != unit: + raise ValueError( + f"Requested analysis unit {unit!r} does not match locked count bundle unit " + f"{bundle_unit!r}" + ) + + counts_path = (manifest_path.parent / "counts.tsv.gz").resolve(strict=True) + if source.is_file() and source != manifest_path.resolve(strict=True) and source != counts_path: + raise ValueError( + f"Input is inside a count bundle but is not its locked counts table: {source}" + ) + counts_record, counts_signature = _file_record(counts_path) + manifest_record, manifest_signature = _file_record(manifest_path) + return CountInput( + counts=counts_path, + counts_record=counts_record, + counts_signature=counts_signature, + manifest=manifest_path.resolve(strict=True), + manifest_record=manifest_record, + manifest_signature=manifest_signature, + manifest_data=manifest_data, + ) + + +def _read_header(path: Path) -> list[str]: + opener = gzip.open if path.name.endswith(".gz") else open + with opener(path, "rt", newline="") as handle: + header = next(csv.reader(handle, delimiter="\t"), None) + if not header: + raise ValueError("Count table is empty or has no header") + if any(not column for column in header): + raise ValueError("Count table contains an empty column name") + duplicate_columns = sorted({column for column in header if header.count(column) > 1}) + if duplicate_columns: + raise ValueError(f"Count table contains duplicate columns: {duplicate_columns}") + return header + + +def _coerce_nonnegative_integer(frame: pd.DataFrame, column: str, maximum: int) -> None: + values = frame[column].astype(str) + valid = values.str.fullmatch(r"[0-9]+") + if not valid.all(): + raise ValueError(f"Column {column!r} contains invalid integer values") + integers = values.map(int) + if integers.gt(maximum).any(): + raise ValueError(f"Column {column!r} contains values larger than {maximum}") + frame[column] = integers.astype("uint64") + + +def _validate_text_column(frame: pd.DataFrame, column: str) -> None: + values = frame[column].astype(str) + invalid = values.eq("") | values.str.contains(r"[\t\r\n]", regex=True) + if invalid.any(): + raise ValueError(f"Column {column!r} contains missing, empty, or multiline values") + frame[column] = values + + +def _genotype_qc(frame: pd.DataFrame) -> dict[str, int]: + gt = frame["GT"] if "GT" in frame.columns else pd.Series(dtype=str) + return { + "forward": int(gt.eq("0|1").sum()), + "reverse": int(gt.eq("1|0").sum()), + "converted_from_alleles": 0, + } + + +def _phase_fallback_reason(frame: pd.DataFrame, requested: bool) -> str | None: + """Apply archived Python's checks after donor-local depth filtering.""" + if not requested: + return "not_requested" + if "GT" not in frame.columns: + return "missing_gt" + gt = frame["GT"] + if gt.nunique(dropna=False) <= 1: + return "identical_gt" + if not gt.isin(["0|1", "1|0"]).any(): + return "no_numeric_phased_gt" + # The archive parses the first numeric allele only after these checks. + # Malformed mixed genotypes must not be silently treated as informative. + gt.str.split("|", n=1).str[0].astype("uint8") + return None + + +def _first_conflicting_group( + frame: pd.DataFrame, + identity: Sequence[str], + compared: Sequence[str], +) -> tuple[Any, ...] | None: + conflicts = ( + frame.groupby(list(identity), sort=False, dropna=False)[list(compared)] + .nunique(dropna=False) + .gt(1) + .any(axis=1) + ) + if not conflicts.any(): + return None + key = conflicts.index[conflicts][0] + return key if isinstance(key, tuple) else (key,) + + +def _load_counts( + count_input: CountInput, + *, + unit: AnalysisUnit, + region_col: str | None, + phased: bool, +) -> tuple[pd.DataFrame, str | None, dict[str, Any]]: + header = _read_header(count_input.counts) + has_donor_id = "donor_id" in header + has_sample = "sample" in header + has_snv_id = "snv_id" in header + if has_donor_id and has_sample: + raise ValueError("Count table cannot contain both donor_id and legacy sample columns") + if count_input.bundled and not has_donor_id: + raise ValueError("Locked count bundles require the canonical donor_id column") + if not has_donor_id and not has_sample: + raise ValueError("Count table requires a donor_id column") + + selected_region = region_col + if unit == "snv" and region_col is not None: + raise ValueError( + "SNV analysis tests variants independently and does not accept --region-col" + ) + if unit == "feature": + if count_input.bundled and region_col not in {None, "region"}: + raise ValueError("Locked feature bundles require the canonical region column") + selected_region = "region" if region_col is None and count_input.bundled else region_col + if not selected_region: + raise ValueError( + "Standalone feature analysis requires --region-col; locked feature bundles use region" + ) + if selected_region in COMMON_COLUMNS or selected_region in {"sample", "snv_id"}: + raise ValueError( + f"Feature region column conflicts with a reserved column: {selected_region}" + ) + + required = set(COMMON_COLUMNS) + if not has_donor_id: + required.remove("donor_id") + required.add("sample") + if selected_region is not None: + required.add(selected_region) + missing = sorted(required.difference(header)) + if missing: + raise ValueError(f"Count table is missing required columns: {missing}") + + frame = pd.read_csv( + count_input.counts, + sep="\t", + dtype=str, + keep_default_na=False, + na_filter=False, + low_memory=False, + ) + if frame.empty: + raise ValueError("Count table contains no observations") + if has_sample: + frame = frame.rename(columns={"sample": "donor_id"}) + + text_columns = ["donor_id", "chrom", "ref", "alt"] + if has_snv_id: + text_columns.append("snv_id") + if selected_region is not None: + text_columns.append(selected_region) + for column in text_columns: + _validate_text_column(frame, column) + for column in ["pos", "ref_count", "alt_count"]: + _coerce_nonnegative_integer(frame, column, 2**32 - 1) + if frame["pos"].eq(0).any(): + raise ValueError("SNV positions must be one-based") + invalid_alleles = ( + frame["ref"].eq(frame["alt"]) + | frame["ref"].str.len().ne(1) + | frame["alt"].str.len().ne(1) + | frame["alt"].str.contains(",", regex=False) + ) + if invalid_alleles.any(): + raise ValueError("Donor-local analysis requires biallelic SNV observations") + + site_identity = ["donor_id", "chrom", "pos"] + allele_conflict = _first_conflicting_group(frame, site_identity, ["ref", "alt"]) + if allele_conflict is not None: + donor_id, chrom, pos = allele_conflict + raise ValueError(f"Conflicting alleles for donor/SNV {donor_id} {chrom}:{pos}") + + if has_snv_id: + snv_id_conflict = _first_conflicting_group( + frame, + ["donor_id", "snv_id"], + ["chrom", "pos", "ref", "alt"], + ) + if snv_id_conflict is not None: + donor_id, snv_id = snv_id_conflict + raise ValueError(f"snv_id {snv_id!r} maps to conflicting variants for donor {donor_id}") + identity_conflict = _first_conflicting_group(frame, SNV_COLUMNS, ["snv_id"]) + if identity_conflict is not None: + donor_id, chrom, pos, ref, alt = identity_conflict + raise ValueError( + f"Multiple snv_id values map to donor/SNV {donor_id} {chrom}:{pos}:{ref}>{alt}" + ) + else: + frame["snv_id"] = ( + frame["chrom"] + + ":" + + frame["pos"].astype(str) + + ":" + + frame["ref"] + + ":" + + frame["alt"] + ) + + gt_qc = _genotype_qc(frame) if phased else None + raw_rows = len(frame) + identity = [*SNV_COLUMNS, "ref_count", "alt_count"] + if selected_region is not None: + identity.append(selected_region) + if phased and "GT" in frame.columns: + identity.append("GT") + frame = frame.drop_duplicates(identity, keep="first").copy() + frame["pos"] = frame["pos"].astype("uint32") + frame["ref_count"] = frame["ref_count"].astype("uint32") + frame["alt_count"] = frame["alt_count"].astype("uint32") + frame["_total_count"] = frame["ref_count"].astype("uint64") + frame["alt_count"].astype( + "uint64" + ) + return ( + frame, + selected_region, + { + "raw_rows": raw_rows, + "validated_rows": len(frame), + "exact_duplicate_rows_removed": raw_rows - len(frame), + "donors": int(frame["donor_id"].nunique()), + "unique_donor_snv_rows": int(frame.drop_duplicates(SNV_COLUMNS).shape[0]), + "phase": gt_qc, + "legacy_sample_normalized": has_sample, + "snv_id_source": "input" if has_snv_id else "synthesized", + }, + ) + + +def _artifact_paths(result_path: Path) -> dict[str, Path]: + if result_path.suffix != ".tsv": + raise ValueError("Donor-local output must use a .tsv filename") + stem = result_path.stem + return { + "results": result_path, + "dispersion": result_path.with_name(f"{stem}.dispersion.tsv"), + "qc": result_path.with_name(f"{stem}.qc.tsv"), + "provenance": result_path.with_name(f"{stem}.provenance.json"), + } + + +def _write_tsv(frame: pd.DataFrame, path: Path, columns: Sequence[str]) -> None: + frame.loc[:, list(columns)].to_csv(path, sep="\t", header=True, index=False) + + +def _finite_float(value: Any, label: str) -> float: + if isinstance(value, bool): + raise RuntimeError(f"Rust returned an invalid {label}") + try: + result = float(value) + except (TypeError, ValueError) as error: + raise RuntimeError(f"Rust returned an invalid {label}") from error + if not math.isfinite(result): + raise RuntimeError(f"Rust returned a non-finite {label}") + return result + + +def _fit_metadata( + raw_fit: Mapping[str, Any], *, add_parameters_returned_status: bool +) -> dict[str, FitMetadataValue]: + metadata: dict[str, FitMetadataValue] = {} + for name, value in raw_fit.items(): + if name != "fit_status" and name != "optimizer" and not name.startswith("optimizer_"): + continue + if value is None: + continue + if isinstance(value, str): + if not value or any(character in value for character in "\t\r\n"): + raise RuntimeError(f"Rust returned an invalid {name}") + metadata[name] = value + elif isinstance(value, (bool, int)): + metadata[name] = value + elif isinstance(value, float): + if not math.isfinite(value): + raise RuntimeError(f"Rust returned a non-finite {name}") + metadata[name] = value + else: + raise RuntimeError(f"Rust returned a non-scalar {name}") + if add_parameters_returned_status and "fit_status" not in metadata: + # The API currently proves that parameters were returned, but does not + # expose optimizer convergence diagnostics. + metadata["fit_status"] = "parameters_returned" + return metadata + + +def _validate_fit( + raw_fit: Mapping[str, Any], + *, + model: Model, + expected_rows: int, + allow_legacy_linear_mock: bool = False, +) -> DispersionFit: + fit_id = raw_fit.get("fit_id") + if not isinstance(fit_id, str) or not fit_id: + raise RuntimeError("Rust dispersion fit did not return a non-empty fit_id") + returned_method = raw_fit.get("method", raw_fit.get("model")) + if returned_method != model: + raise RuntimeError( + f"Rust dispersion fit method {returned_method!r} does not match requested {model!r}" + ) + try: + n_observations = int(raw_fit["n_observations"]) + except (KeyError, TypeError, ValueError) as error: + raise RuntimeError("Rust dispersion fit did not return n_observations") from error + if n_observations != expected_rows: + raise RuntimeError( + f"Rust dispersion fit used {n_observations} observations; expected {expected_rows}" + ) + + if model == "single": + rho = _finite_float(raw_fit.get("rho"), "rho") + linear_d1 = None + linear_d2 = None + linear_depth_center = None + linear_depth_scale = None + linear_values = ( + raw_fit.get("linear_d1"), + raw_fit.get("linear_d2"), + raw_fit.get("linear_depth_center"), + raw_fit.get("linear_depth_scale"), + ) + if any(value is not None for value in linear_values): + raise RuntimeError("Rust single-dispersion fit returned linear parameters") + extended_contract = True + else: + rho = None + linear_d1 = _finite_float(raw_fit.get("linear_d1"), "linear_d1") + linear_d2 = _finite_float(raw_fit.get("linear_d2"), "linear_d2") + if raw_fit.get("rho") is not None: + raise RuntimeError("Rust linear-dispersion fit returned scalar rho") + raw_center = raw_fit.get("linear_depth_center") + raw_scale = raw_fit.get("linear_depth_scale") + if (raw_center is None) != (raw_scale is None): + raise RuntimeError( + "Rust linear-dispersion fit returned incomplete depth standardization" + ) + if raw_center is None: + if not allow_legacy_linear_mock: + raise RuntimeError( + "Rust linear-dispersion fit omitted linear_depth_center and linear_depth_scale" + ) + linear_depth_center = None + linear_depth_scale = None + extended_contract = False + else: + linear_depth_center = _finite_float(raw_center, "linear_depth_center") + linear_depth_scale = _finite_float(raw_scale, "linear_depth_scale") + if linear_depth_scale <= 0: + raise RuntimeError("Rust returned a non-positive linear_depth_scale") + extended_contract = True + metadata = _fit_metadata( + raw_fit, + add_parameters_returned_status=extended_contract, + ) + return DispersionFit( + payload=dict(raw_fit), + fit_id=fit_id, + method=model, + rho=rho, + linear_d1=linear_d1, + linear_d2=linear_d2, + linear_depth_center=linear_depth_center, + linear_depth_scale=linear_depth_scale, + n_observations=n_observations, + metadata=metadata, + extended_contract=extended_contract, + ) + + +def _generated_fit_id( + path: Path, + fit: Mapping[str, Any], + *, + model: Model, + min_count: int, + pseudocount: int, +) -> str: + def float_identity(value: Any) -> str | None: + return None if value is None else _finite_float(value, "dispersion parameter").hex() + + identity = { + "input_sha256": _sha256(path), + "method": model, + "min_count": min_count, + "pseudocount": pseudocount, + "rho": float_identity(fit.get("rho")), + "linear_d1": float_identity(fit.get("linear_d1")), + "linear_d2": float_identity(fit.get("linear_d2")), + "linear_depth_center": float_identity(fit.get("linear_depth_center")), + "linear_depth_scale": float_identity(fit.get("linear_depth_scale")), + "n_observations": fit.get("n_observations"), + } + encoded = json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _fit_dispersion( + frame: pd.DataFrame, + path: Path, + *, + model: Model, + min_count: int, + pseudocount: int, + region_col: str | None = None, + phased: bool = False, +) -> DispersionFit: + fitter = rust_fit_imbalance_dispersion + if fitter is None: + raise RuntimeError("Rust dispersion-fitting API is not available") + fit_columns = [*SNV_COLUMNS, "ref_count", "alt_count"] + fit_frame = frame.loc[:, fit_columns].copy() + # Rust deduplicates within its region key. Preserve Python's SNP-feature + # observations and keep identical sites in separate donors distinct. + memberships = frame[region_col] if region_col is not None else [None] * len(frame) + phases = pd.Series(None, index=frame.index, dtype=object) + if phased: + for _, donor_frame in frame.groupby("donor_id", sort=False): + if _phase_fallback_reason(donor_frame, True) is None: + phases.loc[donor_frame.index] = donor_frame["GT"] + fit_frame["_fit_identity"] = [ + json.dumps([donor, region, gt], ensure_ascii=True, separators=(",", ":")) + for donor, region, gt in zip(frame["donor_id"], memberships, phases, strict=True) + ] + _write_tsv(fit_frame, path, [*fit_columns, "_fit_identity"]) + base_kwargs: dict[str, Any] = { + "min_count": min_count, + "pseudocount": pseudocount, + "method": model, + } + full_kwargs = {**base_kwargs, "phased": False, "region_col": "_fit_identity"} + is_pyo3 = _is_known_pyo3_api(fitter) + if is_pyo3: + # The donor-feature key retains the same observation multiplicity in + # global and per-donor fits; only the fitting scope changes. + raw_fit = fitter(str(path), **full_kwargs) + else: + parameters = inspect.signature(fitter).parameters + accepts_keywords = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() + ) + supports_full_call = accepts_keywords or {"phased", "region_col"}.issubset(parameters) + raw_fit = fitter(str(path), **(full_kwargs if supports_full_call else base_kwargs)) + if not isinstance(raw_fit, Mapping): + raise RuntimeError("Rust dispersion-fitting API returned an invalid payload") + payload = dict(raw_fit) + payload.setdefault( + "fit_id", + _generated_fit_id( + path, + payload, + model=model, + min_count=min_count, + pseudocount=pseudocount, + ), + ) + return _validate_fit( + payload, + model=model, + expected_rows=len(frame), + allow_legacy_linear_mock=not is_pyo3, + ) + + +def _is_known_pyo3_api(callback: Any) -> bool: + module = getattr(callback, "__module__", "") + return module == "wasp2_rust" or module.startswith("wasp2_rust.") + + +def _parameter_values(source: Mapping[str, Any]) -> tuple[Any, Any, Any, Any, Any]: + return ( + source.get("rho"), + source.get("linear_d1"), + source.get("linear_d2"), + source.get("linear_depth_center"), + source.get("linear_depth_scale"), + ) + + +def _parameter_identity(values: Sequence[Any]) -> tuple[str | None, ...]: + return tuple( + None if value is None else _finite_float(value, "dispersion parameter").hex() + for value in values + ) + + +def _call_analyzer_with_fit( + path: Path, + *, + fit: DispersionFit, + model: Model, + phased: bool, + region_col: str | None, + min_count: int, + pseudocount: int, +) -> Mapping[str, Any]: + analyzer = rust_analyze_imbalance_run + if analyzer is None: + raise RuntimeError("Rust donor-analysis API is not available") + base_kwargs: dict[str, Any] = { + "min_count": min_count, + "pseudocount": pseudocount, + "phased": phased, + "region_col": region_col, + "method": model, + } + explicit_kwargs = { + **base_kwargs, + "rho": fit.rho, + "linear_d1": fit.linear_d1, + "linear_d2": fit.linear_d2, + "linear_depth_center": fit.linear_depth_center, + "linear_depth_scale": fit.linear_depth_scale, + } + if _is_known_pyo3_api(analyzer): + raw_run = analyzer(str(path), **explicit_kwargs) + else: + parameters = inspect.signature(analyzer).parameters + if "dispersion_fit" in parameters or not { + "rho", + "linear_d1", + "linear_d2", + }.intersection(parameters): + run = analyzer(str(path), **base_kwargs, dispersion_fit=fit.payload) + if not isinstance(run, Mapping): + raise RuntimeError("Rust donor-analysis API returned an invalid payload") + return run + raw_run = analyzer(str(path), **explicit_kwargs) + + if not isinstance(raw_run, Mapping): + raise RuntimeError("Rust donor-analysis API returned an invalid payload") + if _parameter_identity(_parameter_values(raw_run)) != _parameter_identity( + _parameter_values(fit.payload) + ): + raise RuntimeError("Rust donor-analysis API changed the fixed dispersion parameters") + if raw_run.get("nuisance_source") not in {None, "fixed"}: + raise RuntimeError("Rust donor-analysis API refitted the supplied dispersion parameters") + run = {**raw_run, "dispersion_fit": fit.payload} + if not isinstance(run, Mapping): + raise RuntimeError("Rust donor-analysis API returned an invalid payload") + return run + + +def _run_fit_values(run: Mapping[str, Any]) -> tuple[Any, Any, Any, Any, Any, Any]: + nested = run.get("dispersion_fit") + source = nested if isinstance(nested, Mapping) else run + fit_id = source.get("fit_id", source.get("dispersion_fit_id")) + return fit_id, *_parameter_values(source) + + +def _require_exact_fit_echo(run: Mapping[str, Any], fit: DispersionFit, donor_id: str) -> None: + observed = _run_fit_values(run) + expected = ( + fit.fit_id, + fit.rho, + fit.linear_d1, + fit.linear_d2, + fit.linear_depth_center, + fit.linear_depth_scale, + ) + if observed[0] != expected[0] or _parameter_identity(observed[1:]) != _parameter_identity( + expected[1:] + ): + raise RuntimeError( + f"Rust donor run for {donor_id} did not reuse the requested dispersion fit exactly: " + f"expected {expected!r}, observed {observed!r}" + ) + + +def _validate_rust_results(run: Mapping[str, Any], donor_id: str) -> pd.DataFrame: + raw_results = run.get("results") + if not isinstance(raw_results, list) or not raw_results: + raise RuntimeError(f"Rust returned no results for included donor {donor_id}") + results = pd.DataFrame(raw_results) + missing = sorted(set(RUST_RESULT_COLUMNS).difference(results.columns)) + if missing: + raise RuntimeError(f"Rust results for {donor_id} are missing columns: {missing}") + if results["region"].astype(str).duplicated().any(): + raise RuntimeError(f"Rust returned duplicate result identities for donor {donor_id}") + return results.loc[:, RUST_RESULT_COLUMNS].copy() + + +def _prepare_analysis_table( + donor_frame: pd.DataFrame, + path: Path, + *, + unit: AnalysisUnit, + region_col: str | None, + phased: bool, +) -> tuple[str | None, pd.DataFrame]: + columns = ["chrom", "pos", "ref", "alt"] + rust_region_col = None + if phased and "GT" in donor_frame.columns: + columns.append("GT") + if unit == "feature": + if region_col is None: + raise RuntimeError("Feature analysis has no normalized region column") + columns.append(region_col) + rust_region_col = region_col + columns.extend(["ref_count", "alt_count"]) + _write_tsv(donor_frame, path, columns) + return rust_region_col, donor_frame + + +def _format_donor_results( + run: Mapping[str, Any], + donor_frame: pd.DataFrame, + *, + donor_id: str, + unit: AnalysisUnit, + region_col: str | None, +) -> pd.DataFrame: + results = _validate_rust_results(run, donor_id) + try: + n_observations = int(run["n_observations"]) + except (KeyError, TypeError, ValueError) as error: + raise RuntimeError(f"Rust donor run for {donor_id} omitted n_observations") from error + if n_observations != len(donor_frame): + raise RuntimeError( + f"Rust donor run for {donor_id} used {n_observations} rows; expected {len(donor_frame)}" + ) + + if unit == "snv": + annotation_columns = ["snv_id", "chrom", "pos", "ref", "alt"] + annotations = donor_frame.groupby(annotation_columns, sort=False, as_index=False).agg( + expected_ref_count=("ref_count", "sum"), + expected_alt_count=("alt_count", "sum"), + expected_snp_count=("pos", "size"), + ) + annotations["region"] = annotations["chrom"] + "_" + annotations["pos"].astype(str) + results = annotations.merge(results, on="region", how="inner", validate="one_to_one") + if len(results) != len(annotations): + raise RuntimeError(f"Rust omitted eligible SNV results for donor {donor_id}") + mismatch = ( + results["ref_count"].ne(results["expected_ref_count"]) + | results["alt_count"].ne(results["expected_alt_count"]) + | results["N"].ne(results["expected_ref_count"] + results["expected_alt_count"]) + | results["snp_count"].ne(results["expected_snp_count"]) + ) + if mismatch.any(): + raise RuntimeError(f"Rust SNV counts differ from input for donor {donor_id}") + leading_columns = ["donor_id", "snv_id", "chrom", "pos", "ref", "alt"] + results = results.drop( + columns=["region", "expected_ref_count", "expected_alt_count", "expected_snp_count"] + ) + else: + if region_col is None: + raise RuntimeError("Feature analysis has no normalized region column") + expected = ( + donor_frame.groupby(region_col, sort=False) + .agg( + expected_ref_count=("ref_count", "sum"), + expected_alt_count=("alt_count", "sum"), + expected_snp_count=("pos", "size"), + ) + .reset_index() + .rename(columns={region_col: "region"}) + ) + results = expected.merge(results, on="region", how="inner", validate="one_to_one") + if len(results) != len(expected): + raise RuntimeError(f"Rust omitted eligible feature results for donor {donor_id}") + mismatch = ( + results["ref_count"].ne(results["expected_ref_count"]) + | results["alt_count"].ne(results["expected_alt_count"]) + | results["snp_count"].ne(results["expected_snp_count"]) + ) + if mismatch.any(): + raise RuntimeError(f"Rust feature aggregation differs from input for donor {donor_id}") + results = results.drop( + columns=["expected_ref_count", "expected_alt_count", "expected_snp_count"] + ).rename(columns={"region": "feature_id"}) + leading_columns = ["donor_id", "feature_id"] + + results.insert(0, "donor_id", donor_id) + results["significant_q05"] = results["fdr_pval"] < 0.05 + results["significant_q10"] = results["fdr_pval"] < 0.10 + return results.loc[ + :, [*leading_columns, *STATISTIC_COLUMNS, "significant_q05", "significant_q10"] + ] + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as handle: + os.fsync(handle.fileno()) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _same_inode(left: Path, right: Path) -> bool: + try: + left_stat = left.stat() + right_stat = right.stat() + except FileNotFoundError: + return False + return (left_stat.st_dev, left_stat.st_ino) == (right_stat.st_dev, right_stat.st_ino) + + +def _publish_no_replace(staged: Mapping[str, Path], destinations: Mapping[str, Path]) -> None: + """Publish sidecars first and the result commit marker last without clobbering.""" + + published: list[tuple[Path, Path]] = [] + try: + for name in ["dispersion", "qc", "provenance", "results"]: + source = staged[name] + destination = destinations[name] + os.link(source, destination, follow_symlinks=False) + published.append((source, destination)) + _fsync_directory(destinations["results"].parent) + except BaseException: + for source, destination in reversed(published): + if _same_inode(source, destination): + destination.unlink() + _fsync_directory(destinations["results"].parent) + raise + + +def _require_unchanged(count_input: CountInput) -> None: + if _signature(count_input.counts) != count_input.counts_signature: + raise RuntimeError("Count input changed while donor-local analysis was running") + if count_input.counts_record["sha256"] != _sha256(count_input.counts): + raise RuntimeError("Count input content changed while donor-local analysis was running") + if count_input.manifest is not None: + if count_input.manifest_signature is None or count_input.manifest_record is None: + raise RuntimeError("Internal error: bundle manifest lock is incomplete") + if _signature(count_input.manifest) != count_input.manifest_signature: + raise RuntimeError("Count manifest changed while donor-local analysis was running") + if count_input.manifest_record["sha256"] != _sha256(count_input.manifest): + raise RuntimeError( + "Count manifest content changed while donor-local analysis was running" + ) + + +def run_per_donor_analysis( + count_file: str | Path, + out_file: str | Path, + *, + unit: AnalysisUnitInput, + model: Model = "single", + dispersion_scope: DispersionScope = "per-donor", + region_col: str | None = None, + phased: bool = False, + min_count: int = 10, + pseudocount: int = 1, + min_donor_observations: int = 50, + expected_manifest_sha256: str | None = None, +) -> dict[str, Path]: + """Run independent donor tests with an explicit reusable dispersion fit.""" + + normalized_unit = _normalize_unit(unit) + if model not in {"single", "linear"}: + raise ValueError("model must be 'single' or 'linear'") + if dispersion_scope not in {"global", "per-donor"}: + raise ValueError("dispersion_scope must be 'global' or 'per-donor'") + if min_count < 0 or pseudocount < 0: + raise ValueError("min_count and pseudocount must be nonnegative") + if min_donor_observations < 1: + raise ValueError("min_donor_observations must be positive") + if normalized_unit == "snv" and phased: + raise ValueError("Per-donor SNV analysis is unphased; --phased is not valid") + + count_input = _resolve_count_input( + count_file, + unit=normalized_unit, + expected_manifest_sha256=expected_manifest_sha256, + ) + frame, normalized_region_col, input_qc = _load_counts( + count_input, + unit=normalized_unit, + region_col=region_col, + phased=phased, + ) + + result_path = Path(os.path.abspath(Path(out_file).expanduser())) + paths = _artifact_paths(result_path) + if not result_path.parent.is_dir(): + raise ValueError(f"Output parent directory does not exist: {result_path.parent}") + existing = [str(path) for path in paths.values() if os.path.lexists(path)] + if existing: + raise FileExistsError(f"Refusing to overwrite donor-local analysis artifacts: {existing}") + if rust_fit_imbalance_dispersion is None or rust_analyze_imbalance_run is None: + raise RuntimeError( + "Rust donor-local analysis APIs are not available. Build the extension with " + "`maturin develop --release` in the WASP2 environment." + ) + + eligible_rows = frame.loc[frame["_total_count"] >= min_count].copy() + retained_groups = [] + phase_decisions: dict[str, str | None] = {} + phase_source_frames: dict[str, pd.DataFrame] = {} + for donor_id, donor_rows in eligible_rows.groupby("donor_id", sort=False): + fallback_reason = _phase_fallback_reason(donor_rows, phased) + phase_decisions[str(donor_id)] = fallback_reason + identity = [*SNV_COLUMNS, "ref_count", "alt_count"] + if normalized_region_col is not None: + identity.append(normalized_region_col) + if phased and fallback_reason is None: + # Python parses the first allele before phase-aware deduplication. + # Distinct raw tokens such as 0|0/0|1 can become the same key. + normalized = donor_rows.loc[:, identity].copy() + # A private object label cannot overwrite a user-supplied column. + normalized[object()] = donor_rows["GT"].str.split("|", n=1).str[0].astype("uint8") + retained = donor_rows.loc[~normalized.duplicated(keep="first")] + if len(retained) != len(donor_rows.drop_duplicates([*identity, "GT"])): + # Preserve the original tokens for Rust's pre-dedup phase check. + # Rust then removes the same normalized duplicates; formatting + # and nuisance fitting use the retained observations below. + phase_source_frames[str(donor_id)] = donor_rows.copy() + else: + retained = donor_rows.drop_duplicates(identity, keep="first") + retained_groups.append(retained) + if retained_groups: + eligible_rows = pd.concat(retained_groups).sort_index(kind="stable") + fit_rows = eligible_rows.copy() + donor_qc = ( + frame.groupby("donor_id", sort=True) + .agg( + input_rows=("donor_id", "size"), + unique_snv_rows=("pos", "size"), + ) + .reset_index() + ) + unique_snv_counts = ( + frame.drop_duplicates(SNV_COLUMNS) + .groupby("donor_id", sort=True) + .size() + .rename("unique_snv_rows") + ) + eligible_counts = ( + eligible_rows.drop_duplicates(SNV_COLUMNS) + .groupby("donor_id", sort=True) + .size() + .rename("eligible_snv_observations") + ) + eligible_test_rows = ( + eligible_rows.groupby("donor_id", sort=True).size().rename("eligible_test_rows") + ) + donor_qc = donor_qc.drop(columns="unique_snv_rows").join(unique_snv_counts, on="donor_id") + donor_qc = donor_qc.join(eligible_counts, on="donor_id").join(eligible_test_rows, on="donor_id") + donor_qc[["eligible_snv_observations", "eligible_test_rows"]] = ( + donor_qc[["eligible_snv_observations", "eligible_test_rows"]].fillna(0).astype(int) + ) + donor_qc["analyzed"] = donor_qc["eligible_snv_observations"] >= min_donor_observations + donor_qc["status"] = donor_qc["analyzed"].map( + {True: "included", False: "excluded_min_observations"} + ) + included_donors = donor_qc.loc[donor_qc["analyzed"], "donor_id"].tolist() + if not included_donors: + raise RuntimeError("No donors met the minimum eligible-SNV observation requirement") + + eligible_rows = eligible_rows.loc[eligible_rows["donor_id"].isin(included_donors)].copy() + fit_rows = fit_rows.loc[fit_rows["donor_id"].isin(included_donors)].copy() + staging = Path(tempfile.mkdtemp(prefix=f".{result_path.stem}.staging-", dir=result_path.parent)) + work = staging / "_work" + work.mkdir() + try: + global_fit = None + if dispersion_scope == "global": + global_fit = _fit_dispersion( + fit_rows, + work / "global_fit.tsv", + model=model, + min_count=min_count, + pseudocount=pseudocount, + region_col=normalized_region_col, + phased=phased, + ) + + result_frames: list[pd.DataFrame] = [] + dispersion_rows: list[dict[str, Any]] = [] + provenance_fits: dict[str, dict[str, Any]] = {} + donor_phasing: list[dict[str, Any]] = [] + for donor_id in included_donors: + donor_id = str(donor_id) + donor_rows = eligible_rows.loc[eligible_rows["donor_id"] == donor_id].copy() + donor_fit_rows = fit_rows.loc[fit_rows["donor_id"] == donor_id].copy() + fit = global_fit or _fit_dispersion( + donor_fit_rows, + work / "donor_fit.tsv", + model=model, + min_count=min_count, + pseudocount=pseudocount, + region_col=normalized_region_col, + phased=phased, + ) + fallback_reason = phase_decisions[donor_id] + rust_region_col, _ = _prepare_analysis_table( + phase_source_frames.get(donor_id, donor_rows), + work / "donor_counts.tsv", + unit=normalized_unit, + region_col=normalized_region_col, + phased=phased, + ) + run = _call_analyzer_with_fit( + work / "donor_counts.tsv", + fit=fit, + model=model, + phased=phased, + region_col=rust_region_col, + min_count=min_count, + pseudocount=pseudocount, + ) + _require_exact_fit_echo(run, fit, donor_id) + if bool(run.get("requested_phased")) != phased: + raise RuntimeError(f"Rust donor run for {donor_id} changed requested_phased") + effective_phased = bool(run.get("effective_phased")) + if effective_phased != (phased and fallback_reason is None): + raise RuntimeError(f"Rust phasing decision disagrees with Python for {donor_id}") + donor_phasing.append( + { + "donor_id": donor_id, + "requested_phased": phased, + "effective_phased": effective_phased, + "fallback_reason": fallback_reason, + } + ) + donor_results = _format_donor_results( + run, + donor_rows, + donor_id=donor_id, + unit=normalized_unit, + region_col=normalized_region_col, + ) + result_frames.append(donor_results) + dispersion_row = { + "donor_id": donor_id, + "dispersion_scope": dispersion_scope, + "fit_id": fit.fit_id, + "model": model, + "rho": fit.rho, + "linear_d1": fit.linear_d1, + "linear_d2": fit.linear_d2, + } + if fit.extended_contract: + dispersion_row.update( + { + "linear_depth_center": fit.linear_depth_center, + "linear_depth_scale": fit.linear_depth_scale, + } + ) + dispersion_row.update(fit.metadata) + dispersion_row.update( + { + "fit_observations": fit.n_observations, + "donor_eligible_snv_observations": len( + donor_fit_rows.drop_duplicates(SNV_COLUMNS) + ), + "result_rows": len(donor_results), + } + ) + dispersion_rows.append(dispersion_row) + + if fit.extended_contract or fit.metadata: + fit_record = { + "fit_id": fit.fit_id, + "model": model, + "rho": fit.rho, + "linear_d1": fit.linear_d1, + "linear_d2": fit.linear_d2, + } + if fit.extended_contract: + fit_record.update( + { + "linear_depth_center": fit.linear_depth_center, + "linear_depth_scale": fit.linear_depth_scale, + } + ) + fit_record.update(fit.metadata) + fit_record["fit_observations"] = fit.n_observations + existing_fit = provenance_fits.get(fit.fit_id) + if existing_fit is None: + provenance_fits[fit.fit_id] = {**fit_record, "donor_ids": [donor_id]} + else: + comparable = { + key: value for key, value in existing_fit.items() if key != "donor_ids" + } + if comparable != fit_record: + raise RuntimeError(f"Fit ID {fit.fit_id} mapped to inconsistent metadata") + existing_fit["donor_ids"].append(donor_id) + + _require_unchanged(count_input) + sort_columns = ( + ["donor_id", "snv_id"] + if normalized_unit == "snv" + else [ + "donor_id", + "feature_id", + ] + ) + results = pd.concat(result_frames, ignore_index=True).sort_values( + sort_columns, kind="mergesort" + ) + dispersion = pd.DataFrame(dispersion_rows).sort_values("donor_id", kind="mergesort") + qc = donor_qc.sort_values("donor_id", kind="mergesort") + + staged_paths = {name: staging / path.name for name, path in paths.items()} + results.to_csv(staged_paths["results"], sep="\t", header=True, index=False) + dispersion.to_csv(staged_paths["dispersion"], sep="\t", header=True, index=False) + qc.to_csv(staged_paths["qc"], sep="\t", header=True, index=False) + for name in ["results", "dispersion", "qc"]: + _fsync_file(staged_paths[name]) + + output_records: dict[str, dict[str, str | int]] = {} + for name in ["results", "dispersion", "qc"]: + record, _ = _file_record(staged_paths[name]) + output_records[name] = {**record, "path": paths[name].name} + provenance = { + "schema_version": SCHEMA_VERSION, + "created_at_utc": datetime.now(UTC).isoformat(), + "software": {"package": "wasp2", "version": _package_version()}, + "analysis": { + "scope": "per-donor", + "unit": normalized_unit, + "model": model, + "dispersion_scope": dispersion_scope, + "region_col": normalized_region_col, + "phased": phased, + "multiple_testing": "Benjamini-Hochberg independently within each donor", + "pvalue_method": "chi_square_lrt", + }, + "parameters": { + "min_count": min_count, + "pseudocount": pseudocount, + "min_donor_observations": min_donor_observations, + }, + "inputs": { + "counts": count_input.counts_record, + "count_manifest": count_input.manifest_record, + "count_bundle_schema": ( + count_input.manifest_data.get("schema_version") + if count_input.manifest_data is not None + else None + ), + "qc": input_qc, + }, + "donors": { + "total": len(qc), + "included": int(qc["analyzed"].sum()), + "excluded": int((~qc["analyzed"]).sum()), + "phasing": donor_phasing, + }, + "outputs": output_records, + } + if provenance_fits: + provenance["dispersion_fits"] = list(provenance_fits.values()) + staged_paths["provenance"].write_text( + json.dumps(provenance, indent=2, sort_keys=True) + "\n" + ) + _fsync_file(staged_paths["provenance"]) + _fsync_directory(staging) + _require_unchanged(count_input) + _publish_no_replace(staged_paths, paths) + shutil.rmtree(staging, ignore_errors=True) + return paths + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise diff --git a/src/counting/count_alleles.py b/src/counting/count_alleles.py index f5fe94a0..85965851 100644 --- a/src/counting/count_alleles.py +++ b/src/counting/count_alleles.py @@ -92,6 +92,19 @@ def make_count_df(bam_file: str, df: pl.DataFrame, use_rust: bool = True) -> pl. chrom_list = df.get_column("chrom").unique(maintain_order=True) + multi_allelic = ( + df.select(["chrom", "pos", "ref", "alt"]) + .unique() + .group_by(["chrom", "pos"]) + .len() + .filter(pl.col("len") > 1) + ) + if multi_allelic.height: + raise ValueError( + "Bulk allele counting supports biallelic positions only; " + "remove multi-allelic sites before counting" + ) + # Require Rust path (no Python fallback) if not (use_rust and RUST_AVAILABLE): raise RuntimeError( @@ -117,9 +130,7 @@ def make_count_df(bam_file: str, df: pl.DataFrame, use_rust: bool = True) -> pl. chrom_df = df.filter(pl.col("chrom") == chrom) snp_list = ( - chrom_df.select(["pos", "ref", "alt"]) - .unique(subset=["pos"], maintain_order=True) - .iter_rows() + chrom_df.select(["pos", "ref", "alt"]).unique(maintain_order=True).iter_rows() ) start = timeit.default_timer() diff --git a/src/wasp2_rust/__init__.pyi b/src/wasp2_rust/__init__.pyi index b6b932aa..73d71b8b 100644 --- a/src/wasp2_rust/__init__.pyi +++ b/src/wasp2_rust/__init__.pyi @@ -4,6 +4,8 @@ This module provides high-performance implementations of bottleneck functions for allele-specific analysis in WASP2. """ +__version__: str + from typing import TypedDict class UnifiedStats(TypedDict): @@ -43,6 +45,45 @@ class ImbalanceResult(TypedDict): pval: float fdr_pval: float +class _DispersionFitDiagnostics(TypedDict, total=False): + """Optional optimizer diagnostics supplied by the Rust backend.""" + + fit_status: str + optimizer: str + optimizer_status: str | int + optimizer_converged: bool + optimizer_iterations: int + optimizer_evaluations: int + optimizer_objective: float + +class DispersionFit(_DispersionFitDiagnostics): + """Dispersion parameters fitted from independent observations.""" + + method: str + rho: float | None + linear_d1: float | None + linear_d2: float | None + linear_depth_center: float | None + linear_depth_scale: float | None + n_observations: int + +class ImbalanceRun(TypedDict): + """One independent analysis run with fitted or fixed nuisance parameters.""" + + results: list[ImbalanceResult] + method: str + rho: float | None + linear_d1: float | None + linear_d2: float | None + linear_depth_center: float | None + linear_depth_scale: float | None + n_observations: int + requested_phased: bool + effective_phased: bool + nuisance_source: str + exact_snv: bool + pvalue_method: str + class VariantSpan(TypedDict): """Variant span information from intersection parsing.""" @@ -561,6 +602,8 @@ def analyze_imbalance( min_count: int = 10, pseudocount: int = 1, method: str = "single", + phased: bool = False, + region_col: str | None = None, ) -> list[ImbalanceResult]: """Analyze allelic imbalance using beta-binomial model. @@ -581,3 +624,34 @@ def analyze_imbalance( List of imbalance results per region. """ ... + +def analyze_imbalance_run( + tsv_path: str, + min_count: int = 10, + pseudocount: int = 1, + phased: bool = False, + region_col: str | None = None, + method: str = "single", + rho: float | None = None, + linear_d1: float | None = None, + linear_d2: float | None = None, + linear_depth_center: float | None = None, + linear_depth_scale: float | None = None, + exact_snv: bool | None = None, +) -> ImbalanceRun: + """Analyze with optional fixed dispersion and archived chi-square LRT p-values. + + exact_snv=True is rejected; None/False preserve the archived method. + """ + ... + +def fit_imbalance_dispersion( + tsv_path: str, + min_count: int = 10, + pseudocount: int = 1, + method: str = "single", + phased: bool = False, + region_col: str | None = None, +) -> DispersionFit: + """Fit reusable nuisance parameters from independent SNV observations.""" + ... diff --git a/tests/analysis/fixtures/public_atac/README.md b/tests/analysis/fixtures/public_atac/README.md new file mode 100644 index 00000000..0a413be4 --- /dev/null +++ b/tests/analysis/fixtures/public_atac/README.md @@ -0,0 +1,22 @@ +# Public ATAC bulk compatibility data + +These compressed tables contain measured allele counts from GRCh38 chromosome 21 +of GM12878 ATAC experiment ENCSR637XSC, replicate-2 BAM +[ENCFF962FMH](https://www.encodeproject.org/files/ENCFF962FMH/), intersected with +[ENCFF452ZBX peaks](https://www.encodeproject.org/files/ENCFF452ZBX/) and matched +[HG001 GIAB NISTv3.3.1 phased genotypes](https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/NA12878_HG001/NISTv3.3.1/GRCh38/). + +Source downloads passed MD5 checks. Counting was independently compared with the +unmodified archived Python function at thread settings 1 and 4. There are 27,182 +SNP rows and 556 peak-overlapping rows; the complete counting CLI reproduced all +556 peak rows exactly. `metadata.json` records source and table checksums. + +Tests execute archived Python commit `41354fc` on these inputs rather than storing +Rust outputs as expected results. Phased comparisons select truly phased SNPs. +Linear-reference shape repairs and forwarding of its ignored phased flag are +explicit in the test. Strict statistical tolerances remain `rtol=atol=5e-5`; +counts must match exactly. Sixteen legacy runs plus sixteen donor feature runs +form the 32-run matrix, with NumPy 1.26.4 and SciPy 1.13.1 pinned in Python 3.11 CI. + +This is one donor and one chromosome, using published aligned reads. It does not +validate a new WASP remapping pass, multi-donor calibration or biological claims. diff --git a/tests/analysis/fixtures/public_atac/metadata.json b/tests/analysis/fixtures/public_atac/metadata.json new file mode 100644 index 00000000..fd10e59e --- /dev/null +++ b/tests/analysis/fixtures/public_atac/metadata.json @@ -0,0 +1,29 @@ +{ + "experiment": "ENCSR637XSC", + "bam": "ENCFF962FMH", + "peaks": "ENCFF452ZBX", + "genotypes": "HG001 GIAB NISTv3.3.1 GRCh38", + "chromosome": "chr21", + "archive_commit": "41354fc087404ba1fa99542803ee9cb9b7e0e6bf", + "reference_environment": { + "numpy": "1.26.4", + "scipy": "1.13.1" + }, + "sources": { + "bam_md5": "9d7b262ae771a5e638473f801840772e", + "vcf_md5": "3ddb1305f14e39a04c93ba2db3f7247e", + "peaks_md5": "9be61c6cb1158d47516fdedd54cfb333" + }, + "counts": { + "snv": { + "file": "snp_counts.tsv.gz", + "sha256": "d024e628c1e3b734317f04c59e5af680d9c13ce45a6a507cc05f48255677f043", + "rows": 27182 + }, + "peak": { + "file": "peak_counts.tsv.gz", + "sha256": "4d6769bb4725b5ad973dddf9fb2dfe3f091725ecbbeebb70be476408c965a9b3", + "rows": 556 + } + } +} diff --git a/tests/analysis/fixtures/public_atac/peak_counts.tsv.gz b/tests/analysis/fixtures/public_atac/peak_counts.tsv.gz new file mode 100644 index 00000000..91b8050d Binary files /dev/null and b/tests/analysis/fixtures/public_atac/peak_counts.tsv.gz differ diff --git a/tests/analysis/fixtures/public_atac/snp_counts.tsv.gz b/tests/analysis/fixtures/public_atac/snp_counts.tsv.gz new file mode 100644 index 00000000..c8fa38f1 Binary files /dev/null and b/tests/analysis/fixtures/public_atac/snp_counts.tsv.gz differ diff --git a/tests/analysis/fixtures/real_atac_optimizer_reference.json b/tests/analysis/fixtures/real_atac_optimizer_reference.json new file mode 100644 index 00000000..1ccff5de --- /dev/null +++ b/tests/analysis/fixtures/real_atac_optimizer_reference.json @@ -0,0 +1,2860 @@ +{ + "provenance": { + "archive": "41354fc087404ba1fa99542803ee9cb9b7e0e6bf", + "bam": "ENCFF962FMH", + "peaks": "ENCFF452ZBX", + "vcf": "HG001 NISTv3.3.1 GRCh38 phased", + "subset": "real chr21 boundary observations and balanced-first-SNP peaks", + "reference": "Archived get_imbalance with independent archived full-data SciPy dispersion fixed, to isolate allelic-proportion optimization. No Rust output is used.", + "adaptations": [ + "fixed nuisance: replace only balanced-null fitting with archived full-data SciPy estimates", + "linear: explicit scalar/dispersion-array shape repairs", + "phased linear: forward ignored phased flag" + ], + "limitations": "Balanced-first-SNP unphased proportions may select mirrored, equally likely optima; raw mu equality remains checked in the full real-data report.", + "source_checksums": { + "ENCFF962FMH.bam": { + "md5": "9d7b262ae771a5e638473f801840772e", + "bytes": 7681789216 + }, + "HG001.phased.vcf.gz": { + "md5": "3ddb1305f14e39a04c93ba2db3f7247e", + "bytes": 106366697 + }, + "HG001.phased.vcf.gz.tbi": { + "md5": "1eb071673783d96cbbca3d9ca6f19e36", + "bytes": 1519511 + }, + "peaks.bed.gz": { + "md5": "9be61c6cb1158d47516fdedd54cfb333", + "bytes": 4134636 + } + } + }, + "cases": [ + { + "name": "boundary/single/False/0", + "family": "boundary", + "method": "single", + "phased": false, + "pseudocount": 0, + "fixed_dispersion": { + "rho": 0.04104616438385224 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -6.389225553037658, + "alt_ll": -5.874991948573438e-05, + "mu": 0.9999940391390134, + "lrt": 12.778333606236345, + "pval": 0.0003506571602214173, + "fdr_pval": 0.0003506571602214173 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -6.389225553037658, + "alt_ll": -5.874991946264174e-05, + "mu": 5.9608609865491405e-06, + "lrt": 12.77833360623639, + "pval": 0.00035065716022140845, + "fdr_pval": 0.0003506571602214173 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -7.5526012240973905, + "alt_ll": -7.024515342557436e-05, + "mu": 0.9999940391390134, + "lrt": 15.10506195788793, + "pval": 0.00010169024576284532, + "fdr_pval": 0.00023727724011330577 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -7.5526012240973905, + "alt_ll": -7.024515342557436e-05, + "mu": 0.9999940391390134, + "lrt": 15.10506195788793, + "pval": 0.00010169024576284532, + "fdr_pval": 0.00023727724011330577 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -6.389225553037658, + "alt_ll": -5.874991948573438e-05, + "mu": 0.9999940391390134, + "lrt": 12.778333606236345, + "pval": 0.0003506571602214173, + "fdr_pval": 0.0003506571602214173 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -9.263918812887031, + "alt_ll": -8.751774048754157e-05, + "mu": 0.9999940391390134, + "lrt": 18.527662590293087, + "pval": 1.6745618771578033e-05, + "fdr_pval": 0.00011721933140104623 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -6.790196131686823, + "alt_ll": -6.268803245745858e-05, + "mu": 5.9608609865491405e-06, + "lrt": 13.580266887308731, + "pval": 0.00022857573482696643, + "fdr_pval": 0.0003506571602214173 + } + ], + "archived_full_data_fit": { + "python_rho": 0.04104616438385224 + } + }, + { + "name": "boundary/single/False/1", + "family": "boundary", + "method": "single", + "phased": false, + "pseudocount": 1, + "fixed_dispersion": { + "rho": 0.031241338194282212 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.597171990310841, + "alt_ll": -1.1473986582418778, + "mu": 0.9156961652672377, + "lrt": 8.899546664137926, + "pval": 0.002852414634994033, + "fdr_pval": 0.002852414634994033 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -5.597171990310841, + "alt_ll": -1.1473986582418778, + "mu": 0.08430383473366374, + "lrt": 8.899546664137926, + "pval": 0.002852414634994033, + "fdr_pval": 0.002852414634994033 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.736573736332222, + "alt_ll": -1.1845073074386976, + "mu": 0.9280744517074505, + "lrt": 11.104132857787048, + "pval": 0.0008613555380084625, + "fdr_pval": 0.0020098295886864124 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.736573736332222, + "alt_ll": -1.1845073074386976, + "mu": 0.9280744517074505, + "lrt": 11.104132857787048, + "pval": 0.0008613555380084625, + "fdr_pval": 0.0020098295886864124 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.597171990310841, + "alt_ll": -1.1473986582418778, + "mu": 0.9156961652672377, + "lrt": 8.899546664137926, + "pval": 0.002852414634994033, + "fdr_pval": 0.002852414634994033 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -8.483817804847927, + "alt_ll": -1.2375382896551264, + "mu": 0.9412854790074102, + "lrt": 14.492559030385602, + "pval": 0.0001407142762752784, + "fdr_pval": 0.0009849999339269489 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -5.984979021124989, + "alt_ll": -1.160332184789425, + "mu": 0.07963337356719068, + "lrt": 9.649293672671128, + "pval": 0.0018942440703495743, + "fdr_pval": 0.002852414634994033 + } + ], + "archived_full_data_fit": { + "python_rho": 0.031241338194282212 + } + }, + { + "name": "boundary/single/True/0", + "family": "boundary", + "method": "single", + "phased": true, + "pseudocount": 0, + "fixed_dispersion": { + "rho": 0.041307162864532974 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -6.380341601363806, + "alt_ll": -5.868513188644897e-05, + "mu": 0.9999940391390134, + "lrt": 12.76056583246384, + "pval": 0.0003540037917148921, + "fdr_pval": 0.0003540037917148921 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -6.380341601363806, + "alt_ll": -5.868513188644897e-05, + "mu": 5.9608609865491405e-06, + "lrt": 12.76056583246384, + "pval": 0.0003540037917148921, + "fdr_pval": 0.0003540037917148921 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -7.5403641628088955, + "alt_ll": -7.015322184500405e-05, + "mu": 0.9999940391390134, + "lrt": 15.080588019174101, + "pval": 0.00010301723214466304, + "fdr_pval": 0.00024037354167088043 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -7.5403641628088955, + "alt_ll": -7.015322184500405e-05, + "mu": 0.9999940391390134, + "lrt": 15.080588019174101, + "pval": 0.00010301723214466304, + "fdr_pval": 0.00024037354167088043 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -6.380341601363806, + "alt_ll": -5.868513188644897e-05, + "mu": 0.9999940391390134, + "lrt": 12.76056583246384, + "pval": 0.0003540037917148921, + "fdr_pval": 0.0003540037917148921 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -9.246021344611592, + "alt_ll": -8.737785443813095e-05, + "mu": 0.9999940391390134, + "lrt": 18.491867933514307, + "pval": 1.706307936868434e-05, + "fdr_pval": 0.00011944155558079038 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -6.7802050689212905, + "alt_ll": -6.261440120169937e-05, + "mu": 5.9608609865491405e-06, + "lrt": 13.560284909040178, + "pval": 0.00023102202708643588, + "fdr_pval": 0.0003540037917148921 + } + ], + "archived_full_data_fit": { + "python_rho": 0.041307162864532974 + } + }, + { + "name": "boundary/single/True/1", + "family": "boundary", + "method": "single", + "phased": true, + "pseudocount": 1, + "fixed_dispersion": { + "rho": 0.03143939447095322 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.590394401920598, + "alt_ll": -1.1483895581132888, + "mu": 0.9156172386975225, + "lrt": 8.884009687614618, + "pval": 0.002876790416394797, + "fdr_pval": 0.002876790416394797 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -5.590394401920598, + "alt_ll": -1.1483895581132888, + "mu": 0.08438276130295859, + "lrt": 8.884009687614618, + "pval": 0.002876790416394797, + "fdr_pval": 0.002876790416394797 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.726891923863885, + "alt_ll": -1.1856208329780253, + "mu": 0.9279946875397128, + "lrt": 11.08254218177172, + "pval": 0.0008714424933497622, + "fdr_pval": 0.0020333658178161122 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.726891923863885, + "alt_ll": -1.1856208329780253, + "mu": 0.9279946875397128, + "lrt": 11.08254218177172, + "pval": 0.0008714424933497622, + "fdr_pval": 0.0020333658178161122 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.590394401920598, + "alt_ll": -1.1483895581132888, + "mu": 0.9156172386975225, + "lrt": 8.884009687614618, + "pval": 0.002876790416394797, + "fdr_pval": 0.002876790416394797 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -8.46901393882688, + "alt_ll": -1.238817635425435, + "mu": 0.9412058705174338, + "lrt": 14.46039260680289, + "pval": 0.00014313787117334125, + "fdr_pval": 0.0010019650982133889 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -5.977254607315782, + "alt_ll": -1.1613662182049467, + "mu": 0.0797126593628677, + "lrt": 9.63177677822167, + "pval": 0.0019123951599815383, + "fdr_pval": 0.002876790416394797 + } + ], + "archived_full_data_fit": { + "python_rho": 0.03143939447095322 + } + }, + { + "name": "boundary/linear/False/0", + "family": "boundary", + "method": "linear", + "phased": false, + "pseudocount": 0, + "fixed_dispersion": { + "linear_d1": -2.593059736583777, + "linear_d2": -0.010492431501324003, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.760760836871878, + "alt_ll": -5.403463685560439e-05, + "mu": 0.9999940391390134, + "lrt": 11.521413604470045, + "pval": 0.0006879905267612963, + "fdr_pval": 0.0006879905267612963 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -5.760760836871878, + "alt_ll": -5.403463685560439e-05, + "mu": 5.9608609865491405e-06, + "lrt": 11.521413604470045, + "pval": 0.0006879905267612963, + "fdr_pval": 0.0006879905267612963 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.76757910442724, + "alt_ll": -6.418607752323169e-05, + "mu": 0.9999940391390134, + "lrt": 13.535029836699433, + "pval": 0.00023415166085433029, + "fdr_pval": 0.0005463538753267707 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.76757910442724, + "alt_ll": -6.418607752323169e-05, + "mu": 0.9999940391390134, + "lrt": 13.535029836699433, + "pval": 0.00023415166085433029, + "fdr_pval": 0.0005463538753267707 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.760760836871878, + "alt_ll": -5.403463685560439e-05, + "mu": 0.9999940391390134, + "lrt": 11.521413604470045, + "pval": 0.0006879905267612963, + "fdr_pval": 0.0006879905267612963 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -8.291370765538963, + "alt_ll": -7.973514018111416e-05, + "mu": 0.9999940391390134, + "lrt": 16.582582060797563, + "pval": 4.657683078357323e-05, + "fdr_pval": 0.00032603781548501263 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -6.106138698146697, + "alt_ll": -5.750332318399387e-05, + "mu": 5.9608609865491405e-06, + "lrt": 12.212162389647027, + "pval": 0.00047478970679398677, + "fdr_pval": 0.0006879905267612963 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.593059736583777, + "python_slope": -0.010492431501324003 + } + }, + { + "name": "boundary/linear/False/1", + "family": "boundary", + "method": "linear", + "phased": false, + "pseudocount": 1, + "fixed_dispersion": { + "linear_d1": -2.975483969206107, + "linear_d2": -0.008125105440394378, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.1955904121383245, + "alt_ll": -1.2089534582944177, + "mu": 0.9105705969830463, + "lrt": 7.973273907687814, + "pval": 0.00474730010636262, + "fdr_pval": 0.00474730010636262 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -5.1955904121383245, + "alt_ll": -1.208953458294392, + "mu": 0.08942940301675331, + "lrt": 7.973273907687865, + "pval": 0.004747300106362484, + "fdr_pval": 0.00474730010636262 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.209991197647268, + "alt_ll": -1.2479189349492277, + "mu": 0.9233158555279374, + "lrt": 9.92414452539608, + "pval": 0.001631246482824818, + "fdr_pval": 0.0038062417932579087 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.209991197647268, + "alt_ll": -1.2479189349492277, + "mu": 0.9233158555279374, + "lrt": 9.92414452539608, + "pval": 0.001631246482824818, + "fdr_pval": 0.0038062417932579087 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.1955904121383245, + "alt_ll": -1.2089534582944177, + "mu": 0.9105705969830463, + "lrt": 7.973273907687814, + "pval": 0.00474730010636262, + "fdr_pval": 0.00474730010636262 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -7.7895130267974135, + "alt_ll": -1.300144735168458, + "mu": 0.9371887789608115, + "lrt": 12.978736583257911, + "pval": 0.0003150484858043847, + "fdr_pval": 0.002205339400630693 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -5.540111147939907, + "alt_ll": -1.222744984782235, + "mu": 0.0846441826505917, + "lrt": 8.634732326315344, + "pval": 0.0032981369619646747, + "fdr_pval": 0.00474730010636262 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.9592337583253183, + "python_slope": -0.008125105440394378 + } + }, + { + "name": "boundary/linear/True/0", + "family": "boundary", + "method": "linear", + "phased": true, + "pseudocount": 0, + "fixed_dispersion": { + "linear_d1": -2.5862627684737776, + "linear_d2": -0.0105182950874047, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.750665732409811, + "alt_ll": -5.395671651164946e-05, + "mu": 0.9999940391390134, + "lrt": 11.501223551386598, + "pval": 0.0006955039201319832, + "fdr_pval": 0.0006955039201319845 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -5.750665732409811, + "alt_ll": -5.395671651342582e-05, + "mu": 5.9608609865491405e-06, + "lrt": 11.501223551386595, + "pval": 0.0006955039201319845, + "fdr_pval": 0.0006955039201319845 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.754413049497881, + "alt_ll": -6.408167225657735e-05, + "mu": 0.9999940391390134, + "lrt": 13.508697935651249, + "pval": 0.00023746024524887906, + "fdr_pval": 0.0005540739055807178 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.754413049497881, + "alt_ll": -6.408167225657735e-05, + "mu": 0.9999940391390134, + "lrt": 13.508697935651249, + "pval": 0.00023746024524887906, + "fdr_pval": 0.0005540739055807178 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.750665732409811, + "alt_ll": -5.395671651164946e-05, + "mu": 0.9999940391390134, + "lrt": 11.501223551386598, + "pval": 0.0006955039201319832, + "fdr_pval": 0.0006955039201319845 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -8.273403375022747, + "alt_ll": -7.958790521911396e-05, + "mu": 0.9999940391390134, + "lrt": 16.546647574235056, + "pval": 4.7467829431069906e-05, + "fdr_pval": 0.0003322748060174893 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -6.095005896635046, + "alt_ll": -5.7416547379673943e-05, + "mu": 5.9608609865491405e-06, + "lrt": 12.189896960175332, + "pval": 0.0004804903731737427, + "fdr_pval": 0.0006955039201319845 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.5862627684737776, + "python_slope": -0.0105182950874047 + } + }, + { + "name": "boundary/linear/True/1", + "family": "boundary", + "method": "linear", + "phased": true, + "pseudocount": 1, + "fixed_dispersion": { + "linear_d1": -2.9682839533839394, + "linear_d2": -0.008153245836530314, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 16419450, + "ref": "A", + "alt": "G", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_16419450" + }, + { + "chrom": "chr21", + "pos": 19667014, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 0, + "alt_count": 12, + "other_count": 0, + "region": "chr21_19667014" + }, + { + "chrom": "chr21", + "pos": 42381603, + "ref": "C", + "alt": "A", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42381603" + }, + { + "chrom": "chr21", + "pos": 42389309, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 15, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389309" + }, + { + "chrom": "chr21", + "pos": 42389385, + "ref": "G", + "alt": "C", + "GT": "0|1", + "ref_count": 12, + "alt_count": 0, + "other_count": 0, + "region": "chr21_42389385" + }, + { + "chrom": "chr21", + "pos": 45483107, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 20, + "alt_count": 0, + "other_count": 0, + "region": "chr21_45483107" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21_46295649" + } + ], + "expected": [ + { + "region": "chr21_16419450", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.1874939257202275, + "alt_ll": -1.2102578684520156, + "mu": 0.9104567402190756, + "lrt": 7.954472114536424, + "pval": 0.0047968697957990756, + "fdr_pval": 0.0047968697957990756 + }, + { + "region": "chr21_19667014", + "ref_count": 0, + "alt_count": 12, + "N": 12, + "snp_count": 1, + "null_ll": -5.1874939257202275, + "alt_ll": -1.2102578684520156, + "mu": 0.08954325978262004, + "lrt": 7.954472114536424, + "pval": 0.0047968697957990756, + "fdr_pval": 0.0047968697957990756 + }, + { + "region": "chr21_42381603", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.1989096271204325, + "alt_ll": -1.2493196867622505, + "mu": 0.9232053615637866, + "lrt": 9.899179880716364, + "pval": 0.0016535244511698482, + "fdr_pval": 0.0038582237193963125 + }, + { + "region": "chr21_42389309", + "ref_count": 15, + "alt_count": 0, + "N": 15, + "snp_count": 1, + "null_ll": -6.1989096271204325, + "alt_ll": -1.2493196867622505, + "mu": 0.9232053615637866, + "lrt": 9.899179880716364, + "pval": 0.0016535244511698482, + "fdr_pval": 0.0038582237193963125 + }, + { + "region": "chr21_42389385", + "ref_count": 12, + "alt_count": 0, + "N": 12, + "snp_count": 1, + "null_ll": -5.1874939257202275, + "alt_ll": -1.2102578684520156, + "mu": 0.9104567402190756, + "lrt": 7.954472114536424, + "pval": 0.0047968697957990756, + "fdr_pval": 0.0047968697957990756 + }, + { + "region": "chr21_45483107", + "ref_count": 20, + "alt_count": 0, + "N": 20, + "snp_count": 1, + "null_ll": -7.7735334086592704, + "alt_ll": -1.3016520140508776, + "mu": 0.9370848672619511, + "lrt": 12.943762789216786, + "pval": 0.0003209891670852516, + "fdr_pval": 0.002246924169596761 + }, + { + "region": "chr21_46295649", + "ref_count": 0, + "alt_count": 13, + "N": 13, + "snp_count": 1, + "null_ll": -5.531021056825821, + "alt_ll": -1.2240849095076465, + "mu": 0.0847568789913965, + "lrt": 8.61387229463635, + "pval": 0.003336122833595065, + "fdr_pval": 0.0047968697957990756 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.9519774617108787, + "python_slope": -0.008153245836530314 + } + }, + { + "name": "balanced_anchor/single/False/0", + "family": "balanced_anchor", + "method": "single", + "phased": false, + "pseudocount": 0, + "fixed_dispersion": { + "rho": 0.03295621518540146 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.730375447080839, + "alt_ll": -4.699791043887814, + "mu": 0.41804483019409894, + "lrt": 0.06116880638604982, + "pval": 0.8046580410471728, + "fdr_pval": 0.8046580410471728 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -20.05985517962762, + "alt_ll": -16.211054204245176, + "mu": 0.2550470457103756, + "lrt": 7.697601950764884, + "pval": 0.005529424020491886, + "fdr_pval": 0.011058848040983772 + } + ], + "archived_full_data_fit": { + "python_rho": 0.03295621518540146 + } + }, + { + "name": "balanced_anchor/single/False/1", + "family": "balanced_anchor", + "method": "single", + "phased": false, + "pseudocount": 1, + "fixed_dispersion": { + "rho": 0.028099536389707032 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.7493742671525645, + "alt_ll": -4.740788728734382, + "mu": 0.5565148187766159, + "lrt": 0.017171076836364207, + "pval": 0.8957448865315235, + "fdr_pval": 0.8957448865315235 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -18.784723355306976, + "alt_ll": -16.021364256101982, + "mu": 0.2934713090368672, + "lrt": 5.526718198409988, + "pval": 0.018728201248774133, + "fdr_pval": 0.037456402497548266 + } + ], + "archived_full_data_fit": { + "python_rho": 0.028099536389707032 + } + }, + { + "name": "balanced_anchor/single/True/0", + "family": "balanced_anchor", + "method": "single", + "phased": true, + "pseudocount": 0, + "fixed_dispersion": { + "rho": 0.033056123608989116 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.7301187627048265, + "alt_ll": -4.118217657407499, + "mu": 0.38179142066123356, + "lrt": 1.2238022105946555, + "pval": 0.2686157553087597, + "fdr_pval": 0.2686157553087597 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -20.053415540712095, + "alt_ll": -12.810678949704371, + "mu": 0.25128088994567566, + "lrt": 14.485473182015447, + "pval": 0.00014124458999520496, + "fdr_pval": 0.0002824891799904099 + } + ], + "archived_full_data_fit": { + "python_rho": 0.033056123608989116 + } + }, + { + "name": "balanced_anchor/single/True/1", + "family": "balanced_anchor", + "method": "single", + "phased": true, + "pseudocount": 1, + "fixed_dispersion": { + "rho": 0.02818183091705621 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.749254004335004, + "alt_ll": -4.194251526845608, + "mu": 0.3942713717876904, + "lrt": 1.1100049549787911, + "pval": 0.2920808714350239, + "fdr_pval": 0.2920808714350239 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -18.77987837775621, + "alt_ll": -12.67297087907981, + "mu": 0.28726793301366754, + "lrt": 12.213814997352799, + "pval": 0.0004743693170833306, + "fdr_pval": 0.0009487386341666612 + } + ], + "archived_full_data_fit": { + "python_rho": 0.02818183091705621 + } + }, + { + "name": "balanced_anchor/linear/False/0", + "family": "balanced_anchor", + "method": "linear", + "phased": false, + "pseudocount": 0, + "fixed_dispersion": { + "linear_d1": -2.2896600988853835, + "linear_d2": -0.013397521759567543, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.744634953565037, + "alt_ll": -4.744634953565037, + "mu": 0.5000000000000017, + "lrt": -0.0, + "pval": 1.0, + "fdr_pval": 1.0 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -17.966232263157128, + "alt_ll": -15.922549744741294, + "mu": 0.26486392399336306, + "lrt": 4.087365036831667, + "pval": 0.04320494704484253, + "fdr_pval": 0.08640989408968507 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.2896600988853835, + "python_slope": -0.013397521759567543 + } + }, + { + "name": "balanced_anchor/linear/False/1", + "family": "balanced_anchor", + "method": "linear", + "phased": false, + "pseudocount": 1, + "fixed_dispersion": { + "linear_d1": -2.5581565149445686, + "linear_d2": -0.011773137230599792, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.7846650377208135, + "alt_ll": -4.784665037720812, + "mu": 0.5000000000000001, + "lrt": 3.552713678800501e-15, + "pval": 0.9999999524423742, + "fdr_pval": 0.9999999524423742 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -17.347274194860468, + "alt_ll": -15.985020795752241, + "mu": 0.3071512458291038, + "lrt": 2.724506798216453, + "pval": 0.09881864251465043, + "fdr_pval": 0.19763728502930086 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.534610240483369, + "python_slope": -0.011773137230599792 + } + }, + { + "name": "balanced_anchor/linear/True/0", + "family": "balanced_anchor", + "method": "linear", + "phased": true, + "pseudocount": 0, + "fixed_dispersion": { + "linear_d1": -2.288664881048436, + "linear_d2": -0.013380676923467512, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.744772385907499, + "alt_ll": -4.3474897457811235, + "mu": 0.38736461820875234, + "lrt": 0.7945652802527512, + "pval": 0.37272324505208654, + "fdr_pval": 0.37272324505208654 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -17.963444990605318, + "alt_ll": -12.671604709944702, + "mu": 0.25273415191599713, + "lrt": 10.58368056132123, + "pval": 0.001140902933929827, + "fdr_pval": 0.002281805867859654 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.288664881048436, + "python_slope": -0.013380676923467512 + } + }, + { + "name": "balanced_anchor/linear/True/1", + "family": "balanced_anchor", + "method": "linear", + "phased": true, + "pseudocount": 1, + "fixed_dispersion": { + "linear_d1": -2.557020377999981, + "linear_d2": -0.011759819173369684, + "linear_depth_center": 0.0, + "linear_depth_scale": 1.0 + }, + "counts": [ + { + "chrom": "chr21", + "pos": 37166764, + "ref": "A", + "alt": "G", + "GT": "1|0", + "ref_count": 7, + "alt_count": 7, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 37167055, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 13, + "other_count": 0, + "region": "chr21:37166752-37167128" + }, + { + "chrom": "chr21", + "pos": 46294770, + "ref": "A", + "alt": "T", + "GT": "1|0", + "ref_count": 5, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46294784, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295192, + "ref": "T", + "alt": "G", + "GT": "1|0", + "ref_count": 9, + "alt_count": 16, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295382, + "ref": "T", + "alt": "C", + "GT": "1|0", + "ref_count": 2, + "alt_count": 9, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295569, + "ref": "C", + "alt": "G", + "GT": "0|1", + "ref_count": 10, + "alt_count": 2, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295649, + "ref": "G", + "alt": "C", + "GT": "1|0", + "ref_count": 0, + "alt_count": 13, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295680, + "ref": "C", + "alt": "T", + "GT": "1|0", + "ref_count": 0, + "alt_count": 5, + "other_count": 0, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295862, + "ref": "T", + "alt": "C", + "GT": "0|1", + "ref_count": 7, + "alt_count": 3, + "other_count": 1, + "region": "chr21:46294723-46295998" + }, + { + "chrom": "chr21", + "pos": 46295949, + "ref": "G", + "alt": "A", + "GT": "0|1", + "ref_count": 4, + "alt_count": 1, + "other_count": 0, + "region": "chr21:46294723-46295998" + } + ], + "expected": [ + { + "region": "chr21:37166752-37167128", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "null_ll": -4.7848305371927236, + "alt_ll": -4.402304147994078, + "mu": 0.39804403318979276, + "lrt": 0.7650527783972905, + "pval": 0.38175187956066203, + "fdr_pval": 0.38175187956066203 + }, + { + "region": "chr21:46294723-46295998", + "ref_count": 33, + "alt_count": 48, + "N": 81, + "snp_count": 6, + "null_ll": -17.34507316816004, + "alt_ll": -12.824000565869431, + "mu": 0.29032495767393324, + "lrt": 9.042145204581221, + "pval": 0.0026382587926049032, + "fdr_pval": 0.0052765175852098065 + } + ], + "archived_full_data_fit": { + "python_intercept": -2.533500739653242, + "python_slope": -0.011759819173369684 + } + } + ] +} diff --git a/tests/analysis/test_bulk_edge_parity.py b/tests/analysis/test_bulk_edge_parity.py new file mode 100644 index 00000000..921585ff --- /dev/null +++ b/tests/analysis/test_bulk_edge_parity.py @@ -0,0 +1,165 @@ +"""Execute archived Python on mixed GT and interior dispersion boundaries.""" + +import contextlib +import io +import json + +import numpy as np +import pandas as pd +import pytest +from scipy.stats import betabinom +from scripts.compare_archive_analysis import bulk_counts, compare_frames, load_archive + +from analysis.run_analysis_per_donor import run_per_donor_analysis + + +@pytest.mark.rust +@pytest.mark.parametrize("gt", ["0|0", "1|1"]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("scope", [None, "per-donor", "global"]) +def test_mixed_numeric_genotypes_match_python(tmp_path, gt, pc, scope): + import wasp2_rust + + frame = bulk_counts().drop(columns="gene") + frame["GT"] = "0|1" if gt == "0|0" else "1|0" + frame.loc[frame.index[1::2], "GT"] = gt + with contextlib.redirect_stdout(io.StringIO()): + expected = load_archive()["as_analysis"].get_imbalance( + frame.copy(), phased=True, region_col="region", pseudocount=pc + ) + path = tmp_path / "counts.tsv" + if scope is None: + frame.to_csv(path, sep="\t", index=False) + output = wasp2_rust.analyze_imbalance_run( + str(path), phased=True, region_col="region", pseudocount=pc + ) + assert output["effective_phased"] is True + actual = pd.DataFrame(output["results"]) + else: + frame.assign(donor_id="A").to_csv(path, sep="\t", index=False) + paths = run_per_donor_analysis( + path, + tmp_path / "result.tsv", + unit="feature", + region_col="region", + phased=True, + pseudocount=pc, + dispersion_scope=scope, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + comparison = compare_frames(expected, actual) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.rust +@pytest.mark.parametrize("gt", ["0/1", "A|G", ".|."]) +@pytest.mark.parametrize("scope", [None, "per-donor", "global"]) +def test_malformed_mixed_genotypes_fail_like_python(tmp_path, gt, scope): + import wasp2_rust + + frame = bulk_counts().drop(columns="gene") + frame.loc[0, "GT"] = gt + with contextlib.redirect_stdout(io.StringIO()), pytest.raises((ValueError, TypeError)): + load_archive()["as_analysis"].get_imbalance(frame.copy(), phased=True, region_col="region") + path = tmp_path / "counts.tsv" + if scope is None: + frame.to_csv(path, sep="\t", index=False) + with pytest.raises(RuntimeError, match="Invalid GT"): + wasp2_rust.analyze_imbalance_run(str(path), phased=True, region_col="region") + else: + frame.assign(donor_id="A").to_csv(path, sep="\t", index=False) + with pytest.raises((ValueError, TypeError)): + run_per_donor_analysis( + path, + tmp_path / "result.tsv", + unit="feature", + region_col="region", + phased=True, + dispersion_scope=scope, + min_donor_observations=4, + ) + + +@pytest.mark.rust +@pytest.mark.parametrize("scope", ["per-donor", "global"]) +@pytest.mark.parametrize("orientation", ["0|1", "1|0"]) +@pytest.mark.parametrize("model", ["single", "linear"]) +@pytest.mark.parametrize("pc", [0, 1]) +def test_numeric_gt_duplicates_preserve_pre_dedup_phase_decision( + tmp_path, scope, orientation, model, pc +): + frame = bulk_counts().drop(columns="gene") + frame["GT"] = orientation + duplicate = frame.iloc[:1].copy() + duplicate["GT"] = "0|0" if orientation == "0|1" else "1|1" + frame = pd.concat([frame, duplicate], ignore_index=True) + from scripts.compare_archive_analysis import adapt_linear_reference + + modules = load_archive() + if model == "linear": + adapt_linear_reference(modules) + original = modules["as_analysis"].parse_opt + modules["as_analysis"].parse_opt = lambda df, disp=None, phased=False: original( + df, disp=disp, phased=True + ) + with contextlib.redirect_stdout(io.StringIO()): + expected = modules["as_analysis"].get_imbalance( + frame.copy(), + phased=True, + region_col="region", + method=model, + pseudocount=pc, + ) + path = tmp_path / "counts.tsv" + frame.assign(donor_id="A").to_csv(path, sep="\t", index=False) + paths = run_per_donor_analysis( + path, + tmp_path / "result.tsv", + unit="feature", + region_col="region", + phased=True, + model=model, + pseudocount=pc, + dispersion_scope=scope, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + comparison = compare_frames(expected, actual) + assert comparison["status"] == "PASS", comparison + fit = pd.read_csv(paths["dispersion"], sep="\t").iloc[0] + assert fit["fit_observations"] == len(frame) - 1 + qc = pd.read_csv(paths["qc"], sep="\t").iloc[0] + assert qc["input_rows"] == len(frame) + assert qc["eligible_test_rows"] == len(frame) - 1 + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["donors"]["phasing"][0]["effective_phased"] is True + + +@pytest.mark.rust +@pytest.mark.parametrize("rho", [1e-12, 1e-10, 1e-8, 1.0 - 1e-12]) +@pytest.mark.parametrize("ref_count", [0, 10, 20]) +def test_interior_dispersion_is_not_clipped(tmp_path, rho, ref_count): + import wasp2_rust + + path = tmp_path / "counts.tsv" + pd.DataFrame( + [ + { + "chrom": "chr1", + "pos": 101, + "ref": "A", + "alt": "G", + "region": "peak", + "ref_count": ref_count, + "alt_count": 20 - ref_count, + } + ] + ).to_csv(path, sep="\t", index=False) + output = wasp2_rust.analyze_imbalance_run( + str(path), pseudocount=0, region_col="region", rho=rho + ) + shape = 0.5 * (1.0 - rho) / rho + expected = betabinom.logpmf(ref_count, 20, shape, shape) + assert output["rho"] == rho + np.testing.assert_allclose(output["results"][0]["null_ll"], expected, rtol=5e-5, atol=5e-5) diff --git a/tests/analysis/test_bulk_equivalence.py b/tests/analysis/test_bulk_equivalence.py new file mode 100644 index 00000000..bae17c06 --- /dev/null +++ b/tests/analysis/test_bulk_equivalence.py @@ -0,0 +1,184 @@ +"""Verification must distinguish symmetric optima from arbitrary mu mismatches.""" + +import numpy as np +import pandas as pd +import pytest +from scripts.compare_archive_analysis import ( + compare_bulk_equivalence, + prepare_bulk_reference_counts, +) + + +def example(): + expected = pd.DataFrame( + [ + { + "region": "peak", + "ref_count": 12, + "alt_count": 20, + "N": 32, + "snp_count": 2, + "mu": 0.3, + "null_ll": -5.0, + "alt_ll": -4.0, + "lrt": 2.0, + "pval": 0.157, + "fdr_pval": 0.157, + } + ] + ) + observed = expected.copy() + observed["mu"] = 0.7 + counts = pd.DataFrame( + [ + {"chrom": "chr21", "pos": 1, "region": "peak", "ref_count": 7, "alt_count": 7}, + {"chrom": "chr21", "pos": 2, "region": "peak", "ref_count": 5, "alt_count": 13}, + ] + ) + retained = prepare_bulk_reference_counts( + counts, min_count=10, pseudocount=0, effective_phased=False, region_col="region" + ) + return expected, observed, retained + + +def verify(expected, observed, retained, **kwargs): + return compare_bulk_equivalence( + expected, + observed, + retained_counts=retained, + effective_phased=kwargs.pop("effective_phased", False), + reference_log_likelihood=kwargs.pop("objective", lambda region, mu: -4.0), + dispersion_comparison=kwargs.pop("dispersion", {"status": "PASS"}), + **kwargs, + ) + + +def test_mirrored_optimum_keeps_strict_failure_and_records_evidence(): + expected, observed, retained = example() + before = observed.copy(deep=True) + result = verify(expected, observed, retained) + assert result["strict_comparison"]["status"] == "FAIL" + assert result["strict_comparison"]["failures"] == ["mu"] + assert result["statistical_equivalence"] == "PASS" + evidence = result["regions"][0] + assert evidence["status"] == "SYMMETRIC_EQUIVALENT" + assert evidence["reference_log_likelihoods"] == [-4.0, -4.0, -4.0] + pd.testing.assert_frame_equal(observed, before) + + +@pytest.mark.parametrize( + "failure", + [ + "asymmetric", + "phased", + "not_complementary", + "nonfinite_mu", + "wrong_likelihood", + "nonfinite_objective", + "incorrect_reported_likelihood", + "counts", + "dispersion", + "missing_anchor", + "missing_mu", + "different_regions", + "wrong_observation_count", + "invalid_retained_depth", + ], +) +def test_invalid_equivalence_is_rejected(failure): + expected, observed, retained = example() + kwargs = {} + if failure == "asymmetric": + retained.loc[0, "ref_count"] += 1 + elif failure == "phased": + kwargs["effective_phased"] = True + elif failure == "not_complementary": + observed["mu"] = 0.8 + elif failure == "nonfinite_mu": + observed["mu"] = np.nan + elif failure == "wrong_likelihood": + kwargs["objective"] = lambda region, mu: -4.0 + mu + elif failure == "nonfinite_objective": + kwargs["objective"] = lambda region, mu: np.nan + elif failure == "incorrect_reported_likelihood": + kwargs["objective"] = lambda region, mu: -8.0 + elif failure == "counts": + observed["ref_count"] += 1 + elif failure == "dispersion": + kwargs["dispersion"] = {"status": "FAIL"} + elif failure == "missing_anchor": + retained = retained.drop(columns="ref_count") + elif failure == "missing_mu": + observed = observed.drop(columns="mu") + elif failure == "different_regions": + observed["region"] = "other" + elif failure == "wrong_observation_count": + retained = pd.concat([retained, retained.iloc[[1]]]) + elif failure == "invalid_retained_depth": + retained.loc[0, "N"] += 1 + assert verify(expected, observed, retained, **kwargs)["statistical_equivalence"] == "FAIL" + + +def test_matching_mu_stays_strict_without_symmetry_evidence(): + expected, _, retained = example() + retained.loc[0, "ref_count"] += 1 + result = verify(expected, expected.copy(), retained, effective_phased=True) + assert result["strict_comparison"]["status"] == "PASS" + assert result["statistical_equivalence"] == "PASS" + assert result["regions"][0]["status"] == "STRICT_MATCH" + + +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("phased", [False, True]) +def test_retained_anchor_is_selected_after_filtering_and_phase_aware_deduplication(pc, phased): + counts = pd.DataFrame( + [ + { + "chrom": "chr21", + "pos": 10, + "region": "peak", + "ref_count": 1, + "alt_count": 2, + "GT": "0|1", + }, + { + "chrom": "chr21", + "pos": 20, + "region": "peak", + "ref_count": 5, + "alt_count": 5, + "GT": "0|1", + }, + { + "chrom": "chr21", + "pos": 20, + "region": "peak", + "ref_count": 5, + "alt_count": 5, + "GT": "0|1", + }, + { + "chrom": "chr21", + "pos": 20, + "region": "peak", + "ref_count": 5, + "alt_count": 5, + "GT": "1|0", + }, + { + "chrom": "chr21", + "pos": 30, + "region": "peak", + "ref_count": 3, + "alt_count": 8, + "GT": "0|1", + }, + ] + ) + retained = prepare_bulk_reference_counts( + counts, min_count=10, pseudocount=pc, effective_phased=phased, region_col="region" + ) + assert retained.iloc[0].pos == 20 + assert retained.iloc[0].ref_count == retained.iloc[0].alt_count == 5 + pc + assert retained.pos.tolist() == ([20, 20, 30] if phased else [20, 30]) + assert "GT" in retained if phased else "GT" not in retained diff --git a/tests/analysis/test_donor_archive_parity.py b/tests/analysis/test_donor_archive_parity.py new file mode 100644 index 00000000..542f51f9 --- /dev/null +++ b/tests/analysis/test_donor_archive_parity.py @@ -0,0 +1,489 @@ +"""Donor inference must match separate archived Python invocations. + +Use two identical donor count tables so a shared dispersion fit also has the +same optimum as separate fits. References come from Python, not from Rust. +""" + +import json +from pathlib import Path + +import pandas as pd +import pytest +from scripts.compare_archive_analysis import bulk_counts, compare_frames + +from analysis.run_analysis_per_donor import run_per_donor_analysis + +REFERENCE = json.loads( + (Path(__file__).parent / "fixtures/archive_python_reference.json").read_text() +) + + +@pytest.mark.rust +@pytest.mark.parametrize("unit,phased", [("snv", False), ("feature", False), ("feature", True)]) +@pytest.mark.parametrize("dispersion_scope", ["global", "per-donor"]) +@pytest.mark.parametrize("pc", [0, 1]) +def test_donor_results_match_separate_archived_python(tmp_path, unit, phased, dispersion_scope, pc): + base = bulk_counts().drop(columns="gene") + counts = pd.concat([base.assign(donor_id="donorA"), base.assign(donor_id="donorB")]) + count_file = tmp_path / "counts.tsv" + counts.to_csv(count_file, sep="\t", index=False) + paths = run_per_donor_analysis( + count_file, + tmp_path / "results.tsv", + unit=unit, + model="single", + dispersion_scope=dispersion_scope, + region_col="region" if unit == "feature" else None, + phased=phased, + pseudocount=pc, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t") + reference_unit = "atac" if unit == "feature" else "snv" + expected = pd.DataFrame(REFERENCE["bulk"][f"{reference_unit}/{phased}/{pc}"]) + assert set(actual["donor_id"]) == {"donorA", "donorB"} + for _, donor_results in actual.groupby("donor_id"): + donor_results = donor_results.copy() + if unit == "feature": + donor_results = donor_results.rename(columns={"feature_id": "region"}) + else: + donor_results["region"] = ( + donor_results["chrom"] + "_" + donor_results["pos"].astype(str) + ) + comparison = compare_frames(expected, donor_results) + assert comparison["status"] == "PASS", comparison + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["analysis"]["pvalue_method"] == "chi_square_lrt" + + +@pytest.mark.rust +def test_donor_default_pseudocount_matches_python(tmp_path): + counts = bulk_counts().drop(columns="gene").assign(donor_id="donorA") + count_file = tmp_path / "counts.tsv" + counts.to_csv(count_file, sep="\t", index=False) + paths = run_per_donor_analysis( + count_file, + tmp_path / "default.tsv", + unit="feature", + region_col="region", + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + comparison = compare_frames(pd.DataFrame(REFERENCE["bulk"]["atac/False/1"]), actual) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.rust +@pytest.mark.parametrize("dispersion_scope", ["global", "per-donor"]) +@pytest.mark.parametrize("phased", [False, True]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("model", ["single", "linear"]) +def test_overlapping_donor_features_match_python(tmp_path, dispersion_scope, phased, pc, model): + """Preserve Python observation weights with heterogeneous donors and overlaps.""" + import contextlib + import io + + import numpy as np + from scipy.optimize import OptimizeResult, minimize, minimize_scalar + from scipy.special import expit + from scipy.stats import betabinom + from scripts.compare_archive_analysis import adapt_linear_reference, load_archive + + first = bulk_counts().drop(columns="gene") + second = first.copy() + second["ref_count"] = [11, 12, 21, 20, 17, 22, 8, 11, 19, 21, 9, 12] + second["alt_count"] = [9, 8, 19, 20, 23, 18, 12, 9, 21, 19, 11, 8] + # An identical observation in both donors must survive the global fit. + second.loc[2, ["ref_count", "alt_count"]] = first.loc[2, ["ref_count", "alt_count"]] + donors = {} + for donor, base in [("donorA", first), ("donorB", second)]: + extra = base.iloc[:2].copy() + extra["region"] = "overlap" + # Reverse order to exercise the original first-SNP anchor as well. + donors[donor] = pd.concat([base, extra], ignore_index=True).iloc[::-1].copy() + retained = pd.concat([base.assign(donor_id=donor) for donor, base in donors.items()]) + counts = pd.concat([retained, retained.iloc[:1]], ignore_index=True) + count_file = tmp_path / "counts.tsv" + counts.to_csv(count_file, sep="\t", index=False) + paths = run_per_donor_analysis( + count_file, + tmp_path / "results.tsv", + unit="feature", + region_col="region", + model=model, + dispersion_scope=dispersion_scope, + phased=phased, + pseudocount=pc, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + fits = pd.read_csv(paths["dispersion"], sep="\t").set_index("donor_id") + qc = pd.read_csv(paths["qc"], sep="\t").set_index("donor_id") + modules = load_archive() + if model == "linear": + adapt_linear_reference(modules) + archive = modules["as_analysis"] + if model == "linear" and phased: + original_parse = archive.parse_opt + archive.parse_opt = lambda df, disp=None, phased=False: original_parse( + df, disp=disp, phased=True + ) + original_optimizer = archive.minimize_scalar + + def independent_fit(frame): + ref = frame["ref_count"].to_numpy() + pc + depth = frame["ref_count"].to_numpy() + frame["alt_count"].to_numpy() + 2 * pc + if model == "linear": + return minimize( + archive.opt_linear, x0=(0, 0), method="Nelder-Mead", args=(ref, depth) + ).x + + def balanced_null(rho): + shape = 0.5 * (1 - rho) / rho + return -np.sum(betabinom.logpmf(ref, depth, shape, shape)) + + return minimize_scalar(balanced_null, method="bounded", bounds=(0, 1)).x + + shared_fit = independent_fit(retained) if dispersion_scope == "global" else None + if shared_fit is not None and model == "single": + + def shared_null_optimizer(fun, *args, **kwargs): + # Only replace the balanced-null dispersion fit. Effect optimizers + # still execute the archived Python functions separately per donor. + if len(kwargs.get("args", ())) == 2: + return OptimizeResult(x=shared_fit) + return original_optimizer(fun, *args, **kwargs) + + archive.minimize_scalar = shared_null_optimizer + elif shared_fit is not None: + # The archive calls minimize only for linear nuisance fitting. Effects + # still use its original minimize_scalar and phased/unphased objective. + archive.minimize = lambda *args, **kwargs: OptimizeResult(x=shared_fit) + for donor, base in donors.items(): + with contextlib.redirect_stdout(io.StringIO()): + expected = archive.get_imbalance( + base.copy(), + region_col="region", + phased=phased, + pseudocount=pc, + method=model, + ) + comparison = compare_frames(expected, actual.loc[actual["donor_id"] == donor]) + expected_fit = shared_fit if shared_fit is not None else independent_fit(base) + assert comparison["status"] == "PASS", json.dumps( + { + "donor": donor, + "comparison": comparison, + "python_fit": np.asarray(expected_fit).tolist(), + "rust_fit": fits.loc[donor].to_dict(), + } + ) + assert fits.loc[donor, "fit_observations"] == (28 if dispersion_scope == "global" else 14) + assert fits.loc[donor, "donor_eligible_snv_observations"] == 12 + assert qc.loc[donor, "eligible_snv_observations"] == 12 + assert qc.loc[donor, "eligible_test_rows"] == 14 + if model == "single": + np.testing.assert_allclose(fits.loc[donor, "rho"], expected_fit, rtol=5e-5, atol=5e-5) + else: + depth = base["ref_count"].to_numpy() + base["alt_count"].to_numpy() + fit = fits.loc[donor] + actual_rho = expit( + fit.linear_d1 + + ((depth - fit.linear_depth_center) / fit.linear_depth_scale) * fit.linear_d2 + ) + expected_rho = expit(expected_fit[0] + (depth + 2 * pc) * expected_fit[1]) + np.testing.assert_allclose(actual_rho, expected_rho, rtol=5e-5, atol=5e-5) + if dispersion_scope == "global": + assert fits["fit_id"].nunique() == 1 + else: + assert fits["fit_id"].nunique() == 2 + + +@pytest.mark.rust +@pytest.mark.parametrize("scope", ["global", "per-donor"]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize( + "case,reason", + [ + ("missing", "missing_gt"), + ("same_forward", "identical_gt"), + ("same_reverse", "identical_gt"), + ("alleles", "no_numeric_phased_gt"), + ("unphased", "no_numeric_phased_gt"), + ("filtered", "identical_gt"), + ("informative", None), + ], +) +def test_donor_phasing_matches_archive_fallback(tmp_path, scope, pc, case, reason): + import contextlib + import io + + from scripts.compare_archive_analysis import load_archive + + base = bulk_counts().drop(columns="gene") + if case == "missing": + base = base.drop(columns="GT") + elif case in {"same_forward", "same_reverse"}: + base["GT"] = "0|1" if case == "same_forward" else "1|0" + elif case == "alleles": + base["GT"] = base["GT"].replace({"0|1": "A|G", "1|0": "G|A"}) + elif case == "unphased": + base["GT"] = base["GT"].replace({"0|1": "0/1", "1|0": "1/0"}) + elif case == "filtered": + base.loc[base["GT"] == "1|0", ["ref_count", "alt_count"]] = 0 + file = tmp_path / "counts.tsv" + base.assign(donor_id="donorA").to_csv(file, sep="\t", index=False) + with contextlib.redirect_stdout(io.StringIO()): + expected = load_archive()["as_analysis"].get_imbalance( + base.copy(), + region_col="region", + phased=True, + pseudocount=pc, + ) + paths = run_per_donor_analysis( + file, + tmp_path / "results.tsv", + unit="feature", + region_col="region", + phased=True, + pseudocount=pc, + dispersion_scope=scope, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + comparison = compare_frames(expected, actual) + assert comparison["status"] == "PASS", comparison + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["analysis"]["phased"] is True + assert provenance["donors"]["phasing"] == [ + { + "donor_id": "donorA", + "requested_phased": True, + "effective_phased": reason is None, + "fallback_reason": reason, + } + ] + + +@pytest.mark.rust +def test_donor_phasing_decisions_are_independent(tmp_path): + base = bulk_counts().drop(columns="gene") + first = base.assign(donor_id="informative") + second = base.assign(donor_id="fallback", GT="0|1") + file = tmp_path / "counts.tsv" + pd.concat([first, second]).to_csv(file, sep="\t", index=False) + paths = run_per_donor_analysis( + file, + tmp_path / "results.tsv", + unit="feature", + region_col="region", + phased=True, + dispersion_scope="global", + min_donor_observations=4, + ) + provenance = json.loads(paths["provenance"].read_text()) + decisions = {row["donor_id"]: row for row in provenance["donors"]["phasing"]} + assert decisions["informative"]["effective_phased"] is True + assert decisions["fallback"]["effective_phased"] is False + assert decisions["fallback"]["fallback_reason"] == "identical_gt" + + +@pytest.mark.rust +@pytest.mark.parametrize("unit,phased", [("snv", False), ("feature", False), ("feature", True)]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("scope", ["global", "per-donor"]) +def test_distinct_repeated_snp_observations_match_archive(tmp_path, unit, phased, pc, scope): + import contextlib + import io + + from scripts.compare_archive_analysis import load_archive + + base = bulk_counts().drop(columns="gene") + changed = base.iloc[:1].copy() + changed["ref_count"] += 3 + opposite = base.iloc[:1].copy() + opposite["GT"] = "1|0" + parts = [base, changed, opposite, changed.copy()] + if unit == "feature": + overlapping = changed.copy() + overlapping["region"] = "overlap" + parts.append(overlapping) + base = pd.concat(parts, ignore_index=True) + if unit == "snv": + base = base.drop(columns="region") + counts = pd.concat([base.assign(donor_id=donor) for donor in ["A", "B"]]) + file = tmp_path / "counts.tsv" + counts.to_csv(file, sep="\t", index=False) + with contextlib.redirect_stdout(io.StringIO()): + expected = load_archive()["as_analysis"].get_imbalance( + base.copy(), + region_col="region" if unit == "feature" else None, + phased=phased, + pseudocount=pc, + ) + if unit == "snv": + expected = expected.rename(columns={"variant": "region"}) + paths = run_per_donor_analysis( + file, + tmp_path / "results.tsv", + unit=unit, + region_col="region" if unit == "feature" else None, + phased=phased, + pseudocount=pc, + dispersion_scope=scope, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t") + for _, observed in actual.groupby("donor_id"): + observed = observed.copy() + if unit == "feature": + observed = observed.rename(columns={"feature_id": "region"}) + else: + observed["region"] = observed["chrom"] + "_" + observed["pos"].astype(str) + comparison = compare_frames(expected, observed) + assert comparison["status"] == "PASS", comparison + + +@pytest.mark.rust +def test_fallback_duplicate_gt_is_ignored_after_phase_decision(tmp_path): + import contextlib + import io + + from scripts.compare_archive_analysis import load_archive + + base = bulk_counts().drop(columns="gene").assign(GT="A|G") + duplicate = base.iloc[:1].copy() + duplicate["GT"] = "G|A" + base = pd.concat([base, duplicate], ignore_index=True) + file = tmp_path / "counts.tsv" + base.assign(donor_id="A").to_csv(file, sep="\t", index=False) + with contextlib.redirect_stdout(io.StringIO()): + expected = load_archive()["as_analysis"].get_imbalance( + base.copy(), + region_col="region", + phased=True, + ) + paths = run_per_donor_analysis( + file, + tmp_path / "results.tsv", + unit="feature", + region_col="region", + phased=True, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + assert compare_frames(expected, actual)["status"] == "PASS" + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["donors"]["phasing"][0]["fallback_reason"] == "no_numeric_phased_gt" + assert pd.read_csv(paths["dispersion"], sep="\t")["fit_observations"].item() == 12 + + +@pytest.mark.rust +@pytest.mark.parametrize("depth_case", ["variable", "constant", "overlapping"]) +@pytest.mark.parametrize("scope", ["global", "per-donor"]) +@pytest.mark.parametrize("pc", [0, 1]) +@pytest.mark.parametrize("phased", [False, True]) +def test_linear_donor_matches_archive_fitting(tmp_path, depth_case, scope, pc, phased): + """Explicit array repairs; phased cases also repair the archive's ignored flag.""" + import contextlib + import io + + import numpy as np + from scipy.optimize import OptimizeResult + from scipy.special import expit + from scripts.compare_archive_analysis import adapt_linear_reference, load_archive + + base = bulk_counts().drop(columns="gene") + if depth_case == "constant": + base["alt_count"] = 40 - base["ref_count"] + elif depth_case == "overlapping": + extra = base.iloc[:2].copy() + extra["region"] = "overlap" + base = pd.concat([base, extra], ignore_index=True) + second = base.copy() + depths = second["ref_count"] + second["alt_count"] + second["ref_count"] = np.minimum(second["ref_count"] + 2, depths - 1) + second["alt_count"] = depths - second["ref_count"] + donors = {"A": base, "B": second} + counts = pd.concat( + [frame.assign(donor_id=donor) for donor, frame in donors.items()], ignore_index=True + ) + file = tmp_path / "counts.tsv" + counts.to_csv(file, sep="\t", index=False) + archive_modules = load_archive() + adapt_linear_reference(archive_modules) + archive = archive_modules["as_analysis"] + original_minimize = archive.minimize + if phased: + original_parse_opt = archive.parse_opt + + def forward_ignored_phase(frame, disp=None, phased=False): + # Explicitly repair only linear_model's missing phased argument. + return original_parse_opt(frame, disp=disp, phased=True) + + archive.parse_opt = forward_ignored_phase + pooled_parameters = None + if scope == "global": + pooled_parameters = original_minimize( + archive.opt_linear, + x0=(0, 0), + method="Nelder-Mead", + args=( + counts["ref_count"].to_numpy() + pc, + counts["ref_count"].to_numpy() + counts["alt_count"].to_numpy() + 2 * pc, + ), + ).x + reference_parameters = [] + + def reference_fit(*args, **kwargs): + result = ( + OptimizeResult(x=pooled_parameters) + if pooled_parameters is not None + else original_minimize(*args, **kwargs) + ) + reference_parameters.append(result.x.copy()) + return result + + archive.minimize = reference_fit + expected_frames = {} + coefficients = {} + with contextlib.redirect_stdout(io.StringIO()): + for donor, donor_counts in donors.items(): + expected_frames[donor] = archive.get_imbalance( + donor_counts.copy(), + method="linear", + region_col="region", + phased=phased, + pseudocount=pc, + ) + coefficients[donor] = reference_parameters[-1] + paths = run_per_donor_analysis( + file, + tmp_path / "results.tsv", + unit="feature", + model="linear", + region_col="region", + pseudocount=pc, + phased=phased, + dispersion_scope=scope, + min_donor_observations=4, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename(columns={"feature_id": "region"}) + fits = pd.read_csv(paths["dispersion"], sep="\t") + for _, fit in fits.iterrows(): + donor = fit["donor_id"] + donor_counts = donors[donor] + raw_depth = (donor_counts["ref_count"] + donor_counts["alt_count"]).to_numpy() + intercept, slope = coefficients[donor] + expected_rho = expit(intercept + slope * (raw_depth + 2 * pc)) + actual_rho = expit( + fit["linear_d1"] + + fit["linear_d2"] + * ((raw_depth - fit["linear_depth_center"]) / fit["linear_depth_scale"]) + ) + np.testing.assert_allclose(actual_rho, expected_rho, rtol=5e-5, atol=5e-5) + if depth_case == "constant": + assert fit["linear_depth_scale"] == 1 + for donor, observed in actual.groupby("donor_id"): + comparison = compare_frames(expected_frames[donor], observed) + assert comparison["status"] == "PASS", comparison diff --git a/tests/analysis/test_donor_local_contract.py b/tests/analysis/test_donor_local_contract.py new file mode 100644 index 00000000..3077ef90 --- /dev/null +++ b/tests/analysis/test_donor_local_contract.py @@ -0,0 +1,716 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import pytest +from scipy.stats import false_discovery_control +from typer.testing import CliRunner + +from analysis import __main__ as analysis_cli +from analysis import run_analysis +from analysis import run_analysis_per_donor as donor_analysis + + +def _count_frame(*, overlap: bool = False, exact_duplicate: bool = False) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + donor_counts = { + "donor_a": [(18, 2), (10, 10), (14, 6), (7, 13)], + "donor_b": [(16, 4), (9, 11), (13, 7), (8, 12)], + } + for donor_id, counts in donor_counts.items(): + for index, (ref_count, alt_count) in enumerate(counts, start=1): + rows.append( + { + "donor_id": donor_id, + "chrom": "chr1", + "pos": index * 100, + "ref": "A", + "alt": "G", + "GT": "0|1" if index % 2 else "1|0", + "region": f"feature_{(index + 1) // 2}", + "ref_count": ref_count, + "alt_count": alt_count, + } + ) + if overlap: + for donor_id in donor_counts: + repeated = next(row.copy() for row in rows if row["donor_id"] == donor_id) + repeated["region"] = "feature_overlap" + rows.append(repeated) + if exact_duplicate: + rows.append(rows[0].copy()) + return pd.DataFrame(rows) + + +def _write_counts( + path: Path, + *, + donor_column: str = "donor_id", + overlap: bool = False, + exact_duplicate: bool = False, +) -> pd.DataFrame: + frame = _count_frame(overlap=overlap, exact_duplicate=exact_duplicate) + if donor_column != "donor_id": + frame = frame.rename(columns={"donor_id": donor_column}) + frame.to_csv(path, sep="\t", index=False) + return frame + + +def _default_pvalues(frame: pd.DataFrame, groups: list[tuple[str, pd.DataFrame]]) -> list[float]: + del frame + return [min(0.95, 0.01 * (index + 1)) for index in range(len(groups))] + + +def _install_backend( + monkeypatch: pytest.MonkeyPatch, + *, + pvalues: Callable[[pd.DataFrame, list[tuple[str, pd.DataFrame]]], list[float]] = ( + _default_pvalues + ), + scalar_rho: float = float.fromhex("0x1.23456789abcdep-5"), + linear_d1: float = float.fromhex("-0x1.3456789abcdep+1"), + linear_d2: float = float.fromhex("-0x1.23456789abcdep-6"), +) -> dict[str, list[dict[str, Any]]]: + calls: dict[str, list[dict[str, Any]]] = {"fit": [], "analyze": []} + + def fake_fit(path: str, *, min_count: int, pseudocount: int, method: str): + frame = pd.read_csv(path, sep="\t") + donors = tuple(frame["donor_id"].drop_duplicates()) + fit_id = f"{method}:{','.join(donors)}" + payload = { + "fit_id": fit_id, + "method": method, + "rho": scalar_rho if method == "single" else None, + "linear_d1": linear_d1 if method == "linear" else None, + "linear_d2": linear_d2 if method == "linear" else None, + "n_observations": len(frame), + } + calls["fit"].append( + { + "frame": frame.copy(), + "min_count": min_count, + "pseudocount": pseudocount, + "method": method, + "payload": payload, + } + ) + return payload + + def fake_analyze( + path: str, + *, + min_count: int, + pseudocount: int, + phased: bool, + region_col: str | None, + method: str, + dispersion_fit: dict[str, Any], + ): + frame = pd.read_csv(path, sep="\t") + if region_col is None: + groups = [ + (f"{row.chrom}_{row.pos}", frame.iloc[[index]]) + for index, row in enumerate(frame.itertuples(index=False)) + ] + else: + groups = [ + (str(region), group.copy()) + for region, group in frame.groupby(region_col, sort=False) + ] + raw_pvalues = pvalues(frame, groups) + adjusted = false_discovery_control(raw_pvalues, method="bh") + results = [] + for (region, group), pval, fdr_pval in zip(groups, raw_pvalues, adjusted): + ref_count = int(group["ref_count"].sum()) + alt_count = int(group["alt_count"].sum()) + results.append( + { + "region": region, + "ref_count": ref_count, + "alt_count": alt_count, + "N": ref_count + alt_count, + "snp_count": len(group), + "null_ll": -3.0, + "alt_ll": -2.0, + "mu": 0.6, + "lrt": 2.0, + "pval": float(pval), + "fdr_pval": float(fdr_pval), + } + ) + calls["analyze"].append( + { + "frame": frame.copy(), + "min_count": min_count, + "pseudocount": pseudocount, + "phased": phased, + "region_col": region_col, + "method": method, + "dispersion_fit": dispersion_fit, + } + ) + return { + "results": results, + "n_observations": len(frame), + "requested_phased": phased, + "effective_phased": phased, + "dispersion_fit": dispersion_fit, + } + + monkeypatch.setattr(donor_analysis, "rust_fit_imbalance_dispersion", fake_fit) + monkeypatch.setattr(donor_analysis, "rust_analyze_imbalance_run", fake_analyze) + return calls + + +@pytest.mark.unit +@pytest.mark.parametrize("unit,region_col", [("snv", None), ("feature", "region")]) +@pytest.mark.parametrize("dispersion_scope", ["global", "per-donor"]) +@pytest.mark.parametrize("model", ["single", "linear"]) +def test_model_scope_unit_matrix( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + unit: str, + region_col: str | None, + dispersion_scope: str, + model: str, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts) + calls = _install_backend(monkeypatch) + + paths = donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit=unit, + model=model, + dispersion_scope=dispersion_scope, + region_col=region_col, + phased=False, + min_count=0, + pseudocount=0, + min_donor_observations=1, + ) + + assert len(calls["fit"]) == (1 if dispersion_scope == "global" else 2) + assert len(calls["analyze"]) == 2 + assert {call["method"] for call in calls["fit"]} == {model} + assert {call["method"] for call in calls["analyze"]} == {model} + assert {call["region_col"] for call in calls["analyze"]} == {region_col} + assert all("donor_id" not in call["frame"] for call in calls["analyze"]) + assert all(call["pseudocount"] == 0 for call in calls["fit"] + calls["analyze"]) + + fit_ids = [call["dispersion_fit"]["fit_id"] for call in calls["analyze"]] + assert (len(set(fit_ids)) == 1) is (dispersion_scope == "global") + results = pd.read_csv(paths["results"], sep="\t") + identity = "snv_id" if unit == "snv" else "feature_id" + assert results.groupby("donor_id")[identity].count().to_dict() == { + "donor_a": 4 if unit == "snv" else 2, + "donor_b": 4 if unit == "snv" else 2, + } + dispersion = pd.read_csv(paths["dispersion"], sep="\t") + assert dispersion["model"].eq(model).all() + assert dispersion["dispersion_scope"].eq(dispersion_scope).all() + + +@pytest.mark.unit +@pytest.mark.parametrize("model", ["single", "linear"]) +def test_global_fit_payload_is_reused_exactly( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + model: str, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts) + calls = _install_backend(monkeypatch) + + donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="snv", + model=model, + dispersion_scope="global", + min_count=0, + pseudocount=0, + min_donor_observations=1, + ) + + fitted = calls["fit"][0]["payload"] + reused = [call["dispersion_fit"] for call in calls["analyze"]] + assert reused[0] is reused[1] + assert reused[0] == fitted + assert reused[0]["fit_id"] == fitted["fit_id"] + parameter_names = ["rho"] if model == "single" else ["linear_d1", "linear_d2"] + for parameter in parameter_names: + assert float(reused[0][parameter]).hex() == float(fitted[parameter]).hex() + assert float(reused[1][parameter]).hex() == float(fitted[parameter]).hex() + + +@pytest.mark.unit +def test_bh_correction_and_likelihood_calls_are_donor_local( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts) + expected = { + "donor_a": np.array([0.001, 0.04, 0.2, 0.8]), + "donor_b": np.array([0.03, 0.031, 0.032, 0.9]), + } + + def donor_pvalues(frame: pd.DataFrame, groups: list[tuple[str, pd.DataFrame]]) -> list[float]: + donor_id = "donor_a" if int(frame.iloc[0]["ref_count"]) == 18 else "donor_b" + assert len(groups) == len(expected[donor_id]) + return expected[donor_id].tolist() + + calls = _install_backend(monkeypatch, pvalues=donor_pvalues) + paths = donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="snv", + model="single", + dispersion_scope="per-donor", + min_count=0, + pseudocount=0, + min_donor_observations=1, + ) + + assert len(calls["analyze"]) == 2 + assert all(len(call["frame"]) == 4 for call in calls["analyze"]) + results = pd.read_csv(paths["results"], sep="\t") + observed_qvalues = [] + for donor_id, raw_pvalues in expected.items(): + donor_results = results.loc[results["donor_id"] == donor_id].sort_values("snv_id") + np.testing.assert_allclose(donor_results["pval"], raw_pvalues, rtol=0, atol=0) + expected_qvalues = false_discovery_control(raw_pvalues, method="bh") + np.testing.assert_allclose(donor_results["fdr_pval"], expected_qvalues, rtol=0, atol=1e-15) + observed_qvalues.extend(donor_results["fdr_pval"].tolist()) + pooled_qvalues = false_discovery_control(np.concatenate(list(expected.values())), method="bh") + assert not np.allclose(observed_qvalues, pooled_qvalues) + + +@pytest.mark.unit +@pytest.mark.parametrize("donor_column", ["donor_id", "sample"]) +def test_canonical_and_legacy_donor_columns_normalize_to_donor_id( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + donor_column: str, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts, donor_column=donor_column) + _install_backend(monkeypatch) + + paths = donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="snv", + min_count=0, + pseudocount=0, + min_donor_observations=1, + ) + + results = pd.read_csv(paths["results"], sep="\t") + assert "donor_id" in results + assert "sample" not in results + assert sorted(results["donor_id"].unique()) == ["donor_a", "donor_b"] + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["inputs"]["qc"]["legacy_sample_normalized"] is (donor_column == "sample") + + +@pytest.mark.unit +def test_both_donor_columns_fail_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + counts = tmp_path / "counts.tsv" + frame = _write_counts(counts) + frame["sample"] = frame["donor_id"] + frame.to_csv(counts, sep="\t", index=False) + _install_backend(monkeypatch) + + with pytest.raises(ValueError, match="both donor_id and legacy sample"): + donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="snv", + min_donor_observations=1, + ) + + +@pytest.mark.unit +def test_overlapping_feature_membership_is_preserved_in_dispersion_fit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts, overlap=True, exact_duplicate=True) + calls = _install_backend(monkeypatch) + + paths = donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="feature", + model="single", + dispersion_scope="global", + region_col="region", + min_count=0, + pseudocount=0, + min_donor_observations=1, + ) + + assert len(calls["fit"]) == 1 + assert len(calls["fit"][0]["frame"]) == 10 + assert [len(call["frame"]) for call in calls["analyze"]] == [5, 5] + results = pd.read_csv(paths["results"], sep="\t") + assert results.groupby("donor_id")["feature_id"].apply(set).to_dict() == { + "donor_a": {"feature_1", "feature_2", "feature_overlap"}, + "donor_b": {"feature_1", "feature_2", "feature_overlap"}, + } + provenance = json.loads(paths["provenance"].read_text()) + input_qc = provenance["inputs"]["qc"] + assert input_qc["raw_rows"] == 11 + assert input_qc["validated_rows"] == 10 + assert input_qc["exact_duplicate_rows_removed"] == 1 + assert input_qc["unique_donor_snv_rows"] == 8 + + +@pytest.mark.unit +@pytest.mark.parametrize("conflict_column,replacement", [("alt", "T")]) +def test_conflicting_repeated_feature_rows_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + conflict_column: str, + replacement: Any, +) -> None: + counts = tmp_path / "counts.tsv" + frame = _count_frame(overlap=True) + overlap_index = frame.index[ + (frame["donor_id"] == "donor_a") & (frame["region"] == "feature_overlap") + ][0] + frame.loc[overlap_index, conflict_column] = replacement + frame.to_csv(counts, sep="\t", index=False) + calls = _install_backend(monkeypatch) + + with pytest.raises(ValueError, match="Conflicting"): + donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="feature", + region_col="region", + min_count=0, + min_donor_observations=1, + ) + assert calls == {"fit": [], "analyze": []} + + +@pytest.mark.unit +def test_phased_snv_rejected_before_input_io(monkeypatch: pytest.MonkeyPatch) -> None: + calls = _install_backend(monkeypatch) + with pytest.raises(ValueError, match="SNV analysis is unphased"): + donor_analysis.run_per_donor_analysis( + "does-not-exist.tsv", + "results.tsv", + unit="snv", + phased=True, + ) + assert calls == {"fit": [], "analyze": []} + + +@pytest.mark.unit +@pytest.mark.parametrize("bad_model", ["per_donor", "per-donor", "single-global", "mystery"]) +def test_unknown_and_legacy_pooled_models_fail_before_input_io( + monkeypatch: pytest.MonkeyPatch, + bad_model: str, +) -> None: + calls = _install_backend(monkeypatch) + with pytest.raises(ValueError, match="single.*linear"): + donor_analysis.run_per_donor_analysis( + "does-not-exist.tsv", + "results.tsv", + unit="snv", + model=bad_model, + ) + assert calls == {"fit": [], "analyze": []} + + +@pytest.mark.unit +def test_per_donor_route_defaults_to_raw_counts_without_pseudocount( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, Any] = {} + + def fake_run(count_file: str | Path, out_file: str | Path, **kwargs: Any): + observed.update({"count_file": count_file, "out_file": out_file, **kwargs}) + return {"results": Path(out_file)} + + monkeypatch.setattr(run_analysis, "run_per_donor_analysis", fake_run) + run_analysis.run_ai_analysis( + "counts.tsv", + out_file="results.tsv", + scope="per-donor", + unit="feature", + region_col="region", + model="linear", + dispersion_scope="global", + ) + + assert observed["unit"] == "feature" + assert observed["model"] == "linear" + assert observed["dispersion_scope"] == "global" + assert observed["pseudocount"] == 1 + + +@pytest.mark.unit +@pytest.mark.parametrize("unit,region_col", [("snv", None), ("feature", "region")]) +@pytest.mark.parametrize("dispersion_scope", ["global", "per-donor"]) +@pytest.mark.parametrize("model", ["single", "linear"]) +def test_run_ai_analysis_forwards_model_scope_unit_matrix( + monkeypatch: pytest.MonkeyPatch, + unit: str, + region_col: str | None, + dispersion_scope: str, + model: str, +) -> None: + observed: dict[str, Any] = {} + + def fake_run(count_file: str | Path, out_file: str | Path, **kwargs: Any): + observed.update({"count_file": count_file, "out_file": out_file, **kwargs}) + return {"results": Path(out_file)} + + monkeypatch.setattr(run_analysis, "run_per_donor_analysis", fake_run) + run_analysis.run_ai_analysis( + "counts.tsv", + out_file="results.tsv", + scope="per-donor", + unit=unit, + region_col=region_col, + model=model, + dispersion_scope=dispersion_scope, + ) + + assert observed["unit"] == unit + assert observed["region_col"] == region_col + assert observed["model"] == model + assert observed["dispersion_scope"] == dispersion_scope + assert observed["pseudocount"] == 1 + + +@pytest.mark.unit +def test_console_interface_forwards_donor_local_options( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, Any] = {} + + def fake_run(**kwargs: Any) -> None: + observed.update(kwargs) + + monkeypatch.setattr(analysis_cli, "run_ai_analysis", fake_run) + manifest_digest = "a" * 64 + result = CliRunner().invoke( + analysis_cli.app, + [ + "find-imbalance", + "count-bundle", + "--scope", + "per-donor", + "--unit", + "feature", + "--model", + "linear", + "--dispersion-scope", + "global", + "--region-col", + "region", + "--pseudocount", + "0", + "--min-donor-observations", + "12", + "--expected-manifest-sha256", + manifest_digest, + "-o", + "results.tsv", + ], + ) + + assert result.exit_code == 0, result.output + assert observed["count_file"] == "count-bundle" + assert observed["out_file"] == "results.tsv" + assert observed["scope"] == "per-donor" + assert observed["unit"] == "feature" + assert observed["model"] == "linear" + assert observed["dispersion_scope"] == "global" + assert observed["pseudocount"] == 0 + assert observed["min_donor_observations"] == 12 + assert observed["expected_manifest_sha256"] == manifest_digest + + +@pytest.mark.unit +@pytest.mark.parametrize("bad_model", ["per_donor", "per-donor", "single-global", "mystery"]) +def test_legacy_global_route_rejects_unknown_models( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + bad_model: str, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts) + monkeypatch.setattr(run_analysis, "rust_analyze_imbalance", lambda *args, **kwargs: []) + + with pytest.raises(ValueError, match="single.*linear"): + run_analysis.run_ai_analysis( + counts, + out_file=tmp_path / "results.tsv", + model=bad_model, + per_variant=True, + ) + + +@pytest.mark.unit +def test_retired_cohort_shared_surface_is_absent() -> None: + assert importlib.util.find_spec("analysis.run_analysis_cohort_shared") is None + assert not hasattr(run_analysis, "run_cohort_shared_snv_analysis") + with pytest.raises(ValueError, match="scope must be 'per-donor'"): + run_analysis.run_ai_analysis( + "does-not-exist.tsv", + scope="cohort-shared", + unit="snv", + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("unit,region_col", [("snv", None), ("feature", "region")]) +def test_sidecar_and_provenance_schema( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + unit: str, + region_col: str | None, +) -> None: + counts = tmp_path / "counts.tsv" + _write_counts(counts) + _install_backend(monkeypatch) + paths = donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit=unit, + model="linear", + dispersion_scope="global", + region_col=region_col, + min_count=0, + pseudocount=0, + min_donor_observations=1, + ) + + assert set(paths) == {"results", "dispersion", "qc", "provenance"} + assert all(path.is_file() for path in paths.values()) + result_prefix = ( + ["donor_id", "snv_id", "chrom", "pos", "ref", "alt"] + if unit == "snv" + else ["donor_id", "feature_id"] + ) + assert pd.read_csv(paths["results"], sep="\t").columns.tolist() == [ + *result_prefix, + "ref_count", + "alt_count", + "N", + "snp_count", + "null_ll", + "alt_ll", + "mu", + "lrt", + "pval", + "fdr_pval", + "significant_q05", + "significant_q10", + ] + assert pd.read_csv(paths["dispersion"], sep="\t").columns.tolist() == [ + "donor_id", + "dispersion_scope", + "fit_id", + "model", + "rho", + "linear_d1", + "linear_d2", + "fit_observations", + "donor_eligible_snv_observations", + "result_rows", + ] + assert pd.read_csv(paths["qc"], sep="\t").columns.tolist() == [ + "donor_id", + "input_rows", + "unique_snv_rows", + "eligible_snv_observations", + "eligible_test_rows", + "analyzed", + "status", + ] + + provenance = json.loads(paths["provenance"].read_text()) + assert set(provenance) == { + "schema_version", + "created_at_utc", + "software", + "analysis", + "parameters", + "inputs", + "donors", + "outputs", + } + assert provenance["schema_version"] == donor_analysis.SCHEMA_VERSION + datetime.fromisoformat(provenance["created_at_utc"]) + assert provenance["analysis"] == { + "scope": "per-donor", + "unit": unit, + "model": "linear", + "dispersion_scope": "global", + "region_col": region_col, + "phased": False, + "multiple_testing": "Benjamini-Hochberg independently within each donor", + "pvalue_method": "chi_square_lrt", + } + assert provenance["parameters"] == { + "min_count": 0, + "pseudocount": 0, + "min_donor_observations": 1, + } + assert provenance["donors"] == { + "total": 2, + "included": 2, + "excluded": 0, + "phasing": [ + { + "donor_id": donor, + "requested_phased": False, + "effective_phased": False, + "fallback_reason": "not_requested", + } + for donor in ["donor_a", "donor_b"] + ], + } + assert set(provenance["outputs"]) == {"results", "dispersion", "qc"} + for name, record in provenance["outputs"].items(): + assert set(record) == {"path", "sha256", "size_bytes", "mtime_utc"} + assert record["path"] == paths[name].name + assert record["sha256"] == hashlib.sha256(paths[name].read_bytes()).hexdigest() + assert record["size_bytes"] == paths[name].stat().st_size + datetime.fromisoformat(record["mtime_utc"]) + + +@pytest.mark.unit +def test_overlapping_features_do_not_inflate_donor_eligibility(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + _write_counts(counts, overlap=True) + calls = _install_backend(monkeypatch) + # Each donor has five SNP-feature rows but only four unique SNPs. + with pytest.raises(RuntimeError, match="No donors met"): + donor_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="feature", + region_col="region", + min_count=0, + min_donor_observations=5, + ) + assert calls["fit"] == [] diff --git a/tests/analysis/test_donor_local_statistics.py b/tests/analysis/test_donor_local_statistics.py new file mode 100644 index 00000000..ec490cbb --- /dev/null +++ b/tests/analysis/test_donor_local_statistics.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import pytest +from scipy.optimize import minimize_scalar +from scipy.special import expit, logsumexp +from scipy.stats import betabinom, chi2, false_discovery_control + +from analysis import run_analysis_per_donor as donor_analysis + +RHO_EPSILON = 1e-10 + + +def _statistical_counts() -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + counts = { + "donor_a": [(18, 2), (22, 8), (32, 8), (14, 6), (24, 26), (11, 9)], + "donor_b": [(16, 4), (18, 12), (29, 11), (8, 12), (27, 23), (9, 11)], + } + for donor_id, donor_counts in counts.items(): + for index, (ref_count, alt_count) in enumerate(donor_counts, start=1): + rows.append( + { + "donor_id": donor_id, + "chrom": "chr1", + "pos": index * 100, + "ref": "A", + "alt": "G", + "GT": "0|1" if index % 2 else "1|0", + "region": f"feature_{(index + 1) // 2}", + "ref_count": ref_count, + "alt_count": alt_count, + } + ) + return pd.DataFrame(rows) + + +@pytest.fixture +def rust_backend(monkeypatch: pytest.MonkeyPatch): + wasp2_rust = pytest.importorskip("wasp2_rust") + required = ["fit_imbalance_dispersion", "analyze_imbalance_run", "analyze_imbalance"] + if not all(hasattr(wasp2_rust, name) for name in required): + pytest.skip("worktree Rust extension needs rebuilding") + monkeypatch.setattr( + donor_analysis, + "rust_fit_imbalance_dispersion", + wasp2_rust.fit_imbalance_dispersion, + ) + monkeypatch.setattr( + donor_analysis, + "rust_analyze_imbalance_run", + wasp2_rust.analyze_imbalance_run, + ) + return wasp2_rust + + +def _rho_at_depth(dispersion: pd.Series, total: int) -> float: + if dispersion["model"] == "single": + rho = float(dispersion["rho"]) + else: + center = float(dispersion["linear_depth_center"]) + scale = float(dispersion["linear_depth_scale"]) + assert np.isfinite(center) + assert np.isfinite(scale) and scale > 0 + standardized_depth = (total - center) / scale + linear = float(dispersion["linear_d1"]) + standardized_depth * float( + dispersion["linear_d2"] + ) + rho = float(expit(linear)) + return float(np.clip(rho, RHO_EPSILON, 1.0 - RHO_EPSILON)) + + +def _logpmf(probability: float, rho: float, ref_count: int, total: int) -> float: + if probability <= 0.0 or probability >= 1.0: + return -np.inf + alpha = probability * (1.0 - rho) / rho + beta = (1.0 - probability) * (1.0 - rho) / rho + return float(betabinom.logpmf(ref_count, total, alpha, beta)) + + +def _single_snv_oracle(row: pd.Series, dispersion: pd.Series) -> dict[str, float]: + ref_count = int(row["ref_count"]) + alt_count = int(row["alt_count"]) + total = ref_count + alt_count + rho = _rho_at_depth(dispersion, total) + null_ll = _logpmf(0.5, rho, ref_count, total) + fit = minimize_scalar( + lambda probability: -_logpmf(probability, rho, ref_count, total), + bounds=(0.0, 1.0), + method="bounded", + options={"xatol": 1e-10}, + ) + assert fit.success + alt_ll = -float(fit.fun) + lrt = max(0.0, 2.0 * (alt_ll - null_ll)) + return { + "null_ll": null_ll, + "alt_ll": alt_ll, + "mu": float(fit.x), + "lrt": lrt, + "pval": float(chi2.sf(lrt, 1)), + } + + +def _unphased_feature_oracle(frame: pd.DataFrame, dispersion: pd.Series) -> dict[str, float]: + observations = [] + for row in frame.itertuples(): + total = int(row.ref_count + row.alt_count) + observations.append((int(row.ref_count), total, _rho_at_depth(dispersion, total))) + null_ll = sum(_logpmf(0.5, rho, ref_count, total) for ref_count, total, rho in observations) + + def negative_log_likelihood(probability: float) -> float: + first_ref, first_total, first_rho = observations[0] + log_likelihood = _logpmf(probability, first_rho, first_ref, first_total) + for ref_count, total, rho in observations[1:]: + orientations = [ + _logpmf(probability, rho, ref_count, total), + _logpmf(1.0 - probability, rho, ref_count, total), + ] + log_likelihood += float(logsumexp(orientations) - np.log(2.0)) + return -log_likelihood + + fit = minimize_scalar( + negative_log_likelihood, + bounds=(0.0, 1.0), + method="bounded", + options={"xatol": 1e-10}, + ) + assert fit.success + alt_ll = -float(fit.fun) + lrt = max(0.0, 2.0 * (alt_ll - null_ll)) + return { + "null_ll": null_ll, + "alt_ll": alt_ll, + "mu": float(fit.x), + "lrt": lrt, + "pval": float(chi2.sf(lrt, 1)), + } + + +def _oracle_results( + counts: pd.DataFrame, + dispersion: pd.DataFrame, + *, + unit: str, +) -> pd.DataFrame: + rows = [] + for donor_id, donor_counts in counts.groupby("donor_id", sort=True): + donor_dispersion = dispersion.loc[dispersion["donor_id"] == donor_id].iloc[0] + if unit == "snv": + for row in donor_counts.sort_values(["chrom", "pos"]).to_dict("records"): + series = pd.Series(row) + rows.append( + { + "donor_id": donor_id, + "identity": (f"{row['chrom']}:{row['pos']}:{row['ref']}:{row['alt']}"), + **_single_snv_oracle(series, donor_dispersion), + } + ) + else: + for feature_id, feature in donor_counts.groupby("region", sort=True): + rows.append( + { + "donor_id": donor_id, + "identity": feature_id, + **_unphased_feature_oracle(feature, donor_dispersion), + } + ) + oracle = pd.DataFrame(rows) + oracle["fdr_pval"] = oracle.groupby("donor_id", sort=False)["pval"].transform( + lambda values: false_discovery_control(values.to_numpy(), method="bh") + ) + return oracle.sort_values(["donor_id", "identity"]).reset_index(drop=True) + + +@pytest.mark.rust +@pytest.mark.parametrize("unit,region_col", [("snv", None), ("feature", "region")]) +@pytest.mark.parametrize("dispersion_scope", ["global", "per-donor"]) +@pytest.mark.parametrize("model", ["single", "linear"]) +def test_model_scope_unit_matrix_matches_independent_scipy_oracle( + tmp_path: Path, + rust_backend: Any, + unit: str, + region_col: str | None, + dispersion_scope: str, + model: str, +) -> None: + del rust_backend + counts = _statistical_counts() + count_path = tmp_path / "counts.tsv" + counts.to_csv(count_path, sep="\t", index=False) + paths = donor_analysis.run_per_donor_analysis( + count_path, + tmp_path / f"{unit}-{model}-{dispersion_scope}.tsv", + unit=unit, + model=model, + dispersion_scope=dispersion_scope, + region_col=region_col, + phased=False, + min_count=0, + pseudocount=0, + min_donor_observations=4, + ) + + observed = pd.read_csv(paths["results"], sep="\t") + identity_column = "snv_id" if unit == "snv" else "feature_id" + observed = observed.rename(columns={identity_column: "identity"}).sort_values( + ["donor_id", "identity"] + ) + observed = observed.reset_index(drop=True) + dispersion = pd.read_csv(paths["dispersion"], sep="\t") + oracle = _oracle_results(counts, dispersion, unit=unit) + + assert observed[["donor_id", "identity"]].equals(oracle[["donor_id", "identity"]]) + for column in ["null_ll", "alt_ll", "mu", "lrt", "pval", "fdr_pval"]: + np.testing.assert_allclose( + observed[column], + oracle[column], + rtol=3e-5, + atol=3e-5, + err_msg=f"{model}/{dispersion_scope}/{unit}: {column}", + ) + assert observed["ref_count"].sum() == counts["ref_count"].sum() + assert observed["alt_count"].sum() == counts["alt_count"].sum() + if dispersion_scope == "global": + assert dispersion["fit_id"].nunique() == 1 + parameters = ( + ["rho"] + if model == "single" + else [ + "linear_d1", + "linear_d2", + "linear_depth_center", + "linear_depth_scale", + ] + ) + assert all(dispersion[column].nunique() == 1 for column in parameters) + else: + assert dispersion["fit_id"].nunique() == 2 + + +def _run_legacy( + rust_backend: Any, + path: Path, + *, + method: str, + phased: bool, + region_col: str | None, +) -> pd.DataFrame: + results = rust_backend.analyze_imbalance( + str(path), + min_count=0, + pseudocount=0, + method=method, + phased=phased, + region_col=region_col, + ) + return pd.DataFrame(results).sort_values("region").reset_index(drop=True) + + +@pytest.mark.rust +def test_unphased_feature_keeps_archived_first_snp_orientation(tmp_path, rust_backend): + counts = _statistical_counts().loc[lambda f: f["donor_id"] == "donor_a"].copy() + counts = counts.drop(columns="donor_id") + dispersion = pd.Series({"model": "single", "rho": 0.08}) + for frame in [counts, counts.iloc[::-1]]: + path = tmp_path / "counts.tsv" + frame.to_csv(path, sep="\t", index=False) + run = rust_backend.analyze_imbalance_run( + str(path), min_count=0, pseudocount=0, method="single", rho=0.08, region_col="region" + ) + assert run["pvalue_method"] == "chi_square_lrt" + assert run["exact_snv"] is False + actual = pd.DataFrame(run["results"]).set_index("region") + for region, group in frame.groupby("region", sort=False): + expected = _unphased_feature_oracle(group, dispersion) + for column, value in expected.items(): + np.testing.assert_allclose(actual.loc[region, column], value, rtol=3e-5, atol=3e-5) + + +@pytest.mark.rust +@pytest.mark.parametrize("method", ["single", "linear"]) +def test_allele_swap_preserves_tests_and_mirrors_single_snp_mu( + tmp_path: Path, + rust_backend: Any, + method: str, +) -> None: + counts = ( + _statistical_counts() + .loc[lambda frame: frame["donor_id"] == "donor_a"] + .drop(columns=["donor_id", "region"]) + ) + original_path = tmp_path / "original.tsv" + swapped_path = tmp_path / "swapped.tsv" + counts.to_csv(original_path, sep="\t", index=False) + swapped = counts.copy() + swapped[["ref", "alt"]] = swapped[["alt", "ref"]].to_numpy() + swapped[["ref_count", "alt_count"]] = swapped[["alt_count", "ref_count"]].to_numpy() + swapped["GT"] = swapped["GT"].map({"0|1": "1|0", "1|0": "0|1"}) + swapped.to_csv(swapped_path, sep="\t", index=False) + + original = _run_legacy( + rust_backend, original_path, method=method, phased=False, region_col=None + ) + swapped_results = _run_legacy( + rust_backend, swapped_path, method=method, phased=False, region_col=None + ) + for column in ["null_ll", "alt_ll", "lrt", "pval", "fdr_pval"]: + np.testing.assert_allclose(original[column], swapped_results[column], rtol=0, atol=2e-10) + np.testing.assert_allclose( + original["mu"] + swapped_results["mu"], np.ones(len(original)), rtol=0, atol=2e-5 + ) + + +@pytest.mark.rust +@pytest.mark.parametrize("method", ["single", "linear"]) +def test_single_snp_results_are_phase_invariant( + tmp_path: Path, + rust_backend: Any, + method: str, +) -> None: + counts = ( + _statistical_counts() + .loc[lambda frame: frame["donor_id"] == "donor_a"] + .drop(columns=["donor_id", "region"]) + ) + path = tmp_path / "counts.tsv" + counts.to_csv(path, sep="\t", index=False) + unphased = _run_legacy(rust_backend, path, method=method, phased=False, region_col=None) + phased = _run_legacy(rust_backend, path, method=method, phased=True, region_col=None) + + assert unphased[["region", "ref_count", "alt_count", "N", "snp_count"]].equals( + phased[["region", "ref_count", "alt_count", "N", "snp_count"]] + ) + for column in ["null_ll", "alt_ll", "mu", "lrt", "pval", "fdr_pval"]: + np.testing.assert_allclose(unphased[column], phased[column], rtol=0, atol=1e-12) + + +@pytest.mark.rust +@pytest.mark.parametrize("method", ["per_donor", "per-donor", "single-global", "mystery"]) +def test_rust_rejects_unknown_and_retired_pooled_methods( + tmp_path: Path, + rust_backend: Any, + method: str, +) -> None: + path = tmp_path / "counts.tsv" + _statistical_counts().drop(columns="donor_id").to_csv(path, sep="\t", index=False) + with pytest.raises(RuntimeError, match="removed|Unknown method"): + rust_backend.analyze_imbalance(str(path), method=method) + + +@pytest.mark.rust +def test_rust_has_no_cohort_shared_entry_point(rust_backend: Any) -> None: + assert not hasattr(rust_backend, "analyze_cohort_snvs") diff --git a/tests/analysis/test_per_donor_peak.py b/tests/analysis/test_per_donor_peak.py new file mode 100644 index 00000000..ae5ba907 --- /dev/null +++ b/tests/analysis/test_per_donor_peak.py @@ -0,0 +1,133 @@ +import json +from pathlib import Path + +import pandas as pd +import pytest + +from analysis import run_analysis_per_donor as peak_analysis + + +def _write_peak_counts(path: Path, *, bad_gt: bool = False) -> None: + rows = [] + for donor in ["donor_a", "donor_b", "donor_low"]: + count = 50 if donor != "donor_low" else 2 + for index in range(count): + ref, alt = ("A", "G") + gt = "0|1" if index % 2 == 0 else "1|0" + if bad_gt and donor == "donor_a" and index == 0: + gt = "A/G" + rows.append( + { + "chrom": "chr1", + "pos": index + 1, + "ref": ref, + "alt": alt, + "GT": gt, + "capeak_id": f"peak_{index // 2}", + "ref_count": 8, + "alt_count": 4, + "sample": donor, + "is_caqtl": index // 2 % 2 == 0, + } + ) + pd.DataFrame(rows).to_csv(path, sep="\t", index=False) + + +@pytest.mark.unit +def test_per_donor_peak_analysis_preserves_numeric_phase_and_isolates_models(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + _write_peak_counts(counts) + calls = [] + + def fake_fit(path, **kwargs): + frame = pd.read_csv(path, sep="\t") + donor = ",".join(frame["donor_id"].drop_duplicates()) + return { + "fit_id": f"linear:{donor}", + "method": "linear", + "rho": None, + "linear_d1": -3.0, + "linear_d2": -0.04, + "n_observations": len(frame), + } + + def fake_rust(path, *, dispersion_fit, **kwargs): + frame = pd.read_csv(path, sep="\t") + assert set(frame["GT"]) == {"0|1", "1|0"} + assert kwargs["region_col"] == "capeak_id" + assert kwargs["method"] == "linear" + calls.append((len(frame), kwargs["phased"])) + results = [] + for region, group in frame.groupby("capeak_id", sort=True): + ref_count = int(group["ref_count"].sum()) + alt_count = int(group["alt_count"].sum()) + results.append( + { + "region": region, + "ref_count": ref_count, + "alt_count": alt_count, + "N": ref_count + alt_count, + "snp_count": len(group), + "null_ll": -3.0, + "alt_ll": -2.0, + "mu": 0.6, + "lrt": 2.0, + "pval": 0.02, + "fdr_pval": 0.04, + } + ) + return { + "results": results, + "n_observations": len(frame), + "requested_phased": True, + "effective_phased": True, + "dispersion_fit": dispersion_fit, + } + + monkeypatch.setattr(peak_analysis, "rust_fit_imbalance_dispersion", fake_fit) + monkeypatch.setattr(peak_analysis, "rust_analyze_imbalance_run", fake_rust) + paths = peak_analysis.run_per_donor_analysis( + counts, + tmp_path / "peak_linear_phased.tsv", + unit="feature", + region_col="capeak_id", + model="linear", + phased=True, + ) + + assert calls == [(50, True), (50, True)] + results = pd.read_csv(paths["results"], sep="\t") + assert results.groupby("donor_id").size().to_dict() == {"donor_a": 25, "donor_b": 25} + assert results["significant_q05"].all() + dispersion = pd.read_csv(paths["dispersion"], sep="\t") + assert dispersion["rho"].isna().all() + assert dispersion["linear_d1"].eq(-3.0).all() + qc = pd.read_csv(paths["qc"], sep="\t").set_index("donor_id") + assert qc.loc["donor_low", "status"] == "excluded_min_observations" + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["analysis"]["scope"] == "per-donor" + assert provenance["analysis"]["phased"] is True + + +@pytest.mark.unit +def test_per_donor_peak_analysis_rejects_nonphased_gt(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + _write_peak_counts(counts, bad_gt=True) + frame = pd.read_csv(counts, sep="\t") + with pytest.raises(ValueError, match="invalid literal"): + peak_analysis._phase_fallback_reason(frame.loc[frame["sample"] == "donor_a"], True) + + +@pytest.mark.unit +def test_per_donor_peak_analysis_requires_explicit_region(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + _write_peak_counts(counts) + monkeypatch.setattr(peak_analysis, "rust_fit_imbalance_dispersion", lambda *a, **k: None) + monkeypatch.setattr(peak_analysis, "rust_analyze_imbalance_run", lambda *a, **k: None) + with pytest.raises(ValueError, match="requires --region-col"): + peak_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="feature", + region_col="", + ) diff --git a/tests/analysis/test_per_donor_snv.py b/tests/analysis/test_per_donor_snv.py new file mode 100644 index 00000000..cd35b5bb --- /dev/null +++ b/tests/analysis/test_per_donor_snv.py @@ -0,0 +1,153 @@ +import json +from pathlib import Path + +import pandas as pd +import pytest + +from analysis import run_analysis_per_donor as cohort_analysis +from analysis.run_analysis import run_ai_analysis + + +def _write_counts(path: Path) -> None: + rows = [] + for sample, count in [("donor_a", 50), ("donor_b", 51), ("donor_low", 2)]: + for index in range(count): + rows.append( + { + "chrom": "chr1", + "pos": index + 1, + "ref": "A", + "alt": "G", + "ref_count": 8, + "alt_count": 4, + "sample": sample, + "snv_id": f"chr1:{index + 1}:A:G", + "capeak_id": f"peak_{index // 2}", + "is_caqtl": index % 2 == 0, + } + ) + pd.DataFrame(rows).to_csv(path, sep="\t", index=False) + + +@pytest.mark.unit +def test_per_donor_analysis_isolates_rho_and_bh(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + _write_counts(counts) + calls = [] + + def fake_fit(path, **kwargs): + frame = pd.read_csv(path, sep="\t") + donor = ",".join(frame["donor_id"].drop_duplicates()) + return { + "fit_id": f"single:{donor}", + "method": "single", + "rho": len(frame) / 1000, + "linear_d1": None, + "linear_d2": None, + "n_observations": len(frame), + } + + def fake_rust(path, *, dispersion_fit, **kwargs): + frame = pd.read_csv(path, sep="\t") + assert "sample" not in frame.columns + assert "donor_id" not in frame.columns + calls.append(len(frame)) + results = [] + for row in frame.itertuples(index=False): + results.append( + { + "region": f"{row.chrom}_{row.pos}", + "ref_count": row.ref_count, + "alt_count": row.alt_count, + "N": row.ref_count + row.alt_count, + "snp_count": 1, + "null_ll": -2.0, + "alt_ll": -1.0, + "mu": 0.6, + "lrt": 2.0, + "pval": 0.02, + "fdr_pval": 0.04, + } + ) + return { + "results": results, + "n_observations": len(frame), + "requested_phased": False, + "effective_phased": False, + "dispersion_fit": dispersion_fit, + } + + monkeypatch.setattr(cohort_analysis, "rust_fit_imbalance_dispersion", fake_fit) + monkeypatch.setattr(cohort_analysis, "rust_analyze_imbalance_run", fake_rust) + output = tmp_path / "results.tsv" + paths = cohort_analysis.run_per_donor_analysis( + counts, + output, + unit="snv", + min_donor_observations=50, + ) + + assert sorted(calls) == [50, 51] + results = pd.read_csv(paths["results"], sep="\t") + assert results.groupby("donor_id").size().to_dict() == {"donor_a": 50, "donor_b": 51} + assert results["significant_q05"].all() + qc = pd.read_csv(paths["qc"], sep="\t").set_index("donor_id") + assert qc.loc["donor_low", "status"] == "excluded_min_observations" + assert not bool(qc.loc["donor_low", "analyzed"]) + dispersion = pd.read_csv(paths["dispersion"], sep="\t") + assert dispersion["donor_id"].tolist() == ["donor_a", "donor_b"] + provenance = json.loads(paths["provenance"].read_text()) + assert provenance["analysis"]["scope"] == "per-donor" + assert provenance["analysis"]["multiple_testing"].endswith("within each donor") + assert provenance["donors"] == { + "excluded": 1, + "included": 2, + "total": 3, + "phasing": [ + { + "donor_id": donor, + "requested_phased": False, + "effective_phased": False, + "fallback_reason": "not_requested", + } + for donor in ["donor_a", "donor_b"] + ], + } + + +@pytest.mark.unit +def test_per_donor_analysis_rejects_conflicting_site_rows(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + pd.DataFrame( + [ + ["chr1", 1, "A", "G", 8, 4, "donor_a"], + ["chr1", 1, "A", "T", 8, 4, "donor_a"], + ], + columns=["chrom", "pos", "ref", "alt", "ref_count", "alt_count", "sample"], + ).to_csv(counts, sep="\t", index=False) + monkeypatch.setattr(cohort_analysis, "rust_fit_imbalance_dispersion", lambda *a, **k: None) + monkeypatch.setattr(cohort_analysis, "rust_analyze_imbalance_run", lambda *a, **k: None) + with pytest.raises(ValueError, match="Conflicting alleles"): + cohort_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="snv", + min_donor_observations=1, + ) + + +@pytest.mark.unit +def test_per_donor_analysis_enforces_manifest_hash_and_contract(tmp_path, monkeypatch): + counts = tmp_path / "counts.tsv" + _write_counts(counts) + monkeypatch.setattr(cohort_analysis, "rust_fit_imbalance_dispersion", lambda *a, **k: None) + monkeypatch.setattr(cohort_analysis, "rust_analyze_imbalance_run", lambda *a, **k: None) + with pytest.raises(ValueError, match="requires a locked count bundle"): + cohort_analysis.run_per_donor_analysis( + counts, + tmp_path / "results.tsv", + unit="snv", + expected_manifest_sha256="0" * 64, + ) + with pytest.raises(ValueError, match="unphased"): + run_ai_analysis(counts, scope="per-donor", unit="snv", phased=True) diff --git a/tests/analysis/test_public_atac_matrix.py b/tests/analysis/test_public_atac_matrix.py new file mode 100644 index 00000000..62d0e597 --- /dev/null +++ b/tests/analysis/test_public_atac_matrix.py @@ -0,0 +1,142 @@ +"""Repeat the full 32-run real ATAC matrix against executed archived Python. + +Count tables are frozen measurements from public reads, not Rust-generated +expectations. Linux Python 3.11 CI pins the original numerical environment. +""" + +import contextlib +import gzip +import hashlib +import io +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import scipy +from scipy.special import expit +from scripts.compare_archive_analysis import adapt_linear_reference, compare_frames, load_archive + +from analysis.run_analysis_per_donor import run_per_donor_analysis + +FIXTURES = Path(__file__).parent / "fixtures/public_atac" +METADATA = json.loads((FIXTURES / "metadata.json").read_text()) + + +@pytest.mark.rust +@pytest.mark.skipif( + (np.__version__, scipy.__version__) != ("1.26.4", "1.13.1"), + reason="Full historical matrix uses the pinned NumPy/SciPy environment (Python 3.11 CI)", +) +@pytest.mark.parametrize("unit", ["snv", "peak"]) +@pytest.mark.parametrize("model", ["single", "linear"]) +@pytest.mark.parametrize("phased", [False, True]) +@pytest.mark.parametrize("pc", [0, 1]) +def test_full_public_atac_matrix(tmp_path, unit, model, phased, pc): + import wasp2_rust + + source = METADATA["counts"][unit] + raw = gzip.decompress((FIXTURES / source["file"]).read_bytes()) + assert hashlib.sha256(raw).hexdigest() == source["sha256"] + frame = pd.read_csv(io.BytesIO(raw), sep="\t") + assert len(frame) == source["rows"] + if phased: + frame = frame.loc[frame["GT"].isin(["0|1", "1|0"])].copy() + region = "region" if unit == "peak" else None + modules = load_archive(METADATA["archive_commit"]) + if model == "linear": + adapt_linear_reference(modules) + archive = modules["as_analysis"] + fitted = [] + optimizer_name = "minimize_scalar" if model == "single" else "minimize" + original_optimizer = getattr(archive, optimizer_name) + + def capture_fit(*args, **kwargs): + result = original_optimizer(*args, **kwargs) + if not fitted: + fitted.append(result.x) + return result + + setattr(archive, optimizer_name, capture_fit) + if model == "linear" and phased: + # Explicit reference repair: archived linear_model ignores this flag. + original_parse = archive.parse_opt + archive.parse_opt = lambda df, disp=None, phased=False: original_parse( + df, disp=disp, phased=True + ) + with contextlib.redirect_stdout(io.StringIO()): + expected = archive.get_imbalance( + frame.copy(), method=model, phased=phased, region_col=region, pseudocount=pc + ) + expected = expected.rename(columns={"variant": "region"}) + path = tmp_path / "counts.tsv" + frame.to_csv(path, sep="\t", index=False) + output = wasp2_rust.analyze_imbalance_run( + str(path), method=model, phased=phased, region_col=region, pseudocount=pc + ) + comparison = compare_frames(expected, pd.DataFrame(output["results"])) + assert comparison["status"] == "PASS", json.dumps( + { + "comparison": comparison, + "python_fit": np.asarray(fitted[0]).tolist(), + "rust_fit": {key: output[key] for key in ["rho", "linear_d1", "linear_d2"]}, + } + ) + + def check_dispersion(fit): + if model == "single": + np.testing.assert_allclose(fit["rho"], fitted[0], rtol=5e-5, atol=5e-5) + else: + depth = (frame["ref_count"] + frame["alt_count"]).to_numpy() + depth = depth[depth >= 10] + actual = expit( + fit["linear_d1"] + + ((depth - fit["linear_depth_center"]) / fit["linear_depth_scale"]) + * fit["linear_d2"] + ) + expected_rho = expit(fitted[0][0] + (depth + 2 * pc) * fitted[0][1]) + np.testing.assert_allclose(actual, expected_rho, rtol=5e-5, atol=5e-5) + + check_dispersion(output) + cli_path = tmp_path / "cli.tsv" + command = [ + sys.executable, + "-c", + "from analysis.__main__ import app; app()", + "find-imbalance", + str(path), + "--model", + model, + "--pseudocount", + str(pc), + "--out_file", + str(cli_path), + ] + command += ["--region_col", "region"] if region else ["--per-variant"] + if phased: + command.append("--phased") + subprocess.run(command, check=True, capture_output=True, text=True) + comparison = compare_frames(expected, pd.read_csv(cli_path, sep="\t")) + assert comparison["status"] == "PASS", ("CLI", comparison) + if unit == "peak": + frame.assign(donor_id="HG001").to_csv(path, sep="\t", index=False) + for scope in ["global", "per-donor"]: + paths = run_per_donor_analysis( + path, + tmp_path / f"{scope}.tsv", + unit="feature", + model=model, + region_col="region", + phased=phased, + pseudocount=pc, + dispersion_scope=scope, + ) + actual = pd.read_csv(paths["results"], sep="\t").rename( + columns={"feature_id": "region"} + ) + comparison = compare_frames(expected, actual) + assert comparison["status"] == "PASS", (scope, comparison) + check_dispersion(pd.read_csv(paths["dispersion"], sep="\t").iloc[0]) diff --git a/tests/analysis/test_real_atac_optimizer.py b/tests/analysis/test_real_atac_optimizer.py new file mode 100644 index 00000000..b684f99e --- /dev/null +++ b/tests/analysis/test_real_atac_optimizer.py @@ -0,0 +1,92 @@ +"""Conditional bulk inference against archived Python on real ATAC observations. + +The independently frozen Python fixture fixes the nuisance dispersion to isolate +allelic-proportion optimization. Full-data comparisons separately retain strict +mu checks: this test does not certify orientation for symmetric unphased peaks. +""" + +import json +from pathlib import Path + +import pandas as pd +import pytest +from scipy.special import expit +from scripts.compare_archive_analysis import ( + adapt_linear_reference, + compare_bulk_equivalence, + load_archive, + prepare_bulk_reference_counts, +) + +REFERENCE = json.loads( + (Path(__file__).parent / "fixtures/real_atac_optimizer_reference.json").read_text() +) + + +@pytest.mark.rust +@pytest.mark.parametrize("case", REFERENCE["cases"], ids=lambda case: case["name"]) +def test_real_atac_conditional_optimizer_matches_python(tmp_path, case): + import wasp2_rust + + counts = pd.DataFrame(case["counts"]) + count_file = tmp_path / "counts.tsv" + counts.to_csv(count_file, sep="\t", index=False) + output = wasp2_rust.analyze_imbalance_run( + str(count_file), + method=case["method"], + phased=case["phased"], + pseudocount=case["pseudocount"], + region_col="region", + **case["fixed_dispersion"], + ) + actual = pd.DataFrame(output["results"]) + expected = pd.DataFrame(case["expected"]) + retained = prepare_bulk_reference_counts( + counts, + min_count=10, + pseudocount=case["pseudocount"], + effective_phased=output["effective_phased"], + region_col="region", + ) + modules = load_archive() + if case["method"] == "linear": + adapt_linear_reference(modules) + archive = modules["as_analysis"] + + def reference_likelihood(region, mu): + rows = retained.loc[retained.region == region] + ref = rows.ref_count.to_numpy() + depth = rows.N.to_numpy() + fixed = case["fixed_dispersion"] + rho = ( + fixed["rho"] + if case["method"] == "single" + else expit( + fixed["linear_d1"] + + fixed["linear_d2"] + * ( + (depth - 2 * case["pseudocount"] - fixed["linear_depth_center"]) + / fixed["linear_depth_scale"] + ) + ) + ) + if len(rows) == 1: + value = archive.opt_prob(mu, rho, ref[0], depth[0]) + else: + value = archive.opt_unphased_dp(mu, rho, ref[:1], depth[:1], ref[1:], depth[1:]) + return -float(value) + + # Fixed nuisance parameters are identical by construction in this conditional + # test; the full-data runner separately verifies fitted dispersion predictions. + comparison = compare_bulk_equivalence( + expected, + actual, + retained_counts=retained, + effective_phased=output["effective_phased"], + reference_log_likelihood=reference_likelihood, + dispersion_comparison={"status": "PASS", "source": "identical fixed nuisance"}, + ) + assert comparison["statistical_equivalence"] == "PASS", comparison + if comparison["strict_comparison"]["status"] != "PASS": + assert comparison["strict_comparison"]["failures"] == ["mu"] + assert any(row["status"] == "SYMMETRIC_EQUIVALENT" for row in comparison["regions"]) diff --git a/tests/analysis/test_reference_adapters.py b/tests/analysis/test_reference_adapters.py new file mode 100644 index 00000000..d4465eab --- /dev/null +++ b/tests/analysis/test_reference_adapters.py @@ -0,0 +1,40 @@ +"""Shape repairs must preserve the original probability recurrence exactly.""" + +import numpy as np +import pytest +from scripts.compare_archive_analysis import adapt_linear_reference, load_archive + + +@pytest.mark.parametrize("mu", [0.06, 0.25, 0.38196601125, 0.7, 0.94]) +@pytest.mark.parametrize("rho", [0.03, 0.99]) +def test_scalar_shape_adapter_preserves_archive_arithmetic(mu, rho): + modules = load_archive() + original = modules["as_analysis"].opt_unphased_dp + first_ref, first_n = np.array([10]), np.array([20]) + rest_ref, rest_n = np.array([3, 18, 0]), np.array([20, 24, 10]) + expected = original(mu, rho, first_ref, first_n, rest_ref, rest_n) + adapt_linear_reference(modules) + actual = modules["as_analysis"].opt_unphased_dp(mu, rho, first_ref, first_n, rest_ref, rest_n) + assert actual == expected + + +@pytest.mark.parametrize("mu", [0.06, 0.25, 0.38196601125, 0.7, 0.94]) +def test_vector_adapter_only_slices_dispersion(mu): + modules = load_archive() + probability = modules["as_analysis"].opt_prob + dispersions = np.array([0.01, 0.03, 0.1, 0.99]) + first_ref, first_n = np.array([10]), np.array([20]) + rest_ref, rest_n = np.array([3, 18, 0]), np.array([20, 24, 10]) + first = probability(mu, dispersions[0], first_ref[0], first_n[0]) + forward = probability(mu, dispersions[1:], rest_ref, rest_n, log=False) + reverse = probability(1 - mu, dispersions[1:], rest_ref, rest_n, log=False) + previous = 1 + for p1, p2 in zip(forward, reverse, strict=True): + phase1, phase2 = previous * p1, previous * p2 + previous = 0.5 * phase1 + 0.5 * phase2 + expected = float(first - np.log(previous)) + adapt_linear_reference(modules) + actual = modules["as_analysis"].opt_unphased_dp( + mu, dispersions, first_ref, first_n, rest_ref, rest_n + ) + assert actual == expected diff --git a/tests/analysis/test_region_col_modes.py b/tests/analysis/test_region_col_modes.py index 9982b95a..92213202 100644 --- a/tests/analysis/test_region_col_modes.py +++ b/tests/analysis/test_region_col_modes.py @@ -23,18 +23,15 @@ def _write_counts(path, *, with_region: bool = True): """Write a tiny phased count TSV (two het SNVs in one peak). - ``with_region=True`` emits the current 10-column layout carrying a ``region`` column; + ``with_region=True`` emits a count layout that carries a ``region`` column; ``with_region=False`` omits it. Only the header is consulted by ``WaspAnalysisData``. """ if with_region: - header = "chrom\tpos0\tpos\tref\talt\tGT\tregion\tref_count\talt_count\tother_count\n" - rows = ( - "chr1\t100\t101\tA\tG\t0|1\tpeakX\t10\t8\t0\n" - "chr1\t200\t201\tC\tT\t1|0\tpeakX\t7\t9\t0\n" - ) + header = "chrom\tstart\tpos\tref\talt\tregion\tGT\tref_count\talt_count\tN\n" + rows = "chr1\t100\t101\tA\tG\tpeakX\t0|1\t10\t8\t18\nchr1\t200\t201\tC\tT\tpeakX\t1|0\t7\t9\t16\n" else: - header = "chrom\tpos0\tpos\tref\talt\tGT\tref_count\talt_count\tother_count\n" - rows = "chr1\t100\t101\tA\tG\t0|1\t10\t8\t0\nchr1\t200\t201\tC\tT\t1|0\t7\t9\t0\n" + header = "chrom\tstart\tpos\tref\talt\tGT\tref_count\talt_count\tN\n" + rows = "chr1\t100\t101\tA\tG\t0|1\t10\t8\t18\nchr1\t200\t201\tC\tT\t1|0\t7\t9\t16\n" path.write_text(header + rows) return path @@ -61,6 +58,28 @@ def test_rust_feature_mode_groups_by_region(tmp_path): assert "mu" in res[0] # phased model engaged +@pytest.mark.rust +def test_rust_run_api_reports_model_specific_dispersion(tmp_path): + """The independent-run API exposes scalar or linear fit metadata, never both.""" + wasp2_rust = pytest.importorskip("wasp2_rust") + tsv = _write_counts(tmp_path / "counts.tsv") + + single = wasp2_rust.analyze_imbalance_run( + str(tsv), min_count=0, pseudocount=1, region_col="region", method="single" + ) + linear = wasp2_rust.analyze_imbalance_run( + str(tsv), min_count=0, pseudocount=1, region_col="region", method="linear" + ) + + assert single["rho"] is not None + assert single["linear_d1"] is None + assert single["linear_d2"] is None + assert linear["rho"] is None + assert linear["linear_d1"] is not None + assert linear["linear_d2"] is not None + assert linear["method"] == "linear" + + @pytest.mark.rust def test_rust_solo_mode_per_variant_labels(tmp_path): """region_col=None tests each SNV independently, labelled {chrom}_{pos}.""" @@ -136,31 +155,6 @@ def test_cli_forwards_phased_and_region_col(tmp_path, spy_rust): assert spy_rust["kwargs"].get("region_col") == "region" -@pytest.mark.unit -def test_cli_auto_detects_feature_column(tmp_path, spy_rust): - from analysis.run_analysis import run_ai_analysis - - tsv = _write_counts(tmp_path / "counts.tsv", with_region=True) - run_ai_analysis(str(tsv), out_file=str(tmp_path / "out.tsv")) - - assert spy_rust["kwargs"].get("region_col") == "region" - - -@pytest.mark.unit -def test_cli_does_not_group_bare_counts_by_gt_or_sample(tmp_path, spy_rust): - from analysis.run_analysis import run_ai_analysis - - tsv = _write_counts(tmp_path / "counts.tsv", with_region=False) - lines = tsv.read_text().splitlines() - lines[0] += "\tsample" - lines[1:] = [f"{line}\tSAMPLE1" for line in lines[1:]] - tsv.write_text("\n".join(lines) + "\n") - - run_ai_analysis(str(tsv), out_file=str(tmp_path / "out.tsv")) - - assert spy_rust["kwargs"].get("region_col") is None - - @pytest.mark.unit def test_cli_groupby_fails_closed(tmp_path, spy_rust): from analysis.run_analysis import run_ai_analysis diff --git a/tests/analysis/test_scipy_numerical_kernel.py b/tests/analysis/test_scipy_numerical_kernel.py new file mode 100644 index 00000000..871612ad --- /dev/null +++ b/tests/analysis/test_scipy_numerical_kernel.py @@ -0,0 +1,137 @@ +"""Strict Python compatibility and independent numerical-stability checks. + +Conditional expectations execute archived Python in the current environment. +The original frozen Mac expectations remain unchanged and are additionally +checked in their pinned environment; neither oracle is generated from Rust. +The high-depth checks use SciPy's log PMF as an independent mathematical oracle; +they do not claim compatibility with the archive's underflowing product loop. +""" + +import json +import platform +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import scipy +from scipy.stats import betabinom +from scripts.compare_archive_analysis import compare_frames, fixed_bulk_reference + +REFERENCE = json.loads( + (Path(__file__).parent / "fixtures/real_atac_optimizer_reference.json").read_text() +) + + +@pytest.mark.rust +@pytest.mark.parametrize("case", REFERENCE["cases"], ids=lambda case: case["name"]) +def test_fixed_real_atac_matches_native_python_and_frozen_mac(tmp_path, case): + import wasp2_rust + + counts_file = tmp_path / "counts.tsv" + pd.DataFrame(case["counts"]).to_csv(counts_file, sep="\t", index=False) + output = wasp2_rust.analyze_imbalance_run( + str(counts_file), + method=case["method"], + phased=case["phased"], + pseudocount=case["pseudocount"], + region_col="region", + **case["fixed_dispersion"], + ) + expected = fixed_bulk_reference(case) + comparison = compare_frames(expected, pd.DataFrame(output["results"])) + assert comparison["status"] == "PASS", comparison + # Preserve the original frozen Mac expectations without applying its + # compiler/library bit patterns to another platform's archived Python. + if (platform.system(), platform.machine(), np.__version__, scipy.__version__) == ( + "Darwin", + "arm64", + "1.26.4", + "1.13.1", + ): + frozen = compare_frames(pd.DataFrame(case["expected"]), pd.DataFrame(output["results"])) + assert frozen["status"] == "PASS", frozen + + +@pytest.mark.rust +@pytest.mark.parametrize("rho", [1e-8, 0.1, 0.99]) +@pytest.mark.parametrize("ref_count,total", [(0, 10), (10, 10), (1, 1000), (500, 1000)]) +def test_fixed_dispersion_null_logpmf_matches_scipy(tmp_path, rho, ref_count, total): + import wasp2_rust + + counts_file = tmp_path / "counts.tsv" + pd.DataFrame( + [ + { + "chrom": "chr1", + "pos": 101, + "ref": "A", + "alt": "G", + "region": "peak", + "ref_count": ref_count, + "alt_count": total - ref_count, + } + ] + ).to_csv(counts_file, sep="\t", index=False) + output = wasp2_rust.analyze_imbalance_run( + str(counts_file), + method="single", + phased=False, + pseudocount=0, + region_col="region", + rho=rho, + ) + actual = output["results"][0]["null_ll"] + shape = 0.5 * (1.0 - rho) / rho + expected = betabinom.logpmf(ref_count, total, shape, shape) + assert np.isfinite(actual) + assert np.isclose(actual, expected, rtol=5e-5, atol=5e-5) + + +@pytest.mark.rust +def test_many_high_depth_unphased_observations_remain_finite(tmp_path): + import wasp2_rust + + # Unique positions ensure these are distinct retained observations. A long + # product underflows even though the equivalent log likelihood is finite. + total = 100_000 + ref_counts = np.arange(200, dtype=np.int64) + 2000 + counts_file = tmp_path / "counts.tsv" + pd.DataFrame( + { + "chrom": "chr1", + "pos": np.arange(200) + 101, + "ref": "A", + "alt": "G", + "region": "peak", + "ref_count": ref_counts, + "alt_count": total - ref_counts, + } + ).to_csv(counts_file, sep="\t", index=False) + rho = 0.1 + output = wasp2_rust.analyze_imbalance_run( + str(counts_file), + method="single", + phased=False, + pseudocount=0, + region_col="region", + rho=rho, + ) + result = output["results"][0] + assert result["snp_count"] == len(ref_counts) + assert all(np.isfinite(result[key]) for key in ["mu", "null_ll", "alt_ll", "pval"]) + assert 0.0 < result["mu"] < 1.0 + shape = 0.5 * (1.0 - rho) / rho + expected_null = np.sum(betabinom.logpmf(ref_counts, total, shape, shape)) + assert np.isclose(result["null_ll"], expected_null, rtol=5e-5, atol=5e-5) + + alpha = result["mu"] * (1.0 - rho) / rho + beta = (1.0 - result["mu"]) * (1.0 - rho) / rho + forward = betabinom.logpmf(ref_counts, total, alpha, beta) + reverse = betabinom.logpmf(ref_counts, total, beta, alpha) + expected_alt = forward[0] + np.sum(np.logaddexp(forward[1:], reverse[1:]) - np.log(2.0)) + assert np.isclose(result["alt_ll"], expected_alt, rtol=5e-5, atol=5e-5) + product = 1.0 + for p1, p2 in zip(np.exp(forward[1:]), np.exp(reverse[1:]), strict=True): + product = 0.5 * (product * p1) + 0.5 * (product * p2) + assert product == 0.0 diff --git a/tests/counting/test_count_alleles.py b/tests/counting/test_count_alleles.py index bc1795cb..6d067942 100644 --- a/tests/counting/test_count_alleles.py +++ b/tests/counting/test_count_alleles.py @@ -44,3 +44,19 @@ def test_bulk_counting_preserves_counts_above_uint16(monkeypatch): assert result.schema["ref_count"] == pl.UInt32 assert result.schema["alt_count"] == pl.UInt32 assert result.schema["other_count"] == pl.UInt32 + + +@pytest.mark.unit +def test_bulk_counting_rejects_multiple_alleles_at_one_position(monkeypatch): + monkeypatch.setattr(count_alleles, "RUST_AVAILABLE", True) + variants = pl.DataFrame( + { + "chrom": ["chr1", "chr1"], + "pos": [10, 10], + "ref": ["A", "A"], + "alt": ["G", "T"], + } + ).with_columns(pl.col("chrom").cast(pl.Categorical)) + + with pytest.raises(ValueError, match="biallelic positions only"): + count_alleles.make_count_df("sample.bam", variants) diff --git a/tests/test_extension_version.py b/tests/test_extension_version.py new file mode 100644 index 00000000..68f73afd --- /dev/null +++ b/tests/test_extension_version.py @@ -0,0 +1,7 @@ +import importlib.metadata + +import wasp2_rust + + +def test_rust_extension_version_matches_distribution() -> None: + assert wasp2_rust.__version__ == importlib.metadata.version("wasp2")