From e8c21c2f9dc14c6bc9a87eca4ac12d54c3cb3874 Mon Sep 17 00:00:00 2001 From: Jeff Jaureguy <67065808+Jaureguy760@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:29:26 -0700 Subject: [PATCH] docs: remove obsolete development research and AI workflow notes --- docs/CI_CD_BEST_PRACTICES.md | 245 ------------- docs/DOCKER_BEST_PRACTICES_2025.md | 478 ------------------------- docs/WASP2_ECOSYSTEM.md | 6 - docs/internal/seqera_ai_integration.md | 361 ------------------- scripts/benchmark_docker_build.sh | 194 ---------- 5 files changed, 1284 deletions(-) delete mode 100644 docs/CI_CD_BEST_PRACTICES.md delete mode 100644 docs/DOCKER_BEST_PRACTICES_2025.md delete mode 100644 docs/internal/seqera_ai_integration.md delete mode 100755 scripts/benchmark_docker_build.sh diff --git a/docs/CI_CD_BEST_PRACTICES.md b/docs/CI_CD_BEST_PRACTICES.md deleted file mode 100644 index 14ad303f..00000000 --- a/docs/CI_CD_BEST_PRACTICES.md +++ /dev/null @@ -1,245 +0,0 @@ -# WASP2 CI/CD Best Practices Guide - -> Based on analysis of GenVarLoader, pysam, rust-bio, polars, and uv projects - -## Architecture Overview - -### Runner Configuration - -WASP2 uses **3 specialized self-hosted runners** on Mac M3 Max for optimal parallelization: - -| Runner | Labels | Purpose | -|--------|--------|---------| -| `wasp2-python-runner` | `python, testing, lint, fast` | Fast Python tests, linting | -| `wasp2-rust-runner` | `rust, build, maturin` | Rust builds, wheel building | -| `wasp2-analysis-runner` | `analysis, bioinformatics, docker, slow` | Heavy analysis, Docker | - -### Workflow Selection - -```yaml -# Fast Python tasks -runs-on: [self-hosted, macOS, ARM64, python] - -# Rust compilation -runs-on: [self-hosted, macOS, ARM64, rust] - -# Heavy analysis/Docker -runs-on: [self-hosted, macOS, ARM64, analysis] -``` - -## Workflow Schedule - -All times in UTC. Staggered to prevent resource contention. - -| Workflow | Schedule | Purpose | -|----------|----------|---------| -| Security | Monday 2:05am | Weekly security scans | -| CodeQL | Monday 2:15am | Static analysis | -| Nightly | Daily 3:00am | Extended tests, benchmarks | -| Dependabot | Monday 3:00am | Dependency updates | - -## Caching Strategy - -### Rust (using Swatinem/rust-cache) - -```yaml -- uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - save-if: github.ref == 'refs/heads/main' -``` - -**Benefits:** -- 50-70% faster incremental builds -- Only saves cache on main branch (prevents cache pollution) - -### Python - -```yaml -- uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} - restore-keys: | - pip-${{ runner.os }}- -``` - -### Docker (BuildKit) - -```yaml -- uses: docker/build-push-action@v6 - with: - cache-from: type=gha,scope=wasp2 - cache-to: type=gha,mode=max,scope=wasp2 -``` - -## Matrix Testing Strategy - -### Python Version Matrix - -```yaml -strategy: - fail-fast: false - matrix: - python-version: ['3.10', '3.11', '3.12'] -``` - -### Platform Matrix (Release) - -```yaml -matrix: - include: - - os: ubuntu-latest - target: x86_64 - - os: ubuntu-latest - target: aarch64 - - os: macos-13 - target: x86_64-apple-darwin - - os: macos-14 - target: aarch64-apple-darwin -``` - -## Concurrency Control - -Prevent parallel runs on same branch: - -```yaml -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -``` - -## Dependabot Automation - -### Auto-approve + Auto-merge for Patches - -The `dependabot-auto-merge.yml` workflow: -- **Auto-approves** patch and minor updates -- **Auto-merges** patch updates only (safer) -- **Labels** major updates for manual review - -### Dependabot Configuration - -```yaml -# .github/dependabot.yml -version: 2 -updates: - - package-ecosystem: "pip" - schedule: - interval: "weekly" - day: "monday" - time: "03:00" - commit-message: - prefix: "chore(deps)" -``` - -## Security Scanning - -### Tools - -| Tool | Language | Purpose | -|------|----------|---------| -| `pip-audit` | Python | Dependency vulnerabilities | -| `bandit` | Python | Code security issues | -| `cargo-audit` | Rust | Dependency vulnerabilities | -| `gitleaks` | All | Secret detection | -| `CodeQL` | All | Static analysis | - -### Blocking vs Informational - -- **Blocking:** gitleaks (secrets must never be committed) -- **Informational:** pip-audit, bandit, cargo-audit (logged but don't fail PR) - -## Release Workflow - -### Multi-Platform Wheel Building - -Using `PyO3/maturin-action@v1`: - -```yaml -- uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist -m rust/Cargo.toml - manylinux: "2014" - before-script-linux: | - yum install -y bzip2-devel xz-devel zlib-devel -``` - -### Trusted Publishing (OIDC) - -No PyPI tokens needed! Configure in PyPI settings: - -```yaml -permissions: - id-token: write # OIDC - -- uses: pypa/gh-action-pypi-publish@release/v1 - # No token required - uses OIDC -``` - -## Performance Optimization Tips - -### M3 Max Specific - -1. **6-8 parallel jobs** maximum (leaving cores for OS) -2. **Separate runners** for different workload types -3. **sccache** for Rust incremental compilation -4. **Cache benchmark data** to avoid regeneration - -### Expected Performance Gains - -| Stage | Without Optimization | With Optimization | -|-------|---------------------|-------------------| -| Lint/Format | 2-3 min | 30-45 sec | -| Unit Tests | 5-8 min | 2-3 min | -| Rust Build (cold) | 8-12 min | 4-6 min | -| Rust Build (warm) | 2-3 min | 30-60 sec | -| Full Pipeline | 35-45 min | 8-12 min | - -## Nightly Testing - -The `nightly.yml` workflow runs: -- Extended unit tests (all Python versions) -- Integration tests -- Performance benchmarks -- Nextflow pipeline tests (optional) - -### Running Manually - -```bash -gh workflow run nightly.yml -f test_set=benchmarks -``` - -## Runner Management - -### Setup Multi-Runners - -```bash -./scripts/setup-multi-runners.sh -``` - -### Management Commands - -```bash -# Check all runners -for d in ~/wasp2-runners/*/; do (cd "$d" && ./svc.sh status); done - -# Stop all runners -for d in ~/wasp2-runners/*/; do (cd "$d" && ./svc.sh stop); done - -# Start all runners -for d in ~/wasp2-runners/*/; do (cd "$d" && ./svc.sh start); done -``` - -### View on GitHub - -https://github.com/mcvickerlab/WASP2/settings/actions/runners - -## References - -- [GenVarLoader CI/CD](https://github.com/mcvickerlab/GenVarLoader) - Pixi, commitizen, multi-platform -- [uv workflows](https://github.com/astral-sh/uv) - Planning jobs, caching -- [pysam CI](https://github.com/pysam-developers/pysam) - maturin, multi-platform wheels -- [rust-bio CI](https://github.com/rust-bio/rust-bio) - Rust best practices -- [GitHub Actions Best Practices](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions) diff --git a/docs/DOCKER_BEST_PRACTICES_2025.md b/docs/DOCKER_BEST_PRACTICES_2025.md deleted file mode 100644 index 70deedcb..00000000 --- a/docs/DOCKER_BEST_PRACTICES_2025.md +++ /dev/null @@ -1,478 +0,0 @@ -# Docker Best Practices for Python Applications (2025-2026) - -**Research Date:** January 2025 -**Target Project:** WASP2 (Python + Rust bioinformatics) -**Author:** Performance Analysis - ---- - -## Table of Contents - -1. [Base Image Selection](#1-base-image-selection) -2. [Package Manager Comparison](#2-package-manager-comparison) -3. [Multi-Stage Build Patterns](#3-multi-stage-build-patterns) -4. [Image Size Reduction](#4-image-size-reduction) -5. [Build Cache Optimization](#5-build-cache-optimization) -6. [Runtime Performance](#6-runtime-performance) -7. [Security Hardening](#7-security-hardening) -8. [WASP2 Specific Recommendations](#8-wasp2-specific-recommendations) - ---- - -## 1. Base Image Selection - -### Comparison Matrix - -| Image | Size | glibc | Shells | CVEs* | Scientific Computing | Bioinformatics | -|-------|------|-------|--------|-------|---------------------|----------------| -| `python:3.11-slim-bookworm` | 221MB | Yes | bash | Medium | Excellent | Excellent | -| `python:3.11-alpine` | 89MB | No (musl) | sh | Low | Poor | Poor | -| `gcr.io/distroless/python3-debian12` | 95MB | Yes | None | Very Low | Limited | Poor | -| `cgr.dev/chainguard/python` | ~80MB | Yes (Wolfi) | None | Near Zero | Good | Limited | -| `ghcr.io/astral-sh/uv:python3.11-bookworm-slim` | 288MB | Yes | bash | Medium | Excellent | Excellent | - -*CVEs = Common Vulnerabilities and Exposures (typical count) - -### Recommendations by Use Case - -**Scientific/Bioinformatics (WASP2):** -- **Use:** `python:3.11-slim-bookworm` -- **Reason:** glibc compatibility for numpy/scipy wheels, apt for samtools/bcftools - -**Minimal API Services:** -- **Use:** `gcr.io/distroless/python3-debian12` -- **Reason:** Smallest attack surface, no shell access - -**Enterprise with SLA requirements:** -- **Use:** `cgr.dev/chainguard/python` (paid) -- **Reason:** Daily CVE patches, signed builds, SBOM included - -**Development/CI:** -- **Use:** `ghcr.io/astral-sh/uv:python3.11-bookworm-slim` -- **Reason:** UV pre-installed, fast dependency resolution - -### Why NOT Alpine for Scientific Python - -Alpine uses musl libc instead of glibc, causing: - -1. **No pre-built wheels:** numpy, scipy, pandas require compilation -2. **Build time:** 10-30 minutes vs 30 seconds with glibc wheels -3. **Binary compatibility:** Some C extensions fail or have subtle bugs -4. **Performance:** musl can be slower for numerical workloads (5-15%) - -```dockerfile -# BAD for scientific Python -FROM python:3.11-alpine -RUN pip install numpy # Compiles from source, may fail - -# GOOD for scientific Python -FROM python:3.11-slim-bookworm -RUN pip install numpy # Uses pre-built wheel, instant -``` - ---- - -## 2. Package Manager Comparison - -### UV vs pip vs Poetry (2025) - -| Feature | UV (v0.9.26) | pip (24.x) | Poetry (1.8.x) | -|---------|--------------|------------|----------------| -| **Install Speed** | 10-100x faster | Baseline | 0.3-0.5x | -| **Lockfile** | `uv.lock` (native) | None (pip-tools) | `poetry.lock` | -| **Resolution** | Parallel, incremental | Sequential | Sequential | -| **Docker Integration** | Excellent | Basic | Limited | -| **Cache Efficiency** | Excellent | Good | Moderate | -| **Binary Size** | 10MB | Built-in | ~50MB | -| **Rust Builds** | Native (maturin) | Manual | Manual | - -### UV Docker Features - -```dockerfile -# 1. Copy UV binary (10MB, pinned version) -COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /bin/ - -# 2. Cache mounts for 10-100x faster rebuilds -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked - -# 3. Bytecode compilation for 20-30% faster startup -ENV UV_COMPILE_BYTECODE=1 - -# 4. Non-editable install for production -RUN uv sync --locked --no-editable - -# 5. Skip dev dependencies -ENV UV_NO_DEV=1 -``` - -### Migration from pip to UV - -```dockerfile -# BEFORE (pip) -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -RUN pip install --no-cache-dir . - -# AFTER (UV) - 10-100x faster -COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /bin/uv -COPY pyproject.toml uv.lock ./ -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-install-project -COPY . . -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-editable -``` - ---- - -## 3. Multi-Stage Build Patterns - -### Pattern for Python + Rust Projects (WASP2) - -```dockerfile -# syntax=docker/dockerfile:1.7 - -# ============================================================ -# Stage 1: Build Rust extension -# ============================================================ -FROM rust:1.75-bookworm AS rust-builder - -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3-dev libclang-dev libhts-dev libbz2-dev liblzma-dev \ - zlib1g-dev pkg-config && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --break-system-packages maturin>=1.4 - -WORKDIR /build -COPY rust/ rust/ -COPY pyproject.toml README.md LICENSE ./ - -RUN --mount=type=cache,target=/root/.cargo/registry \ - --mount=type=cache,target=/root/.cargo/git \ - maturin build --release -m rust/Cargo.toml -o /wheels - -# ============================================================ -# Stage 2: Build Python dependencies -# ============================================================ -FROM python:3.11-slim-bookworm AS python-builder - -COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /bin/uv -ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_NO_DEV=1 - -WORKDIR /app - -# Install dependencies (cached layer) -COPY pyproject.toml uv.lock ./ -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-install-project - -# Install Rust wheel -COPY --from=rust-builder /wheels/*.whl /tmp/ -RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install /tmp/*.whl - -# Install project (non-editable) -COPY src/ ./src/ -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-editable - -# ============================================================ -# Stage 3: Minimal runtime -# ============================================================ -FROM python:3.11-slim-bookworm AS runtime - -# Runtime dependencies only -RUN apt-get update && apt-get install -y --no-install-recommends \ - samtools bcftools bedtools tabix libhts3 procps \ - && rm -rf /var/lib/apt/lists/* - -# Non-root user -RUN groupadd -g 1000 app && useradd -u 1000 -g app -m app -USER app - -# Copy venv from builder -COPY --from=python-builder --chown=app:app /app/.venv /app/.venv -ENV PATH="/app/.venv/bin:$PATH" - -WORKDIR /data -CMD ["python", "--version"] -``` - -### Key Benefits - -| Aspect | Improvement | -|--------|-------------| -| Final image size | 50-70% smaller (no Rust toolchain) | -| Build cache hits | 80-95% on dependency changes | -| Security surface | Minimal (no compilers in runtime) | -| Rebuild time | 10-100x faster with UV caches | - ---- - -## 4. Image Size Reduction - -### Techniques Ranked by Impact - -| Technique | Typical Savings | Implementation | -|-----------|-----------------|----------------| -| Multi-stage builds | 1-2GB | Separate build/runtime stages | -| Remove build tools | 500MB-1GB | Don't install gcc, make, etc. in runtime | -| Use slim base | 300-500MB | `-slim` variant instead of full | -| UV vs pip | 50-100MB | Better dependency resolution | -| `--no-cache-dir` | 50-200MB | `pip install --no-cache-dir` | -| Combine RUN layers | 20-50MB | Chain commands with `&&` | -| `.dockerignore` | Variable | Exclude tests, docs, .git | -| Strip binaries | 10-30MB | `strip` on compiled extensions | -| Remove apt lists | 30-50MB | `rm -rf /var/lib/apt/lists/*` | - -### .dockerignore Best Practices - -```gitignore -# Version control -.git -.gitignore - -# Development -tests/ -docs/ -*.md -!README.md -.vscode/ -.idea/ - -# Python artifacts -__pycache__/ -*.pyc -.venv/ -dist/ -build/ -*.egg-info/ - -# Rust artifacts (if not using multi-stage properly) -rust/target/ - -# Security sensitive -.env -.env.* -*.pem -*.key - -# Large data files -*.bam -*.vcf -*.fastq -``` - ---- - -## 5. Build Cache Optimization - -### BuildKit Cache Mounts - -```dockerfile -# Python package cache -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked - -# Rust crates cache -RUN --mount=type=cache,target=/root/.cargo/registry \ - --mount=type=cache,target=/root/.cargo/git \ - cargo build --release - -# apt cache (Debian/Ubuntu) -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && apt-get install -y package -``` - -### Layer Ordering Strategy - -```dockerfile -# GOOD: Least-changing files first -COPY pyproject.toml uv.lock ./ # 1. Rarely changes -RUN uv sync --locked --no-install-project # 2. Cached if deps unchanged -COPY src/ ./src/ # 3. Changes frequently -RUN uv sync --locked # 4. Only rebuilds app - -# BAD: Invalidates cache on any change -COPY . . -RUN uv sync --locked -``` - -### Bind Mounts for Config Files - -```dockerfile -# Avoid COPY cache invalidation for config-only operations -RUN --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - uv sync --locked --no-install-project -``` - ---- - -## 6. Runtime Performance - -### Environment Variables - -```dockerfile -# Bytecode compilation (20-30% faster startup) -ENV UV_COMPILE_BYTECODE=1 - -# Prevent runtime .pyc generation -ENV PYTHONDONTWRITEBYTECODE=1 - -# Unbuffered output for logging -ENV PYTHONUNBUFFERED=1 - -# Memory optimization for long-running processes -ENV MALLOC_ARENA_MAX=2 - -# Disable hash randomization (reproducible, slightly faster) -ENV PYTHONHASHSEED=0 -``` - -### Startup Time Optimization - -| Technique | Impact | Trade-off | -|-----------|--------|-----------| -| Bytecode compilation | -30% startup | +10% image size | -| Lazy imports | -50% startup | Code complexity | -| Module preloading | -20% startup | Memory usage | -| Static linking | -10% startup | Larger binaries | - -### Memory Optimization - -```dockerfile -# For memory-constrained environments -ENV MALLOC_ARENA_MAX=2 -ENV PYTHONMALLOC=malloc - -# For numpy/scipy -ENV OMP_NUM_THREADS=4 -ENV MKL_NUM_THREADS=4 -``` - ---- - -## 7. Security Hardening - -### Non-Root User - -```dockerfile -# Create user with specific UID/GID -RUN groupadd --system --gid 1000 app && \ - useradd --system --gid 1000 --uid 1000 \ - --create-home --shell /sbin/nologin app - -# Switch before running application -USER app - -# Verify permissions -RUN whoami && id -``` - -### Read-Only Filesystem - -```bash -# Runtime flag (not in Dockerfile) -docker run --read-only --tmpfs /tmp --tmpfs /app/cache myimage -``` - -### Capability Dropping - -```bash -# Runtime flag (not in Dockerfile) -docker run --cap-drop=ALL --cap-add=NET_BIND_SERVICE myimage -``` - -### Security Scanning - -```bash -# Trivy (recommended) -trivy image --severity HIGH,CRITICAL wasp2:latest - -# Grype -grype wasp2:latest - -# Snyk -snyk container test wasp2:latest -``` - -### Image Signing and Verification - -```bash -# Sign with cosign -cosign sign --key cosign.key wasp2:latest - -# Verify -cosign verify --key cosign.pub wasp2:latest -``` - -### Security Checklist - -- [ ] Non-root user (`USER` directive) -- [ ] No secrets in image (use runtime secrets) -- [ ] Base image pinned (digest or specific tag) -- [ ] Regular vulnerability scanning -- [ ] HEALTHCHECK defined -- [ ] Minimal installed packages -- [ ] No shell in production (consider distroless) -- [ ] Read-only filesystem where possible -- [ ] Capabilities dropped - ---- - -## 8. WASP2 Specific Recommendations - -### Summary - -| Aspect | Current | Recommended | -|--------|---------|-------------| -| Base image | `python:3.11-slim-bookworm` | Keep (optimal for bioinformatics) | -| Package manager | pip | **UV** (10-100x faster) | -| Build stages | 2 | 3 (separate Python builder) | -| Cache mounts | None | **Add BuildKit mounts** | -| Non-root user | Yes | Keep | -| Bytecode compilation | No | **Enable** | -| Image signing | No | Consider for releases | - -### Migration Path - -1. **Phase 1 (Quick Win):** Add BuildKit cache mounts to existing Dockerfile -2. **Phase 2 (Medium):** Replace pip with UV, add bytecode compilation -3. **Phase 3 (Full):** Use optimized 3-stage Dockerfile - -### Expected Improvements - -| Metric | Current (est.) | Optimized | Improvement | -|--------|----------------|-----------|-------------| -| Image size | 800-1000MB | 500-600MB | 40% smaller | -| Fresh build | 10-15 min | 5-8 min | 50% faster | -| Cached rebuild | 3-5 min | 30-60 sec | 80% faster | -| Startup time | 2-3 sec | 1-2 sec | 40% faster | -| CVE count | Varies | Baseline-10% | Reduced | - -### Files Created - -- `/Users/jeffjaureguy/Projects/WASP2/Dockerfile.optimized` - New optimized Dockerfile -- `/Users/jeffjaureguy/Projects/WASP2/scripts/benchmark_docker_build.sh` - Build comparison script -- `/Users/jeffjaureguy/Projects/WASP2/.dockerignore` - Updated ignore patterns - ---- - -## References - -- [UV Docker Guide](https://docs.astral.sh/uv/guides/integration/docker/) - Official UV documentation -- [uv-docker-example](https://github.com/astral-sh/uv-docker-example) - Official example repository -- [Google Distroless](https://github.com/GoogleContainerTools/distroless) - Minimal base images -- [Chainguard Images](https://edu.chainguard.dev/chainguard/chainguard-images/getting-started/python/) - Zero-CVE images -- [Docker BuildKit](https://docs.docker.com/build/buildkit/) - Advanced build features -- [Maturin Docker](https://github.com/PyO3/maturin) - Rust/Python wheel building - ---- - -*Document generated: January 2025* -*UV Version: 0.9.26* -*Python Target: 3.11* diff --git a/docs/WASP2_ECOSYSTEM.md b/docs/WASP2_ECOSYSTEM.md index 34bcb350..653774b0 100644 --- a/docs/WASP2_ECOSYSTEM.md +++ b/docs/WASP2_ECOSYSTEM.md @@ -21,7 +21,6 @@ | ML Output Formats | [#36](../../issues/36) | ✅ Complete | | GenVarLoader | [#37](../../issues/37) | ✅ Complete | | nf-core Compliance | [#38](../../issues/38) | ✅ Complete | -| Seqera AI | [#39](../../issues/39) | ✅ Complete | ## Module Inventory @@ -84,9 +83,6 @@ pipelines/ │ ▼ nf-core Compliance ✅ - │ - ▼ - Seqera AI ✅ ``` ## Implementation Roadmap @@ -105,7 +101,6 @@ pipelines/ ### Phase 3: Integration ✅ - [x] GenVarLoader integration (#37) - Via Zarr output format - [x] nf-core compliance (#38) - Pipeline structure compliance -- [x] Seqera AI compatibility (#39) - [Integration guide](./source/seqera_ai_integration.md) ## ML Output Formats @@ -156,7 +151,6 @@ nextflow run . -profile test_stub -stub-run - [nf-core/drop](https://nf-co.re/drop/dev/) - Reference OUTRIDER implementation - [GenVarLoader](https://genvarloader.readthedocs.io/) - ML variant loading -- [Seqera AI](https://seqera.io/blog/seqera-ai-new-features-june-2025/) - Pipeline AI assistant --- *Milestone: v1.3.0 - Pipeline Ecosystem* diff --git a/docs/internal/seqera_ai_integration.md b/docs/internal/seqera_ai_integration.md deleted file mode 100644 index fd8e7e1d..00000000 --- a/docs/internal/seqera_ai_integration.md +++ /dev/null @@ -1,361 +0,0 @@ -# Seqera AI Development Integration - -> WASP2 Pipeline Development with AI-Assisted Tooling - -This guide documents the integration of Seqera AI tools into the WASP2 pipeline development workflow. It complements Claude Code for complex logic while leveraging Seqera AI's specialized Nextflow DSL2 capabilities. - -## Overview - -WASP2 pipeline development benefits from a multi-tool AI strategy: - -| Tool | Strength | Use Case | -|------|----------|----------| -| **Claude Code** | Architecture, complex logic, code review | Design decisions, debugging, refactoring | -| **Seqera AI** | Nextflow DSL2 syntax, nf-test generation | Process definitions, pipeline scaffolding | -| **Nextflow Tooling** | Environment validation, nf-test, nf-core lint | Pre-flight checks, compliance verification | - -## Seqera AI Capabilities - -### 1. VS Code Integration - -The Seqera AI VS Code extension provides: - -- **@Seqera chat**: Nextflow code generation via chat interface -- **Pipeline Mode**: Contextual debugging with error explanations -- **nf-test generation**: Automatic test template creation - -Installation: -```bash -# VS Code Extension Marketplace -# Search: "Seqera AI" -# Or visit: https://marketplace.visualstudio.com/items?itemName=seqera.seqera-ai -``` - -### 2. Chat Features - -Use `@Seqera` in VS Code chat for: - -``` -@Seqera Create a process that runs STAR alignment with WASP filtering -@Seqera Generate nf-test for this module -@Seqera Debug this error: "WASP2_COUNT failed with exit code 1" -``` - -### 3. Pipeline Mode - -Enable Pipeline Mode when working on WASP2 pipelines for: - -- Contextual understanding of pipeline structure -- Error message interpretation -- Fix suggestions with DSL2 syntax - -## Development Workflow - -### Recommended 4-Phase Approach - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Phase 1: DESIGN │ Phase 2: GENERATE │ -│ ─────────────────── │ ────────────────── │ -│ Tool: Claude Code │ Tool: Seqera AI │ -│ │ │ -│ • Architecture design │ • DSL2 process definitions │ -│ • Module structure │ • Workflow scaffolding │ -│ • Error handling │ • nf-test templates │ -│ • Complex algorithms │ • Config generation │ -├─────────────────────────┼───────────────────────────────────│ -│ Phase 3: VALIDATE │ Phase 4: REVIEW │ -│ ─────────────────── │ ────────────────── │ -│ Tool: Nextflow/nf-test │ Tool: Claude Code │ -│ │ │ -│ • nextflow -preview │ • Security review │ -│ • nf-test execution │ • Code review │ -│ • nf-core lint │ • Performance optimization │ -└─────────────────────────┴───────────────────────────────────┘ -``` - -### Phase 1: Design with Claude Code - -Start by designing the module structure: - -```bash -# Example: Planning a new WASP2 subworkflow -claude "Design a subworkflow for allelic imbalance analysis that: -- Takes BAM + VCF input -- Runs WASP2_COUNT for allele counting -- Runs WASP2_ANALYZE for statistical testing -- Outputs TSV with beta-binomial p-values" -``` - -Output: Architecture document with file structure, input/output specs, error handling strategy. - -### Phase 2: Generate DSL2 with Seqera AI - -Use Seqera AI for Nextflow-specific code: - -``` -@Seqera Create a DSL2 process called WASP2_COUNT that: -- Takes meta map, bam, bai, vcf as input -- Runs wasp2-count count-variants -- Outputs meta map and counts TSV -- Uses container 'ghcr.io/mcvickerlab/wasp2:latest' -``` - -Example generated output: -```nextflow -process WASP2_COUNT { - tag "$meta.id" - label 'process_medium' - - container 'ghcr.io/mcvickerlab/wasp2:latest' - - input: - tuple val(meta), path(bam), path(bai), path(vcf) - - output: - tuple val(meta), path("*.counts.tsv"), emit: counts - path "versions.yml" , emit: versions - - script: - def prefix = task.ext.prefix ?: "${meta.id}" - """ - wasp2-count count-variants \\ - ${bam} \\ - ${vcf} \\ - --output ${prefix}.counts.tsv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - wasp2: \$(wasp2-count --version | head -n1 | cut -d' ' -f2) - END_VERSIONS - """ - - stub: - def prefix = task.ext.prefix ?: "${meta.id}" - """ - touch ${prefix}.counts.tsv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - wasp2: 1.0.0 - END_VERSIONS - """ -} -``` - -Generate nf-test: -``` -@Seqera Generate nf-test for the WASP2_COUNT process -``` - -### Phase 3: Validate Environment - -Before running the pipeline, validate the environment using standard Nextflow checks: - -```bash -# Verify Nextflow installation and version -nextflow -v - -# Check Docker/Singularity availability -docker --version || singularity --version - -# Validate pipeline configuration (dry-run) -nextflow run . -profile test -preview - -# Verify container accessibility -docker pull ghcr.io/mcvickerlab/wasp2:latest -``` - -> **Note**: The [Anthropic life-sciences](https://github.com/anthropics/life-sciences) plugin provides additional validation scripts (`check_environment.py`, `generate_samplesheet.py`, `manage_genomes.py`) when installed. See their documentation for setup instructions. - -### Phase 4: Review with Claude Code - -Final review and integration: - -```bash -# Code review -claude "Review this Nextflow module for nf-core compliance: -$(cat modules/local/wasp2_count/main.nf)" - -# Integration testing -claude "Help debug this nf-test failure: -$(nf-test test modules/local/wasp2_count/tests/main.nf.test 2>&1)" -``` - -## Tool Comparison Matrix - -| Capability | Claude Code | Seqera AI | Nextflow Tooling | -|------------|:-----------:|:---------:|:----------------:| -| Architecture design | ★★★ | ★ | - | -| DSL2 syntax | ★★ | ★★★ | - | -| nf-test generation | ★★ | ★★★ | ★★ | -| Error debugging | ★★★ | ★★★ | ★ | -| Environment validation | ★ | ★★ | ★★★ | -| Samplesheet validation | ★★ | ★ | ★★★ | -| Code review | ★★★ | ★ | - | -| Complex algorithms | ★★★ | ★ | - | -| nf-core compliance | ★★★ | ★★ | ★★★ | - -## Example Workflows - -### Creating a New WASP2 Module - -1. **Design** (Claude Code): - ```bash - claude "Design a WASP2 module for differential allelic imbalance - between conditions. Include input/output specs and algorithm outline." - ``` - -2. **Generate** (Seqera AI): - ``` - @Seqera Create DSL2 process for differential_allelic_imbalance - that compares two conditions using beta-binomial regression - ``` - -3. **Test** (Seqera AI): - ``` - @Seqera Generate nf-test with test data for this module - ``` - -4. **Validate** (nf-test): - ```bash - nextflow run . -profile test -preview - nf-test test modules/local/differential_ai/tests/ - ``` - -5. **Review** (Claude Code): - ```bash - claude "Review for nf-core compliance and optimize performance" - ``` - -### Debugging Pipeline Failures - -1. **Enable Pipeline Mode** in VS Code Seqera AI settings - -2. **Ask Seqera AI**: - ``` - @Seqera Pipeline Mode: Debug this error from WASP2_COUNT: - "Error: VCF index not found for sample001.vcf.gz" - ``` - -3. **Complex fixes** with Claude Code: - ```bash - claude "The VCF index issue suggests a race condition in our - subworkflow. Review the channel logic in subworkflows/allelic_analysis.nf" - ``` - -## Configuration - -### Seqera Platform Integration - -For running WASP2 pipelines on Seqera Platform: - -```yaml -# seqera.yml -manifest: - name: 'wasp2/nf-rnaseq' - version: '1.0.0' - description: 'WASP2 RNA-seq allelic imbalance pipeline' - -compute: - aws: - region: 'us-west-2' - queue: 'wasp2-production' - -params: - outdir: 's3://wasp2-results/rnaseq' - genome: 'GRCh38' -``` - -### VS Code Settings - -```json -{ - "seqera.ai.pipelineMode": true, - "seqera.ai.workspace": "wasp2-pipelines", - "seqera.nextflow.path": "~/.nextflow/bin/nextflow" -} -``` - -## Best Practices - -### 1. Use the Right Tool for the Task - -- **Architecture and design** → Claude Code -- **Nextflow syntax and boilerplate** → Seqera AI -- **Complex debugging** → Both tools together -- **Environment validation** → Standard Nextflow tooling - -### 2. Leverage Pipeline Mode - -Enable Seqera AI Pipeline Mode when: -- Debugging pipeline failures -- Writing new processes -- Understanding error messages - -### 3. Maintain nf-core Compliance - -All WASP2 modules should follow nf-core standards: -- Meta map propagation -- Proper container definitions -- versions.yml output -- Stub run support -- nf-test coverage - -### 4. Document AI Assistance - -When using AI-generated code, document the source: - -```nextflow -/* - * Module: WASP2_DIFFERENTIAL_AI - * - * Generated: Seqera AI (VS Code) - * Reviewed: Claude Code - * Author: WASP2 Team - */ -``` - -### 5. Security Considerations - -When using AI-assisted development: - -- **Review all generated code** before committing - AI may introduce insecure patterns -- **Never commit credentials** - Use Nextflow secrets or environment variables for sensitive data -- **Pin container versions** - Avoid `latest` tags in production (`ghcr.io/mcvickerlab/wasp2:1.3.0`) -- **Validate inputs** - AI-generated processes may not include proper input validation -- **Use signed containers** - Enable container signature verification when available -- **Audit dependencies** - Review any new dependencies suggested by AI tools - -```nextflow -// SECURE: Pin container version, use secrets -process SECURE_EXAMPLE { - container 'ghcr.io/mcvickerlab/wasp2:1.3.0' - secret 'API_KEY' - - // ... -} - -// INSECURE: Avoid these patterns -// container 'ghcr.io/mcvickerlab/wasp2:latest' // Unpinned -// script: "curl ${params.api_key}" // Exposed credential -``` - -## References - -- [Seqera AI Features](https://seqera.io/blog/seqera-ai-new-features-june-2025/) -- [Seqera AI VS Code Extension](https://seqera.io/blog/seqera-ai--nextflow-vs-code/) -- [Anthropic life-sciences Plugin](https://github.com/anthropics/life-sciences) -- [nf-core Developer Docs](https://nf-co.re/docs/contributing/modules) -- [WASP2 Pipeline Documentation](./WASP2_ECOSYSTEM.md) - -## Related Issues - -- **Parent**: [EPIC #25 - Nextflow Pipeline Ecosystem](https://github.com/mcvickerlab/WASP2/issues/25) -- **Supports**: [#58 - nf-core subworkflow compliance](https://github.com/mcvickerlab/WASP2/issues/58) -- **Complements**: [#95 - Anthropic life-sciences integration](https://github.com/mcvickerlab/WASP2/issues/95) - ---- - -*Milestone: v1.3.0 - Pipeline Ecosystem* -*Last updated: 2026-02-03* diff --git a/scripts/benchmark_docker_build.sh b/scripts/benchmark_docker_build.sh deleted file mode 100755 index d650f41c..00000000 --- a/scripts/benchmark_docker_build.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# WASP2 Docker Build Benchmark Script -# Compares original vs optimized Dockerfile builds -# Usage: ./scripts/benchmark_docker_build.sh - -set -e - -PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$PROJECT_DIR" - -echo "========================================" -echo "WASP2 Docker Build Benchmark" -echo "========================================" -echo "" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Function to format time -format_time() { - local seconds=$1 - printf "%dm %ds" $((seconds/60)) $((seconds%60)) -} - -# Function to format size -format_size() { - local size=$1 - if [[ $size =~ ^[0-9]+$ ]]; then - printf "%.1fMB" $(echo "scale=1; $size/1048576" | bc) - else - echo "$size" - fi -} - -# Ensure BuildKit is enabled -export DOCKER_BUILDKIT=1 - -echo "Build Configuration:" -echo " - BuildKit: enabled" -echo " - Project: $PROJECT_DIR" -echo "" - -# Benchmark function -benchmark_build() { - local name=$1 - local dockerfile=$2 - local tag=$3 - - echo "----------------------------------------" - echo "Building: $name" - echo " Dockerfile: $dockerfile" - echo " Tag: $tag" - echo "----------------------------------------" - - # Clear build cache for fair comparison - docker builder prune -f --filter type=exec.cachemount 2>/dev/null || true - - # Build with timing - local start_time=$(date +%s) - - if docker buildx build \ - -f "$dockerfile" \ - -t "$tag" \ - --progress=plain \ - --no-cache \ - . 2>&1 | tee /tmp/build_${name}.log; then - - local end_time=$(date +%s) - local duration=$((end_time - start_time)) - - # Get image size - local size=$(docker images "$tag" --format "{{.Size}}") - - echo "" - echo -e "${GREEN}SUCCESS:${NC} $name" - echo " Build time: $(format_time $duration)" - echo " Image size: $size" - - # Store results - echo "$name,$duration,$size" >> /tmp/benchmark_results.csv - else - echo -e "${RED}FAILED:${NC} $name" - echo "$name,FAILED,FAILED" >> /tmp/benchmark_results.csv - fi - echo "" -} - -# Benchmark cached rebuild -benchmark_cached_build() { - local name=$1 - local dockerfile=$2 - local tag=$3 - - echo "----------------------------------------" - echo "Cached Rebuild: $name" - echo "----------------------------------------" - - # Make a small change to trigger rebuild - touch src/counting/__init__.py - - local start_time=$(date +%s) - - if docker buildx build \ - -f "$dockerfile" \ - -t "$tag" \ - --progress=plain \ - . 2>&1 | tee /tmp/cached_${name}.log; then - - local end_time=$(date +%s) - local duration=$((end_time - start_time)) - - echo -e "${GREEN}Cached rebuild time:${NC} $(format_time $duration)" - echo "$name-cached,$duration,-" >> /tmp/benchmark_results.csv - fi - echo "" -} - -# Initialize results file -echo "build,duration_seconds,size" > /tmp/benchmark_results.csv - -# Check if both Dockerfiles exist -if [[ ! -f "Dockerfile" ]]; then - echo -e "${RED}ERROR:${NC} Original Dockerfile not found" - exit 1 -fi - -if [[ ! -f "Dockerfile.optimized" ]]; then - echo -e "${YELLOW}WARNING:${NC} Dockerfile.optimized not found, skipping optimized build" - SKIP_OPTIMIZED=1 -fi - -# Run benchmarks -echo "" -echo "========================================" -echo "Phase 1: Fresh Builds (no cache)" -echo "========================================" - -benchmark_build "original" "Dockerfile" "wasp2:original" - -if [[ -z "$SKIP_OPTIMIZED" ]]; then - benchmark_build "optimized" "Dockerfile.optimized" "wasp2:optimized" -fi - -echo "" -echo "========================================" -echo "Phase 2: Cached Rebuilds" -echo "========================================" - -benchmark_cached_build "original" "Dockerfile" "wasp2:original" - -if [[ -z "$SKIP_OPTIMIZED" ]]; then - benchmark_cached_build "optimized" "Dockerfile.optimized" "wasp2:optimized" -fi - -echo "" -echo "========================================" -echo "Results Summary" -echo "========================================" -echo "" -cat /tmp/benchmark_results.csv | column -t -s ',' -echo "" - -# Security scan comparison -echo "========================================" -echo "Security Scan (Trivy)" -echo "========================================" - -if command -v trivy &> /dev/null; then - echo "" - echo "Scanning wasp2:original..." - trivy image --severity HIGH,CRITICAL --quiet wasp2:original 2>/dev/null || echo "Scan failed or no issues" - - if [[ -z "$SKIP_OPTIMIZED" ]]; then - echo "" - echo "Scanning wasp2:optimized..." - trivy image --severity HIGH,CRITICAL --quiet wasp2:optimized 2>/dev/null || echo "Scan failed or no issues" - fi -else - echo "Trivy not installed. Install with: brew install trivy" -fi - -echo "" -echo "========================================" -echo "Benchmark Complete" -echo "========================================" -echo "" -echo "Full build logs available at:" -echo " /tmp/build_original.log" -if [[ -z "$SKIP_OPTIMIZED" ]]; then - echo " /tmp/build_optimized.log" -fi