MetaPathways 4.0.0: Nextflow local/Slurm workflows, PGDB builds and explorer

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,28 @@
+name: Bug report
+description: Report a problem running MetaPathways or using its reports.
+title: "[Bug]: "
+body:
+  - type: input
+    id: version
+    attributes:
+      label: MetaPathways version
+      description: Shown at the bottom of the explorer or by metapathways version.
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Installation and execution environment
+      description: For example, Mamba installation on Linux; local execution or Slurm.
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: What happened?
+      description: Include the command or explorer steps, expected result, and actual result.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant logs or screenshots
+      description: Paste relevant error messages or attach a small example.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,18 @@
+name: Feature request
+description: Suggest an improvement to MetaPathways or its explorer.
+title: "[Feature]: "
+body:
+  - type: textarea
+    id: need
+    attributes:
+      label: What would you like to do?
+      description: Describe the task and what currently makes it difficult.
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Suggested improvement
+      description: Describe the behavior or output you would find useful.
+    validations:
+      required: true

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+## Problem and resulting behavior
+
+Describe the concrete change and any compatibility implications.
+
+## Validation
+
+- Tested commit:
+- Local tests and installed-package checks:
+- Single/two-sample test workflow and checkpoint reuse:
+- Report/explorer and CSV export:
+- Optional Pathway Tools and Slurm checks (or explicitly untested):
+- Nonpublishing Release workflow/artifact links:
+
+See [the tester checklist](https://hallamlab-metapathways.readthedocs.io/en/latest/pr-testing.html). Feature PRs target `dev`; keep them in draft until single-server and Slurm benchmark review plus independent tester sign-off are complete. Production promotion uses a separate `dev` → `main` PR. Do not include licensed installers/images, credentials, or production outputs.

.github/workflows/release.yml

@@ -5,10 +5,26 @@ on:
     tags: ['v*']
   workflow_dispatch:
     inputs:
+      publish_anaconda:
+        description: 'Publish validated Conda package to Anaconda.org'
+        type: boolean
+        default: false
+      publish_quay:
+        description: 'Publish validated container to Quay'
+        type: boolean
+        default: false
+      publish_github:
+        description: 'Publish GitHub release (may trigger linked Zenodo archiving)'
+        type: boolean
+        default: false
       release_tag:
-        description: 'Existing tag to publish/recover (blank = build only)'
+        description: 'Existing version tag to build or publish (required for publishing from a branch)'
         type: string
         default: ''
+      test_containers:
+        description: 'Build/test Docker and SIF without enabling publication'
+        type: boolean
+        default: false
       source_run_id:
         description: 'Optional successful build run to reuse; requires release_tag'
         type: string
@@ -23,7 +39,21 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  controls:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Validate publication selections
+        env:
+          PUBLISH_ANACONDA: ${{ inputs.publish_anaconda || false }}
+          PUBLISH_QUAY: ${{ inputs.publish_quay || false }}
+          PUBLISH_GITHUB: ${{ inputs.publish_github || false }}
+          RELEASE_TAG_INPUT: ${{ inputs.release_tag }}
+        run: python3 scripts/check_release_controls.py
   build:
+    needs: controls
     runs-on: ubuntu-22.04
     timeout-minutes: 90
     defaults:
@@ -57,10 +87,13 @@ jobs:
         if: inputs.source_run_id == ''
         run: |
           conda install --yes --override-channels -c conda-forge conda-build conda-index pyyaml 'setuptools>=83,<85' wheel
-          python -m pip install build
+          python -m pip install build pandas pybedtools pyfastx tqdm scipy
       - name: Test release controls
         if: inputs.source_run_id == ''
         run: python -m unittest discover -s tests/release -v
+      - name: Test scheduling, containers, and read mapping
+        if: inputs.source_run_id == ''
+        run: python -m unittest discover -s tests -p 'test_*.py' -v
       - name: Build and test release
         if: inputs.source_run_id == ''
         env:
@@ -102,8 +135,9 @@ jobs:
           if-no-files-found: ignore
 
   github-release:
-    needs: build
-    if: github.ref_type == 'tag' || inputs.release_tag != ''
+    environment: release
+    needs: [build, containers]
+    if: github.event_name == 'workflow_dispatch' && inputs.publish_github
     runs-on: ubuntu-24.04
     permissions:
       contents: write
@@ -129,8 +163,9 @@ jobs:
         run: python scripts/release.py github-release "$RELEASE_TAG"
 
   anaconda:
-    needs: [build, github-release]
-    if: (github.ref_type == 'tag' || inputs.release_tag != '') && vars.PUBLISH_CONDA == 'true'
+    needs: build
+    if: github.event_name == 'workflow_dispatch' && inputs.publish_anaconda
+    environment: release
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v7
@@ -154,18 +189,19 @@ jobs:
       - name: Upload validated package
         shell: bash -el {0}
         env:
+          ANACONDA_OWNER: ${{ vars.ANACONDA_OWNER || 'hallamlab' }}
           BINSTAR_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
           RELEASE_TAG: ${{ inputs.release_tag || github.ref_name }}
         run: |
           if [[ -z "$BINSTAR_API_TOKEN" ]]; then
-            echo "::error::Set the ANACONDA_API_TOKEN secret before enabling PUBLISH_CONDA."
+            echo "::error::Set the ANACONDA_API_TOKEN secret before selecting publish_anaconda."
             exit 1
           fi
           python scripts/release.py upload-conda --ref "$RELEASE_TAG"
 
   containers:
-    needs: [build, github-release]
-    if: github.ref_type == 'tag' || inputs.release_tag != ''
+    needs: build
+    if: inputs.test_containers || inputs.publish_quay || inputs.publish_github
     runs-on: ubuntu-24.04
     timeout-minutes: 90
     permissions:
@@ -190,7 +226,14 @@ jobs:
       - name: Build Docker and Apptainer; validate containers
         env:
           RELEASE_TAG: ${{ inputs.release_tag || github.ref_name }}
-        run: python scripts/release.py container-build --ref "$RELEASE_TAG"
+          RELEASE_REF_TYPE: ${{ github.ref_type }}
+          RELEASE_TAG_INPUT: ${{ inputs.release_tag }}
+        run: |
+          if [[ "$RELEASE_REF_TYPE" == tag || -n "$RELEASE_TAG_INPUT" ]]; then
+            python scripts/release.py container-build --ref "$RELEASE_TAG"
+          else
+            python scripts/release.py container-build
+          fi
       - name: Scan container for fixable high and critical vulnerabilities
         run: |
           mkdir -p dist/security
@@ -216,28 +259,87 @@ jobs:
           name: container-security-report
           path: dist/security/
           if-no-files-found: ignore
+      - name: Export tested Docker image for independent publication
+        run: |
+          set -o pipefail
+          image=$(python -c 'import json; print(json.load(open("dist/containers/container-validation.json"))["image"])')
+          docker save "$image" | gzip > dist/containers/docker-image.tar.gz
+          cd dist/containers
+          sha256sum docker-image.tar.gz >> container-SHA256SUMS
       - name: Retain container assets and diagnostics
         if: always()
         uses: actions/upload-artifact@v7.0.1
         with:
           name: container-assets
           path: dist/containers/
           if-no-files-found: warn
+
+  github-containers:
+    needs: [containers, github-release]
+    if: github.event_name == 'workflow_dispatch' && inputs.publish_github
+    permissions:
+      contents: write
+    environment: release
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - uses: actions/download-artifact@v8.0.1
+        with:
+          name: release-assets
+          path: dist/release
+      - uses: actions/download-artifact@v8.0.1
+        with:
+          name: container-assets
+          path: dist/containers
       - name: Attach validated SIF and checksums to GitHub release
         env:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           RELEASE_TAG: ${{ inputs.release_tag || github.ref_name }}
         run: python scripts/release.py container-attach --ref "$RELEASE_TAG"
+
+  quay:
+    needs: containers
+    if: github.event_name == 'workflow_dispatch' && inputs.publish_quay
+    environment: release
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - uses: actions/download-artifact@v8.0.1
+        with:
+          name: release-assets
+          path: dist/release
+      - uses: actions/download-artifact@v8.0.1
+        with:
+          name: container-assets
+          path: dist/containers
+      - name: Verify and load the tested Docker image
+        env:
+          RELEASE_TAG: ${{ inputs.release_tag || github.ref_name }}
+        run: |
+          python scripts/release.py container-verify --ref "$RELEASE_TAG"
+          docker load -i dist/containers/docker-image.tar.gz
       - name: Publish Docker image to Quay
-        if: vars.PUBLISH_QUAY == 'true'
         env:
+          QUAY_REPOSITORY: ${{ vars.QUAY_REPOSITORY || 'quay.io/hallamlab/metapathways' }}
           QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
           QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
           RELEASE_TAG: ${{ inputs.release_tag || github.ref_name }}
         run: |
           if [[ -z "$QUAY_USERNAME" || -z "$QUAY_PASSWORD" ]]; then
-            echo "::error::Set QUAY_USERNAME and QUAY_PASSWORD secrets before enabling PUBLISH_QUAY."
+            echo "::error::Set QUAY_USERNAME and QUAY_PASSWORD secrets before selecting publish_quay."
             exit 1
           fi
           export DOCKER_CONFIG="$RUNNER_TEMP/metapathways-docker-auth"
@@ -246,7 +348,6 @@ jobs:
           printf '%s' "$QUAY_PASSWORD" | docker login quay.io --username "$QUAY_USERNAME" --password-stdin
           python scripts/release.py container-push --ref "$RELEASE_TAG"
       - name: Synchronize Quay repository overview
-        if: vars.PUBLISH_QUAY == 'true'
         env:
           QUAY_API_TOKEN: ${{ secrets.QUAY_API_TOKEN }}
         run: |

.github/workflows/smoke.yml

@@ -2,9 +2,9 @@ name: Package and documentation smoke tests
 
 on:
   push:
-    branches: [dev]
+    branches: [master, main, dev, "feat/**"]
   pull_request:
-    branches: [dev]
+    branches: [master, main, dev]
   workflow_dispatch:
 
 permissions:
@@ -21,21 +21,47 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: '3.11'
-      - name: Install build, documentation, and CLI import dependencies
+      - name: Install build and CLI import dependencies
         run: >-
           python -m pip install build 'setuptools>=83,<85' wheel
-          pandas pybedtools pyfastx tqdm scipy -r docs/requirements.txt
+          pandas pybedtools pyfastx tqdm scipy pyyaml cffconvert
+      - name: Validate software citation metadata
+        run: cffconvert --validate
       - name: Test release controls
         run: python -m unittest discover -s tests/release -v
+      - name: Test scheduling, containers, and read mapping
+        run: python -m unittest discover -s tests -p 'test_*.py' -v
       - name: Build source and wheel distributions
         run: python -m build --no-isolation
       - name: Check installed CLI outside the source directory
         run: |
-          python -m pip install --no-deps dist/*.whl
+          python -m pip install dist/*.whl
+          cp scripts/check_installed_assets.py "$RUNNER_TEMP/check_installed_assets.py"
           cd "$RUNNER_TEMP"
+          python check_installed_assets.py
           metapathways version
-          for command in run build_db mag_split ptools; do
+          magsplitter --help
+          python -c "import camelot_frs"
+          metapathways prepare_test -o test
+          for command in prepare_test run analysis_wf build_db mag_split build_pt ptools report; do
             metapathways "$command" -h
           done
-      - name: Build documentation with warnings treated as errors
-        run: sphinx-build -W --keep-going -b html docs/src docs/build
+      - name: Check GitHub documentation and CLI reference
+        run: python scripts/check_docs.py
+
+  documentation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - name: Install documentation dependencies
+        run: python -m pip install -r docs/requirements.txt
+      - name: Build the complete guide
+        run: python -m sphinx -W --keep-going -b html docs docs/_build/html
+      - name: Check rendered links
+        run: python scripts/check_docs_html.py docs/_build/html --readme README.md

.github/workflows/workflow-style.yml

@@ -0,0 +1,13 @@
+name: Workflow figure style
+on:
+  pull_request:
+  push:
+permissions:
+  contents: read
+jobs:
+  mp-nodal-style:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check MP nodal workflow figures
+        run: python3 scripts/check_workflow_style.py

.gitignore

@@ -85,3 +85,7 @@ global_errors_warnings.txt
 
 # Local manuscript figures and their export tools
 /docs/figures/
+
+# Nextflow local metadata; invocation copies are retained with analysis outputs.
+/.nextflow/
+/.nextflow.log*