From 890d79b5539842bd8d4f78c2b47596d5fb0921bb Mon Sep 17 00:00:00 2001 From: amandazhu Date: Mon, 14 Sep 2026 10:28:09 +1000 Subject: [PATCH 01/25] fix: show SU usage from NCI on admin dashboard --- app/services/results_utils.py | 9 ++++- tests/test_services_results_utils.py | 52 ++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/app/services/results_utils.py b/app/services/results_utils.py index f4b0217e..172d54bd 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -92,7 +92,14 @@ class WorkflowResultsSpec: def get_transfer_prefixes(self, run: WorkflowRun) -> list[str]: """Return run-scoped output prefixes that should be transferred from Gadi.""" - return _non_root_output_prefixes(run, self.get_prefixes(run)) + prefixes = _non_root_output_prefixes(run, self.get_prefixes(run)) + # UsageReport.csv is published straight to the run root by every workflow's + # `gadi.publishTo` (see e.g. bindflow.config), so it's excluded by the check + # above along with the rest of the root/work dir. Transfer it individually + # (non-recursive, since it doesn't end in "/") or NCI SU never reaches S3. + if run.id: + prefixes.append(f"{run.id}/UsageReport.csv") + return prefixes def get_transfer_items( self, diff --git a/tests/test_services_results_utils.py b/tests/test_services_results_utils.py index b1ebe81b..71222680 100644 --- a/tests/test_services_results_utils.py +++ b/tests/test_services_results_utils.py @@ -295,10 +295,17 @@ def test_workflow_results_spec_get_transfer_items_maps_output_prefixes(mock_sett destination_location=f"s3://test-s3-bucket/{run.id}/generate/report.html", recursive=False, ), + OutputTransferItem( + source_location=f"/test/output/de-novo-design/{run.id}/UsageReport.csv", + destination_location=f"s3://test-s3-bucket/{run.id}/UsageReport.csv", + recursive=False, + ), ] -def test_builtin_specs_get_transfer_prefixes_excludes_run_root(mock_settings): +def test_builtin_specs_get_transfer_prefixes_excludes_run_root_but_keeps_usage_report( + mock_settings, +): run = WorkflowRun(id=uuid4(), owner_user_id=uuid4(), sample_id="T1024") bindcraft_spec = WORKFLOW_OUTPUT_SPECS["de-novo-design"]["bindcraft"] @@ -312,23 +319,27 @@ def test_builtin_specs_get_transfer_prefixes_excludes_run_root(mock_settings): f"{run.id}/ranker/", f"{run.id}/generate/", f"{run.id}/bindcraft/T1024_0_output/", + f"{run.id}/UsageReport.csv", ] assert boltz_spec.get_transfer_prefixes(run) == [ f"{run.id}/reports/", f"{run.id}/boltz/top_ranked_structures/", f"{run.id}/mmseqs/", f"{run.id}/boltz/T1024/", + f"{run.id}/UsageReport.csv", ] assert alphafold2_spec.get_transfer_prefixes(run) == [ f"{run.id}/reports/", f"{run.id}/alphafold2/split_msa_prediction/top_ranked_structures/", f"{run.id}/alphafold2/split_msa_prediction/T1024/", + f"{run.id}/UsageReport.csv", ] assert colabfold_spec.get_transfer_prefixes(run) == [ f"{run.id}/reports/", f"{run.id}/colabfold/top_ranked_structures/", f"{run.id}/mmseqs/", f"{run.id}/colabfold/T1024/", + f"{run.id}/UsageReport.csv", ] assert wisps_spec.get_transfer_prefixes(run) == [ f"{run.id}/multiqc/", @@ -338,9 +349,11 @@ def test_builtin_specs_get_transfer_prefixes_excludes_run_root(mock_settings): f"{run.id}/boltz_predictions/pae/", f"{run.id}/colabfold_predictions/pdb/", f"{run.id}/colabfold_predictions/pae/", + f"{run.id}/UsageReport.csv", ] assert rfdiffusion_spec.get_transfer_prefixes(run) == [ f"{run.id}/results/", + f"{run.id}/UsageReport.csv", ] assert boltz_spec.get_transfer_items(run, settings=mock_settings) == [ @@ -366,6 +379,11 @@ def test_builtin_specs_get_transfer_prefixes_excludes_run_root(mock_settings): destination_location=f"s3://test-s3-bucket/{run.id}/boltz/T1024/", recursive=True, ), + OutputTransferItem( + source_location=f"/test/output/single-prediction/{run.id}/UsageReport.csv", + destination_location=f"s3://test-s3-bucket/{run.id}/UsageReport.csv", + recursive=False, + ), ] assert colabfold_spec.get_transfer_items(run, settings=mock_settings) == [ @@ -391,6 +409,11 @@ def test_builtin_specs_get_transfer_prefixes_excludes_run_root(mock_settings): destination_location=f"s3://test-s3-bucket/{run.id}/colabfold/T1024/", recursive=True, ), + OutputTransferItem( + source_location=f"/test/output/single-prediction/{run.id}/UsageReport.csv", + destination_location=f"s3://test-s3-bucket/{run.id}/UsageReport.csv", + recursive=False, + ), ] @@ -423,16 +446,21 @@ def test_workflow_results_spec_create_output_transfers_is_idempotent( DataTransfer.direction == "output", ) ).all() - assert len(first_result) == 4 - assert len(second_result) == 4 - assert len(output_transfers) == 4 + assert len(first_result) == 5 + assert len(second_result) == 5 + assert len(output_transfers) == 5 assert first_result[0].id == existing_transfer.id assert first_result[0].status == "in_progress" assert first_result[0].transfer_id == "task-existing" assert first_result[0].recursive is True assert [transfer.id for transfer in second_result] == [transfer.id for transfer in first_result] - assert [transfer.status for transfer in first_result[1:]] == ["pending", "pending", "pending"] - assert [transfer.recursive for transfer in first_result] == [True, True, True, True] + assert [transfer.status for transfer in first_result[1:]] == [ + "pending", + "pending", + "pending", + "pending", + ] + assert [transfer.recursive for transfer in first_result] == [True, True, True, True, False] def test_colabfold_create_output_transfers_creates_expected_rows( @@ -451,10 +479,10 @@ def test_colabfold_create_output_transfers_creates_expected_rows( DataTransfer.direction == "output", ) ).all() - assert len(result) == 4 - assert len(output_transfers) == 4 - assert [transfer.status for transfer in result] == ["pending"] * 4 - assert [transfer.recursive for transfer in result] == [True] * 4 + assert len(result) == 5 + assert len(output_transfers) == 5 + assert [transfer.status for transfer in result] == ["pending"] * 5 + assert [transfer.recursive for transfer in result] == [True, True, True, True, False] assert [(transfer.source_location, transfer.destination_location) for transfer in result] == [ ( f"/test/output/single-prediction/{run.id}/reports/", @@ -472,6 +500,10 @@ def test_colabfold_create_output_transfers_creates_expected_rows( f"/test/output/single-prediction/{run.id}/colabfold/T1024/", f"s3://test-s3-bucket/{run.id}/colabfold/T1024/", ), + ( + f"/test/output/single-prediction/{run.id}/UsageReport.csv", + f"s3://test-s3-bucket/{run.id}/UsageReport.csv", + ), ] From 4cbd0f7d2f6e94942c1c3b28f3af9d10cfc2f3de Mon Sep 17 00:00:00 2001 From: amandazhu Date: Mon, 14 Sep 2026 10:31:33 +1000 Subject: [PATCH 02/25] docs: simpler comments --- app/services/results_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/services/results_utils.py b/app/services/results_utils.py index 172d54bd..cbd73826 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -93,10 +93,7 @@ class WorkflowResultsSpec: def get_transfer_prefixes(self, run: WorkflowRun) -> list[str]: """Return run-scoped output prefixes that should be transferred from Gadi.""" prefixes = _non_root_output_prefixes(run, self.get_prefixes(run)) - # UsageReport.csv is published straight to the run root by every workflow's - # `gadi.publishTo` (see e.g. bindflow.config), so it's excluded by the check - # above along with the rest of the root/work dir. Transfer it individually - # (non-recursive, since it doesn't end in "/") or NCI SU never reaches S3. + # Also include the UsageReport.csv if run.id: prefixes.append(f"{run.id}/UsageReport.csv") return prefixes From 63dfd7ec8ad3d127ca40405840a2c34ac60fdb23 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:11:25 +1000 Subject: [PATCH 03/25] fix: update single prediction cost multiplier --- app/schemas/workflows/single_prediction.py | 2 +- app/services/credits.py | 2 +- tests/test_routes_workflows.py | 6 +++--- tests/test_schemas.py | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/schemas/workflows/single_prediction.py b/app/schemas/workflows/single_prediction.py index 210b4132..8721652a 100644 --- a/app/schemas/workflows/single_prediction.py +++ b/app/schemas/workflows/single_prediction.py @@ -18,7 +18,7 @@ SINGLE_PREDICTION_MAX_ENTITIES = 52 SINGLE_PREDICTION_LIGAND_SIZE = 30 SINGLE_PREDICTION_SIZE_LIMITS: dict[str, int] = { - "alphafold2": 2000, + "alphafold2": 1000, "colabfold": 4000, "boltz": 4000, } diff --git a/app/services/credits.py b/app/services/credits.py index 9c3d4d3d..9c1df94f 100644 --- a/app/services/credits.py +++ b/app/services/credits.py @@ -91,7 +91,7 @@ class WorkflowCreditsResponse(BaseModel): category="single-prediction", displayName="Single Prediction", basis=CreditBasis.CONSTANT, - toolMultipliers={"boltz": 50, "colabfold": 50, "alphafold2": 200}, + toolMultipliers={"boltz": 50, "colabfold": 50, "alphafold2": 50}, ), WorkflowCreditConfig( category="bulk-prediction", diff --git a/tests/test_routes_workflows.py b/tests/test_routes_workflows.py index 70b49a86..cca80dec 100644 --- a/tests/test_routes_workflows.py +++ b/tests/test_routes_workflows.py @@ -891,10 +891,10 @@ def test_launch_single_prediction_requires_protein(client: TestClient, test_engi def test_launch_single_prediction_rejects_oversized_alphafold2(client: TestClient, test_engine): _add_proteinfold_workflow(test_engine) - payload = _single_prediction_payload([_protein_entity(sequence="A" * 2000)], tool="alphafold2") + payload = _single_prediction_payload([_protein_entity(sequence="A" * 1000)], tool="alphafold2") response = client.post("/api/workflows/launch", json=payload) assert response.status_code == 422 - assert "less than 2000" in response.json()["detail"] + assert "less than 1000" in response.json()["detail"] @patch("app.routes.workflows.upload_csv_to_s3") @@ -1319,7 +1319,7 @@ def test_get_workflow_credits_multipliers_match_spec(client: TestClient): single = by_category["single-prediction"] assert single["basis"] == CreditBasis.CONSTANT.value - assert single["toolMultipliers"] == {"boltz": 50, "colabfold": 50, "alphafold2": 200} + assert single["toolMultipliers"] == {"boltz": 50, "colabfold": 50, "alphafold2": 50} bulk = by_category["bulk-prediction"] assert bulk["basis"] == CreditBasis.FASTA_ENTRY_COUNT.value diff --git a/tests/test_schemas.py b/tests/test_schemas.py index b7d85d7c..8a453a48 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -493,7 +493,7 @@ def _protein(sequence="ACDEFGHIK", copy_number=1): def test_size_limit_defaults_and_boltz_potentials(): assert single_prediction_size_limit("colabfold", False) == 4000 assert single_prediction_size_limit("boltz", False) == 4000 - assert single_prediction_size_limit("alphafold2", False) == 2000 + assert single_prediction_size_limit("alphafold2", False) == 1000 assert single_prediction_size_limit("boltz", True) == 2000 assert single_prediction_size_limit("colabfold", True) == 4000 @@ -550,9 +550,9 @@ def test_validate_single_prediction_ligand_uses_fixed_size(): def test_validate_single_prediction_size_limit_is_exclusive(): - with pytest.raises(ValueError, match="less than 2000"): - validate_single_prediction_entities([_protein(sequence="A" * 2000)], "alphafold2") - validate_single_prediction_entities([_protein(sequence="A" * 1999)], "alphafold2") + with pytest.raises(ValueError, match="less than 1000"): + validate_single_prediction_entities([_protein(sequence="A" * 1000)], "alphafold2") + validate_single_prediction_entities([_protein(sequence="A" * 999)], "alphafold2") # Short lists of valid/invalid examples per molecule type. Parametrized below From ab2844ba4c36aa57dbd67aef4b5c44b6dc3e2d9b Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:45:15 +1000 Subject: [PATCH 04/25] feat: force result syncing for completed workflow --- app/db/admin.py | 93 ++++++++++++++++++ app/services/job_sync.py | 42 ++++++++ tests/db/test_db_admin.py | 166 ++++++++++++++++++++++++++++++++ tests/test_services_job_sync.py | 53 ++++++++++ 4 files changed, 354 insertions(+) diff --git a/app/db/admin.py b/app/db/admin.py index 36ea57fa..e9daa703 100644 --- a/app/db/admin.py +++ b/app/db/admin.py @@ -10,6 +10,7 @@ import hmac import io import json +import logging import os import secrets from datetime import UTC, datetime @@ -36,8 +37,10 @@ from ..auth.validator import fetch_userinfo_claims, verify_access_token_claims from ..config import Settings, get_settings from ..routes.dependencies import get_db +from ..schemas.workflows.shared import PipelineStatus from ..services.credits import launch_credit_cost from ..services.globus_transfer import reset_failed_output_transfers +from ..services.job_sync import force_resync_run_outputs from . import engine from .models import job_queue from .models.core import ( @@ -51,6 +54,8 @@ WorkflowRun, ) +logger = logging.getLogger(__name__) + _ADMIN_TEMPLATES_DIR = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "templates" ) @@ -289,6 +294,94 @@ def get_search_query(self, request: Request, term: str) -> Any: ) return or_(base_clause, owner_clause) + @action( + name="force_resync_outputs", + text="Force resync outputs", + confirmation=( + "Re-scan S3 and resync result outputs for the selected completed " + "runs, even though they're already marked as synced? Submits a " + "new output transfer first for any prefix the current spec " + "requires but hasn't been transferred yet. Use this after a " + "results-utils change adds a new output category or classifier, " + "so existing runs pick up newly recognised files." + ), + submit_btn_text="Force resync", + submit_btn_class="btn-warning", + icon_class="fa-solid fa-arrows-rotate", + ) + async def force_resync_outputs_action(self, request: Request, pks: list[Any]) -> str: + runs = await self.find_by_pks(request, pks) + return await _force_resync_workflow_runs(request.state.session, runs) + + @row_action( + name="force_resync_outputs", + text="Force resync outputs", + confirmation=( + "Re-scan S3 and resync result outputs for this run, even though " + "it's already marked as synced? Submits a new output transfer " + "first if the current spec requires a prefix that hasn't been " + "transferred yet." + ), + submit_btn_text="Force resync", + submit_btn_class="btn-warning", + action_btn_class="btn-warning", + icon_class="fa-solid fa-arrows-rotate", + ) + async def row_action_force_resync_outputs(self, request: Request, pk: Any) -> str: + run = await self.find_by_pk(request, pk) + if run is None: + raise ActionFailed("Workflow run not found.") + return await _force_resync_workflow_runs(request.state.session, [run]) + + +async def _force_resync_workflow_runs(db: Session, runs: list[WorkflowRun]) -> str: + """Force-resync result outputs for the given completed workflow runs. + + Bypasses the usual sync_completed_at short-circuit so runs pick up any + S3 keys that a *new* results-utils category/classifier now recognises but + didn't exist (from this run's perspective) at the time it last synced - + submitting a new output transfer first if the spec now requires a prefix + that was never transferred out in the first place (see + force_resync_run_outputs). + """ + processed = 0 + submitted = 0 + skipped = 0 + errored = 0 + outputs_synced = 0 + + for run in runs: + if run.seqera_final_status != PipelineStatus.SUCCEEDED.value: + skipped += 1 + continue + try: + outcome = await force_resync_run_outputs(db, run) + except Exception: + db.rollback() + logger.exception("Force resync of outputs failed for workflow run %s", run.id) + errored += 1 + continue + + if outcome.ready: + processed += 1 + outputs_synced += outcome.outputs_synced + else: + submitted += 1 + + if processed == 0 and submitted == 0 and errored == 0: + raise ActionFailed( + "No completed runs were selected - only SUCCEEDED runs can be force-resynced." + ) + + summary = [f"{processed} run(s) resynced ({outputs_synced} output(s) found)"] + if submitted: + summary.append(f"{submitted} submitted new output transfer(s), not yet ready to resync") + if skipped: + summary.append(f"{skipped} skipped (not completed)") + if errored: + summary.append(f"{errored} errored") + return ", ".join(summary) + "." + def _build_workflow_runs_csv(session: Session) -> str: """Build a CSV dump of every workflow run row. diff --git a/app/services/job_sync.py b/app/services/job_sync.py index e09b4e49..2b98299e 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -318,6 +318,48 @@ async def finalize_completed_workflow_run( return outputs_synced +@dataclass(frozen=True) +class ForceResyncOutcome: + """Outcome of a force-resync attempt for one completed run.""" + + # False if a new output transfer was just submitted (or an existing one is + # still in flight) and results weren't re-synced this pass. + ready: bool + outputs_synced: int + + +async def force_resync_run_outputs( + db: Session, + run: WorkflowRun, + *, + suppress_s3_errors: bool = True, + settings: Settings | None = None, +) -> ForceResyncOutcome: + """ + Force a completed run to pick up results-utils spec changes: submit any + output transfer a prefix added since this run last synced now requires + (`_ensure_completed_run_output_transfers` is idempotent - it only creates + a transfer for a source/destination pair that doesn't already have one), + then re-scan S3 and resync result metadata even if already synced. + + Unlike `sync_workflow_run(..., force=True)`, this never re-polls Seqera - + callers must have already confirmed `run.seqera_final_status` is + SUCCEEDED. + """ + output_transfer_state = _ensure_completed_run_output_transfers(db, run, settings=settings) + if not output_transfer_state.ready: + return ForceResyncOutcome(ready=False, outputs_synced=0) + + outputs_synced = await finalize_completed_workflow_run( + db, + run, + force=True, + suppress_s3_errors=suppress_s3_errors, + settings=settings, + ) + return ForceResyncOutcome(ready=True, outputs_synced=outputs_synced) + + def check_all_output_transfers_completed(db: Session, run: WorkflowRun) -> bool: output_transfers = db.scalars( select(DataTransfer).where( diff --git a/tests/db/test_db_admin.py b/tests/db/test_db_admin.py index 35259ab1..4fcf843b 100644 --- a/tests/db/test_db_admin.py +++ b/tests/db/test_db_admin.py @@ -5,6 +5,7 @@ import os from collections.abc import Generator from types import SimpleNamespace +from unittest.mock import AsyncMock from uuid import uuid4 import pytest @@ -34,6 +35,7 @@ ) from app.db.models.core import AppUser, DataTransfer, RunInput, RunOutput, S3Object, WorkflowRun from app.routes.dependencies import get_db +from app.services.job_sync import ForceResyncOutcome from tests.conftest import SettingsNoEnv DB_ADMIN_REQUIRED_ENV = { @@ -390,6 +392,170 @@ def test_workflow_run_admin_sbp_credit_not_sortable() -> None: assert "sbp_credit" not in WorkflowRunAdmin.sortable_fields +def _admin_action_request(action: RequestAction) -> Request: + request = Request( + { + "type": "http", + "method": "POST", + "path": "/", + "headers": [], + "query_string": b"", + "server": ("testserver", 80), + "scheme": "http", + "client": ("testclient", 123), + } + ) + request.state.action = action + return request + + +async def test_workflow_run_admin_force_resync_row_action_resyncs_succeeded_run( + test_db, mocker +) -> None: + user = AppUser( + id=uuid4(), + auth0_user_id="auth0|force-resync-row", + name="Force Resync Row", + email="force-resync-row@example.com", + ) + run = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="force-resync-row-run", + work_dir="/tmp/force-resync-row-run", + seqera_final_status="SUCCEEDED", + ) + test_db.add_all([user, run]) + test_db.commit() + + force_resync = mocker.patch( + "app.db.admin.force_resync_run_outputs", + new_callable=AsyncMock, + return_value=ForceResyncOutcome(ready=True, outputs_synced=3), + ) + + request = _admin_action_request(RequestAction.ROW_ACTION) + request.state.session = test_db + + view = WorkflowRunAdmin(WorkflowRun) + message = await view.handle_row_action(request, str(run.id), "force_resync_outputs") + + assert message == "1 run(s) resynced (3 output(s) found)." + force_resync.assert_called_once() + args, _kwargs = force_resync.call_args + assert args[0] is test_db + assert args[1].id == run.id + + +async def test_workflow_run_admin_force_resync_row_action_rejects_non_succeeded_run( + test_db, mocker +) -> None: + user = AppUser( + id=uuid4(), + auth0_user_id="auth0|force-resync-row-reject", + name="Force Resync Row Reject", + email="force-resync-row-reject@example.com", + ) + run = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="force-resync-row-reject-run", + work_dir="/tmp/force-resync-row-reject-run", + seqera_final_status="RUNNING", + ) + test_db.add_all([user, run]) + test_db.commit() + + force_resync = mocker.patch( + "app.db.admin.force_resync_run_outputs", new_callable=AsyncMock + ) + + request = _admin_action_request(RequestAction.ROW_ACTION) + request.state.session = test_db + + view = WorkflowRunAdmin(WorkflowRun) + with pytest.raises(ActionFailed, match="only SUCCEEDED runs"): + await view.handle_row_action(request, str(run.id), "force_resync_outputs") + + force_resync.assert_not_called() + + +async def test_workflow_run_admin_force_resync_batch_action_reports_counts(test_db, mocker) -> None: + user = AppUser( + id=uuid4(), + auth0_user_id="auth0|force-resync-batch", + name="Force Resync Batch", + email="force-resync-batch@example.com", + ) + succeeded_ok = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="force-resync-batch-ok", + work_dir="/tmp/force-resync-batch-ok", + seqera_final_status="SUCCEEDED", + ) + succeeded_err = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="force-resync-batch-err", + work_dir="/tmp/force-resync-batch-err", + seqera_final_status="SUCCEEDED", + ) + succeeded_new_transfer = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="force-resync-batch-new-transfer", + work_dir="/tmp/force-resync-batch-new-transfer", + seqera_final_status="SUCCEEDED", + ) + still_running = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="force-resync-batch-running", + work_dir="/tmp/force-resync-batch-running", + seqera_final_status="RUNNING", + ) + test_db.add_all( + [user, succeeded_ok, succeeded_err, succeeded_new_transfer, still_running] + ) + test_db.commit() + + async def fake_force_resync(db, run, *, suppress_s3_errors=True, settings=None): + if run.id == succeeded_ok.id: + return ForceResyncOutcome(ready=True, outputs_synced=2) + if run.id == succeeded_err.id: + raise RuntimeError("s3 boom") + if run.id == succeeded_new_transfer.id: + return ForceResyncOutcome(ready=False, outputs_synced=0) + raise AssertionError("force_resync_run_outputs called for a skipped run") + + mocker.patch( + "app.db.admin.force_resync_run_outputs", + new=AsyncMock(side_effect=fake_force_resync), + ) + + request = _admin_action_request(RequestAction.ACTION) + request.state.session = test_db + + view = WorkflowRunAdmin(WorkflowRun) + message = await view.handle_action( + request, + [ + str(succeeded_ok.id), + str(succeeded_err.id), + str(succeeded_new_transfer.id), + str(still_running.id), + ], + "force_resync_outputs", + ) + + assert message == ( + "1 run(s) resynced (2 output(s) found), " + "1 submitted new output transfer(s), not yet ready to resync, " + "1 skipped (not completed), 1 errored." + ) + + @pytest.mark.parametrize( ("raw", "expected"), [ diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index 501e0d54..b825cb7d 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -219,6 +219,59 @@ async def test_sync_workflow_run_sync_completed_reflects_finalized_state( assert run.sync_completed_at is None +@pytest.mark.asyncio +async def test_force_resync_run_outputs_submits_new_transfer_before_resyncing( + test_db, persistent_models, monkeypatch +): + """A results-utils spec update that now requires a prefix never + transferred before must submit that transfer, not just re-list whatever + is already in S3.""" + run = _create_run( + seqera_final_status="SUCCEEDED", + sync_completed_at=datetime.now(tz=UTC), + ) + ensure_transfers = MagicMock(return_value=job_sync.OutputTransferState(ready=False)) + finalize = AsyncMock(return_value=0) + + monkeypatch.setattr(job_sync, "_ensure_completed_run_output_transfers", ensure_transfers) + monkeypatch.setattr(job_sync, "finalize_completed_workflow_run", finalize) + + outcome = await job_sync.force_resync_run_outputs(test_db, run) + + ensure_transfers.assert_called_once_with(test_db, run, settings=None) + finalize.assert_not_awaited() + assert outcome.ready is False + assert outcome.outputs_synced == 0 + + +@pytest.mark.asyncio +async def test_force_resync_run_outputs_resyncs_once_transfers_are_ready( + test_db, persistent_models, monkeypatch +): + run = _create_run( + seqera_final_status="SUCCEEDED", + sync_completed_at=datetime.now(tz=UTC), + ) + ensure_transfers = MagicMock(return_value=job_sync.OutputTransferState(ready=True)) + finalize = AsyncMock(return_value=4) + + monkeypatch.setattr(job_sync, "_ensure_completed_run_output_transfers", ensure_transfers) + monkeypatch.setattr(job_sync, "finalize_completed_workflow_run", finalize) + + outcome = await job_sync.force_resync_run_outputs(test_db, run) + + ensure_transfers.assert_called_once_with(test_db, run, settings=None) + finalize.assert_awaited_once_with( + test_db, + run, + force=True, + suppress_s3_errors=True, + settings=None, + ) + assert outcome.ready is True + assert outcome.outputs_synced == 4 + + @pytest.mark.asyncio @pytest.mark.parametrize("seqera_status", ["FAILED", "CANCELLED"]) async def test_sync_workflow_run_terminal_non_success_records_status_without_result_sync( From 71fef33aa07f0628ec3d130a69c619cb8dc49913 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:32:47 +1000 Subject: [PATCH 05/25] chore: lint --- tests/db/test_db_admin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/db/test_db_admin.py b/tests/db/test_db_admin.py index 4fcf843b..5eb83a3e 100644 --- a/tests/db/test_db_admin.py +++ b/tests/db/test_db_admin.py @@ -466,9 +466,7 @@ async def test_workflow_run_admin_force_resync_row_action_rejects_non_succeeded_ test_db.add_all([user, run]) test_db.commit() - force_resync = mocker.patch( - "app.db.admin.force_resync_run_outputs", new_callable=AsyncMock - ) + force_resync = mocker.patch("app.db.admin.force_resync_run_outputs", new_callable=AsyncMock) request = _admin_action_request(RequestAction.ROW_ACTION) request.state.session = test_db @@ -515,9 +513,7 @@ async def test_workflow_run_admin_force_resync_batch_action_reports_counts(test_ work_dir="/tmp/force-resync-batch-running", seqera_final_status="RUNNING", ) - test_db.add_all( - [user, succeeded_ok, succeeded_err, succeeded_new_transfer, still_running] - ) + test_db.add_all([user, succeeded_ok, succeeded_err, succeeded_new_transfer, still_running]) test_db.commit() async def fake_force_resync(db, run, *, suppress_s3_errors=True, settings=None): From 417ebee190cb6de1b5d8089ce3fb552f7f273540 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Tue, 15 Sep 2026 18:21:16 +1000 Subject: [PATCH 06/25] fix: interpret PBS qtime/stime as Australia/Sydney, not UTC --- app/services/gadi_pbs_jobs.py | 13 +++++++++++-- tests/test_services_gadi_pbs_jobs.py | 29 ++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/app/services/gadi_pbs_jobs.py b/app/services/gadi_pbs_jobs.py index 05a5e9d8..d7d616c8 100644 --- a/app/services/gadi_pbs_jobs.py +++ b/app/services/gadi_pbs_jobs.py @@ -31,6 +31,7 @@ from dataclasses import dataclass from datetime import UTC, datetime from typing import Any +from zoneinfo import ZoneInfo from ..config import Settings, get_settings from .s3 import read_s3_file @@ -55,6 +56,13 @@ # "Mon Jun 1 03:00:00 2026". _PBS_DATETIME_FORMAT = "%a %b %d %H:%M:%S %Y" +# qtime/stime are the Gadi server's local wall-clock time, not UTC - confirmed +# against real values that only made sense as Australia/Sydney (Gadi/Canberra +# share the same zone and DST rules). Labeling them UTC directly, as this used +# to do, left every timestamp off by the Sydney UTC offset once the dashboard +# converted them to Sydney time for display a second time. +_PBS_TIMEZONE = ZoneInfo("Australia/Sydney") + class GadiPbsJobsError(RuntimeError): """Raised when the pushed jobs object is missing, unreadable, or malformed.""" @@ -108,9 +116,10 @@ def _parse_pbs_datetime(raw: Any) -> datetime | None: if not isinstance(raw, str) or not raw.strip(): return None try: - return datetime.strptime(raw.strip(), _PBS_DATETIME_FORMAT).replace(tzinfo=UTC) + naive = datetime.strptime(raw.strip(), _PBS_DATETIME_FORMAT) except ValueError: return None + return naive.replace(tzinfo=_PBS_TIMEZONE).astimezone(UTC) def _decode_account_name(raw: Any) -> str | None: @@ -132,7 +141,7 @@ def _decode_account_name(raw: Any) -> str | None: try: decoded_user = base64.b64decode(user_part, validate=True).decode("utf-8") base64.b64decode(ip_part, validate=True) # confirms the pattern; IP itself is discarded - except binascii.Error, ValueError, UnicodeDecodeError: + except (binascii.Error, ValueError, UnicodeDecodeError): return raw return decoded_user diff --git a/tests/test_services_gadi_pbs_jobs.py b/tests/test_services_gadi_pbs_jobs.py index cfc5249d..e70c9298 100644 --- a/tests/test_services_gadi_pbs_jobs.py +++ b/tests/test_services_gadi_pbs_jobs.py @@ -72,8 +72,11 @@ async def test_get_pbs_jobs_parses_pushed_snapshot(): assert running.state_label == "Running" assert running.queue == "normal" assert running.account == "yz52" - assert running.submitted_at == datetime(2026, 6, 1, 2, 55, 0, tzinfo=UTC) - assert running.started_at == datetime(2026, 6, 1, 3, 0, 0, tzinfo=UTC) + # qtime/stime are Gadi's local (Australia/Sydney) wall-clock time, not + # UTC - "02:55:00"/"03:00:00" on 1 Jun (AEST, UTC+10, no DST) is + # 16:55:00/17:00:00 UTC on 31 May. + assert running.submitted_at == datetime(2026, 5, 31, 16, 55, 0, tzinfo=UTC) + assert running.started_at == datetime(2026, 5, 31, 17, 0, 0, tzinfo=UTC) queued = by_id["12346.gadi-pbs"] assert queued.state == "Q" @@ -286,6 +289,28 @@ async def test_get_pbs_jobs_tolerates_unparseable_timestamps(): assert snapshot.jobs[0].submitted_at is None +@pytest.mark.asyncio +async def test_get_pbs_jobs_converts_local_sydney_time_across_dst(): + """qtime/stime are Gadi's local Australia/Sydney wall-clock time, not + UTC - this must apply the correct offset on both sides of DST (AEST in + winter, AEDT in summer), not a fixed one.""" + payload = json.dumps( + { + "generatedAt": "2026-06-01T03:00:00Z", + "qstatJobs": { + "Jobs": {"1.gadi-pbs": {"job_state": "Q", "qtime": "Thu Jan 15 10:00:00 2026"}} + }, + } + ) + with patch( + "app.services.gadi_pbs_jobs.read_s3_file", new_callable=AsyncMock, return_value=payload + ): + snapshot = await get_pbs_jobs() + + # 10:00 AEDT (UTC+11, daylight saving in January) -> 23:00 UTC the prior day. + assert snapshot.jobs[0].submitted_at == datetime(2026, 1, 14, 23, 0, 0, tzinfo=UTC) + + @pytest.mark.asyncio async def test_get_pbs_jobs_raises_when_object_missing(): with patch( From a54f8c07d31c31a68c7a97ac177625047447d056 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Tue, 15 Sep 2026 18:24:44 +1000 Subject: [PATCH 07/25] fix: lint --- app/services/gadi_pbs_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/gadi_pbs_jobs.py b/app/services/gadi_pbs_jobs.py index d7d616c8..ddedcdc0 100644 --- a/app/services/gadi_pbs_jobs.py +++ b/app/services/gadi_pbs_jobs.py @@ -141,7 +141,7 @@ def _decode_account_name(raw: Any) -> str | None: try: decoded_user = base64.b64decode(user_part, validate=True).decode("utf-8") base64.b64decode(ip_part, validate=True) # confirms the pattern; IP itself is discarded - except (binascii.Error, ValueError, UnicodeDecodeError): + except binascii.Error, ValueError, UnicodeDecodeError: return raw return decoded_user From 40fcb0ff65620b63243a144cc6aab76d8c266737 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:07:43 +1000 Subject: [PATCH 08/25] fix: force missing transfer files --- app/services/job_sync.py | 14 ++++++--- app/services/job_utils.py | 18 +++++++++--- tests/test_services_job_sync.py | 4 +-- tests/test_services_job_utils.py | 49 ++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/app/services/job_sync.py b/app/services/job_sync.py index 2b98299e..bb137176 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -311,6 +311,7 @@ async def finalize_completed_workflow_run( run, suppress_s3_errors=suppress_s3_errors, settings=settings, + force=force, ) run.sync_completed_at = datetime.now(tz=UTC) db.add(run) @@ -381,6 +382,7 @@ async def _sync_completed_run_results( *, suppress_s3_errors: bool, settings: Settings | None = None, + force: bool = False, ) -> int: try: spec = get_output_spec(run) @@ -395,8 +397,8 @@ async def _sync_completed_run_results( spec=spec, suppress_s3_errors=suppress_s3_errors, ) - await ensure_completed_run_score(db, run, UIStatus.COMPLETED.value) - await sync_service_usage(db, run, UIStatus.COMPLETED.value) + await ensure_completed_run_score(db, run, UIStatus.COMPLETED.value, force=force) + await sync_service_usage(db, run, UIStatus.COMPLETED.value, force=force) else: synced_keys = await sync_workflow_outputs( db, @@ -405,8 +407,12 @@ async def _sync_completed_run_results( suppress_s3_errors=suppress_s3_errors, settings=settings, ) - await ensure_completed_run_score(db, run, UIStatus.COMPLETED.value, settings=settings) - await sync_service_usage(db, run, UIStatus.COMPLETED.value, settings=settings) + await ensure_completed_run_score( + db, run, UIStatus.COMPLETED.value, settings=settings, force=force + ) + await sync_service_usage( + db, run, UIStatus.COMPLETED.value, settings=settings, force=force + ) return len(synced_keys) diff --git a/app/services/job_utils.py b/app/services/job_utils.py index 6bc67fcf..49cae5c1 100644 --- a/app/services/job_utils.py +++ b/app/services/job_utils.py @@ -304,13 +304,18 @@ def _get_sample_id_for_score(run: WorkflowRun) -> str | None: async def ensure_completed_run_score( - db: Session, run: WorkflowRun, ui_status: str, settings: Settings | None = None + db: Session, + run: WorkflowRun, + ui_status: str, + settings: Settings | None = None, + *, + force: bool = False, ) -> float | None: if ui_status != "Completed": return None existing = db.execute(select(RunMetric).where(RunMetric.run_id == run.id)).scalar_one_or_none() - if existing and existing.max_score is not None: + if not force and existing and existing.max_score is not None: return _round_score(existing.max_score) # Score computation is best-effort: a run with an unknown workflow/tool (e.g. a @@ -346,11 +351,16 @@ async def ensure_completed_run_score( async def sync_service_usage( - db: Session, run: WorkflowRun, ui_status: str, settings: Settings | None = None + db: Session, + run: WorkflowRun, + ui_status: str, + settings: Settings | None = None, + *, + force: bool = False, ) -> float | None: if ui_status != "Completed": return None - if run.service_usage is not None: + if not force and run.service_usage is not None: return run.service_usage try: diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index b825cb7d..64c93072 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -139,8 +139,8 @@ async def test_sync_workflow_run_succeeded_finalizes_after_output_transfers_comp spec=spec, suppress_s3_errors=True, ) - ensure_score.assert_awaited_once_with(test_db, run, "Completed") - sync_usage.assert_awaited_once_with(test_db, run, "Completed") + ensure_score.assert_awaited_once_with(test_db, run, "Completed", force=False) + sync_usage.assert_awaited_once_with(test_db, run, "Completed", force=False) assert result.terminal is True assert result.sync_completed is True assert result.outputs_synced == 2 diff --git a/tests/test_services_job_utils.py b/tests/test_services_job_utils.py index c1278856..7b6d6f8f 100644 --- a/tests/test_services_job_utils.py +++ b/tests/test_services_job_utils.py @@ -275,6 +275,55 @@ async def test_ensure_completed_run_score_branches(): fake_fail_spec.get_max_score.assert_awaited_once_with(db_fail, run) +@pytest.mark.asyncio +async def test_ensure_completed_run_score_force_recomputes_despite_existing_value(): + """force=True must re-derive the score even though a cached one already + exists - e.g. after a results-utils change, the underlying result file's + content may have changed since the score was first computed.""" + run = SimpleNamespace( + id="rid", + seqera_run_id="wf-1", + workflow=SimpleNamespace(name="de-novo-design"), + tool="bindcraft", + ) + db = _DB(scalar=SimpleNamespace(max_score=0.5)) + fake_spec = SimpleNamespace(get_max_score=AsyncMock(return_value=0.95)) + + with ( + patch("app.services.job_utils.get_output_spec", return_value=fake_spec), + patch("app.services.job_utils.sync_workflow_outputs", new_callable=AsyncMock), + ): + score = await job_utils.ensure_completed_run_score(db, run, "Completed", force=True) + + assert score == 0.95 + fake_spec.get_max_score.assert_awaited_once_with(db, run) + assert db.committed is True + + +@pytest.mark.asyncio +async def test_sync_service_usage_force_recomputes_despite_existing_value(): + run = SimpleNamespace( + id="rid", + seqera_run_id="wf-1", + workflow=SimpleNamespace(name="de-novo-design"), + tool="bindcraft", + service_usage=1.0, + ) + db = _DB() + fake_spec = SimpleNamespace(get_service_units=AsyncMock(return_value=2.5)) + + with ( + patch("app.services.job_utils.get_output_spec", return_value=fake_spec), + patch("app.services.job_utils.sync_workflow_outputs", new_callable=AsyncMock), + ): + usage = await job_utils.sync_service_usage(db, run, "Completed", force=True) + + assert usage == 2.5 + assert run.service_usage == 2.5 + fake_spec.get_service_units.assert_awaited_once_with(db, run) + assert db.committed is True + + @pytest.mark.asyncio async def test_ensure_completed_run_score_persists_spec_score(test_db): user = AppUser( From b672a9d7ee510c00de8ecd03bc73f5ef222435ba Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:46:46 +1000 Subject: [PATCH 09/25] fix: run outputs table in admin dashboard --- app/db/admin.py | 33 ++++--- app/services/job_sync.py | 20 ++--- app/templates/displays/safe_relation.html | 9 ++ tests/db/test_db_admin.py | 100 ++++++++++++++++++++++ tests/test_services_job_sync.py | 4 +- tests/test_services_job_utils.py | 4 +- 6 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 app/templates/displays/safe_relation.html diff --git a/app/db/admin.py b/app/db/admin.py index e9daa703..d4698847 100644 --- a/app/db/admin.py +++ b/app/db/admin.py @@ -60,6 +60,12 @@ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "templates" ) +# The built-in displays/relation.html builds its href from the foreign model's +# raw pk, which 500s ("May not contain path separators") for a pk containing +# "/" (e.g. an S3 object key). Use for any field pointing at +# S3ObjectAdmin/RunInputAdmin/RunOutputAdmin. +_SAFE_RELATION_TEMPLATE = "displays/safe_relation.html" + DEFAULT_DB_ADMIN_REQUIRED_ROLE = "biocommons/role/sbp/admin" DEFAULT_DB_ADMIN_ROLES_CLAIM = "https://biocommons.org.au/roles" DEFAULT_DB_ADMIN_SESSION_COOKIE = "sbp_admin_session" @@ -337,12 +343,9 @@ async def row_action_force_resync_outputs(self, request: Request, pk: Any) -> st async def _force_resync_workflow_runs(db: Session, runs: list[WorkflowRun]) -> str: """Force-resync result outputs for the given completed workflow runs. - Bypasses the usual sync_completed_at short-circuit so runs pick up any - S3 keys that a *new* results-utils category/classifier now recognises but - didn't exist (from this run's perspective) at the time it last synced - - submitting a new output transfer first if the spec now requires a prefix - that was never transferred out in the first place (see - force_resync_run_outputs). + See force_resync_run_outputs: bypasses the usual "already synced" + short-circuit and submits any output transfer still missing, so runs + pick up categories/files results-utils didn't recognise before. """ processed = 0 submitted = 0 @@ -565,8 +568,8 @@ class S3ObjectAdmin(UrlSafePrimaryKeyModelView): fields = [ "object_key", "uri", - HasMany("run_inputs", identity="run-input"), - HasMany("run_outputs", identity="run-output"), + HasMany("run_inputs", identity="run-input", display_template=_SAFE_RELATION_TEMPLATE), + HasMany("run_outputs", identity="run-output", display_template=_SAFE_RELATION_TEMPLATE), "version_id", "size_bytes", ] @@ -578,16 +581,26 @@ async def repr(self, obj: Any, request: Request) -> str: class RunInputAdmin(UrlSafePrimaryKeyModelView): fields = [ HasOne("run", identity="workflow-run"), - HasOne("s3_object", identity="s3-object"), + HasOne("s3_object", identity="s3-object", display_template=_SAFE_RELATION_TEMPLATE), ] class RunOutputAdmin(UrlSafePrimaryKeyModelView): fields = [ HasOne("run", identity="workflow-run"), - HasOne("s3_object", identity="s3-object"), + HasOne("s3_object", identity="s3-object", display_template=_SAFE_RELATION_TEMPLATE), ] + def get_search_query(self, request: Request, term: str) -> Any: + # `run` is a relationship, not a column, so the default search only + # matches the raw run_id UUID. Add job name via a correlated EXISTS. + base_clause = super().get_search_query(request, term) + run_name_clause = exists().where( + WorkflowRun.id == RunOutput.run_id, + WorkflowRun.run_name.ilike(f"%{term}%"), + ) + return or_(base_clause, run_name_clause) + class DataTransferAdmin(ModelView): fields = [ diff --git a/app/services/job_sync.py b/app/services/job_sync.py index bb137176..c6f9f9be 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -323,8 +323,8 @@ async def finalize_completed_workflow_run( class ForceResyncOutcome: """Outcome of a force-resync attempt for one completed run.""" - # False if a new output transfer was just submitted (or an existing one is - # still in flight) and results weren't re-synced this pass. + # False if a transfer was just submitted (or still in flight) and + # results weren't re-synced this pass. ready: bool outputs_synced: int @@ -336,16 +336,12 @@ async def force_resync_run_outputs( suppress_s3_errors: bool = True, settings: Settings | None = None, ) -> ForceResyncOutcome: - """ - Force a completed run to pick up results-utils spec changes: submit any - output transfer a prefix added since this run last synced now requires - (`_ensure_completed_run_output_transfers` is idempotent - it only creates - a transfer for a source/destination pair that doesn't already have one), - then re-scan S3 and resync result metadata even if already synced. - - Unlike `sync_workflow_run(..., force=True)`, this never re-polls Seqera - - callers must have already confirmed `run.seqera_final_status` is - SUCCEEDED. + """Force a completed run to pick up results-utils spec changes. + + Submits any output transfer a spec change now requires but hasn't + happened yet, then re-scans S3 and resyncs metadata even if already + synced. Unlike sync_workflow_run(force=True), never re-polls Seqera - + callers must confirm seqera_final_status is SUCCEEDED first. """ output_transfer_state = _ensure_completed_run_output_transfers(db, run, settings=settings) if not output_transfer_state.ready: diff --git a/app/templates/displays/safe_relation.html b/app/templates/displays/safe_relation.html new file mode 100644 index 00000000..a3562a3e --- /dev/null +++ b/app/templates/displays/safe_relation.html @@ -0,0 +1,9 @@ + +
+ {% for v in (data if field.multiple else [data]) %} + {{ v["_meta"]["repr"] }} + {% endfor %} +
diff --git a/tests/db/test_db_admin.py b/tests/db/test_db_admin.py index 5eb83a3e..ae26bbb7 100644 --- a/tests/db/test_db_admin.py +++ b/tests/db/test_db_admin.py @@ -11,6 +11,8 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from jinja2 import Environment, FileSystemLoader +from sqlalchemy import select from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import Response @@ -20,9 +22,12 @@ from app.config import get_settings from app.db.admin import ( + _ADMIN_TEMPLATES_DIR, + _SAFE_RELATION_TEMPLATE, AppUserAdmin, DataTransferAdmin, NciServiceUnitsField, + RunInputAdmin, RunOutputAdmin, S3ObjectAdmin, SbpCreditField, @@ -128,6 +133,101 @@ def test_app_user_admin_includes_credit_column() -> None: assert "credit_updated_by" in field_names +def test_run_output_admin_search_matches_by_job_run_name(test_db) -> None: + user = AppUser( + id=uuid4(), + auth0_user_id="auth0|run-output-search", + name="Run Output Search", + email="run-output-search@example.com", + ) + matching_run = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="run-output-search-match", + work_dir="/tmp/run-output-search-match", + run_name="anne-staging-wf-sp", + ) + other_run = WorkflowRun( + id=uuid4(), + owner_user_id=user.id, + seqera_run_id="run-output-search-other", + work_dir="/tmp/run-output-search-other", + run_name="someone-else-job", + ) + matching_object = S3Object( + object_key="run-output-search-match/results/report.html", + uri="s3://bucket/run-output-search-match/results/report.html", + ) + other_object = S3Object( + object_key="run-output-search-other/results/report.html", + uri="s3://bucket/run-output-search-other/results/report.html", + ) + test_db.add_all([user, matching_run, other_run, matching_object, other_object]) + test_db.flush() + test_db.add_all( + [ + RunOutput(run_id=matching_run.id, s3_object_id=matching_object.object_key), + RunOutput(run_id=other_run.id, s3_object_id=other_object.object_key), + ] + ) + test_db.commit() + + request = Request( + { + "type": "http", + "method": "GET", + "path": "/", + "headers": [], + "query_string": b"", + "server": ("testserver", 80), + "scheme": "http", + "client": ("testclient", 123), + } + ) + request.state.action = RequestAction.LIST + + view = RunOutputAdmin(RunOutput) + query = view.get_search_query(request, "staging-wf-sp") + rows = test_db.scalars(select(RunOutput).where(query)).all() + + assert {row.s3_object_id for row in rows} == {matching_object.object_key} + + +def test_s3_object_relation_fields_use_safe_relation_template() -> None: + """Any field pointing at S3Object/RunInput/RunOutput (pks contain "/") + must use the safe template - the default one 500s on those.""" + + def _field(fields: list, name: str): + return next(f for f in fields if getattr(f, "name", None) == name) + + assert _field(RunOutputAdmin.fields, "s3_object").display_template == _SAFE_RELATION_TEMPLATE + assert _field(RunInputAdmin.fields, "s3_object").display_template == _SAFE_RELATION_TEMPLATE + assert _field(S3ObjectAdmin.fields, "run_inputs").display_template == _SAFE_RELATION_TEMPLATE + assert _field(S3ObjectAdmin.fields, "run_outputs").display_template == _SAFE_RELATION_TEMPLATE + # WorkflowRun's pk is a plain UUID, so it doesn't need the override. + assert _field(RunOutputAdmin.fields, "run").display_template != _SAFE_RELATION_TEMPLATE + + +def test_safe_relation_template_renders_href_from_detail_url_not_raw_pk() -> None: + """Must build the href from the pre-computed _meta.detailUrl, not the + foreign model's raw (slash-containing) pk.""" + env = Environment(loader=FileSystemLoader(_ADMIN_TEMPLATES_DIR)) + template = env.get_template(_SAFE_RELATION_TEMPLATE) + + data = { + "object_key": "run-id/colabfold/job/job_report.tsv", + "_meta": { + "repr": "job_report.tsv", + "detailUrl": "http://testserver/admin/s3-object/detail/ENCODED", + }, + } + html = template.render(field=SimpleNamespace(multiple=False), data=data) + + assert 'href="http://testserver/admin/s3-object/detail/ENCODED"' in html + assert "job_report.tsv" in html + assert "run-id/colabfold/job/job_report.tsv" not in html + + def test_data_transfer_admin_includes_expected_columns() -> None: field_names = _admin_field_names(DataTransferAdmin) assert "workflow_run" in field_names diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index 64c93072..065cb75d 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -223,9 +223,7 @@ async def test_sync_workflow_run_sync_completed_reflects_finalized_state( async def test_force_resync_run_outputs_submits_new_transfer_before_resyncing( test_db, persistent_models, monkeypatch ): - """A results-utils spec update that now requires a prefix never - transferred before must submit that transfer, not just re-list whatever - is already in S3.""" + """Must submit a newly-required transfer, not just re-list what's in S3.""" run = _create_run( seqera_final_status="SUCCEEDED", sync_completed_at=datetime.now(tz=UTC), diff --git a/tests/test_services_job_utils.py b/tests/test_services_job_utils.py index 7b6d6f8f..9fb35855 100644 --- a/tests/test_services_job_utils.py +++ b/tests/test_services_job_utils.py @@ -277,9 +277,7 @@ async def test_ensure_completed_run_score_branches(): @pytest.mark.asyncio async def test_ensure_completed_run_score_force_recomputes_despite_existing_value(): - """force=True must re-derive the score even though a cached one already - exists - e.g. after a results-utils change, the underlying result file's - content may have changed since the score was first computed.""" + """force=True must re-derive the score even though one is already cached.""" run = SimpleNamespace( id="rid", seqera_run_id="wf-1", From 4448f7e19d9a4f588f84dc55f4fa34f36e42be34 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:19:53 +1000 Subject: [PATCH 10/25] fix: run id handling in admin dashboard --- app/templates/detail.html | 159 ++++++++++++++++++++++ app/templates/displays/safe_relation.html | 4 +- tests/db/test_db_admin.py | 31 ++++- 3 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 app/templates/detail.html diff --git a/app/templates/detail.html b/app/templates/detail.html new file mode 100644 index 00000000..8bc1a0a1 --- /dev/null +++ b/app/templates/detail.html @@ -0,0 +1,159 @@ +{% extends "layout.html" %} +{% set pk = model.pk_attr %} +{% block header %} +
+

{{ model.label }}

+ +
+{% endblock %} +{% block modal %} + {{ super() }} + {% include "modals/loading.html" %} + {% include "modals/error.html" %} + {% include "modals/delete.html" %} + {% include "modals/actions.html" %} + +{% endblock %} +{% block content %} +
+
+
+
+
+
+ +

+ #{{ obj[pk] }} +

+
+ {% with pk=obj[pk] %} + {% include "row-actions.html" %} + {% endwith %} +
+
+
+
+
+
+ + + + + + + + + +
+
+
+
+
+ {% block details_table %} +
+ + + + + + + + + {% for field in model.get_fields_list(request, 'DETAIL' | ra) %} + + + + + {% endfor %} + +
{{ _("Attribute") }}{{ _("Value") }}
+
+ {{ field.label }} +
+
+ {% with data=obj[field.name] %} + {% if data == None %} + {% include "displays/_null.html" %} + {% elif (data | is_iter) and (data |length) ==0 %} + {% include "displays/_empty.html" %} + {% else %} + {% include field.display_template %} + {% endif %} + {% endwith %} +
+
+ {% endblock %} +
+
+
+
+
+{% endblock %} +{% block head_css %} + {{ super() }} + {% for link in model._additional_css_links(request, "DETAIL" | ra) %} + + {% endfor %} +{% endblock %} +{% block script %} + {{ super() }} + + + + {% for link in model._additional_js_links(request, "DETAIL" | ra) %} + + {% endfor %} + +{% endblock %} diff --git a/app/templates/displays/safe_relation.html b/app/templates/displays/safe_relation.html index a3562a3e..3827002a 100644 --- a/app/templates/displays/safe_relation.html +++ b/app/templates/displays/safe_relation.html @@ -3,7 +3,9 @@ pk - which 500s for a pk containing "/" (e.g. an S3 object key). -->
{% for v in (data if field.multiple else [data]) %} - {{ v["_meta"]["repr"] }} {% endfor %}
diff --git a/tests/db/test_db_admin.py b/tests/db/test_db_admin.py index ae26bbb7..7d261fbf 100644 --- a/tests/db/test_db_admin.py +++ b/tests/db/test_db_admin.py @@ -4,6 +4,7 @@ import os from collections.abc import Generator +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock from uuid import uuid4 @@ -210,22 +211,42 @@ def _field(fields: list, name: str): def test_safe_relation_template_renders_href_from_detail_url_not_raw_pk() -> None: """Must build the href from the pre-computed _meta.detailUrl, not the - foreign model's raw (slash-containing) pk.""" + foreign model's raw (slash-containing) pk. A long repr (a full S3 key) + must be truncated with CSS, not left to overflow the page - full value + stays reachable via the title attribute.""" env = Environment(loader=FileSystemLoader(_ADMIN_TEMPLATES_DIR)) template = env.get_template(_SAFE_RELATION_TEMPLATE) + long_key = "run-id/colabfold/job/some_very_long_descriptive_output_filename.tsv" data = { - "object_key": "run-id/colabfold/job/job_report.tsv", + "object_key": long_key, "_meta": { - "repr": "job_report.tsv", + "repr": long_key, "detailUrl": "http://testserver/admin/s3-object/detail/ENCODED", }, } html = template.render(field=SimpleNamespace(multiple=False), data=data) assert 'href="http://testserver/admin/s3-object/detail/ENCODED"' in html - assert "job_report.tsv" in html - assert "run-id/colabfold/job/job_report.tsv" not in html + assert "text-truncate" in html + assert f'title="{long_key}"' in html + # text-truncate alone doesn't clip inside a flex container (the item's + # default min-width:auto keeps it at full content width) - min-width:0 + # is required for max-width to actually take effect. + assert "min-width: 0" in html + + +def test_detail_template_truncates_card_title_pk() -> None: + """Overridden from starlette_admin's own detail.html: the page heading + (`#{{ obj[pk] }}`) is a flex item next to the Edit/Delete actions and + used to render a long pk (e.g. a composite RunOutput pk with an S3 key + in it) unbounded, pushing those actions out of frame.""" + detail_template_path = Path(_ADMIN_TEMPLATES_DIR) / "detail.html" + html = detail_template_path.read_text() + + assert 'class="card-title text-truncate"' in html + assert "min-width: 0" in html + assert 'title="{{ obj[pk] }}"' in html def test_data_transfer_admin_includes_expected_columns() -> None: From 2fdbcd245199893733e080eaacd7fae2561615f6 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:39:35 +1000 Subject: [PATCH 11/25] fix: turn missing file transfer back to pending --- app/services/job_sync.py | 20 +++++- app/services/results_utils.py | 38 +++++++++++ tests/test_services_job_sync.py | 59 +++++++++++++++++ tests/test_services_results_utils.py | 96 ++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 3 deletions(-) diff --git a/app/services/job_sync.py b/app/services/job_sync.py index c6f9f9be..9767e7ba 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -21,7 +21,12 @@ map_pipeline_status_to_ui, ) from .job_utils import ensure_completed_run_score, extract_pipeline_status, sync_service_usage -from .results_utils import get_output_spec, sync_workflow_outputs +from .results_utils import ( + get_output_spec, + reset_completed_output_transfers, + run_has_missing_required_categories, + sync_workflow_outputs, +) from .seqera import describe_workflow from .seqera_errors import SeqeraAPIError @@ -340,8 +345,13 @@ async def force_resync_run_outputs( Submits any output transfer a spec change now requires but hasn't happened yet, then re-scans S3 and resyncs metadata even if already - synced. Unlike sync_workflow_run(force=True), never re-polls Seqera - - callers must confirm seqera_final_status is SUCCEEDED first. + synced. If a required category is still missing afterwards - e.g. a file + was deleted directly from S3, bypassing this app, so its "completed" + Globus transfer was never told to redo it - every completed output + transfer for the run is reset to pending so the scheduler resubmits a + fresh copy from source. Unlike sync_workflow_run(force=True), never + re-polls Seqera - callers must confirm seqera_final_status is SUCCEEDED + first. """ output_transfer_state = _ensure_completed_run_output_transfers(db, run, settings=settings) if not output_transfer_state.ready: @@ -354,6 +364,10 @@ async def force_resync_run_outputs( suppress_s3_errors=suppress_s3_errors, settings=settings, ) + + if run_has_missing_required_categories(db, run) and reset_completed_output_transfers(db, run): + return ForceResyncOutcome(ready=False, outputs_synced=outputs_synced) + return ForceResyncOutcome(ready=True, outputs_synced=outputs_synced) diff --git a/app/services/results_utils.py b/app/services/results_utils.py index b5835c84..1a9671d4 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -444,6 +444,44 @@ def s3_uri_to_key(uri: str | None) -> str | None: return parts[3].strip() or None +def run_has_missing_required_categories(db: Session, run: WorkflowRun) -> bool: + """Whether this run is still missing a required output category, using + whatever's currently recorded as its RunOutput keys.""" + try: + spec = get_output_spec(run) + except ValueError: + return False + outputs = collect_classified_outputs(db, run, spec) + return bool(missing_required_categories(outputs, spec)) + + +def reset_completed_output_transfers(db: Session, run: WorkflowRun) -> int: + """Reset every completed Globus output transfer for this run to pending. + + A "completed" status only proves a transfer copied successfully at the + time it ran - if a file was later deleted directly from S3 (bypassing + this app), nothing else would ever notice or retry it, since + create_output_transfers reuses any existing transfer for the same + source/destination regardless of status. Force-resync calls this when a + required category is still missing after a resync attempt, so the + scheduler resubmits a fresh copy of each output folder from its source. + Returns the number of transfers reset. + """ + completed_transfers = db.scalars( + select(DataTransfer).where( + DataTransfer.workflow_run_id == run.id, + DataTransfer.provider == "globus", + DataTransfer.direction == "output", + DataTransfer.status == "completed", + ) + ).all() + for transfer in completed_transfers: + transfer.reset_to_pending(session=db, commit=False) + if completed_transfers: + db.commit() + return len(completed_transfers) + + def _non_root_output_prefixes(run: WorkflowRun, prefixes: list[str]) -> list[str]: """Return run-scoped output prefixes, excluding the broad run root prefix.""" if not run.id: diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index 065cb75d..452f6bdb 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -242,6 +242,65 @@ async def test_force_resync_run_outputs_submits_new_transfer_before_resyncing( assert outcome.outputs_synced == 0 +@pytest.mark.asyncio +async def test_force_resync_run_outputs_resets_transfers_when_still_missing_required_category( + test_db, persistent_models, monkeypatch +): + """If a required category is still missing after resyncing - e.g. a file + was deleted directly from S3, so its "completed" Globus transfer was + never told to redo it - every completed output transfer must be reset + so the scheduler resubmits a fresh copy, and the outcome reports + not-ready so callers know to check back later.""" + run = _create_run( + seqera_final_status="SUCCEEDED", + sync_completed_at=datetime.now(tz=UTC), + ) + ensure_transfers = MagicMock(return_value=job_sync.OutputTransferState(ready=True)) + finalize = AsyncMock(return_value=3) + missing_categories = MagicMock(return_value=True) + reset_transfers = MagicMock(return_value=2) + + monkeypatch.setattr(job_sync, "_ensure_completed_run_output_transfers", ensure_transfers) + monkeypatch.setattr(job_sync, "finalize_completed_workflow_run", finalize) + monkeypatch.setattr(job_sync, "run_has_missing_required_categories", missing_categories) + monkeypatch.setattr(job_sync, "reset_completed_output_transfers", reset_transfers) + + outcome = await job_sync.force_resync_run_outputs(test_db, run) + + finalize.assert_awaited_once() + missing_categories.assert_called_once_with(test_db, run) + reset_transfers.assert_called_once_with(test_db, run) + assert outcome.ready is False + assert outcome.outputs_synced == 3 + + +@pytest.mark.asyncio +async def test_force_resync_run_outputs_stays_ready_when_nothing_to_reset( + test_db, persistent_models, monkeypatch +): + """A required category can be missing with no completed transfer left to + retry (e.g. the source itself is gone) - nothing more force-resync can + do, so it must not report not-ready forever.""" + run = _create_run( + seqera_final_status="SUCCEEDED", + sync_completed_at=datetime.now(tz=UTC), + ) + ensure_transfers = MagicMock(return_value=job_sync.OutputTransferState(ready=True)) + finalize = AsyncMock(return_value=3) + missing_categories = MagicMock(return_value=True) + reset_transfers = MagicMock(return_value=0) + + monkeypatch.setattr(job_sync, "_ensure_completed_run_output_transfers", ensure_transfers) + monkeypatch.setattr(job_sync, "finalize_completed_workflow_run", finalize) + monkeypatch.setattr(job_sync, "run_has_missing_required_categories", missing_categories) + monkeypatch.setattr(job_sync, "reset_completed_output_transfers", reset_transfers) + + outcome = await job_sync.force_resync_run_outputs(test_db, run) + + assert outcome.ready is True + assert outcome.outputs_synced == 3 + + @pytest.mark.asyncio async def test_force_resync_run_outputs_resyncs_once_transfers_are_ready( test_db, persistent_models, monkeypatch diff --git a/tests/test_services_results_utils.py b/tests/test_services_results_utils.py index 79f57de7..8da59235 100644 --- a/tests/test_services_results_utils.py +++ b/tests/test_services_results_utils.py @@ -50,9 +50,11 @@ list_workflow_outputs_from_s3, make_wisps_classifier, read_result_output_file, + reset_completed_output_transfers, resolve_fasta_form_data, resolve_pdb_presigned_urls, resolve_submitted_form_data, + run_has_missing_required_categories, s3_uri_to_key, ) from app.services.s3 import S3ServiceError @@ -61,6 +63,7 @@ DataTransferFactory, RunOutputFactory, S3ObjectFactory, + WorkflowFactory, WorkflowRunFactory, ) @@ -507,6 +510,99 @@ def test_colabfold_create_output_transfers_creates_expected_rows( ] +def test_run_has_missing_required_categories_true_when_category_absent( + test_db, persistent_models +): + """A required category (e.g. 'report') with no recorded output at all - + such as when its only source file was deleted directly from S3 along + with its RunOutput row - must be detected as missing.""" + user = AppUserFactory.create_sync() + workflow = WorkflowFactory.create_sync(name="de-novo-design", tool="bindcraft") + run = WorkflowRunFactory.create_sync(owner=user, workflow=workflow, tool="bindcraft") + + stats_object = S3ObjectFactory.create_sync( + object_key=f"{run.id}/ranker/{run.id}_final_design_stats.csv", + uri=f"s3://bucket/{run.id}/ranker/{run.id}_final_design_stats.csv", + ) + RunOutputFactory.create_sync( + run_id=run.id, + s3_object_id=stats_object.object_key, + data_transfer=DataTransferFactory.create_sync(workflow_run=run, direction="output"), + ) + # required_categories for bindcraft also includes "report" and "pdb" - + # neither has any recorded output, so the run stays incomplete. + + assert run_has_missing_required_categories(test_db, run) is True + + +def test_run_has_missing_required_categories_false_when_satisfied(test_db, persistent_models): + user = AppUserFactory.create_sync() + workflow = WorkflowFactory.create_sync(name="de-novo-design", tool="bindcraft") + run = WorkflowRunFactory.create_sync(owner=user, workflow=workflow, tool="bindcraft") + + outputs = { + "report": f"{run.id}/generate/{run.id}_report.html", + "pdb": f"{run.id}/ranker/{run.id}_ranked/{run.id}.pdb", + "stats_csv": f"{run.id}/ranker/{run.id}_final_design_stats.csv", + } + for key in outputs.values(): + s3_object = S3ObjectFactory.create_sync(object_key=key, uri=f"s3://bucket/{key}") + RunOutputFactory.create_sync( + run_id=run.id, + s3_object_id=s3_object.object_key, + data_transfer=DataTransferFactory.create_sync(workflow_run=run, direction="output"), + ) + + assert run_has_missing_required_categories(test_db, run) is False + + +def test_reset_completed_output_transfers_resets_only_completed_globus_rows( + test_db, persistent_models +): + run = WorkflowRunFactory.create_sync() + completed_one = DataTransferFactory.create_sync( + workflow_run=run, + direction="output", + provider="globus", + status="completed", + transfer_id="task-1", + ) + completed_two = DataTransferFactory.create_sync( + workflow_run=run, + direction="output", + provider="globus", + status="completed", + transfer_id="task-2", + ) + failed = DataTransferFactory.create_sync( + workflow_run=run, + direction="output", + provider="globus", + status="failed", + transfer_id="task-3", + ) + s3_provider = DataTransferFactory.create_sync( + workflow_run=run, + direction="output", + provider="s3", + status="pending", + ) + + reset_count = reset_completed_output_transfers(test_db, run) + + assert reset_count == 2 + test_db.refresh(completed_one) + test_db.refresh(completed_two) + test_db.refresh(failed) + test_db.refresh(s3_provider) + assert completed_one.status == "pending" + assert completed_one.transfer_id is None + assert completed_two.status == "pending" + assert completed_two.transfer_id is None + assert failed.status == "failed" + assert s3_provider.status == "pending" + + def test_rfdiffusion_helpers_classify_keys_and_build_prefixes(): run = WorkflowRun(id=uuid4(), owner_user_id=uuid4(), sample_id="sampleZ") From 45f2cfff20ced33fd46645c23ead8f79af54b155 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:32:23 +1000 Subject: [PATCH 12/25] fix: completed bookkeeping data transfer --- app/services/job_sync.py | 15 ++++++--------- app/services/results_utils.py | 20 ++++++++------------ tests/test_services_job_sync.py | 10 ++-------- tests/test_services_job_utils.py | 2 +- tests/test_services_results_utils.py | 9 +++------ 5 files changed, 20 insertions(+), 36 deletions(-) diff --git a/app/services/job_sync.py b/app/services/job_sync.py index 9767e7ba..1efebec4 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -343,15 +343,12 @@ async def force_resync_run_outputs( ) -> ForceResyncOutcome: """Force a completed run to pick up results-utils spec changes. - Submits any output transfer a spec change now requires but hasn't - happened yet, then re-scans S3 and resyncs metadata even if already - synced. If a required category is still missing afterwards - e.g. a file - was deleted directly from S3, bypassing this app, so its "completed" - Globus transfer was never told to redo it - every completed output - transfer for the run is reset to pending so the scheduler resubmits a - fresh copy from source. Unlike sync_workflow_run(force=True), never - re-polls Seqera - callers must confirm seqera_final_status is SUCCEEDED - first. + Submits any output transfer a spec change now requires, then re-scans + S3 and resyncs metadata even if already synced. If a required category + is still missing afterwards (e.g. a file was deleted directly from S3), + every completed output transfer is reset to pending for the scheduler + to redo. Unlike sync_workflow_run(force=True), never re-polls Seqera - + callers must confirm seqera_final_status is SUCCEEDED first. """ output_transfer_state = _ensure_completed_run_output_transfers(db, run, settings=settings) if not output_transfer_state.ready: diff --git a/app/services/results_utils.py b/app/services/results_utils.py index 1a9671d4..98ebb8ac 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -445,8 +445,7 @@ def s3_uri_to_key(uri: str | None) -> str | None: def run_has_missing_required_categories(db: Session, run: WorkflowRun) -> bool: - """Whether this run is still missing a required output category, using - whatever's currently recorded as its RunOutput keys.""" + """Whether this run's currently recorded RunOutputs miss a required category.""" try: spec = get_output_spec(run) except ValueError: @@ -456,16 +455,11 @@ def run_has_missing_required_categories(db: Session, run: WorkflowRun) -> bool: def reset_completed_output_transfers(db: Session, run: WorkflowRun) -> int: - """Reset every completed Globus output transfer for this run to pending. - - A "completed" status only proves a transfer copied successfully at the - time it ran - if a file was later deleted directly from S3 (bypassing - this app), nothing else would ever notice or retry it, since - create_output_transfers reuses any existing transfer for the same - source/destination regardless of status. Force-resync calls this when a - required category is still missing after a resync attempt, so the - scheduler resubmits a fresh copy of each output folder from its source. - Returns the number of transfers reset. + """Reset this run's completed Globus output transfers to pending. + + Used when a required category is still missing after a resync - a + "completed" transfer only proves it copied successfully at the time, + not that the file is still there now. Returns the number reset. """ completed_transfers = db.scalars( select(DataTransfer).where( @@ -1174,6 +1168,8 @@ def _sync_run_output_records( source_location=run_outdir, destination_location=s3_object.uri, recursive=False, + # Bookkeeping link, not a real job - the key was just found in S3. + status="completed", ) db.add(output_transfer) db.add(RunOutput(run_id=run.id, s3_object_id=normalized, data_transfer=output_transfer)) diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index 452f6bdb..5f7ce2b7 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -246,11 +246,7 @@ async def test_force_resync_run_outputs_submits_new_transfer_before_resyncing( async def test_force_resync_run_outputs_resets_transfers_when_still_missing_required_category( test_db, persistent_models, monkeypatch ): - """If a required category is still missing after resyncing - e.g. a file - was deleted directly from S3, so its "completed" Globus transfer was - never told to redo it - every completed output transfer must be reset - so the scheduler resubmits a fresh copy, and the outcome reports - not-ready so callers know to check back later.""" + """Missing category after resync -> reset completed transfers, report not-ready.""" run = _create_run( seqera_final_status="SUCCEEDED", sync_completed_at=datetime.now(tz=UTC), @@ -278,9 +274,7 @@ async def test_force_resync_run_outputs_resets_transfers_when_still_missing_requ async def test_force_resync_run_outputs_stays_ready_when_nothing_to_reset( test_db, persistent_models, monkeypatch ): - """A required category can be missing with no completed transfer left to - retry (e.g. the source itself is gone) - nothing more force-resync can - do, so it must not report not-ready forever.""" + """Nothing left to reset -> must not report not-ready forever.""" run = _create_run( seqera_final_status="SUCCEEDED", sync_completed_at=datetime.now(tz=UTC), diff --git a/tests/test_services_job_utils.py b/tests/test_services_job_utils.py index 9fb35855..641ca5ab 100644 --- a/tests/test_services_job_utils.py +++ b/tests/test_services_job_utils.py @@ -449,7 +449,7 @@ def _list_side_effect(prefix: str, file_extension=None, **_kwargs): assert output_transfer.provider == "s3" assert output_transfer.source_location == f"s3://test-s3-bucket/{run_id}" assert output_transfer.destination_location == persisted.uri - assert output_transfer.status == "pending" + assert output_transfer.status == "completed" link = ( test_db.query(RunOutput).filter_by(run_id=run.id, s3_object_id=snapshot_key).one_or_none() ) diff --git a/tests/test_services_results_utils.py b/tests/test_services_results_utils.py index 8da59235..9a09f88e 100644 --- a/tests/test_services_results_utils.py +++ b/tests/test_services_results_utils.py @@ -513,9 +513,7 @@ def test_colabfold_create_output_transfers_creates_expected_rows( def test_run_has_missing_required_categories_true_when_category_absent( test_db, persistent_models ): - """A required category (e.g. 'report') with no recorded output at all - - such as when its only source file was deleted directly from S3 along - with its RunOutput row - must be detected as missing.""" + """A required category with no recorded output at all must be detected as missing.""" user = AppUserFactory.create_sync() workflow = WorkflowFactory.create_sync(name="de-novo-design", tool="bindcraft") run = WorkflowRunFactory.create_sync(owner=user, workflow=workflow, tool="bindcraft") @@ -529,9 +527,7 @@ def test_run_has_missing_required_categories_true_when_category_absent( s3_object_id=stats_object.object_key, data_transfer=DataTransferFactory.create_sync(workflow_run=run, direction="output"), ) - # required_categories for bindcraft also includes "report" and "pdb" - - # neither has any recorded output, so the run stays incomplete. - + # "report" and "pdb" are also required but have no output here. assert run_has_missing_required_categories(test_db, run) is True @@ -789,6 +785,7 @@ async def fake_list_s3_files(prefix: str, settings=None): data_transfer = test_db.get(DataTransfer, run_output.data_transfer_id) assert data_transfer is not None assert data_transfer.provider == "s3" + assert data_transfer.status == "completed" assert data_transfer.destination_location.endswith(report_key) From d84a6a9f3afdabe907b1620ff0a4ff769e432cb3 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Thu, 17 Sep 2026 13:51:23 +1000 Subject: [PATCH 13/25] fix: completed status for job dashboard --- app/routes/workflow/jobs.py | 17 +++++- tests/test_routes_workflow_jobs.py | 83 +++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/app/routes/workflow/jobs.py b/app/routes/workflow/jobs.py index 6fcc5b50..5053eb95 100644 --- a/app/routes/workflow/jobs.py +++ b/app/routes/workflow/jobs.py @@ -26,6 +26,8 @@ JobDetailsResponse, JobListItem, JobListResponse, + PipelineStatus, + UIStatus, map_pipeline_status_to_ui, ) from ...services.job_utils import ( @@ -80,12 +82,25 @@ def _resolve_stored_score(owned_run: WorkflowRun | None) -> float | None: def _get_stored_terminal_ui_status(run: WorkflowRun) -> str | None: + """Resolve UI status from stored columns alone, no live Seqera call needed. + + A run isn't "Completed" just because Seqera succeeded - it's only Completed + once output transfers have synced too (see `WorkflowRun.results_sync_status`). + Still syncing -> In progress; a permanently failed output transfer -> Failed. + """ if run.seqera_final_status is None: return None status = run.seqera_final_status.strip().upper() if status not in TERMINAL_SEQERA_STATUSES: return None - return map_pipeline_status_to_ui(status) + if status != PipelineStatus.SUCCEEDED.value: + return map_pipeline_status_to_ui(status) + sync_status = run.results_sync_status + if sync_status == "syncing": + return UIStatus.IN_PROGRESS.value + if sync_status == "partial": + return UIStatus.FAILED.value + return UIStatus.COMPLETED.value @router.post("/{run_id}/cancel", response_model=CancelWorkflowResponse) diff --git a/tests/test_routes_workflow_jobs.py b/tests/test_routes_workflow_jobs.py index 53a825ca..7ce5d53d 100644 --- a/tests/test_routes_workflow_jobs.py +++ b/tests/test_routes_workflow_jobs.py @@ -10,7 +10,7 @@ from fastapi import HTTPException from fastapi.testclient import TestClient -from app.db.models.core import RunMetric +from app.db.models.core import DataTransfer, RunMetric from app.main import create_app from app.routes.workflow.jobs import get_job_details, list_jobs from app.services.job_utils import UserJobListRow @@ -589,6 +589,56 @@ async def test_list_jobs_stored_terminal_status_skips_seqera_lookup(mock_db, moc assert response.jobs[0].score == 0.91 +@pytest.mark.asyncio +async def test_list_jobs_permanently_failed_output_transfer_shows_failed(mock_db, mock_user_id): + """Seqera succeeded and sync finished, but a permanently failed output transfer + means the job is Failed overall, not Completed.""" + run = WorkflowRunFactory.build( + seqera_run_id="wf-partial-sync", + seqera_final_status="SUCCEEDED", + sync_completed_at=datetime(2026, 2, 1, 11, 0, tzinfo=UTC), + binder_name=None, + run_name="Partially Synced Job", + submission_timestamp=datetime(2026, 2, 1, 10, 0, tzinfo=UTC), + ) + run.metrics = None + run.data_transfers = [ + DataTransfer( + workflow_run_id=run.id, + direction="output", + provider="globus", + source_location="a", + destination_location="b", + status="failed", + ) + ] + user_run = UserJobListRowFactory.build( + run=run, + run_id="run-partial-sync", + seqera_run_id="wf-partial-sync", + score=0.91, + ) + describe = AsyncMock() + + with ( + patch("app.routes.workflow.jobs.get_user_job_list_page", return_value=([user_run], 1)), + patch("app.routes.workflow.jobs.describe_workflow", describe), + ): + response = await list_jobs( + search=None, + status_filter=None, + limit=50, + offset=0, + current_user_id=mock_user_id, + db=mock_db, + ) + + describe.assert_not_awaited() + assert len(response.jobs) == 1 + assert response.jobs[0].status == "Failed" + assert response.jobs[0].score is None + + @pytest.mark.asyncio async def test_list_jobs_synced_completed_run_skips_score_and_usage_sync(mock_db, mock_user_id): """Completed runs already marked synced don't sync score and usage""" @@ -1137,6 +1187,37 @@ async def test_get_job_details_in_progress_no_score(mock_db, mock_user_id): assert response.score is None +@pytest.mark.asyncio +async def test_get_job_details_succeeded_but_output_sync_pending_shows_in_progress( + mock_db, mock_user_id +): + """Seqera succeeded, but output transfers haven't finished syncing yet - job + should read In progress, not Completed, and needs no live Seqera call.""" + owned_run = WorkflowRunFactory.build( + seqera_run_id="seqera-wf-999", + seqera_final_status="SUCCEEDED", + sync_completed_at=None, + ) + owned_run.metrics = None + owned_run.data_transfers = [] + + describe = AsyncMock() + + with ( + patch("app.routes.workflow.jobs.get_owned_run_by_id", return_value=owned_run), + patch("app.routes.workflow.jobs.describe_workflow", describe), + ): + response = await get_job_details( + run_id="wf-999", + current_user_id=mock_user_id, + db=mock_db, + ) + + describe.assert_not_awaited() + assert response.status == "In progress" + assert response.score is None + + @pytest.mark.asyncio async def test_get_job_details_seqera_error(mock_db, mock_user_id): """Test handling of Seqera API error in job details.""" From 28ab8a3a4c9bf684f2cacfe0da445f096d6197ca Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:53:20 +1000 Subject: [PATCH 14/25] chore: lint --- app/services/job_sync.py | 4 +--- tests/test_services_results_utils.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/services/job_sync.py b/app/services/job_sync.py index 1efebec4..a2c0f3f1 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -417,9 +417,7 @@ async def _sync_completed_run_results( await ensure_completed_run_score( db, run, UIStatus.COMPLETED.value, settings=settings, force=force ) - await sync_service_usage( - db, run, UIStatus.COMPLETED.value, settings=settings, force=force - ) + await sync_service_usage(db, run, UIStatus.COMPLETED.value, settings=settings, force=force) return len(synced_keys) diff --git a/tests/test_services_results_utils.py b/tests/test_services_results_utils.py index 9a09f88e..68f9d1bc 100644 --- a/tests/test_services_results_utils.py +++ b/tests/test_services_results_utils.py @@ -510,9 +510,7 @@ def test_colabfold_create_output_transfers_creates_expected_rows( ] -def test_run_has_missing_required_categories_true_when_category_absent( - test_db, persistent_models -): +def test_run_has_missing_required_categories_true_when_category_absent(test_db, persistent_models): """A required category with no recorded output at all must be detected as missing.""" user = AppUserFactory.create_sync() workflow = WorkflowFactory.create_sync(name="de-novo-design", tool="bindcraft") From 24566a087453edc9bd9d3c227b58e7bd7516e484 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:00:59 +1000 Subject: [PATCH 15/25] fix: update completed sync time --- app/services/job_sync.py | 6 ++++++ tests/test_services_job_sync.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/app/services/job_sync.py b/app/services/job_sync.py index a2c0f3f1..9096693e 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -363,6 +363,12 @@ async def force_resync_run_outputs( ) if run_has_missing_required_categories(db, run) and reset_completed_output_transfers(db, run): + # Undo the "fully synced" mark finalize_completed_workflow_run just set, + # so the run re-enters the scheduler's normal sync pipeline once the + # transfer we just reset completes - otherwise nothing else would notice. + run.sync_completed_at = None + db.add(run) + db.commit() return ForceResyncOutcome(ready=False, outputs_synced=outputs_synced) return ForceResyncOutcome(ready=True, outputs_synced=outputs_synced) diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index 5f7ce2b7..f32fc2d1 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -268,6 +268,8 @@ async def test_force_resync_run_outputs_resets_transfers_when_still_missing_requ reset_transfers.assert_called_once_with(test_db, run) assert outcome.ready is False assert outcome.outputs_synced == 3 + # Must re-enter the scheduler's normal pipeline once the reset transfer completes. + assert run.sync_completed_at is None @pytest.mark.asyncio @@ -293,6 +295,7 @@ async def test_force_resync_run_outputs_stays_ready_when_nothing_to_reset( assert outcome.ready is True assert outcome.outputs_synced == 3 + assert run.sync_completed_at is not None @pytest.mark.asyncio From 8c2571f4b133f69db13d03c67b3b76eb9e7673a1 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:46:09 +1000 Subject: [PATCH 16/25] fix: manually click force again to update sync completed at --- app/services/job_sync.py | 12 ++++++------ tests/test_services_job_sync.py | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/services/job_sync.py b/app/services/job_sync.py index 9096693e..b5f114b2 100644 --- a/app/services/job_sync.py +++ b/app/services/job_sync.py @@ -363,12 +363,12 @@ async def force_resync_run_outputs( ) if run_has_missing_required_categories(db, run) and reset_completed_output_transfers(db, run): - # Undo the "fully synced" mark finalize_completed_workflow_run just set, - # so the run re-enters the scheduler's normal sync pipeline once the - # transfer we just reset completes - otherwise nothing else would notice. - run.sync_completed_at = None - db.add(run) - db.commit() + # Deliberately does NOT clear run.sync_completed_at: every results + # route treats a null sync_completed_at as "nothing is ready yet" + # (is_syncing_results), which would hide this run's other, already- + # synced outputs while the reset transfer is in flight. The tradeoff + # is that this run won't reappear in the scheduler's own queue - + # force-resync must be run again once that transfer completes. return ForceResyncOutcome(ready=False, outputs_synced=outputs_synced) return ForceResyncOutcome(ready=True, outputs_synced=outputs_synced) diff --git a/tests/test_services_job_sync.py b/tests/test_services_job_sync.py index f32fc2d1..460ed8d0 100644 --- a/tests/test_services_job_sync.py +++ b/tests/test_services_job_sync.py @@ -268,8 +268,9 @@ async def test_force_resync_run_outputs_resets_transfers_when_still_missing_requ reset_transfers.assert_called_once_with(test_db, run) assert outcome.ready is False assert outcome.outputs_synced == 3 - # Must re-enter the scheduler's normal pipeline once the reset transfer completes. - assert run.sync_completed_at is None + # Deliberately untouched - clearing it would hide other, already-synced + # outputs from every results route while the reset transfer is in flight. + assert run.sync_completed_at is not None @pytest.mark.asyncio From 23be7d5c6db120909b8ff78452e74162348ccc1c Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:23:55 +1000 Subject: [PATCH 17/25] fix: filter single prediction results --- app/services/results_utils.py | 43 +++++++++++++++++++++- tests/test_services_results_utils.py | 55 ++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/app/services/results_utils.py b/app/services/results_utils.py index b5835c84..9c720712 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -678,6 +678,11 @@ def classify_proteinfold_output_key( return None filename = normalized.rsplit("/", 1)[-1] + # plddt.tsv is no longer surfaced to the portal; plddt is already + # captured in the per-model report/structure outputs. + if filename.lower().endswith("_plddt.tsv"): + return None + if re.search(report_pattern, normalized): return ClassifiedOutput(category="report", label=filename) if re.search(pdb_pattern, normalized): @@ -1076,6 +1081,40 @@ def missing_required_categories( return set(spec.required_categories) - found +_PAE_TSV_RANK_PATTERN = re.compile(r"^(?P.*)_(?P\d+)_pae\.tsv$", re.IGNORECASE) + + +def _drop_non_lowest_rank_pae(outputs: dict[str, ClassifiedOutput]) -> dict[str, ClassifiedOutput]: + """Keep only the lowest-ranked ``*_pae.tsv`` per sample directory. + + Proteinfold tools publish one PAE file per predicted model rank, but the + portal only needs the top-ranked model's PAE. Different tools number + their lowest rank 0 or 1, so pick the minimum found rather than a fixed + number. + """ + lowest_rank_by_group: dict[str, tuple[int, str]] = {} + pae_keys: set[str] = set() + for key, output in outputs.items(): + if output.category != "stats_csv": + continue + match = _PAE_TSV_RANK_PATTERN.match(output.label) + if not match: + continue + pae_keys.add(key) + directory = key.rsplit("/", 1)[0] if "/" in key else "" + group = f"{directory}::{match.group('prefix')}" + rank = int(match.group("rank")) + current = lowest_rank_by_group.get(group) + if current is None or rank < current[0]: + lowest_rank_by_group[group] = (rank, key) + + keep_keys = {key for _, key in lowest_rank_by_group.values()} + drop_keys = pae_keys - keep_keys + if not drop_keys: + return outputs + return {key: output for key, output in outputs.items() if key not in drop_keys} + + def collect_classified_outputs( db: Session, run: WorkflowRun, @@ -1087,7 +1126,7 @@ def collect_classified_outputs( classified = spec.classify_output(key, sample_id) if classified: outputs[key] = classified - return outputs + return _drop_non_lowest_rank_pae(outputs) def _filter_outputs_by_category( @@ -1186,7 +1225,7 @@ async def list_workflow_outputs_from_s3( if classified is not None: outputs[key] = classified - return outputs + return _drop_non_lowest_rank_pae(outputs) async def sync_workflow_outputs( diff --git a/tests/test_services_results_utils.py b/tests/test_services_results_utils.py index 79f57de7..f7dc27d2 100644 --- a/tests/test_services_results_utils.py +++ b/tests/test_services_results_utils.py @@ -640,6 +640,56 @@ async def read_bytes(key: str, **_kwargs) -> bytes: assert {call.args[0] for call in mock_read_s3_bytes.await_args_list} == set(output_contents) +@pytest.mark.asyncio +async def test_get_all_downloads_zipped_keeps_only_lowest_rank_pae_and_drops_plddt( + test_db, persistent_models +): + """Single-prediction proteinfold runs publish one *_pae.tsv per model rank + and a *_plddt.tsv summary; the portal should only ever surface the + top-ranked model's PAE and never the plddt.tsv file. + """ + user = AppUserFactory.create_sync() + run = WorkflowRunFactory.create_sync( + owner=user, + workflow=Workflow( + name="single-prediction", + repo_url="https://github.com/test/single-prediction", + default_revision="main", + config_path="/config/single-prediction.config", + ), + tool="boltz", + sample_id="T1024", + seqera_run_id="wf-boltz-pae-ranks", + ) + + output_contents = { + f"{run.id}/boltz/T1024/paes/T1024_0_pae.tsv": b"rank-0-pae\n", + f"{run.id}/boltz/T1024/paes/T1024_1_pae.tsv": b"rank-1-pae\n", + f"{run.id}/boltz/T1024/T1024_plddt.tsv": b"plddt\n", + f"{run.id}/boltz/T1024/T1024_ptm.tsv": b"0\t0.9\n", + } + outputs = [S3Object(object_key=key, uri=f"s3://bucket/{key}") for key in output_contents] + test_db.add_all([user, run, *outputs]) + test_db.commit() + test_db.add_all([_make_run_output(run, item.object_key) for item in outputs]) + test_db.commit() + + async def read_bytes(key: str, **_kwargs) -> bytes: + return output_contents[key] + + with patch( + "app.services.results_utils.read_s3_bytes", + new=AsyncMock(side_effect=read_bytes), + ): + zip_buffer = await get_all_downloads_zipped(test_db, run) + + with ZipFile(BytesIO(zip_buffer.getvalue())) as zip_file: + assert set(zip_file.namelist()) == { + "stats_csv/T1024_0_pae.tsv", + "stats_csv/T1024_ptm.tsv", + } + + @pytest.mark.asyncio async def test_get_result_report_download_persists_result_found_only_on_retry( test_db, persistent_models @@ -1014,6 +1064,11 @@ def test_boltz_proteinfold_helpers_classify_keys_and_build_prefixes(): f"{run.id}/mmseqs/T1024.a3m", "T1024" ) == ClassifiedOutput("alignment", "T1024.a3m") + # plddt.tsv is never surfaced, regardless of sample_id + assert ( + classify_boltz_proteinfold_output(f"{run.id}/boltz/T1024/T1024_plddt.tsv", "T1024") is None + ) + # "single_prediction" paths do not match when sample_id is set assert ( classify_boltz_proteinfold_output( From 62ad8caa4b1b7d0d0fd7312c707ce73ad2035ef8 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:30:00 +1000 Subject: [PATCH 18/25] fix: override the content block for pk-trunc fix --- app/templates/detail.html | 78 +-------------------------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/app/templates/detail.html b/app/templates/detail.html index 8bc1a0a1..7a1abaf1 100644 --- a/app/templates/detail.html +++ b/app/templates/detail.html @@ -1,27 +1,5 @@ -{% extends "layout.html" %} +{% extends "@starlette-admin/detail.html" %} {% set pk = model.pk_attr %} -{% block header %} -
-

{{ model.label }}

- -
-{% endblock %} -{% block modal %} - {{ super() }} - {% include "modals/loading.html" %} - {% include "modals/error.html" %} - {% include "modals/delete.html" %} - {% include "modals/actions.html" %} - -{% endblock %} {% block content %}
@@ -103,57 +81,3 @@

{% endblock %} -{% block head_css %} - {{ super() }} - {% for link in model._additional_css_links(request, "DETAIL" | ra) %} - - {% endfor %} -{% endblock %} -{% block script %} - {{ super() }} - - - - {% for link in model._additional_js_links(request, "DETAIL" | ra) %} - - {% endfor %} - -{% endblock %} From d0239a6c7b411f670a0ec7103e1fcd00adebede0 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Fri, 18 Sep 2026 10:45:51 +1000 Subject: [PATCH 19/25] feat: no more bindflow --- app/routes/workflows.py | 170 +++++------------------ app/scheduler/jobs.py | 10 +- app/services/bindflow_config.py | 37 ----- app/services/bindflow_executor.py | 185 ------------------------- app/services/proteindj_config.py | 18 ++- app/services/proteindj_executor.py | 4 +- tests/test_services_bindflow_config.py | 168 ---------------------- 7 files changed, 56 insertions(+), 536 deletions(-) delete mode 100644 app/services/bindflow_config.py delete mode 100644 app/services/bindflow_executor.py delete mode 100644 tests/test_services_bindflow_config.py diff --git a/app/routes/workflows.py b/app/routes/workflows.py index 9c14dfbf..dfc2309f 100644 --- a/app/routes/workflows.py +++ b/app/routes/workflows.py @@ -44,7 +44,6 @@ SinglePredictionEntity, validate_single_prediction_entities, ) -from ..services.bindflow_executor import prepare_bindflow_workflow, resolve_bindflow_asset_path from ..services.credits import ( WorkflowCreditsResponse, is_credits_enabled, @@ -149,8 +148,8 @@ def _extract_final_design_count(form_data: WorkflowFormData | None) -> int | Non """Credit-cost quantity for a launch. Sourced from max_trajectories (the "Number of Trajectories" form field) rather than number_of_final_designs, since the latter is no longer user-facing for bindcraft — it's derived - server-side when the bindflow samplesheet is built (see - datasets.upload_csv_to_s3) and isn't present in the launch payload. + server-side when the samplesheet is built (see datasets.upload_csv_to_s3) + and isn't present in the launch payload. """ if not isinstance(form_data, WorkflowFormData): return None @@ -159,7 +158,7 @@ def _extract_final_design_count(form_data: WorkflowFormData | None) -> int | Non return None try: parsed = int(str(value).strip()) - except TypeError, ValueError: + except (TypeError, ValueError): return None return parsed if parsed >= 1 else None @@ -214,11 +213,11 @@ async def _stage_referenced_samplesheet_file( return the s3InputKey of a corrected samplesheet with that column rewritten to the local path. - Some samplesheets (bindcraft's starting_pdb, proteinfold's fasta) carry a raw - S3 URI to a separately-uploaded file. Globus stages the samplesheet CSV to - Gadi as-is, but the pipeline reads that column as a local file path, not an - S3 URI (unlike ProteinDJ, which takes its pdb path as a direct pipeline - param, never via a samplesheet) - so the referenced file must be staged + Some samplesheets (proteinfold's fasta) carry a raw S3 URI to a + separately-uploaded file. Globus stages the samplesheet CSV to Gadi as-is, + but the pipeline reads that column as a local file path, not an S3 URI + (unlike ProteinDJ, which takes its pdb path as a direct pipeline param, + never via a samplesheet) - so the referenced file must be staged separately and the samplesheet corrected to point at where it lands. """ try: @@ -289,64 +288,6 @@ async def _stage_referenced_samplesheet_file( return csv_upload.file_key -async def _rewrite_bindflow_settings_asset_columns( - *, s3_input_key: str, repo_assets_path: str -) -> str: - """Fill in settings_filters/settings_advanced samplesheet columns with the - local Gadi path to bindflow's bundled default JSON files - the frontend - leaves these fields unset (see sbp-portal's de-novo-design.ts), so - resolve_bindflow_asset_path fills in the known default; any other, - genuinely custom value is left untouched. - - Unlike starting_pdb (_stage_referenced_samplesheet_file, above) these - columns don't need their own Globus transfer or RunInput/DataTransfer - bookkeeping - they reference files that are already part of the workflow - repo, staged as a whole. This is a plain string rewrite, re-uploaded only - if something actually changed. - """ - try: - samplesheet_rows = await read_csv_from_s3(s3_input_key) - except S3ConfigurationError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"S3 configuration error: {exc}", - ) from exc - except S3ServiceError as exc: - raise HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, - detail=f"Failed to read samplesheet at s3InputKey: {exc}", - ) from exc - if not samplesheet_rows: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="Samplesheet at s3InputKey is empty.", - ) - samplesheet_row = samplesheet_rows[0] - changed = False - for field_name in ("settings_filters", "settings_advanced"): - resolved = resolve_bindflow_asset_path( - field_name, samplesheet_row.get(field_name), repo_assets_path=repo_assets_path - ) - if resolved is not None and resolved != samplesheet_row.get(field_name): - samplesheet_row[field_name] = resolved - changed = True - if not changed: - return s3_input_key - try: - csv_upload = await upload_csv_to_s3(samplesheet_row) - except S3ConfigurationError as exc: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"S3 configuration error: {exc}", - ) from exc - except S3ServiceError as exc: - raise HTTPException( - status_code=status.HTTP_502_BAD_GATEWAY, - detail=f"Failed to re-upload corrected samplesheet: {exc}", - ) from exc - return csv_upload.file_key - - def _stage_wisps_fasta( *, db_session: Session, @@ -484,21 +425,14 @@ async def launch_workflow( detail=f"Workflow '{workflow.name}' is missing default_revision in workflows table.", ) - # Staged for repo_assets_path below - our own backend code reads - # pipeline-bundled asset files (e.g. bindcraft settings) directly off disk. try: - repo_staging_locations = ensure_repo_staging_requested( - db_session, workflow, settings=settings - ) + ensure_repo_staging_requested(db_session, workflow, settings=settings) except RepoStagingError as exc: raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Failed to resolve workflow repo: {exc}", ) from exc pipeline_url = workflow.repo_url - # gadi_path and assets_gadi_path are the same checkout (see - # build_repo_gadi_path, _rewrite_bindflow_settings_asset_columns). - repo_assets_path = repo_staging_locations.assets_gadi_path user = db_session.execute( select(AppUser.email).where(AppUser.id == current_user_id) @@ -583,28 +517,10 @@ async def launch_workflow( # Validation above must run before any of this, since it involves real S3/Globus # I/O that a malformed request shouldn't pay the cost of (and shouldn't be able # to trigger before its own formData is validated). - is_rfdiffusion_launch = ( - workflow_name in ("de-novo-design", "bindflow", "bindcraft") - and selected_tool.lower() == "rfdiffusion" - ) - is_bindcraft_launch = ( - workflow_name in ("de-novo-design", "bindflow", "bindcraft") and not is_rfdiffusion_launch - ) + is_de_novo_design_launch = workflow_name in ("de-novo-design", "bindflow", "bindcraft") is_proteinfold_launch = workflow_name in ("single-prediction", "proteinfold") is_wisps_launch = workflow_name in ("interaction-screening", "bulk-prediction") - if is_bindcraft_launch: - s3_input_key = await _stage_referenced_samplesheet_file( - db_session=db_session, - s3_input_key=s3_input_key, - field_name="starting_pdb", - run_id=run_id, - workflow_name=workflow_name, - globus_settings=settings.globus, - ) - s3_input_key = await _rewrite_bindflow_settings_asset_columns( - s3_input_key=s3_input_key, repo_assets_path=repo_assets_path - ) - elif is_proteinfold_launch: + if is_proteinfold_launch: s3_input_key = await _stage_referenced_samplesheet_file( db_session=db_session, s3_input_key=s3_input_key, @@ -624,14 +540,15 @@ async def launch_workflow( ) staged_input_location: str | None = None - if not is_rfdiffusion_launch: - # rfdiffusion (ProteinDJ) has no samplesheet: s3InputKey for it is the - # starting PDB's own S3 URI, not a bare key (see de-novo-design.ts, which - # skips the samplesheet upload for this tool and reuses starting_pdb's URI - # directly). prepare_proteindj_workflow stages that PDB itself via its own - # DataTransfer, so staging "s3InputKey" here too would both double-prefix - # the URI (it's already a full s3:// URI, not a bare key) and create a - # second, unused DataTransfer for the same file. + if not is_de_novo_design_launch: + # de-novo-design (ProteinDJ, both tools) has no samplesheet: s3InputKey + # for it is the starting PDB's own S3 URI, not a bare key (see + # de-novo-design.ts, which skips the samplesheet upload for rfdiffusion + # and reuses starting_pdb's URI directly). prepare_proteindj_workflow + # stages that PDB itself via its own DataTransfer, so staging + # "s3InputKey" here too would both double-prefix the URI (it's already + # a full s3:// URI, not a bare key) and create a second, unused + # DataTransfer for the same file. s3_bucket = settings.aws.s3_bucket s3_input_uri = f"s3://{s3_bucket}/{s3_input_key}" if db_session.get(S3Object, s3_input_key) is None: @@ -680,39 +597,22 @@ async def launch_workflow( staged_input_location=staged_input_location, ) elif workflow_name in ("de-novo-design", "bindflow", "bindcraft"): - # de-novo-design → bindflow executor (bindcraft) or proteindj executor - # (rfdiffusion), depending on the chosen algorithm. - tool_mode = selected_tool + # de-novo-design → proteindj executor for both tools, distinguished + # only by the design_mode prepare_proteindj_workflow computes from + # the selected tool (see get_proteindj_design_mode). de_novo_launch_form = payload.launch.model_copy(update={"runName": seqera_run_name}) - if tool_mode.lower() == "rfdiffusion": - queued_job = await prepare_proteindj_workflow( - de_novo_launch_form, - settings=settings, - db_session=db_session, - workflow_run=workflow_run, - pipeline=pipeline_url, - config_path=workflow.config_path, - revision=workflow.default_revision, - output_id=str(run_id), - form_data=payload.formData, - user_details=user_details, - ) - else: - assert staged_input_location is not None - queued_job = await prepare_bindflow_workflow( - de_novo_launch_form, - settings=settings, - db_session=db_session, - workflow_run=workflow_run, - pipeline=pipeline_url, - config_path=workflow.config_path, - revision=workflow.default_revision, - output_id=str(run_id), - form_data=payload.formData, - user_details=user_details, - staged_input_location=staged_input_location, - repo_assets_path=repo_assets_path, - ) + queued_job = await prepare_proteindj_workflow( + de_novo_launch_form, + settings=settings, + db_session=db_session, + workflow_run=workflow_run, + pipeline=pipeline_url, + config_path=workflow.config_path, + revision=workflow.default_revision, + output_id=str(run_id), + form_data=payload.formData, + user_details=user_details, + ) elif workflow_name in ("interaction-screening", "bulk-prediction"): assert wisps_form_data is not None assert staged_input_location is not None diff --git a/app/scheduler/jobs.py b/app/scheduler/jobs.py index 4066893d..a5687864 100644 --- a/app/scheduler/jobs.py +++ b/app/scheduler/jobs.py @@ -17,7 +17,6 @@ from ..routes.dependencies import get_db from ..schemas.workflows.shared import WorkflowName from ..services import globus_transfer, health, seqera, workflow_repo_staging -from ..services.bindflow_executor import launch_bindflow_workflow from ..services.credits import MONTHLY_CREDIT_REFRESH_ACTOR, SBP_USER_CREDIT_ALLOWANCE from ..services.job_sync import get_runs_requiring_sync, sync_workflow_runs from ..services.proteindj_executor import launch_proteindj_workflow @@ -198,12 +197,9 @@ def launch_job(job_id: UUID, dry_run: bool = False, *, db_session: Session | Non elif workflow_name in ("single-prediction", "proteinfold"): launch_func = launch_proteinfold_workflow elif workflow_name in ("de-novo-design", "bindflow", "bindcraft"): - # de-novo-design covers two algorithms (bindcraft vs rfdiffusion), each - # with its own executor; workflow_run.tool holds the one selected at launch. - tool = (job.workflow_run.tool or "").lower() - launch_func = ( - launch_proteindj_workflow if tool == "rfdiffusion" else launch_bindflow_workflow - ) + # de-novo-design covers two algorithms (bindcraft vs rfdiffusion), both + # via the proteindj executor - it derives design_mode from workflow_run.tool. + launch_func = launch_proteindj_workflow else: raise ValueError(f"Unsupported workflow: {job.workflow.name}") try: diff --git a/app/services/bindflow_config.py b/app/services/bindflow_config.py deleted file mode 100644 index 9d19e649..00000000 --- a/app/services/bindflow_config.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Bindflow workflow configuration and executor settings.""" - -from __future__ import annotations - -from typing import Any - -from ..schemas.workflows.shared import WorkflowUserDetails -from .workflow_config_fetcher import fetch_workflow_config - - -def get_bindflow_default_params( - out_dir: str, samplesheet_url: str, gadi_project: str = "yz52" -) -> dict[str, Any]: - """Get default parameters for bindflow workflow.""" - return { - "project": gadi_project, - "outdir": out_dir, - "input": samplesheet_url, - } - - -def get_bindflow_config_profiles() -> list[str]: - """Get config profiles for bindflow workflow.""" - return ["singularity"] - - -def get_bindflow_config_text( - config_file_path: str, - *, - user_details: WorkflowUserDetails, -) -> str: - """Read bindflow base config and append a process override block with runtime values.""" - base = fetch_workflow_config(config_file_path) - - cluster_opts = f"-A {user_details.get_encoded_account_details()}" - override = f'\nprocess {{\n clusterOptions = "{cluster_opts}"\n}}\n' - return base + override diff --git a/app/services/bindflow_executor.py b/app/services/bindflow_executor.py deleted file mode 100644 index ae16a1b6..00000000 --- a/app/services/bindflow_executor.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Bindflow workflow executor for Seqera Platform.""" - -from __future__ import annotations - -import logging -from datetime import UTC, datetime -from typing import Any - -from sqlalchemy.orm import Session - -from ..config import Settings, get_settings -from ..db.models import QueuedJob, WorkflowRun -from ..schemas.workflows.shared import WorkflowFormData, WorkflowLaunchForm, WorkflowUserDetails -from .bindflow_config import ( - get_bindflow_config_profiles, - get_bindflow_config_text, - get_bindflow_default_params, -) -from .globus_transfer import build_gadi_output_path -from .launch_payloads import ( - DEFAULT_MODULE_LOADS, - get_executor_script, - inject_prerun_script, - without_prerun_script, -) -from .seqera import ( - WorkflowLaunchResult, - params_to_yaml_text, - post_seqera_launch, -) -from .seqera_errors import WorkflowLaunchError - -logger = logging.getLogger(__name__) - -# settings_filters/settings_advanced reference default JSON files bundled in -# the bindflow repo itself. The frontend leaves these fields unset (see -# sbp-portal's de-novo-design.ts) and relies on the backend to fill in the -# local Gadi path: resolve_bindflow_asset_path below fills in the known -# default for an empty value; anything else (a genuinely custom value) is -# passed through unchanged rather than guessed at, since staging arbitrary -# user-supplied settings files isn't supported yet. -_BINDFLOW_DEFAULT_ASSET_RELATIVE_PATHS = { - "settings_filters": "assets/bindcraft/default_filters.json", - "settings_advanced": "assets/bindcraft/default_4stage_multimer.json", -} - - -def resolve_bindflow_asset_path( - field_name: str, value: object, *, repo_assets_path: str -) -> str | None: - if isinstance(value, str) and value.strip(): - return value - default_relative_path = _BINDFLOW_DEFAULT_ASSET_RELATIVE_PATHS.get(field_name) - if default_relative_path is None: - return None - return f"{repo_assets_path}/{default_relative_path}" - - -async def prepare_bindflow_workflow( # pylint: disable=too-many-locals - form: WorkflowLaunchForm, - *, - settings: Settings, - db_session: Session, - workflow_run: WorkflowRun, - pipeline: str, - config_path: str, - revision: str | None = None, - output_id: str | None = None, - form_data: WorkflowFormData, - user_details: WorkflowUserDetails, - staged_input_location: str, - repo_assets_path: str, - commit: bool = False, -) -> QueuedJob: - """Build and queue a bindflow launch payload.""" - workspace_id = settings.seqera.work_space - compute_env_id = settings.seqera.compute_id - work_dir = settings.seqera.work_dir - - run_name = (form.runName or "").strip() - if not run_name: - raise WorkflowLaunchError("Missing run name for workflow launch") - # Always use a unique backend-generated ID for outputs to avoid S3 prefix collisions. - output_key = (output_id or "").strip() - if not output_key: - raise WorkflowLaunchError("Missing output identifier for workflow launch") - out_dir = build_gadi_output_path( - output_key, - "de-novo-design", - globus_settings=settings.globus, - ) - - default_params = get_bindflow_default_params(out_dir, staged_input_location) - settings_filters = resolve_bindflow_asset_path( - "settings_filters", - form_data.extra_fields.get("settings_filters"), - repo_assets_path=repo_assets_path, - ) - settings_advanced = resolve_bindflow_asset_path( - "settings_advanced", - form_data.extra_fields.get("settings_advanced"), - repo_assets_path=repo_assets_path, - ) - if settings_filters: - default_params["settings_filters"] = settings_filters - if settings_advanced: - default_params["settings_advanced"] = settings_advanced - - # Serialize to YAML - params_text = params_to_yaml_text(default_params) - - # Add custom paramsText from frontend if provided - if form.paramsText and form.paramsText.strip(): - params_text = f"{params_text}\n{form.paramsText.rstrip()}" - - launch_payload: dict[str, Any] = { - "computeEnvId": compute_env_id, - "runName": run_name, - "pipeline": pipeline, - "workDir": work_dir, - "workspaceId": workspace_id, - "revision": revision or "dev", - "paramsText": params_text, - "configProfiles": get_bindflow_config_profiles(), - "configText": get_bindflow_config_text( - config_path, - user_details=user_details, - ), - "resume": False, - } - - queued_job = QueuedJob( - workflow=workflow_run.workflow, - workflow_run=workflow_run, - launch_payload=without_prerun_script(launch_payload), - status="pending", - next_attempt_at=datetime.now(UTC), - ) - db_session.add(queued_job) - if commit: - db_session.commit() - else: - db_session.flush() - return queued_job - - -async def launch_bindflow_workflow( # pylint: disable=too-many-locals - *, - queued_job: QueuedJob, - settings: Settings | None = None, - dry_run: bool = False, -) -> WorkflowLaunchResult | None: - """Launch a bindflow workflow on the Seqera Platform.""" - settings = settings or get_settings() - launch_payload = queued_job.launch_payload - - # Log the complete params being sent - logger.info("Launch payload paramsText", extra={"paramsText": launch_payload["paramsText"]}) - - logger.info( - "Launching bindflow workflow via Seqera API", - extra={ - "workspaceId": launch_payload["workspaceId"], - "computeEnvId": launch_payload["computeEnvId"], - "pipeline": launch_payload["pipeline"], - "runName": launch_payload["runName"], - }, - ) - - prerun_script = get_executor_script( - prerun_script_path=queued_job.workflow.prerun_script_path, - repo_gadi_path=queued_job.workflow.repo_gadi_path, - repo_url=queued_job.workflow.repo_url, - module_loads=DEFAULT_MODULE_LOADS, - ) - runtime_payload = inject_prerun_script( - launch_payload=launch_payload, prerun_script=prerun_script - ) - - if dry_run: - logger.info("Dry run - not launching bindflow workflow") - return None - return await post_seqera_launch( - {"launch": runtime_payload}, workflow_label="Bindflow", settings=settings - ) diff --git a/app/services/proteindj_config.py b/app/services/proteindj_config.py index d1ce5086..df554d6d 100644 --- a/app/services/proteindj_config.py +++ b/app/services/proteindj_config.py @@ -1,4 +1,4 @@ -"""ProteinDJ workflow configuration and executor settings (modeled after bindflow).""" +"""ProteinDJ workflow configuration and executor settings.""" from __future__ import annotations @@ -8,17 +8,28 @@ from .workflow_config_fetcher import fetch_workflow_config +def get_proteindj_design_mode(tool: str) -> str: + """Map the de-novo-design tool selection to a ProteinDJ design_mode. + + BindCraft uses ProteinDJ's own bindcraft_denovo fold-design mode; + everything else (rfdiffusion) uses standalone binder design. + """ + return "bindcraft_denovo" if tool.lower() == "bindcraft" else "binder_denovo" + + def get_proteindj_default_params( out_dir: str, input_pdb: str, hotspot_residues: str, num_designs: int, design_length: str, + design_mode: str, ) -> dict[str, Any]: """Get default parameters for proteindj workflow. - ProteinDJ (rfdiffusion) takes a single PDB plus design params directly — - no samplesheet — so these are passed straight through as paramsText keys. + ProteinDJ takes a single PDB plus design params directly — no + samplesheet — so these are passed straight through as paramsText keys. + Serves both the rfdiffusion and bindcraft tools; only design_mode differs. """ return { "out_dir": out_dir, @@ -26,6 +37,7 @@ def get_proteindj_default_params( "hotspot_residues": hotspot_residues, "num_designs": num_designs, "design_length": design_length, + "design_mode": design_mode, } diff --git a/app/services/proteindj_executor.py b/app/services/proteindj_executor.py index c2b534aa..c299b1ea 100644 --- a/app/services/proteindj_executor.py +++ b/app/services/proteindj_executor.py @@ -1,4 +1,4 @@ -"""ProteinDJ workflow executor for Seqera Platform (modeled after bindflow).""" +"""ProteinDJ workflow executor for Seqera Platform.""" from __future__ import annotations @@ -26,6 +26,7 @@ get_proteindj_config_profiles, get_proteindj_config_text, get_proteindj_default_params, + get_proteindj_design_mode, ) from .results_utils import s3_uri_to_key from .seqera import ( @@ -125,6 +126,7 @@ async def prepare_proteindj_workflow( # pylint: disable=too-many-locals hotspot_residues=proteindj_fields.target_hotspot_residues, num_designs=proteindj_fields.number_of_final_designs, design_length=_design_length(proteindj_fields), + design_mode=get_proteindj_design_mode(form_data.tool), ) # Serialize to YAML diff --git a/tests/test_services_bindflow_config.py b/tests/test_services_bindflow_config.py deleted file mode 100644 index 61573aa7..00000000 --- a/tests/test_services_bindflow_config.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Tests for the bindflow configuration module.""" - -# pylint: disable=missing-function-docstring -from __future__ import annotations - -from unittest.mock import mock_open, patch - -import httpx -import pytest -import respx - -from app.schemas.workflows.shared import WorkflowUserDetails -from app.services.bindflow_config import ( - get_bindflow_config_profiles, - get_bindflow_config_text, - get_bindflow_default_params, -) -from app.services.launch_payloads import get_executor_script - -_SHEET_URL = "https://api.seqera.test/workspaces/ws1/datasets/ds1/v/1/n/samplesheet.csv" - - -def _user_details(email: str, ip_address: str = "") -> WorkflowUserDetails: - return WorkflowUserDetails(user_email=email, ip_address=ip_address) - - -# ============================================================================= -# Tests for get_bindflow_default_params() -# ============================================================================= - - -def test_get_bindflow_default_params_contains_outdir(): - params = get_bindflow_default_params("s3://bucket/out", _SHEET_URL) - assert params["outdir"] == "s3://bucket/out" - - -def test_get_bindflow_default_params_contains_input(): - params = get_bindflow_default_params("s3://bucket/out", _SHEET_URL) - assert params["input"] == _SHEET_URL - - -def test_get_bindflow_default_params_required_keys(): - params = get_bindflow_default_params("s3://bucket/out", _SHEET_URL) - for key in ("project", "outdir", "input"): - assert key in params - - -def test_get_bindflow_default_params_is_dict(): - assert isinstance(get_bindflow_default_params("s3://out", _SHEET_URL), dict) - - -def test_get_bindflow_default_params_project_value(): - params = get_bindflow_default_params("s3://bucket/out", _SHEET_URL) - assert params["project"] == "yz52" - - -# ============================================================================= -# Tests for get_executor_script() -# ============================================================================= - - -def _bindflow_executor_script() -> str: - return get_executor_script( - prerun_script_path=None, - repo_gadi_path="/test/workflow_repos/owner-repo/abc123.git", - repo_url="https://github.com/test-owner/test-repo", - module_loads=["singularity", "nextflow"], - ) - - -def test_get_executor_script_loads_modules(): - script = _bindflow_executor_script() - assert "module load singularity" in script - assert "module load nextflow" in script - assert "export NXF_OFFLINE=true" in script - assert "export NXF_ASSETS=/test/workflow_repos/" in script - - -# ============================================================================= -# Tests for get_bindflow_config_profiles() -# ============================================================================= - - -def test_get_bindflow_config_profiles_returns_list(): - assert isinstance(get_bindflow_config_profiles(), list) - - -def test_get_bindflow_config_profiles_contains_singularity(): - profiles = get_bindflow_config_profiles() - assert "singularity" in profiles - assert "gadi" not in profiles - - -# ============================================================================= -# Tests for get_bindflow_config_text() -# ============================================================================= - - -def test_get_bindflow_config_text_includes_base_config(): - with patch("builtins.open", mock_open(read_data="base_config_content")): - result = get_bindflow_config_text( - "/fake/bindflow.config", - user_details=_user_details("user@example.com"), - ) - assert "base_config_content" in result - - -def test_get_bindflow_config_text_appends_process_block(): - with patch("builtins.open", mock_open(read_data="")): - result = get_bindflow_config_text( - "/fake/bindflow.config", - user_details=_user_details("user@example.com"), - ) - assert "process {" in result - assert "clusterOptions" in result - - -def test_get_bindflow_config_text_interpolates_email(): - with patch("builtins.open", mock_open(read_data="")): - result = get_bindflow_config_text( - "/fake/bindflow.config", - user_details=_user_details("alice@example.com"), - ) - assert "-A YWxpY2VAZXhhbXBsZS5jb20=" in result - - -def test_get_bindflow_config_text_without_ip_address_omits_encoding(): - with patch("builtins.open", mock_open(read_data="")): - result = get_bindflow_config_text( - "/fake/bindflow.config", - user_details=_user_details("user@example.com"), - ) - assert "-A dXNlckBleGFtcGxlLmNvbQ==" in result - assert ":" not in result.split("clusterOptions = ")[1] - - -def test_get_bindflow_config_text_with_ip_address_appends_encoded_ip(): - with patch("builtins.open", mock_open(read_data="")): - result = get_bindflow_config_text( - "/fake/bindflow.config", - user_details=_user_details("user@example.com", ip_address="1.2.3.4"), - ) - assert "-A dXNlckBleGFtcGxlLmNvbQ==:MS4yLjMuNA==" in result - - -def test_get_bindflow_config_text_url_fetching(): - with respx.mock: - respx.get("https://raw.githubusercontent.com/org/repo/main/bindflow.config").mock( - return_value=httpx.Response(200, text="remote_base_config") - ) - result = get_bindflow_config_text( - "https://raw.githubusercontent.com/org/repo/main/bindflow.config", - user_details=_user_details("user@example.com"), - ) - assert "remote_base_config" in result - assert "clusterOptions" in result - - -def test_get_bindflow_config_text_url_error_raises(): - with respx.mock: - respx.get("https://raw.githubusercontent.com/org/repo/main/bindflow.config").mock( - return_value=httpx.Response(404, text="Not Found") - ) - with pytest.raises(httpx.HTTPStatusError): - get_bindflow_config_text( - "https://raw.githubusercontent.com/org/repo/main/bindflow.config", - user_details=_user_details("user@example.com"), - ) From 53d63ac8c9c119c3cb870905387dabf9ccb93750 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Fri, 18 Sep 2026 10:48:01 +1000 Subject: [PATCH 20/25] fix: use proteindj instead bindflow --- app/schemas/workflows/de_novo_design.py | 8 ++++---- app/services/datasets.py | 14 +++++++------- app/services/proteinfold_config.py | 2 +- app/services/proteinfold_executor.py | 2 +- app/services/results_utils.py | 4 ++-- tests/conftest.py | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/schemas/workflows/de_novo_design.py b/app/schemas/workflows/de_novo_design.py index b315e398..f9c8833b 100644 --- a/app/schemas/workflows/de_novo_design.py +++ b/app/schemas/workflows/de_novo_design.py @@ -1,8 +1,8 @@ """Schemas for the de-novo-design workflow. -BindCraft's form data is schema-driven from a remote JSON schema (see the -frontend's ``de-novo-design`` workflow), so the only concrete schema here is -for ProteinDJ (the rfdiffusion tool). +Both the rfdiffusion and bindcraft tools launch ProteinDJ (see +``proteindj_executor.py``) and share this same form shape - only the +design_mode passed to the pipeline differs between them. """ from __future__ import annotations @@ -17,7 +17,7 @@ class ProteinDjFormData(WorkflowFormData): - """Form data for the ProteinDJ (rfdiffusion) de-novo-design workflow.""" + """Form data for the ProteinDJ de-novo-design workflow (rfdiffusion or bindcraft tool).""" starting_pdb: str = Field(..., description="S3 URI of the uploaded starting PDB file") target_hotspot_residues: str = Field( diff --git a/app/services/datasets.py b/app/services/datasets.py index 8180a93c..589d5a63 100644 --- a/app/services/datasets.py +++ b/app/services/datasets.py @@ -63,22 +63,22 @@ def convert_form_data_to_csv(form_data: dict[str, Any]) -> str: def _apply_bindcraft_design_target( form_data: dict[str, Any], workflow: str | None, tool: str | None ) -> None: - """BindCraft's bindflow samplesheet requires number_of_final_designs, but - the de-novo-design form only collects max_trajectories ("Number of + """BindCraft's samplesheet requires number_of_final_designs, but the + de-novo-design form only collects max_trajectories ("Number of Trajectories") — this derives the QC-pass target as 2x the trajectory count so the run isn't QC-gated below what was requested, without ever exposing it to the user. Scoped to workflow=de-novo-design AND tool=bindcraft specifically — de-novo-design also covers rfdiffusion, - which has no bindflow samplesheet or trajectory-retry concept and must - never have this derivation applied. Also protects other callers of this - generic samplesheet builder (e.g. single-prediction), even if their own - form data happened to contain a field with this name. + which has no samplesheet or trajectory-retry concept and must never have + this derivation applied. Also protects other callers of this generic + samplesheet builder (e.g. single-prediction), even if their own form data + happened to contain a field with this name. """ if workflow != "de-novo-design" or tool != "bindcraft" or "max_trajectories" not in form_data: return try: max_trajectories = int(str(form_data["max_trajectories"]).strip()) - except TypeError, ValueError: + except (TypeError, ValueError): return form_data["number_of_final_designs"] = max_trajectories * 2 diff --git a/app/services/proteinfold_config.py b/app/services/proteinfold_config.py index ae6e6faa..b79ac9e9 100644 --- a/app/services/proteinfold_config.py +++ b/app/services/proteinfold_config.py @@ -1,4 +1,4 @@ -"""Proteinfold workflow configuration and executor settings (modeled after bindflow).""" +"""Proteinfold workflow configuration and executor settings.""" from __future__ import annotations diff --git a/app/services/proteinfold_executor.py b/app/services/proteinfold_executor.py index 53d4d544..a925d7db 100644 --- a/app/services/proteinfold_executor.py +++ b/app/services/proteinfold_executor.py @@ -1,4 +1,4 @@ -"""Proteinfold workflow executor for Seqera Platform (modeled after bindflow).""" +"""Proteinfold workflow executor for Seqera Platform.""" from __future__ import annotations diff --git a/app/services/results_utils.py b/app/services/results_utils.py index b5835c84..1399d1fa 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -1112,8 +1112,8 @@ def _sync_run_output_records( changed = False # Every workflow config publishes its results under this same run-scoped - # prefix, whether the pipeline calls the param "outdir" (bindflow, - # proteinfold, wisps) or "out_dir" (proteindj) - the value is identical. + # prefix, whether the pipeline calls the param "outdir" (proteinfold, + # wisps) or "out_dir" (proteindj) - the value is identical. run_outdir = _build_s3_uri(str(run.id), settings=settings) for key in keys: diff --git a/tests/conftest.py b/tests/conftest.py index 9f3545c1..a158cd13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -223,7 +223,7 @@ def app(test_engine): description="Test workflow", repo_url="https://github.com/test/repo", default_revision="dev", - config_path="/some/bindflow.config", + config_path="/some/proteindj.config", ) ) setup_session.commit() From 865c79bcc724ccc1bae7611b8a9b8bd869d3af68 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Fri, 18 Sep 2026 10:50:02 +1000 Subject: [PATCH 21/25] fix: tests --- tests/test_routes_workflows.py | 158 ++++++--------------------------- 1 file changed, 26 insertions(+), 132 deletions(-) diff --git a/tests/test_routes_workflows.py b/tests/test_routes_workflows.py index cca80dec..a1faec8b 100644 --- a/tests/test_routes_workflows.py +++ b/tests/test_routes_workflows.py @@ -46,22 +46,6 @@ async def _queue_job_for_route_prepare(form, **kwargs): return queued_job -async def _queue_job_for_proteindj_route_prepare(form, **kwargs): - # proteindj has no samplesheet, so prepare_proteindj_workflow takes no - # s3_input_key positional arg (unlike prepare_bindflow_workflow above). - db_session = kwargs["db_session"] - workflow_run = kwargs["workflow_run"] - queued_job = QueuedJob( - workflow=workflow_run.workflow, - workflow_run=workflow_run, - launch_payload={"runName": form.runName}, - status="pending", - ) - db_session.add(queued_job) - db_session.flush() - return queued_job - - @pytest.fixture def role_check_client(test_engine): """Test client with auth bypassed but require_workflow_execution_role active.""" @@ -93,8 +77,7 @@ def role_check_client(test_engine): description="Test workflow", repo_url="https://github.com/test/repo", default_revision="dev", - config_path="/some/bindflow.config", - prerun_script_path="/some/bindflow-prerun.sh", + config_path="/some/proteindj.config", ) ) setup_session.commit() @@ -113,28 +96,24 @@ def _get_db(): yield c -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow", side_effect=_queue_job_for_route_prepare) -def test_launch_success_without_dataset( - mock_prepare, mock_read_csv, mock_upload_csv, client: TestClient, test_engine -): +@patch( + "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare +) +def test_launch_success_without_dataset(mock_prepare, client: TestClient, test_engine): """Test successful workflow launch without dataset.""" - _mock_samplesheet_staging( - mock_read_csv, mock_upload_csv, "starting_pdb", "s3://test-bucket/pdb/target.pdb" - ) payload = { "launch": { "workflow": "de-novo-design", "tool": "bindcraft", "runName": "test-run", }, - "s3InputKey": "inputs/samplesheets/test.csv", + "s3InputKey": "inputs/pdb/target.pdb", "formData": { "workflow": "de-novo-design", "tool": "bindcraft", "id": "s1", "binder_name": "PDL1", + "starting_pdb": "s3://test-bucket/pdb/target.pdb", "max_trajectories": 20, }, } @@ -182,94 +161,23 @@ def test_launch_success_without_dataset( assert queued_job is not None assert queued_job.status == "staging" - run_input = db.scalar(select(RunInput).where(RunInput.run_id == created_run.id)) - assert run_input is not None - assert run_input.data_transfer_id is not None - input_transfer = db.scalar( - select(DataTransfer).where(DataTransfer.id == run_input.data_transfer_id) - ) - assert input_transfer is not None - assert input_transfer.workflow_run_id == created_run.id - assert input_transfer.direction == "input" - assert input_transfer.provider == "globus" - # The samplesheet is re-uploaded with the starting_pdb column corrected to a - # staged Gadi path (_stage_referenced_samplesheet_file), so the main input - # transfer's source is the corrected samplesheet key, not the original upload. - assert input_transfer.source_location.endswith("inputs/samplesheets/corrected.csv") - assert input_transfer.status == "pending" - assert input_transfer.destination_location == ( - f"/test/input/de-novo-design/{created_run.id}/corrected.csv" - ) - - # The starting_pdb file referenced by the samplesheet gets its own transfer. - pdb_transfer = db.scalar( - select(DataTransfer).where( - DataTransfer.workflow_run_id == created_run.id, - DataTransfer.source_location == "s3://test-bucket/pdb/target.pdb", - ) - ) - assert pdb_transfer is not None - assert pdb_transfer.direction == "input" - assert pdb_transfer.provider == "globus" - assert pdb_transfer.destination_location == ( - f"/test/input/de-novo-design/{created_run.id}/target.pdb" + # ProteinDJ (both tools) stages its own PDB inside prepare_proteindj_workflow, + # which is mocked here - so the generic launch route must not also create a + # DataTransfer for it. + transfer_count = db.scalar( + select(func.count()) + .select_from(DataTransfer) + .where(DataTransfer.workflow_run_id == created_run.id) ) + assert transfer_count == 0 -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow", side_effect=_queue_job_for_route_prepare) -def test_launch_bindcraft_fills_in_default_settings_assets_in_samplesheet( - mock_prepare, mock_read_csv, mock_upload_csv, client: TestClient -): - """settings_filters/settings_advanced samplesheet columns are left empty - by the frontend (see sbp-portal's de-novo-design.ts) - confirmed in - production via a real staged samplesheet. _rewrite_bindflow_settings_ - asset_columns must fill them in with the local Gadi path to bindflow's - bundled default JSON files before the samplesheet is staged to Gadi.""" - mock_read_csv.return_value = [ - { - "starting_pdb": "s3://test-bucket/pdb/target.pdb", - "settings_filters": "", - "settings_advanced": "", - } - ] - mock_upload_csv.return_value = S3UploadResult( - success=True, file_key="inputs/samplesheets/corrected.csv", bucket="test-bucket" - ) - - payload = { - "launch": {"workflow": "de-novo-design", "tool": "bindcraft", "runName": "test-run"}, - "s3InputKey": "inputs/samplesheets/test.csv", - "formData": {"workflow": "de-novo-design", "tool": "bindcraft"}, - } - - response = client.post("/api/workflows/launch", json=payload) - - assert response.status_code == 201 - # First call corrects starting_pdb (_stage_referenced_samplesheet_file), - # second call rewrites the settings_* columns - both re-upload the row. - assert mock_upload_csv.call_count == 2 - rewritten_row = mock_upload_csv.call_args_list[1].args[0] - assert rewritten_row["settings_filters"] == ( - "/staged/workflow-repo/assets/assets/bindcraft/default_filters.json" - ) - assert rewritten_row["settings_advanced"] == ( - "/staged/workflow-repo/assets/assets/bindcraft/default_4stage_multimer.json" - ) - - -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow") +@patch("app.routes.workflows.prepare_proteindj_workflow") def test_launch_queue_preparation_configuration_error( - mock_prepare, mock_read_csv, mock_upload_csv, client: TestClient, test_engine + mock_prepare, client: TestClient, test_engine ): """Local queue payload configuration errors should return 500.""" mock_prepare.side_effect = WorkflowLaunchError("Missing output identifier for workflow launch") - _mock_samplesheet_staging( - mock_read_csv, mock_upload_csv, "starting_pdb", "s3://test-bucket/pdb/target.pdb" - ) payload = { "launch": { @@ -292,16 +200,9 @@ def test_launch_queue_preparation_configuration_error( assert count == 0 -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow") -def test_launch_queue_preparation_error( - mock_prepare, mock_read_csv, mock_upload_csv, client: TestClient, test_engine -): +@patch("app.routes.workflows.prepare_proteindj_workflow") +def test_launch_queue_preparation_error(mock_prepare, client: TestClient, test_engine): """Unexpected queue preparation errors are returned as local queue failures.""" - _mock_samplesheet_staging( - mock_read_csv, mock_upload_csv, "starting_pdb", "s3://test-bucket/pdb/target.pdb" - ) mock_prepare.side_effect = RuntimeError("could not build queue payload") payload = { @@ -366,13 +267,12 @@ def _add_rfdiffusion_workflow(test_engine): db.commit() -@patch("app.routes.workflows.prepare_bindflow_workflow") @patch( "app.routes.workflows.prepare_proteindj_workflow", - side_effect=_queue_job_for_proteindj_route_prepare, + side_effect=_queue_job_for_route_prepare, ) def test_launch_de_novo_design_rfdiffusion_routes_to_proteindj( - mock_prepare_proteindj, mock_prepare_bindflow, client: TestClient, test_engine + mock_prepare_proteindj, client: TestClient, test_engine ): """tool='rfdiffusion' on de-novo-design must dispatch to the proteindj executor.""" _add_rfdiffusion_workflow(test_engine) @@ -393,7 +293,6 @@ def test_launch_de_novo_design_rfdiffusion_routes_to_proteindj( data = response.json() assert data["status"] == "staging" mock_prepare_proteindj.assert_called_once() - mock_prepare_bindflow.assert_not_called() assert ( mock_prepare_proteindj.call_args.kwargs["pipeline"] == "https://github.com/test/proteindj" ) @@ -936,18 +835,13 @@ def test_launch_single_prediction_boltz_potentials_reduces_limit( } -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow", side_effect=_queue_job_for_route_prepare) -def test_launch_allowed_with_workflow_role( - mock_prepare, mock_read_csv, mock_upload_csv, role_check_client, monkeypatch -): +@patch( + "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare +) +def test_launch_allowed_with_workflow_role(mock_prepare, role_check_client, monkeypatch): """Users holding the workflow execution role can launch.""" monkeypatch.setenv("DB_ADMIN_ROLES_CLAIM", ROLES_CLAIM) monkeypatch.setenv("WORKFLOW_EXECUTION_ROLE", WORKFLOW_ROLE) - _mock_samplesheet_staging( - mock_read_csv, mock_upload_csv, "starting_pdb", "s3://test-bucket/pdb/target.pdb" - ) with patch( "app.routes.dependencies.verify_access_token_claims", @@ -1034,7 +928,7 @@ def wisps_client(test_engine): description="Test BindCraft workflow", repo_url="https://github.com/test/repo", default_revision="dev", - config_path="/some/bindflow.config", + config_path="/some/proteindj.config", ) ) From 86d0d6f88032fa1a9019ccf95c01e94fad73fbf3 Mon Sep 17 00:00:00 2001 From: amandazhu Date: Fri, 18 Sep 2026 10:50:27 +1000 Subject: [PATCH 22/25] fix: tests --- tests/test_routes_workflows.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/test_routes_workflows.py b/tests/test_routes_workflows.py index a1faec8b..12a01c7d 100644 --- a/tests/test_routes_workflows.py +++ b/tests/test_routes_workflows.py @@ -1229,13 +1229,11 @@ def test_get_workflow_credits_multipliers_match_spec(client: TestClient): TEST_USER_ID = UUID("11111111-1111-1111-1111-111111111111") -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow", side_effect=_queue_job_for_route_prepare) +@patch( + "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare +) def test_launch_deducts_credits_when_enabled( mock_prepare, - mock_read_csv, - mock_upload_csv, client, test_engine, monkeypatch, @@ -1244,9 +1242,6 @@ def test_launch_deducts_credits_when_enabled( """With credits enabled, a successful de-novo launch deducts multiplier × designs.""" mock_settings.enable_credits = True client.app.dependency_overrides[get_settings] = lambda: mock_settings - _mock_samplesheet_staging( - mock_read_csv, mock_upload_csv, "starting_pdb", "s3://test-bucket/pdb/target.pdb" - ) with Session(test_engine) as db: db.execute(update(AppUser).where(AppUser.id == TEST_USER_ID).values(credit=100)) db.commit() @@ -1271,7 +1266,9 @@ def test_launch_deducts_credits_when_enabled( assert credit == 70 # 100 − (10 × 3) -@patch("app.routes.workflows.prepare_bindflow_workflow", side_effect=_queue_job_for_route_prepare) +@patch( + "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare +) def test_launch_rejected_when_insufficient_credits( mock_prepare, client, test_engine, monkeypatch, mock_settings ): @@ -1302,17 +1299,12 @@ def test_launch_rejected_when_insufficient_credits( assert credit == 10 # unchanged -@patch("app.routes.workflows.upload_csv_to_s3") -@patch("app.routes.workflows.read_csv_from_s3") -@patch("app.routes.workflows.prepare_bindflow_workflow", side_effect=_queue_job_for_route_prepare) -def test_launch_does_not_deduct_when_credits_disabled( - mock_prepare, mock_read_csv, mock_upload_csv, client, test_engine, monkeypatch -): +@patch( + "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare +) +def test_launch_does_not_deduct_when_credits_disabled(mock_prepare, client, test_engine, monkeypatch): """With credits disabled (default), launches never touch the balance.""" monkeypatch.delenv("ENABLE_CREDITS", raising=False) - _mock_samplesheet_staging( - mock_read_csv, mock_upload_csv, "starting_pdb", "s3://test-bucket/pdb/target.pdb" - ) with Session(test_engine) as db: db.execute(update(AppUser).where(AppUser.id == TEST_USER_ID).values(credit=5)) db.commit() From fc6e3cd6385c2767559311fd7398b5268429496d Mon Sep 17 00:00:00 2001 From: amandazhu Date: Fri, 18 Sep 2026 13:41:10 +1000 Subject: [PATCH 23/25] fix: tests --- tests/scheduler/test_scheduler_jobs.py | 29 +- tests/test_proteindj_coverage.py | 66 ++++ tests/test_routes_workflows.py | 2 +- tests/test_services_datasets.py | 6 +- tests/test_services_seqera.py | 408 +------------------------ 5 files changed, 82 insertions(+), 429 deletions(-) diff --git a/tests/scheduler/test_scheduler_jobs.py b/tests/scheduler/test_scheduler_jobs.py index 8eaf8bd4..f576cec7 100644 --- a/tests/scheduler/test_scheduler_jobs.py +++ b/tests/scheduler/test_scheduler_jobs.py @@ -238,7 +238,7 @@ async def _successful_launch(**kwargs): monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _successful_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _successful_launch) scheduler_jobs.launch_job(queued_job.id) @@ -260,7 +260,7 @@ async def _successful_launch(**kwargs): monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _successful_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _successful_launch) scheduler_jobs.launch_job(queued_job.id) @@ -270,7 +270,7 @@ async def _successful_launch(**kwargs): assert queued_job.attempts == 0 -def test_launch_job_submits_successful_bindflow_job(test_db, persistent_models, monkeypatch): +def test_launch_job_submits_successful_proteindj_job(test_db, persistent_models, monkeypatch): queued_job = _create_queued_job() calls = [] @@ -280,7 +280,7 @@ async def _successful_launch(**kwargs): monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _successful_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _successful_launch) scheduler_jobs.launch_job(queued_job.id) @@ -308,7 +308,7 @@ async def _successful_launch(**kwargs): monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _successful_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _successful_launch) scheduler_jobs.launch_job(queued_job.id, dry_run=True) @@ -335,13 +335,9 @@ async def _successful_launch(**kwargs): calls.append(kwargs) return WorkflowLaunchResult(workflow_id="seqera-run-rfd", status="submitted") - async def _unexpected_bindflow_launch(**_kwargs): - raise AssertionError("launch_bindflow_workflow should not be called for rfdiffusion") - monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _successful_launch) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _unexpected_bindflow_launch) scheduler_jobs.launch_job(queued_job.id) @@ -354,7 +350,8 @@ async def _unexpected_bindflow_launch(**_kwargs): assert queued_job.status == "submitted" -def test_launch_job_dispatches_bindflow_for_bindcraft_tool(test_db, persistent_models, monkeypatch): +def test_launch_job_dispatches_proteindj_for_bindcraft_tool(test_db, persistent_models, monkeypatch): + """The BindCraft tool also dispatches through the proteindj executor now.""" queued_job = _create_queued_job(tool="bindcraft") calls = [] @@ -362,13 +359,9 @@ async def _successful_launch(**kwargs): calls.append(kwargs) return WorkflowLaunchResult(workflow_id="seqera-run-bc", status="submitted") - async def _unexpected_proteindj_launch(**_kwargs): - raise AssertionError("launch_proteindj_workflow should not be called for bindcraft") - monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _successful_launch) - monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _unexpected_proteindj_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _successful_launch) scheduler_jobs.launch_job(queued_job.id) @@ -440,7 +433,7 @@ def test_launch_job_dry_run_failure_does_not_update_job(test_db, persistent_mode queued_job = _create_queued_job() monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _failing_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _failing_launch) scheduler_jobs.launch_job(queued_job.id, dry_run=True) @@ -458,7 +451,7 @@ def test_launch_job_counts_failed_attempt_and_schedules_retry( queued_job = _create_queued_job() monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _failing_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _failing_launch) scheduler_jobs.launch_job(queued_job.id) @@ -482,7 +475,7 @@ def test_launch_job_marks_failed_after_max_failed_attempts(test_db, persistent_m queued_job = _create_queued_job(attempts=scheduler_jobs.LAUNCH_MAX_ATTEMPTS - 1) monkeypatch.setattr(scheduler_jobs, "get_db", _get_db_override(test_db)) monkeypatch.setattr(scheduler_jobs, "is_seqera_available", lambda _db, **_kwargs: True) - monkeypatch.setattr(scheduler_jobs, "launch_bindflow_workflow", _failing_launch) + monkeypatch.setattr(scheduler_jobs, "launch_proteindj_workflow", _failing_launch) scheduler_jobs.launch_job(queued_job.id) diff --git a/tests/test_proteindj_coverage.py b/tests/test_proteindj_coverage.py index fae40795..aa5d8b9e 100644 --- a/tests/test_proteindj_coverage.py +++ b/tests/test_proteindj_coverage.py @@ -16,6 +16,7 @@ get_proteindj_config_profiles, get_proteindj_config_text, get_proteindj_default_params, + get_proteindj_design_mode, ) from app.services.proteindj_executor import ( _design_length, @@ -179,6 +180,7 @@ def test_get_proteindj_default_params_all_fields(): hotspot_residues="A20,A21", num_designs=5, design_length="100-150", + design_mode="binder_denovo", ) assert params == { "out_dir": "s3://bucket/out", @@ -186,6 +188,7 @@ def test_get_proteindj_default_params_all_fields(): "hotspot_residues": "A20,A21", "num_designs": 5, "design_length": "100-150", + "design_mode": "binder_denovo", } @@ -194,6 +197,23 @@ def test_get_proteindj_default_params_missing_required_field_raises(): get_proteindj_default_params("s3://bucket/out", num_designs=3) # type: ignore[call-arg] +# ============================================================================= +# Tests for get_proteindj_design_mode() +# ============================================================================= + + +def test_get_proteindj_design_mode_bindcraft(): + assert get_proteindj_design_mode("bindcraft") == "bindcraft_denovo" + + +def test_get_proteindj_design_mode_rfdiffusion(): + assert get_proteindj_design_mode("rfdiffusion") == "binder_denovo" + + +def test_get_proteindj_design_mode_is_case_insensitive(): + assert get_proteindj_design_mode("BindCraft") == "bindcraft_denovo" + + # ============================================================================= # Tests for get_proteindj_config_profiles() # ============================================================================= @@ -334,6 +354,7 @@ async def test_prepare_proteindj_workflow_writes_expected_queued_job( assert "hotspot_residues: A20,A21" in params_text assert "num_designs: 5" in params_text assert "design_length: 100-150" in params_text + assert "design_mode: binder_denovo" in params_text # The uploaded starting-pdb file gets its own Globus staging record, separate # from the main samplesheet input handled in the workflows route. @@ -359,6 +380,51 @@ async def test_prepare_proteindj_workflow_writes_expected_queued_job( assert s3_object.uri == "s3://my-bucket/inputs/test.pdb" +@pytest.mark.anyio +async def test_prepare_proteindj_workflow_bindcraft_tool_uses_bindcraft_denovo_mode( + test_db, persistent_models, seqera_env +): + """The BindCraft toggle launches ProteinDJ with design_mode=bindcraft_denovo.""" + user = AppUserFactory.create_sync() + workflow = WorkflowFactory.create_sync(name="de-novo-design") + workflow_run = WorkflowRunFactory.create_sync(workflow=workflow, owner=user) + + form = _make_launch_form(tool="bindcraft", runName="queued-bindcraft-run") + form_data = WorkflowFormData( + workflow="de-novo-design", + tool="bindcraft", + starting_pdb="s3://my-bucket/inputs/test.pdb", + target_hotspot_residues="A20,A21", + number_of_final_designs=5, + min_length=100, + max_length=150, + ) + + with ( + patch("app.services.proteindj_executor.get_proteindj_config_text", return_value=""), + patch( + "app.services.proteindj_executor.get_proteindj_config_profiles", + return_value=["singularity"], + ), + ): + await prepare_proteindj_workflow( + form=form, + settings=seqera_env, + db_session=test_db, + workflow_run=workflow_run, + pipeline="https://github.com/org/proteindj", + config_path="/fake/proteindj.config", + output_id="run-output-id", + form_data=form_data, + user_details=_USER_DETAILS, + ) + + queued_job = test_db.scalar( + select(QueuedJob).where(QueuedJob.workflow_run_id == workflow_run.id) + ) + assert "design_mode: bindcraft_denovo" in queued_job.launch_payload["paramsText"] + + @pytest.mark.anyio async def test_prepare_proteindj_workflow_appends_custom_params_text( test_db, persistent_models, seqera_env diff --git a/tests/test_routes_workflows.py b/tests/test_routes_workflows.py index 12a01c7d..aae4f44b 100644 --- a/tests/test_routes_workflows.py +++ b/tests/test_routes_workflows.py @@ -13,7 +13,7 @@ from app.config import get_settings from app.db.models import QueuedJob -from app.db.models.core import AppUser, DataTransfer, RunInput, RunMetric, Workflow, WorkflowRun +from app.db.models.core import AppUser, DataTransfer, RunMetric, Workflow, WorkflowRun from app.routes.dependencies import get_current_user_id, get_db from app.services.s3 import S3UploadResult from app.services.seqera_errors import WorkflowLaunchError diff --git a/tests/test_services_datasets.py b/tests/test_services_datasets.py index 70023d52..3506777c 100644 --- a/tests/test_services_datasets.py +++ b/tests/test_services_datasets.py @@ -253,9 +253,9 @@ def test_apply_bindcraft_design_target_noop_for_other_workflows(): def test_apply_bindcraft_design_target_noop_for_rfdiffusion(): - """de-novo-design also covers rfdiffusion, which has no bindflow - samplesheet or trajectory-retry concept — the derivation must never - apply to it, even though it shares the same workflow name as bindcraft.""" + """de-novo-design also covers rfdiffusion, which has no samplesheet or + trajectory-retry concept — the derivation must never apply to it, even + though it shares the same workflow name as bindcraft.""" form_data = {"max_trajectories": 5} _apply_bindcraft_design_target(form_data, "de-novo-design", "rfdiffusion") assert "number_of_final_designs" not in form_data diff --git a/tests/test_services_seqera.py b/tests/test_services_seqera.py index 3fe3281a..db517ffe 100644 --- a/tests/test_services_seqera.py +++ b/tests/test_services_seqera.py @@ -2,418 +2,12 @@ from __future__ import annotations -import json -from contextlib import contextmanager -from unittest.mock import Mock, patch - import httpx import pytest import respx -from sqlalchemy import select -from app.db.models import QueuedJob -from app.schemas.workflows.shared import WorkflowFormData, WorkflowLaunchForm, WorkflowUserDetails -from app.services.bindflow_executor import ( - launch_bindflow_workflow, - prepare_bindflow_workflow, -) -from app.services.seqera import ( - WorkflowExecutorError, - WorkflowLaunchResult, - count_active_workflows, - get_queue_status, -) +from app.services.seqera import count_active_workflows, get_queue_status from app.services.seqera_errors import SeqeraAPIError -from tests.datagen import AppUserFactory, QueuedJobFactory, WorkflowFactory, WorkflowRunFactory - -_CONFIG_PATH = "/some/bindflow.config" -_USER_DETAILS = WorkflowUserDetails( - user_email="test@example.com", - ip_address="127.0.0.1", -) - - -def _empty_form_data() -> WorkflowFormData: - return WorkflowFormData(workflow="de-novo-design", tool="bindcraft") - - -def _queued_bindflow_job( - *, - params_text: str | None = None, - prerun_script_path: str | None = None, -) -> QueuedJob: - user = AppUserFactory.create_sync() - workflow = WorkflowFactory.create_sync( - name="de-novo-design", - repo_url="https://github.com/test/repo", - prerun_script_path=prerun_script_path, - ) - workflow_run = WorkflowRunFactory.create_sync(workflow=workflow, owner=user) - launch_payload = { - "computeEnvId": "test_compute_env_id", - "runName": "seqera-test-run", - "pipeline": "https://github.com/test/repo", - "workDir": "/test/work/dir", - "workspaceId": "test_workspace_id", - "revision": "dev", - "paramsText": params_text - or ( - "project: yz52\n" - "outdir: s3://test-s3-bucket/run-out\n" - "input: s3://test-s3-bucket/inputs/samplesheets/test.csv\n" - "mode: bindcraft" - ), - "configProfiles": ["gadi"], - "configText": "", - "resume": False, - } - return QueuedJobFactory.create_sync( - workflow=workflow, - workflow_run=workflow_run, - launch_payload=launch_payload, - status="pending", - ) - - -@contextmanager -def _mock_bindflow_db_context(): - workflow = Mock(name="workflow") - workflow_run = Mock(name="workflow_run") - workflow_run.workflow = workflow - db_session = Mock(name="db_session") - queued_job = Mock(name="queued_job") - with patch( - "app.services.bindflow_executor.QueuedJob", return_value=queued_job - ) as queued_job_cls: - yield db_session, workflow_run, workflow, queued_job_cls, queued_job - - -@pytest.fixture(autouse=True) -def mock_bindflow_config_text(): - """Prevent get_bindflow_config_text from trying to open a real file.""" - with patch("app.services.bindflow_executor.get_bindflow_config_text", return_value=""): - yield - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_success_minimal(persistent_models): - """Test successful workflow launch with minimal parameters.""" - route = respx.post(url__regex=r"https://api\.seqera\.test/workflow/launch.*").mock( - return_value=httpx.Response( - 200, - json={"workflowId": "wf_test_123"}, - ) - ) - - result = await launch_bindflow_workflow(queued_job=_queued_bindflow_job()) - - assert isinstance(result, WorkflowLaunchResult) - assert result.workflow_id == "wf_test_123" - assert result.status == "submitted" - assert route.called - assert route.call_count == 1 - request = route.calls.last.request - payload = json.loads(request.content) - assert "module load singularity" in payload["launch"]["preRunScript"] - assert "module load nextflow" in payload["launch"]["preRunScript"] - - -@pytest.mark.asyncio -async def test_prepare_bindflow_workflow_writes_expected_queued_job( - test_db, persistent_models, mock_settings -): - mock_settings.seqera.work_space = "ws_123" - mock_settings.seqera.compute_id = "ce_456" - mock_settings.seqera.work_dir = "/work/dir" - mock_settings.aws.s3_bucket = "my-bucket" - - user = AppUserFactory.create_sync() - workflow = WorkflowFactory.create_sync() - workflow_run = WorkflowRunFactory.create_sync(workflow=workflow, owner=user) - - form = WorkflowLaunchForm( - workflow="de-novo-design", - tool="bindcraft", - runName="queued-bindflow-run", - paramsText="custom_param: value", - ) - - with ( - patch("app.services.bindflow_executor.get_bindflow_config_profiles", return_value=["gadi"]), - patch( - "app.services.bindflow_executor.get_bindflow_config_text", return_value="config_text" - ), - ): - prepared_job = await prepare_bindflow_workflow( - form=form, - settings=mock_settings, - db_session=test_db, - workflow_run=workflow_run, - pipeline="https://github.com/test/repo", - config_path=_CONFIG_PATH, - revision="main", - output_id="run-output-id", - form_data=_empty_form_data(), - user_details=_USER_DETAILS, - staged_input_location="/test/input/de-novo-design/run-id/test.csv", - repo_assets_path="/test/workflow_repos/test-repo/abc123", - ) - - queued_job = test_db.scalar( - select(QueuedJob).where(QueuedJob.workflow_run_id == workflow_run.id) - ) - assert queued_job is not None - assert queued_job.workflow_id == workflow.id - assert queued_job.workflow_run_id == workflow_run.id - assert queued_job.status == "pending" - assert queued_job.next_attempt_at is not None - assert queued_job.id == prepared_job.id - assert queued_job.launch_payload["computeEnvId"] == "ce_456" - assert queued_job.launch_payload["runName"] == "queued-bindflow-run" - assert queued_job.launch_payload["pipeline"] == "https://github.com/test/repo" - assert queued_job.launch_payload["workDir"] == "/work/dir" - assert queued_job.launch_payload["workspaceId"] == "ws_123" - assert queued_job.launch_payload["revision"] == "main" - assert queued_job.launch_payload["configProfiles"] == ["gadi"] - assert queued_job.launch_payload["configText"] == "config_text" - assert "preRunScript" not in queued_job.launch_payload - assert queued_job.launch_payload["resume"] is False - assert ( - "outdir: /test/output/de-novo-design/run-output-id" - in queued_job.launch_payload["paramsText"] - ) - assert ( - "input: /test/input/de-novo-design/run-id/test.csv" - in queued_job.launch_payload["paramsText"] - ) - assert "mode:" not in queued_job.launch_payload["paramsText"] - assert "custom_param: value" in queued_job.launch_payload["paramsText"] - - -@pytest.mark.asyncio -async def test_prepare_bindflow_workflow_fills_in_default_when_settings_unset( - test_db, persistent_models, mock_settings -): - """The frontend no longer sends any value for settings_filters/ - settings_advanced (see sbp-portal's de-novo-design.ts) - an empty/missing - value must still resolve to the bindflow repo's bundled default file, - not be left blank.""" - user = AppUserFactory.create_sync() - workflow = WorkflowFactory.create_sync() - workflow_run = WorkflowRunFactory.create_sync(workflow=workflow, owner=user) - - form = WorkflowLaunchForm(workflow="de-novo-design", tool="bindcraft", runName="run-1") - - with ( - patch("app.services.bindflow_executor.get_bindflow_config_profiles", return_value=["gadi"]), - patch( - "app.services.bindflow_executor.get_bindflow_config_text", return_value="config_text" - ), - ): - await prepare_bindflow_workflow( - form=form, - settings=mock_settings, - db_session=test_db, - workflow_run=workflow_run, - pipeline="file:/g/data/yz52/sbp_data/workflow_repos/x-bindflow/abc123.git", - config_path=_CONFIG_PATH, - revision="dev", - output_id="run-output-id", - form_data=_empty_form_data(), - user_details=_USER_DETAILS, - staged_input_location="/test/input/de-novo-design/run-id/test.csv", - repo_assets_path="/g/data/yz52/sbp_data/workflow_repos/x-bindflow/abc123.git", - ) - - queued_job = test_db.scalar( - select(QueuedJob).where(QueuedJob.workflow_run_id == workflow_run.id) - ) - assert queued_job is not None - params_text = queued_job.launch_payload["paramsText"] - assert ( - "settings_filters: /g/data/yz52/sbp_data/workflow_repos/x-bindflow/abc123.git/" - "assets/bindcraft/default_filters.json" in params_text - ) - assert ( - "settings_advanced: /g/data/yz52/sbp_data/workflow_repos/x-bindflow/abc123.git/" - "assets/bindcraft/default_4stage_multimer.json" in params_text - ) - - -@pytest.mark.asyncio -async def test_prepare_bindflow_workflow_passes_through_custom_settings_value( - test_db, persistent_models, mock_settings -): - """A value that doesn't match the known default bindflow-repo URL is left - unchanged - there's no support for staging arbitrary user-supplied - settings files yet, so it's passed through as-is rather than guessed at.""" - user = AppUserFactory.create_sync() - workflow = WorkflowFactory.create_sync() - workflow_run = WorkflowRunFactory.create_sync(workflow=workflow, owner=user) - - form = WorkflowLaunchForm(workflow="de-novo-design", tool="bindcraft", runName="run-1") - form_data = WorkflowFormData( - workflow="de-novo-design", - tool="bindcraft", - settings_filters="https://example.com/my-custom-filters.json", - settings_advanced="https://example.com/my-custom-advanced.json", - ) - - with ( - patch("app.services.bindflow_executor.get_bindflow_config_profiles", return_value=["gadi"]), - patch( - "app.services.bindflow_executor.get_bindflow_config_text", return_value="config_text" - ), - ): - await prepare_bindflow_workflow( - form=form, - settings=mock_settings, - db_session=test_db, - workflow_run=workflow_run, - pipeline="file:/g/data/yz52/sbp_data/workflow_repos/x-bindflow/abc123.git", - config_path=_CONFIG_PATH, - revision="dev", - output_id="run-output-id", - form_data=form_data, - user_details=_USER_DETAILS, - staged_input_location="/test/input/de-novo-design/run-id/test.csv", - repo_assets_path="/g/data/yz52/sbp_data/workflow_repos/x-bindflow/abc123.git", - ) - - queued_job = test_db.scalar( - select(QueuedJob).where(QueuedJob.workflow_run_id == workflow_run.id) - ) - assert queued_job is not None - params_text = queued_job.launch_payload["paramsText"] - assert "settings_filters: https://example.com/my-custom-filters.json" in params_text - assert "settings_advanced: https://example.com/my-custom-advanced.json" in params_text - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_success_with_all_params(persistent_models): - """Test successful launch with all parameters.""" - route = respx.post(url__regex=r".*/workflow/launch.*").mock( - return_value=httpx.Response( - 200, - json={"workflowId": "wf_full_456"}, - ) - ) - - with ( - _mock_bindflow_db_context() as (db_session, workflow_run, *_), - patch( - "app.services.bindflow_executor.get_executor_script", - return_value="prerun_body", - ) as mock_script, - ): - result = await launch_bindflow_workflow( - queued_job=_queued_bindflow_job( - params_text=( - "input: s3://test-s3-bucket/inputs/samplesheets/test.csv\n" - "custom_param: value" - ), - prerun_script_path="/some/prerun.sh", - ) - ) - - assert result.workflow_id == "wf_full_456" - assert route.called - request = route.calls.last.request - payload = json.loads(request.content) - assert ( - "input: s3://test-s3-bucket/inputs/samplesheets/test.csv" in payload["launch"]["paramsText"] - ) - assert payload["launch"]["preRunScript"] == "prerun_body" - assert mock_script.call_args.kwargs["prerun_script_path"] == "/some/prerun.sh" - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_includes_default_params(persistent_models): - """Test that default parameters are included.""" - route = respx.post(url__regex=r".*/workflow/launch.*").mock( - return_value=httpx.Response(200, json={"workflowId": "wf_123"}) - ) - - await launch_bindflow_workflow(queued_job=_queued_bindflow_job()) - - request = route.calls.last.request - payload = json.loads(request.content) - params_text = payload["launch"]["paramsText"] - - assert "project: yz52" in params_text - assert "outdir:" in params_text - assert "input:" in params_text - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_with_dataset_adds_input_url(persistent_models): - """Test that providing a dataset ID adds it to launch payload.""" - route = respx.post(url__regex=r".*/workflow/launch.*").mock( - return_value=httpx.Response(200, json={"workflowId": "wf_dataset_999"}) - ) - - await launch_bindflow_workflow(queued_job=_queued_bindflow_job()) - - request = route.calls.last.request - payload = json.loads(request.content) - params_text = payload["launch"]["paramsText"] - - assert "input: s3://test-s3-bucket/inputs/samplesheets/test.csv" in params_text - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_api_error_response(persistent_models): - """Test handling of API error response.""" - respx.post(url__regex=r".*/workflow/launch.*").mock( - return_value=httpx.Response(400, text="Invalid request") - ) - - with pytest.raises(WorkflowExecutorError, match="400"): - await launch_bindflow_workflow(queued_job=_queued_bindflow_job()) - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_missing_workflow_id_in_response(persistent_models): - """Test error handling when API response lacks workflowId.""" - respx.post(url__regex=r".*/workflow/launch.*").mock( - return_value=httpx.Response(200, json={"status": "success"}) - ) - - with pytest.raises(WorkflowExecutorError, match="workflowId"): - await launch_bindflow_workflow(queued_job=_queued_bindflow_job()) - - -@pytest.mark.asyncio -@respx.mock -async def test_launch_with_custom_params_text(persistent_models): - """Test launch with custom paramsText.""" - route = respx.post(url__regex=r".*/workflow/launch.*").mock( - return_value=httpx.Response(200, json={"workflowId": "wf_params_xyz"}) - ) - - await launch_bindflow_workflow( - queued_job=_queued_bindflow_job( - params_text=( - "outdir: s3://test-s3-bucket/run-out\n" - "input: s3://test-s3-bucket/inputs/samplesheets/test.csv\n" - "my_custom_param: 42\n" - "another_param: test" - ) - ) - ) - - request = route.calls.last.request - payload = json.loads(request.content) - params_text = payload["launch"]["paramsText"] - - assert "my_custom_param: 42" in params_text - assert "another_param: test" in params_text def _totals_by_status_handler(totals: dict[str, int]): From 04a0cf0cc7f3c0f8d602d7333124cd3d54ebc4cb Mon Sep 17 00:00:00 2001 From: amandazhu Date: Fri, 18 Sep 2026 14:48:08 +1000 Subject: [PATCH 24/25] fix: lint --- app/routes/workflows.py | 2 +- app/services/datasets.py | 2 +- tests/scheduler/test_scheduler_jobs.py | 4 +++- tests/test_routes_workflows.py | 24 ++++++++---------------- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/app/routes/workflows.py b/app/routes/workflows.py index dfc2309f..b4de37a4 100644 --- a/app/routes/workflows.py +++ b/app/routes/workflows.py @@ -158,7 +158,7 @@ def _extract_final_design_count(form_data: WorkflowFormData | None) -> int | Non return None try: parsed = int(str(value).strip()) - except (TypeError, ValueError): + except TypeError, ValueError: return None return parsed if parsed >= 1 else None diff --git a/app/services/datasets.py b/app/services/datasets.py index 589d5a63..2a36ec83 100644 --- a/app/services/datasets.py +++ b/app/services/datasets.py @@ -78,7 +78,7 @@ def _apply_bindcraft_design_target( return try: max_trajectories = int(str(form_data["max_trajectories"]).strip()) - except (TypeError, ValueError): + except TypeError, ValueError: return form_data["number_of_final_designs"] = max_trajectories * 2 diff --git a/tests/scheduler/test_scheduler_jobs.py b/tests/scheduler/test_scheduler_jobs.py index f576cec7..bdc068c9 100644 --- a/tests/scheduler/test_scheduler_jobs.py +++ b/tests/scheduler/test_scheduler_jobs.py @@ -350,7 +350,9 @@ async def _successful_launch(**kwargs): assert queued_job.status == "submitted" -def test_launch_job_dispatches_proteindj_for_bindcraft_tool(test_db, persistent_models, monkeypatch): +def test_launch_job_dispatches_proteindj_for_bindcraft_tool( + test_db, persistent_models, monkeypatch +): """The BindCraft tool also dispatches through the proteindj executor now.""" queued_job = _create_queued_job(tool="bindcraft") calls = [] diff --git a/tests/test_routes_workflows.py b/tests/test_routes_workflows.py index aae4f44b..4a7ea42f 100644 --- a/tests/test_routes_workflows.py +++ b/tests/test_routes_workflows.py @@ -96,9 +96,7 @@ def _get_db(): yield c -@patch( - "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare -) +@patch("app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare) def test_launch_success_without_dataset(mock_prepare, client: TestClient, test_engine): """Test successful workflow launch without dataset.""" payload = { @@ -835,9 +833,7 @@ def test_launch_single_prediction_boltz_potentials_reduces_limit( } -@patch( - "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare -) +@patch("app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare) def test_launch_allowed_with_workflow_role(mock_prepare, role_check_client, monkeypatch): """Users holding the workflow execution role can launch.""" monkeypatch.setenv("DB_ADMIN_ROLES_CLAIM", ROLES_CLAIM) @@ -1229,9 +1225,7 @@ def test_get_workflow_credits_multipliers_match_spec(client: TestClient): TEST_USER_ID = UUID("11111111-1111-1111-1111-111111111111") -@patch( - "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare -) +@patch("app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare) def test_launch_deducts_credits_when_enabled( mock_prepare, client, @@ -1266,9 +1260,7 @@ def test_launch_deducts_credits_when_enabled( assert credit == 70 # 100 − (10 × 3) -@patch( - "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare -) +@patch("app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare) def test_launch_rejected_when_insufficient_credits( mock_prepare, client, test_engine, monkeypatch, mock_settings ): @@ -1299,10 +1291,10 @@ def test_launch_rejected_when_insufficient_credits( assert credit == 10 # unchanged -@patch( - "app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare -) -def test_launch_does_not_deduct_when_credits_disabled(mock_prepare, client, test_engine, monkeypatch): +@patch("app.routes.workflows.prepare_proteindj_workflow", side_effect=_queue_job_for_route_prepare) +def test_launch_does_not_deduct_when_credits_disabled( + mock_prepare, client, test_engine, monkeypatch +): """With credits disabled (default), launches never touch the balance.""" monkeypatch.delenv("ENABLE_CREDITS", raising=False) with Session(test_engine) as db: From 55b5d528660e2500bceff55da362ca1602ef49c8 Mon Sep 17 00:00:00 2001 From: Anne Phan <53896516+vtnphan@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:36:19 +1000 Subject: [PATCH 25/25] fix: update placeholder max score in rfdiffusion and bindcraft --- app/services/proteindj_config.py | 2 +- app/services/results_utils.py | 124 ++------------ tests/test_proteindj_coverage.py | 8 +- tests/test_routes_results.py | 107 +++++------- tests/test_services_job_utils.py | 235 ++++----------------------- tests/test_services_results_utils.py | 152 +++++------------ 6 files changed, 129 insertions(+), 499 deletions(-) diff --git a/app/services/proteindj_config.py b/app/services/proteindj_config.py index df554d6d..d6a59919 100644 --- a/app/services/proteindj_config.py +++ b/app/services/proteindj_config.py @@ -14,7 +14,7 @@ def get_proteindj_design_mode(tool: str) -> str: BindCraft uses ProteinDJ's own bindcraft_denovo fold-design mode; everything else (rfdiffusion) uses standalone binder design. """ - return "bindcraft_denovo" if tool.lower() == "bindcraft" else "binder_denovo" + return "bindcraft_denovo" if tool.lower() == "bindcraft" else "rfd_denovo" def get_proteindj_default_params( diff --git a/app/services/results_utils.py b/app/services/results_utils.py index ea94fcfa..5a155c8a 100644 --- a/app/services/results_utils.py +++ b/app/services/results_utils.py @@ -629,54 +629,6 @@ def get_output_spec(run: WorkflowRun) -> WorkflowResultsSpec: ) -def classify_bindcraft_output_key( - key: str, sample_id: str | None = None -) -> ClassifiedOutput | None: - normalized = key.strip() - if not normalized or normalized.endswith("/"): - return None - - basename = normalized.rsplit("/", 1)[-1] - lowered = normalized.lower() - - if basename.endswith("_final_design_stats.csv"): - return ClassifiedOutput(category="stats_csv", label=basename) - if "/generate/" in lowered and basename.lower().endswith(".html"): - return ClassifiedOutput(category="report", label=basename) - if "/bindcraft/" in lowered and "_0_output/" in lowered and basename.lower().endswith(".png"): - return ClassifiedOutput(category="snapshot", label=basename) - if "/ranker/" in lowered and "_ranked/" in lowered and basename.lower().endswith(".pdb"): - return ClassifiedOutput(category="pdb", label=basename) - return None - - -def get_bindcraft_score_file(keys: list[str], sample_id: str | None) -> str | None: - for key in keys: - normalized = key.strip() - if not normalized: - continue - basename = normalized.rsplit("/", 1)[-1] - if basename.endswith("_final_design_stats.csv"): - return normalized - return None - - -async def extract_bindcraft_max_score( - score_file: str, settings: Settings | None = None -) -> float | None: - settings = settings or get_settings() - content = await read_s3_file(score_file, settings=settings) - csv_reader = csv.DictReader(StringIO(content)) - values: list[float] = [] - - for row in csv_reader: - value = row.get("Average_i_pTM") - if value and value.strip(): - values.append(float(value)) - - return max(values) if values else None - - def get_proteinfold_score_file(keys: list[str], sample_id: str | None) -> str | None: sample_id_pattern = re.escape(sample_id) if sample_id else "single-prediction" score_pattern = rf"/{sample_id_pattern}/.*{sample_id_pattern}_ptm\.(tsv|csv)" @@ -762,30 +714,6 @@ def classify_colabfold_proteinfold_output( ) -def build_bindcraft_output_listing_prefixes(run: WorkflowRun) -> list[str]: - run_uuid = str(getattr(run, "id", "") or "").strip() - if not run_uuid: - return [] - - # Always include run-UUID-only prefixes; these do not depend on sample_id. - prefixes: list[str] = [ - f"{run_uuid}/", - f"{run_uuid}/ranker/", - f"{run_uuid}/generate/", - ] - - # Append bindcraft sample-specific prefixes only when a sample_id is available. - sample_id = get_sample_id_for_result(run) - if sample_id: - prefixes.extend( - [ - f"{run_uuid}/bindcraft/{sample_id}_0_output/", - ] - ) - - return prefixes - - def build_boltz_proteinfold_output_listing_prefixes(run: WorkflowRun) -> list[str]: run_uuid = str(getattr(run, "id", "") or "").strip() if not run_uuid: @@ -1002,9 +930,9 @@ async def extract_rfdiffusion_max_score( row = next(csv_reader, None) if row is None: return None - value = row.get("af2_plddt_overall") + value = row.get("af2_iptm") if value and value.strip(): - return float(value) / 100 + return float(value) return None @@ -1043,15 +971,18 @@ def _make_bulk_prediction_spec(tool: WorkflowTool) -> WorkflowResultsSpec: WORKFLOW_OUTPUT_SPECS: dict[WorkflowName, dict[WorkflowTool, WorkflowResultsSpec]] = { "de-novo-design": { + # BindCraft and RFdiffusion are both just fold-design front ends for the + # same downstream ProteinDJ pipeline (sequence design, structure + # prediction, ranking), so they publish identical results/ outputs and + # share every output-collection function here. "bindcraft": WorkflowResultsSpec( kind="de-novo-design", tool="bindcraft", - required_categories={"report", "stats_csv", "pdb"}, - get_prefixes=build_bindcraft_output_listing_prefixes, - get_score_file=get_bindcraft_score_file, - extract_max_score=extract_bindcraft_max_score, - classifier=classify_bindcraft_output_key, - supports_snapshots=True, + required_categories={"stats_csv", "pdb"}, + get_prefixes=build_rfdiffusion_output_listing_prefixes, + get_score_file=get_rfdiffusion_score_file, + extract_max_score=extract_rfdiffusion_max_score, + classifier=classify_rfdiffusion_output_key, hidden_download_categories=frozenset({"pdb"}), ), "rfdiffusion": WorkflowResultsSpec( @@ -1285,39 +1216,6 @@ async def sync_workflow_outputs( return keys -async def sync_bindcraft_outputs( - db: Session, run: WorkflowRun, settings: Settings | None = None -) -> list[str]: - """Discover bindcraft result artifacts in S3 and persist them as run outputs.""" - settings = settings or get_settings() - discovered: list[str] = [] - for prefix in build_bindcraft_output_listing_prefixes(run): - try: - files = await list_s3_files(prefix=prefix, settings=settings) - except (S3ConfigurationError, S3ServiceError) as exc: - logger.warning( - "Failed to list bindcraft outputs from S3", - extra={ - "runId": str(run.id), - "seqeraRunId": run.seqera_run_id, - "prefix": prefix, - "error": str(exc), - }, - ) - continue - for item in files: - key = str(item.get("key", "")).strip() - if not key or key in discovered: - continue - if classify_bindcraft_output_key(key): - discovered.append(key) - - if discovered: - _sync_run_output_records(db, run, discovered, settings=settings) - - return discovered - - _CATEGORY_ORDER: dict[str, int] = { "report": 0, "stats_csv": 1, diff --git a/tests/test_proteindj_coverage.py b/tests/test_proteindj_coverage.py index aa5d8b9e..4efa06e8 100644 --- a/tests/test_proteindj_coverage.py +++ b/tests/test_proteindj_coverage.py @@ -180,7 +180,7 @@ def test_get_proteindj_default_params_all_fields(): hotspot_residues="A20,A21", num_designs=5, design_length="100-150", - design_mode="binder_denovo", + design_mode="rfd_denovo", ) assert params == { "out_dir": "s3://bucket/out", @@ -188,7 +188,7 @@ def test_get_proteindj_default_params_all_fields(): "hotspot_residues": "A20,A21", "num_designs": 5, "design_length": "100-150", - "design_mode": "binder_denovo", + "design_mode": "rfd_denovo", } @@ -207,7 +207,7 @@ def test_get_proteindj_design_mode_bindcraft(): def test_get_proteindj_design_mode_rfdiffusion(): - assert get_proteindj_design_mode("rfdiffusion") == "binder_denovo" + assert get_proteindj_design_mode("rfdiffusion") == "rfd_denovo" def test_get_proteindj_design_mode_is_case_insensitive(): @@ -354,7 +354,7 @@ async def test_prepare_proteindj_workflow_writes_expected_queued_job( assert "hotspot_residues: A20,A21" in params_text assert "num_designs: 5" in params_text assert "design_length: 100-150" in params_text - assert "design_mode: binder_denovo" in params_text + assert "design_mode: rfd_denovo" in params_text # The uploaded starting-pdb file gets its own Globus staging record, separate # from the main samplesheet input handled in the workflows route. diff --git a/tests/test_routes_results.py b/tests/test_routes_results.py index be74b259..896dfb8c 100644 --- a/tests/test_routes_results.py +++ b/tests/test_routes_results.py @@ -44,6 +44,18 @@ def _configure_bindcraft_run(run: WorkflowRun) -> None: run.submitted_form_data = {"mode": "bindcraft"} +def _configure_boltz_run(run: WorkflowRun) -> None: + """BindCraft no longer has its own report output (see WORKFLOW_OUTPUT_SPECS), + so report tests exercise that generic behaviour via a tool that still has one.""" + run.workflow = Workflow( + name="single-prediction", + repo_url="https://github.com/test/single-prediction", + default_revision="main", + config_path="/config/single-prediction.config", + ) + run.submitted_form_data = {"mode": "boltz"} + + def _make_run_output(run: WorkflowRun, object_key: str) -> RunOutput: """Build a RunOutput row linked to a throwaway DataTransfer for test fixtures.""" transfer = DataTransfer( @@ -355,25 +367,19 @@ async def test_get_result_downloads_returns_presigned_links_for_tracked_outputs( sample_id="demo2", work_dir="/tmp/wf-downloads-1", ) + test_db.add_all([user, workflow, run]) + test_db.flush() outputs = [ S3Object( - object_key="demo2/ranker/demo2_final_design_stats.csv", - uri="s3://bucket/demo2/ranker/demo2_final_design_stats.csv", + object_key=f"{run.id}/results/ranked_designs.csv", + uri=f"s3://bucket/{run.id}/results/ranked_designs.csv", ), S3Object( - object_key="demo2/ranker/demo2_Ranked/1_PDL1_model1.pdb", - uri="s3://bucket/demo2/ranker/demo2_Ranked/1_PDL1_model1.pdb", - ), - S3Object( - object_key=f"{run.id}/generate/PDL1_l100_s975117.html", - uri=f"s3://bucket/{run.id}/generate/PDL1_l100_s975117.html", - ), - S3Object( - object_key=f"{run.id}/bindcraft/demo2_0_output/demo2_preview.png", - uri=f"s3://bucket/{run.id}/bindcraft/demo2_0_output/demo2_preview.png", + object_key=f"{run.id}/results/ranked_designs/1_PDL1_model1.pdb", + uri=f"s3://bucket/{run.id}/results/ranked_designs/1_PDL1_model1.pdb", ), ] - test_db.add_all([user, run, workflow, *outputs]) + test_db.add_all(outputs) test_db.commit() test_db.add_all([_make_run_output(run, item.object_key) for item in outputs]) test_db.commit() @@ -395,22 +401,17 @@ async def test_get_result_downloads_returns_presigned_links_for_tracked_outputs( assert result.runId == str(run.id) # pdb is flagged hidden but still returned individually. assert [item.category for item in result.downloads] == [ - "report", "stats_csv", "pdb", ] assert [item.label for item in result.downloads] == [ - "PDL1_l100_s975117.html", - "demo2_final_design_stats.csv", + "ranked_designs.csv", "1_PDL1_model1.pdb", ] assert all(item.category != "snapshot" for item in result.downloads) assert result.zipCategories == ["pdb"] - assert ( - result.downloads[1].url - == "https://signed.example/demo2/ranker/demo2_final_design_stats.csv" - ) - assert mock_presign.await_count == 3 + assert result.downloads[0].url == f"https://signed.example/{run.id}/results/ranked_designs.csv" + assert mock_presign.await_count == 2 @pytest.mark.asyncio @@ -491,9 +492,8 @@ async def test_get_result_download_all_returns_valid_zip_file( test_db.flush() output_contents = { - f"{run.id}/generate/result.html": b"report", - f"{run.id}/ranker/download-all_final_design_stats.csv": b"score\n0.9\n", - f"{run.id}/ranker/download-all_Ranked/model.pdb": b"ATOM\n", + f"{run.id}/results/ranked_designs.csv": b"score\n0.9\n", + f"{run.id}/results/ranked_designs/model.pdb": b"ATOM\n", } outputs = [S3Object(object_key=key, uri=f"s3://bucket/{key}") for key in output_contents] test_db.add_all(outputs) @@ -518,12 +518,10 @@ async def read_bytes(key: str, **_kwargs) -> bytes: ) with ZipFile(returned_zip) as zip_file: assert set(zip_file.namelist()) == { - "report/result.html", - "stats_csv/download-all_final_design_stats.csv", + "stats_csv/ranked_designs.csv", "pdb/model.pdb", } - assert zip_file.read("report/result.html") == b"report" - assert zip_file.read("stats_csv/download-all_final_design_stats.csv") == b"score\n0.9\n" + assert zip_file.read("stats_csv/ranked_designs.csv") == b"score\n0.9\n" assert zip_file.read("pdb/model.pdb") == b"ATOM\n" @@ -545,9 +543,8 @@ async def test_get_result_download_category_returns_valid_zip_file( test_db.flush() output_contents = { - f"{run.id}/generate/result.html": b"report", - f"{run.id}/ranker/download-category_final_design_stats.csv": b"score\n0.9\n", - f"{run.id}/ranker/download-category_Ranked/model.pdb": b"ATOM\n", + f"{run.id}/results/ranked_designs.csv": b"score\n0.9\n", + f"{run.id}/results/ranked_designs/model.pdb": b"ATOM\n", } outputs = [S3Object(object_key=key, uri=f"s3://bucket/{key}") for key in output_contents] test_db.add_all(outputs) @@ -927,9 +924,11 @@ async def test_get_result_downloads_maps_s3_service_error_to_502(test_db, mock_s @pytest.mark.asyncio -async def test_get_result_snapshots_returns_presigned_links_for_tracked_outputs( +async def test_get_result_snapshots_returns_empty_for_bindcraft_without_snapshot_support( test_db, mock_settings ): + """BindCraft now shares RFdiffusion's output collection, which has no + snapshot category, so no S3 lookup is attempted even with tracked PNGs.""" user = AppUser( auth0_user_id="auth0|results-user-snapshots-1", name="Results User Snapshots 1", @@ -949,38 +948,18 @@ async def test_get_result_snapshots_returns_presigned_links_for_tracked_outputs( sample_id="demo2", work_dir="/tmp/wf-snapshots-1", ) - outputs = [ - S3Object( - object_key=f"{run.id}/bindcraft/demo2_0_output/demo2_preview.png", - uri=f"s3://bucket/{run.id}/bindcraft/demo2_0_output/demo2_preview.png", - ), - S3Object( - object_key=f"{run.id}/bindcraft/demo2_0_output/demo2_preview_2.png", - uri=f"s3://bucket/{run.id}/bindcraft/demo2_0_output/demo2_preview_2.png", - ), - ] - test_db.add_all([user, run, workflow, *outputs]) - test_db.commit() - test_db.add_all([_make_run_output(run, item.object_key) for item in outputs]) + test_db.add_all([user, run, workflow]) test_db.commit() - with ( - patch( - "app.services.results_utils.generate_presigned_url", - new_callable=AsyncMock, - side_effect=lambda key, **_kwargs: f"https://signed.example/{key}", - ), - patch( - "app.services.results_utils.list_s3_files", - new_callable=AsyncMock, - return_value=[], - ), - ): + with patch( + "app.services.results_utils.list_s3_files", + new_callable=AsyncMock, + ) as mocked_list: result = await get_result_snapshots(str(run.id), user.id, test_db, mock_settings) assert result.runId == str(run.id) - assert [item.category for item in result.snapshots] == ["snapshot", "snapshot"] - assert [item.label for item in result.snapshots] == ["demo2_preview.png", "demo2_preview_2.png"] + assert result.snapshots == [] + mocked_list.assert_not_awaited() @pytest.mark.asyncio @@ -1098,8 +1077,8 @@ async def test_get_result_report_returns_single_presigned_html_for_tracked_outpu sample_id="demo2", work_dir="/tmp/wf-report-1", ) - _configure_bindcraft_run(run) - report_key = f"{run.id}/generate/PDL1_l100_s975117.html" + _configure_boltz_run(run) + report_key = f"{run.id}/reports/PDL1_l100_s975117_report.html" report = S3Object( object_key=report_key, uri=f"s3://bucket/{report_key}", @@ -1178,15 +1157,15 @@ async def test_get_result_report_syncs_run_uuid_prefixed_animation_output(test_d sample_id="s1", work_dir="/tmp/wf-report-3", ) - _configure_bindcraft_run(run) + _configure_boltz_run(run) test_db.add_all([user, run]) test_db.commit() run_id = run.id - real_key = f"{run_id}/generate/PDL1_l79_s800698.html" + real_key = f"{run_id}/reports/PDL1_l79_s800698_report.html" def _list_side_effect(prefix: str, file_extension=None, **_kwargs): - if prefix == f"{run_id}/generate/": + if prefix == f"{run_id}/reports/": return [ { "key": real_key, diff --git a/tests/test_services_job_utils.py b/tests/test_services_job_utils.py index 641ca5ab..96af8cef 100644 --- a/tests/test_services_job_utils.py +++ b/tests/test_services_job_utils.py @@ -6,7 +6,6 @@ from unittest.mock import ANY, AsyncMock, patch import pytest -from sqlalchemy import select from app.db.models.core import ( AppUser, @@ -75,6 +74,20 @@ def _configure_bindcraft_run(run: WorkflowRun) -> None: run.submitted_form_data = {"mode": "bindcraft"} +def _configure_boltz_run(run: WorkflowRun) -> None: + """BindCraft no longer has its own report output (see WORKFLOW_OUTPUT_SPECS), + so report-download tests exercise that generic behaviour via a tool that + still has one.""" + run.workflow = Workflow( + name="single-prediction", + repo_url="https://github.com/test/single-prediction", + default_revision="main", + config_path="/config/single-prediction.config", + ) + run.tool = "boltz" + run.submitted_form_data = {"mode": "boltz"} + + def test_coerce_and_extract_helpers(): payload = {"workflow": {"status": "RUNNING"}} assert job_utils.coerce_workflow_payload(payload) == payload["workflow"] @@ -394,159 +407,6 @@ async def test_ensure_completed_run_score_returns_none_when_spec_has_no_score(te assert test_db.get(RunMetric, run.id) is None -@pytest.mark.asyncio -async def test_sync_bindcraft_outputs_discovers_run_uuid_prefixed_snapshot_png(test_db): - user = AppUser( - auth0_user_id="auth0|snapshot-user", - name="Snapshot User", - email="snapshot-user@example.com", - ) - run = WorkflowRun( - owner=user, - seqera_run_id="seqera-snapshot-1", - sample_id="sampleA", - work_dir="workdir-snapshot-1", - ) - test_db.add_all([user, run]) - test_db.commit() - run_id = run.id - - snapshot_key = f"{run_id}/bindcraft/sampleA_0_output/sampleA_preview.png" - - def _list_side_effect(prefix: str, file_extension=None, **_kwargs): - if prefix == f"{run_id}/bindcraft/sampleA_0_output/": - return [ - { - "key": snapshot_key, - "size": 2048, - "last_modified": "2026-03-12T00:00:00Z", - "bucket": "test-bucket", - } - ] - return [] - - with patch( - "app.services.results_utils.list_s3_files", - new_callable=AsyncMock, - side_effect=_list_side_effect, - ): - discovered = await results_utils.sync_bindcraft_outputs(test_db, run) - - assert snapshot_key in discovered - persisted = test_db.get(S3Object, snapshot_key) - assert persisted is not None - assert persisted.uri.endswith(snapshot_key) - - run_output = test_db.scalar( - select(RunOutput).where(RunOutput.run_id == run_id, RunOutput.s3_object_id == snapshot_key) - ) - assert run_output is not None - assert run_output.data_transfer_id is not None - output_transfer = test_db.get(DataTransfer, run_output.data_transfer_id) - assert output_transfer is not None - assert output_transfer.workflow_run_id == run_id - assert output_transfer.direction == "output" - assert output_transfer.provider == "s3" - assert output_transfer.source_location == f"s3://test-s3-bucket/{run_id}" - assert output_transfer.destination_location == persisted.uri - assert output_transfer.status == "completed" - link = ( - test_db.query(RunOutput).filter_by(run_id=run.id, s3_object_id=snapshot_key).one_or_none() - ) - assert link is not None - - -@pytest.mark.asyncio -async def test_get_result_snapshot_downloads_returns_tracked_snapshots(test_db): - user = AppUser( - auth0_user_id="auth0|snapshot-download-user", - name="Snapshot Download User", - email="snapshot-download-user@example.com", - ) - run = WorkflowRun( - owner=user, - seqera_run_id="seqera-snapshot-download-1", - sample_id="sampleB", - work_dir="workdir-snapshot-download-1", - ) - _configure_bindcraft_run(run) - test_db.add_all([user, run]) - test_db.flush() - run_id = run.id - - snapshot_keys = [ - f"{run_id}/bindcraft/sampleB_0_output/sampleB_preview.png", - f"{run_id}/bindcraft/sampleB_0_output/sampleB_preview_2.png", - ] - snapshots = [S3Object(object_key=key, uri=f"s3://bucket/{key}") for key in snapshot_keys] - test_db.add_all(snapshots) - test_db.add_all([_make_run_output(run, key) for key in snapshot_keys]) - test_db.commit() - - with ( - patch("app.services.results_utils.list_s3_files", new_callable=AsyncMock, return_value=[]), - patch( - "app.services.results_utils.generate_presigned_url", - new_callable=AsyncMock, - side_effect=lambda key, **_kwargs: f"https://signed.example/{key}", - ) as mocked_presign, - ): - result = await results_utils.get_result_snapshot_downloads(test_db, run) - - assert [item.category for item in result] == ["snapshot", "snapshot"] - assert [item.key for item in result] == snapshot_keys - assert mocked_presign.await_count == 2 - - -@pytest.mark.asyncio -async def test_get_result_snapshot_downloads_discovers_snapshot_from_s3(test_db): - user = AppUser( - auth0_user_id="auth0|snapshot-discovery-user", - name="Snapshot Discovery User", - email="snapshot-discovery-user@example.com", - ) - run = WorkflowRun( - owner=user, - seqera_run_id="seqera-snapshot-download-2", - sample_id="sampleC", - work_dir="workdir-snapshot-download-2", - ) - _configure_bindcraft_run(run) - test_db.add_all([user, run]) - test_db.commit() - - snapshot_key = f"{run.id}/bindcraft/sampleC_0_output/sampleC_preview.png" - - def _list_side_effect(prefix: str, file_extension=None, **_kwargs): - if prefix == f"{run.id}/bindcraft/sampleC_0_output/": - return [ - { - "key": snapshot_key, - "size": 2048, - "last_modified": "2026-03-12T00:00:00Z", - "bucket": "test-bucket", - } - ] - return [] - - with ( - patch( - "app.services.results_utils.list_s3_files", - new_callable=AsyncMock, - side_effect=_list_side_effect, - ), - patch( - "app.services.results_utils.generate_presigned_url", - new_callable=AsyncMock, - side_effect=lambda key, **_kwargs: f"https://signed.example/{key}", - ), - ): - result = await results_utils.get_result_snapshot_downloads(test_db, run) - - assert [item.key for item in result] == [snapshot_key] - assert [item.category for item in result] == ["snapshot"] - - @pytest.mark.asyncio async def test_get_result_snapshot_downloads_returns_empty_when_missing(test_db): user = AppUser( @@ -614,11 +474,11 @@ async def test_get_result_report_download_returns_tracked_report(test_db): sample_id="sampleE", work_dir="workdir-report-download-1", ) - _configure_bindcraft_run(run) + _configure_boltz_run(run) test_db.add_all([user, run]) test_db.flush() run_id = run.id - report_key = f"{run_id}/generate/sampleE_report.html" + report_key = f"{run_id}/reports/sampleE_report.html" report = S3Object( object_key=report_key, uri=f"s3://bucket/{report_key}", @@ -661,12 +521,12 @@ async def test_get_result_report_download_skips_sync_when_report_is_already_trac sample_id="sampleFast", work_dir="workdir-report-fast-path-1", ) - _configure_bindcraft_run(run) - report_key = f"{run.id}/generate/sampleFast_report.html" + _configure_boltz_run(run) + report_key = f"{run.id}/reports/sampleFast_report.html" with ( patch("app.services.results_utils._get_run_output_keys", return_value=[report_key]), - patch("app.services.results_utils.sync_bindcraft_outputs", new=AsyncMock()) as mocked_sync, + patch("app.services.results_utils.list_s3_files", new_callable=AsyncMock) as mocked_list, patch( "app.services.results_utils.generate_presigned_url", new_callable=AsyncMock, @@ -677,7 +537,7 @@ async def test_get_result_report_download_skips_sync_when_report_is_already_trac assert result is not None assert result.key == report_key - mocked_sync.assert_not_awaited() + mocked_list.assert_not_awaited() @pytest.mark.asyncio @@ -689,14 +549,13 @@ async def test_get_result_output_downloads_skips_sync_when_required_outputs_are_ ) _configure_bindcraft_run(run) tracked_keys = [ - f"{run.id}/generate/sampleTracked_report.html", - f"{run.id}/ranker/sampleTracked_final_design_stats.csv", - f"{run.id}/ranker/sampleTracked_ranked/sampleTracked_model_1.pdb", + f"{run.id}/results/ranked_designs.csv", + f"{run.id}/results/ranked_designs/sampleTracked_model_1.pdb", ] with ( patch("app.services.results_utils._get_run_output_keys", return_value=tracked_keys), - patch("app.services.results_utils.sync_bindcraft_outputs", new=AsyncMock()) as mocked_sync, + patch("app.services.results_utils.list_s3_files", new_callable=AsyncMock) as mocked_list, patch( "app.services.results_utils.generate_presigned_url", new_callable=AsyncMock, @@ -706,9 +565,9 @@ async def test_get_result_output_downloads_skips_sync_when_required_outputs_are_ result = await results_utils.get_result_output_downloads(test_db, run) # pdb is still returned individually, just flagged as hidden. - assert [item.category for item in result.downloads] == ["report", "stats_csv", "pdb"] + assert [item.category for item in result.downloads] == ["stats_csv", "pdb"] assert result.zip_categories == ["pdb"] - mocked_sync.assert_not_awaited() + mocked_list.assert_not_awaited() @pytest.mark.asyncio @@ -726,13 +585,13 @@ async def test_get_result_report_download_discovers_report_from_s3(test_db): ) test_db.add_all([user, run]) test_db.flush() - _configure_bindcraft_run(run) + _configure_boltz_run(run) test_db.commit() - report_key = f"{run.id}/generate/sampleF_report.html" + report_key = f"{run.id}/reports/sampleF_report.html" def _list_side_effect(prefix: str, file_extension=None, **_kwargs): - if prefix == f"{run.id}/generate/": + if prefix == f"{run.id}/reports/": return [ { "key": report_key, @@ -771,17 +630,16 @@ async def test_get_result_report_download_falls_back_to_listing_when_sync_finds_ sample_id="sampleG", work_dir="workdir-report-fallback-1", ) - _configure_bindcraft_run(run) + _configure_boltz_run(run) test_db.commit() - report_key = f"{run.id}/generate/sampleG_report.html" + report_key = f"{run.id}/reports/sampleG_report.html" with ( - patch("app.services.results_utils.sync_bindcraft_outputs", new=AsyncMock(return_value=[])), patch( "app.services.results_utils.list_s3_files", new_callable=AsyncMock, side_effect=lambda prefix, **_kwargs: ( - [{"key": report_key}] if prefix.endswith("generate/") else [] + [{"key": report_key}] if prefix.endswith("reports/") else [] ), ), patch( @@ -797,37 +655,6 @@ async def test_get_result_report_download_falls_back_to_listing_when_sync_finds_ assert result.category == "report" -@pytest.mark.asyncio -async def test_get_result_snapshot_downloads_fall_back_to_listing_when_sync_finds_nothing(test_db): - run = WorkflowRunFactory.build( - seqera_run_id="seqera-snapshot-fallback-1", - sample_id="sampleH", - work_dir="workdir-snapshot-fallback-1", - ) - _configure_bindcraft_run(run) - snapshot_key = f"{run.id}/bindcraft/sampleH_0_output/sampleH_preview.png" - - with ( - patch("app.services.results_utils.sync_bindcraft_outputs", new=AsyncMock(return_value=[])), - patch( - "app.services.results_utils.list_s3_files", - new_callable=AsyncMock, - side_effect=lambda prefix, **_kwargs: ( - [{"key": snapshot_key}] if prefix.endswith("sampleH_0_output/") else [] - ), - ), - patch( - "app.services.results_utils.generate_presigned_url", - new_callable=AsyncMock, - side_effect=lambda key, **_kwargs: f"https://signed.example/{key}", - ), - ): - result = await results_utils.get_result_snapshot_downloads(test_db, run) - - assert [item.key for item in result] == [snapshot_key] - assert [item.category for item in result] == ["snapshot"] - - @pytest.mark.asyncio async def test_ensure_completed_run_score_returns_none_when_get_output_spec_raises(): run = SimpleNamespace(id="rid", seqera_run_id="wf-no-spec", tool=None, workflow=None) diff --git a/tests/test_services_results_utils.py b/tests/test_services_results_utils.py index 72437942..96d89ac1 100644 --- a/tests/test_services_results_utils.py +++ b/tests/test_services_results_utils.py @@ -19,25 +19,21 @@ WorkflowResultsSpec, _build_s3_uri, build_alphafold2_proteinfold_output_listing_prefixes, - build_bindcraft_output_listing_prefixes, build_boltz_proteinfold_output_listing_prefixes, build_colabfold_proteinfold_output_listing_prefixes, build_rfdiffusion_output_listing_prefixes, build_wisps_output_listing_prefixes, classify_alphafold2_proteinfold_output, - classify_bindcraft_output_key, classify_boltz_proteinfold_output, classify_colabfold_proteinfold_output, classify_rfdiffusion_output_key, classify_shared_outputs, classify_wisps_output_key, - extract_bindcraft_max_score, extract_proteinfold_max_score, extract_rfdiffusion_max_score, extract_wisps_max_score, format_log_entries, get_all_downloads_zipped, - get_bindcraft_score_file, get_category_downloads_zipped, get_proteinfold_score_file, get_result_output_downloads, @@ -213,52 +209,21 @@ def test_get_sample_id_for_result_uses_fallback_order_and_strips(): assert get_sample_id_for_result(run_empty) is None -def test_bindcraft_helpers_classify_keys_and_build_prefixes(mock_settings): - run = WorkflowRun(id=uuid4(), owner_user_id=uuid4(), sample_id="sampleZ") - - assert classify_bindcraft_output_key(" ") is None - assert classify_bindcraft_output_key("folder/") is None - assert classify_bindcraft_output_key(f"{run.id}/Accepted/Animation/report.html") is None - assert classify_bindcraft_output_key( - f"{run.id}/generate/bindcraft_report.html" - ) == ClassifiedOutput( - "report", - "bindcraft_report.html", - ) - assert classify_bindcraft_output_key( - f"{run.id}/bindcraft/sampleZ_0_output/preview.png" - ) == ClassifiedOutput( - "snapshot", - "preview.png", - ) - assert classify_bindcraft_output_key( - f"{run.id}/ranker/sampleZ_ranked/model.pdb" - ) == ClassifiedOutput( - "pdb", - "model.pdb", - ) - assert classify_bindcraft_output_key( - f"{run.id}/ranker/sampleZ_final_design_stats.csv" - ) == ClassifiedOutput( - "stats_csv", - "sampleZ_final_design_stats.csv", - ) +def test_bindcraft_spec_reuses_rfdiffusion_output_collection(): + """BindCraft and RFdiffusion are both fold-design front ends for the same + downstream ProteinDJ pipeline, so their output collection must be identical.""" + bindcraft_spec = WORKFLOW_OUTPUT_SPECS["de-novo-design"]["bindcraft"] + rfdiffusion_spec = WORKFLOW_OUTPUT_SPECS["de-novo-design"]["rfdiffusion"] - prefixes = build_bindcraft_output_listing_prefixes(run) - assert prefixes == [ - f"{run.id}/", - f"{run.id}/ranker/", - f"{run.id}/generate/", - f"{run.id}/bindcraft/sampleZ_0_output/", - ] + assert bindcraft_spec.get_prefixes is rfdiffusion_spec.get_prefixes + assert bindcraft_spec.classifier is rfdiffusion_spec.classifier + assert bindcraft_spec.get_score_file is rfdiffusion_spec.get_score_file + assert bindcraft_spec.extract_max_score is rfdiffusion_spec.extract_max_score + assert bindcraft_spec.required_categories == rfdiffusion_spec.required_categories + assert bindcraft_spec.supports_snapshots == rfdiffusion_spec.supports_snapshots - run_without_sample = SimpleNamespace(id=run.id, sample_id=None, binder_name=None, form_id=None) - assert build_bindcraft_output_listing_prefixes(run_without_sample) == [ - f"{run.id}/", - f"{run.id}/ranker/", - f"{run.id}/generate/", - ] +def test_build_s3_uri_uses_bucket_when_configured(mock_settings): mock_settings.aws.s3_bucket = "test-bucket" assert ( _build_s3_uri("path/to/file.txt", settings=mock_settings) @@ -319,9 +284,7 @@ def test_builtin_specs_get_transfer_prefixes_excludes_run_root_but_keeps_usage_r rfdiffusion_spec = WORKFLOW_OUTPUT_SPECS["de-novo-design"]["rfdiffusion"] assert bindcraft_spec.get_transfer_prefixes(run) == [ - f"{run.id}/ranker/", - f"{run.id}/generate/", - f"{run.id}/bindcraft/T1024_0_output/", + f"{run.id}/results/", f"{run.id}/UsageReport.csv", ] assert boltz_spec.get_transfer_prefixes(run) == [ @@ -517,15 +480,15 @@ def test_run_has_missing_required_categories_true_when_category_absent(test_db, run = WorkflowRunFactory.create_sync(owner=user, workflow=workflow, tool="bindcraft") stats_object = S3ObjectFactory.create_sync( - object_key=f"{run.id}/ranker/{run.id}_final_design_stats.csv", - uri=f"s3://bucket/{run.id}/ranker/{run.id}_final_design_stats.csv", + object_key=f"{run.id}/results/ranked_designs.csv", + uri=f"s3://bucket/{run.id}/results/ranked_designs.csv", ) RunOutputFactory.create_sync( run_id=run.id, s3_object_id=stats_object.object_key, data_transfer=DataTransferFactory.create_sync(workflow_run=run, direction="output"), ) - # "report" and "pdb" are also required but have no output here. + # "pdb" is also required but has no output here. assert run_has_missing_required_categories(test_db, run) is True @@ -535,9 +498,8 @@ def test_run_has_missing_required_categories_false_when_satisfied(test_db, persi run = WorkflowRunFactory.create_sync(owner=user, workflow=workflow, tool="bindcraft") outputs = { - "report": f"{run.id}/generate/{run.id}_report.html", - "pdb": f"{run.id}/ranker/{run.id}_ranked/{run.id}.pdb", - "stats_csv": f"{run.id}/ranker/{run.id}_final_design_stats.csv", + "pdb": f"{run.id}/results/ranked_designs/{run.id}.pdb", + "stats_csv": f"{run.id}/results/ranked_designs.csv", } for key in outputs.values(): s3_object = S3ObjectFactory.create_sync(object_key=key, uri=f"s3://bucket/{key}") @@ -648,7 +610,7 @@ def test_get_rfdiffusion_score_file_uses_ranked_designs_csv(): @pytest.mark.asyncio async def test_extract_rfdiffusion_max_score_reads_first_ranked_design_score(): - csv_text = "design,af2_plddt_overall\nsampleZ_0,91.3\nsampleZ_1,88.1\n" + csv_text = "rank,design,af2_iptm\n1,sampleZ_0,0.913\n2,sampleZ_1,0.881\n" with patch( "app.services.results_utils.read_s3_file", @@ -664,12 +626,12 @@ async def test_extract_rfdiffusion_max_score_reads_first_ranked_design_score(): @pytest.mark.asyncio async def test_extract_rfdiffusion_max_score_returns_none_without_score_value(): """ - Test extract_rfdiffusion_max_score returns None when no af2_plddt_overall score is available + Test extract_rfdiffusion_max_score returns None when no af2_iptm score is available """ for csv_text in [ - "design,af2_plddt_overall\n", - "design,af2_plddt_overall\nsampleZ_0, \n", - "design,other_score\nsampleZ_0,91.3\n", + "rank,design,af2_iptm\n", + "rank,design,af2_iptm\n1,sampleZ_0, \n", + "rank,design,other_score\n1,sampleZ_0,0.913\n", ]: with patch( "app.services.results_utils.read_s3_file", @@ -697,9 +659,8 @@ async def test_get_all_downloads_zipped_writes_category_label_files_and_reads_ea ) output_contents = { - f"{run.id}/generate/result.html": b"result", - f"{run.id}/ranker/sampleZ_final_design_stats.csv": b"score\n0.9\n", - f"{run.id}/ranker/sampleZ_ranked/model.pdb": b"ATOM\n", + f"{run.id}/results/ranked_designs.csv": b"score\n0.9\n", + f"{run.id}/results/ranked_designs/model.pdb": b"ATOM\n", } outputs = [S3Object(object_key=key, uri=f"s3://bucket/{key}") for key in output_contents] test_db.add_all([user, run, *outputs]) @@ -718,12 +679,10 @@ async def read_bytes(key: str, **_kwargs) -> bytes: with ZipFile(BytesIO(zip_buffer.getvalue())) as zip_file: assert set(zip_file.namelist()) == { - "report/result.html", - "stats_csv/sampleZ_final_design_stats.csv", + "stats_csv/ranked_designs.csv", "pdb/model.pdb", } - assert zip_file.read("report/result.html") == b"result" - assert zip_file.read("stats_csv/sampleZ_final_design_stats.csv") == b"score\n0.9\n" + assert zip_file.read("stats_csv/ranked_designs.csv") == b"score\n0.9\n" assert zip_file.read("pdb/model.pdb") == b"ATOM\n" assert mock_read_s3_bytes.await_count == len(output_contents) @@ -792,24 +751,25 @@ async def test_get_result_report_download_persists_result_found_only_on_retry( run = WorkflowRunFactory.create_sync( owner=user, workflow=Workflow( - name="de-novo-design", - repo_url="https://github.com/test/de-novo-design", + name="single-prediction", + repo_url="https://github.com/test/single-prediction", default_revision="main", - config_path="/config/de-novo-design.config", + config_path="/config/single-prediction.config", ), - tool="bindcraft", + tool="boltz", + sample_id="T1024", seqera_run_id="wf-report-retry", ) test_db.commit() - report_key = f"{run.id}/generate/result.html" - generate_prefix = f"{run.id}/generate/" + report_key = f"{run.id}/reports/T1024_report.html" + reports_prefix = f"{run.id}/reports/" calls_per_prefix: dict[str, int] = {} async def fake_list_s3_files(prefix: str, settings=None): calls_per_prefix[prefix] = calls_per_prefix.get(prefix, 0) + 1 # Only found on the second (loud, suppress_s3_errors=False) listing pass. - if prefix == generate_prefix and calls_per_prefix[prefix] >= 2: + if prefix == reports_prefix and calls_per_prefix[prefix] >= 2: return [{"key": report_key}] return [] @@ -857,11 +817,11 @@ async def test_read_result_output_file_persists_newly_discovered_output(test_db, ) test_db.commit() - stats_key = f"{run.id}/ranker/sampleZ_final_design_stats.csv" - ranker_prefix = f"{run.id}/ranker/" + stats_key = f"{run.id}/results/ranked_designs.csv" + results_prefix = f"{run.id}/results/" async def fake_list_s3_files(prefix: str, settings=None): - if prefix == ranker_prefix: + if prefix == results_prefix: return [{"key": stats_key}] return [] @@ -878,46 +838,12 @@ async def fake_list_s3_files(prefix: str, settings=None): content, label = await read_result_output_file(test_db, run, stats_key) assert content == b"score\n0.9\n" - assert label == "sampleZ_final_design_stats.csv" + assert label == "ranked_designs.csv" run_output = test_db.scalars(select(RunOutput).where(RunOutput.run_id == run.id)).one() assert run_output.s3_object_id == stats_key -def test_get_bindcraft_score_file_uses_final_design_stats(): - keys = [ - "run/ranker/model.pdb", - "run/ranker/s1_final_design_stats.csv", - "run/generate/report.html", - ] - - assert get_bindcraft_score_file(keys, "s1") == "run/ranker/s1_final_design_stats.csv" - - -def test_get_bindcraft_score_file_returns_none_without_stats(): - keys = [ - "run/ranker/model.pdb", - "run/generate/report.html", - ] - - assert get_bindcraft_score_file(keys, "s1") is None - - -@pytest.mark.asyncio -async def test_extract_bindcraft_max_score_reads_average_i_ptm(): - csv_text = "design_id,Average_i_pTM\nA,0.12\nB,0.91\nC,\n" - - with patch( - "app.services.results_utils.read_s3_file", - new_callable=AsyncMock, - return_value=csv_text, - ) as read_file: - score = await extract_bindcraft_max_score("run/ranker/s1_final_design_stats.csv") - - assert score == 0.91 - read_file.assert_awaited_once_with("run/ranker/s1_final_design_stats.csv", settings=ANY) - - @pytest.mark.parametrize( ("tool", "key"), [