From 366e13d2078be5fe16baf71bf8db82bed37ad2cd Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 31 Oct 2026 13:22:45 +0000 Subject: [PATCH] fix(storyboard): skip observations after unsupported forcing --- .../unsupported-force-scenario-cascade.md | 5 + src/lib/testing/storyboard/runner.ts | 29 +++- ...ryboard-force-scenario-unsupported.test.js | 164 +++++++++++++++++- 3 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 .changeset/unsupported-force-scenario-cascade.md diff --git a/.changeset/unsupported-force-scenario-cascade.md b/.changeset/unsupported-force-scenario-cascade.md new file mode 100644 index 000000000..fd0899090 --- /dev/null +++ b/.changeset/unsupported-force-scenario-cascade.md @@ -0,0 +1,5 @@ +--- +'@adcp/sdk': patch +--- + +Skip dependent storyboard observations when a stateful test-controller force scenario is unsupported, including when forcing is the only stateful step in its phase. Preserve a not-applicable coverage result without dispatching unforced observations or awarding passing-step credit; independent phases and genuine failures retain their normal grading. diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 28e269b1e..f5df74b40 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -4855,7 +4855,7 @@ async function executeStoryboardPass( stepId: string; reason: RunnerSkipReason | RunnerDetailedSkipReason; substitution_chain?: string; - /** The trigger is an explicit phase capability gate, not a failed setup. */ + /** The trigger is unavailable coverage (capability or forcing), not a failed setup. */ capabilityUnavailable?: boolean; }; const phaseStatefulCascades = new Map(); @@ -6137,7 +6137,9 @@ async function executeStoryboardPass( } if (capabilityUnavailable) { recordUnavailableOutputs(phase.id, step); - if (routingContext) phaseInheritedCapabilityTrigger ??= trigger ?? undefined; + if (routingContext || trigger?.reason === 'force_scenario_unsupported') { + phaseInheritedCapabilityTrigger ??= trigger ?? undefined; + } } else if (routingContext) { recordUnavailableOutputs(phase.id, step, missingPrerequisiteContextKeysByPhase); recordHardPrerequisiteFailure(step, trigger === null); @@ -6392,6 +6394,19 @@ async function executeStoryboardPass( if (step.stateful) { if (isHardMissingStateSkipReason(result.skip_reason)) { recordHardMissingState(step, result); + } else if (result.skip_reason === 'force_scenario_unsupported') { + // An unsupported force_* call never established the controlled state + // its observations require. Unlike ordinary implicit-account setup, + // it cannot use the sole-stateful-step exemption. Skip dependent + // coverage neutrally, while retaining any earlier hard failure. + recordUnavailableOutputs(phase.id, step); + if (!phaseStatefulCascades.has(phase.id)) { + phaseStatefulCascades.set(phase.id, { + stepId: step.id, + reason: result.skip_reason, + capabilityUnavailable: true, + }); + } } else if ( result.skip_reason === 'not_applicable' || result.skip_reason === 'capability_prerequisite_unavailable' @@ -6535,7 +6550,7 @@ async function executeStoryboardPass( if ( phasePendingMissingTool.trigger && !phaseRescuedTargets.has(phasePendingMissingTool.trigger.stepId) && - !phaseStatefulCascades.has(phase.id) + (!phaseStatefulCascades.has(phase.id) || phaseStatefulCascades.get(phase.id)?.capabilityUnavailable === true) ) { const subs = phasePendingMissingTool.trigger.substitutes; const subsList = subs.length === 1 ? `"${subs[0]}"` : subs.map(s => `"${s}"`).join(', '); @@ -6567,7 +6582,13 @@ async function executeStoryboardPass( // simply doesn't use this sync pathway — which is valid. Cascading on a // sole not_applicable would incorrectly penalise adapters that manage // state implicitly and have no list_accounts peer (adcp-client#1146). - if (phasePendingNotApplicable && !phaseEstablishedStatefulState && !phaseStatefulCascades.has(phase.id)) { + if ( + phasePendingNotApplicable && + !phaseEstablishedStatefulState && + (!phaseStatefulCascades.has(phase.id) || + (phasePendingNotApplicable.capabilityUnavailable !== true && + phaseStatefulCascades.get(phase.id)?.capabilityUnavailable === true)) + ) { const hadStatefulPeers = phaseStatefulStepIds.some(id => id !== phasePendingNotApplicable!.stepId); if (hadStatefulPeers) { phaseStatefulCascades.set(phase.id, phasePendingNotApplicable); diff --git a/test/lib/storyboard-force-scenario-unsupported.test.js b/test/lib/storyboard-force-scenario-unsupported.test.js index 02093b9d7..0c18ff01b 100644 --- a/test/lib/storyboard-force-scenario-unsupported.test.js +++ b/test/lib/storyboard-force-scenario-unsupported.test.js @@ -18,7 +18,7 @@ const { describe, test } = require('node:test'); const assert = require('node:assert/strict'); -const { runStoryboardStep } = require('../../dist/lib/testing/storyboard/runner'); +const { runStoryboard, runStoryboardStep } = require('../../dist/lib/testing/storyboard/runner'); const { DETAILED_SKIP_TO_CANONICAL } = require('../../dist/lib/testing/storyboard/types'); function buildControllerStubClient(controllerResponse) { @@ -148,3 +148,165 @@ describe('runStoryboardStep: force_scenario_unsupported (adcp#1805)', () => { assert.notEqual(result.skip_reason, 'force_scenario_unsupported'); }); }); + +async function runForceDependency({ + layout = 'downstream', + controllerResponse = { success: false, error: 'UNKNOWN_SCENARIO' }, + observedStatus = 'completed', + failBeforeForce = false, + statefulForce = true, + deferredSetup = null, +} = {}) { + const force = structuredClone(storyboard.phases[0].steps[0]); + force.stateful = statefulForce; + const observe = id => ({ + id, + title: 'Observe forced outcome', + task: 'get_products', + stateful: true, + sample_request: { buying_mode: 'brief', brief: 'Display inventory' }, + validations: [{ check: 'field_value', path: 'status', value: 'rejected', description: 'Forced rejection' }], + }); + const setup = { id: 'setup', title: 'Set up forced state', steps: [force] }; + if (deferredSetup) { + setup.steps.unshift({ + id: 'missing_setup', + title: 'Prepare account state', + task: 'sync_accounts', + stateful: true, + sample_request: {}, + }); + setup.steps.push({ ...observe('substitute'), provides_state_for: 'missing_setup' }); + } + const phases = [setup]; + if (failBeforeForce) { + phases.unshift({ id: 'failure', title: 'Independent failure', depends_on: [], steps: [observe('real_failure')] }); + setup.depends_on = []; + } + if (layout === 'same_phase') setup.steps.push(observe('observe')); + else { + phases.push({ + id: 'observe', + title: 'Observe forced state', + depends_on: layout === 'independent' ? [] : failBeforeForce ? ['setup', 'failure'] : ['setup'], + steps: [observe('observe')], + }); + if (layout === 'chain') { + phases.push({ id: 'readback', title: 'Read back', depends_on: ['observe'], steps: [observe('readback')] }); + } + } + const calls = []; + const tools = ['comply_test_controller', 'get_products']; + const client = { + executeTask: async task => { + calls.push(task); + return { + success: true, + data: task === 'comply_test_controller' ? controllerResponse : { status: observedStatus }, + }; + }, + }; + const result = await runStoryboard( + 'https://stub.example/mcp', + { ...storyboard, phases }, + { + protocol: 'mcp', + agentTools: tools, + skip_controller_seeding: true, + _client: client, + _profile: { + name: 'stub', + tools, + ...(deferredSetup === 'not_applicable' && { + raw_capabilities: { account: { require_operator_auth: true } }, + }), + }, + } + ); + return { result, calls, steps: result.phases.flatMap(phase => phase.steps) }; +} + +describe('runStoryboard: unsupported force setup cascades (adcp-client#3549)', () => { + for (const layout of ['same_phase', 'downstream', 'chain']) { + test(`${layout}: observations skip without dispatch or seller failures`, async () => { + const { result, calls, steps } = await runForceDependency({ layout }); + assert.deepEqual(calls, ['comply_test_controller'], 'unforced observations must never dispatch'); + assert.equal(steps[0].skip_reason, 'force_scenario_unsupported'); + for (const step of steps.slice(1)) { + assert.equal(step.skipped, true); + assert.equal(step.passed, true); + assert.equal(step.skip.reason, 'not_applicable'); + assert.match(step.skip.detail, /force_creative.*force_scenario_unsupported/); + assert.deepEqual(step.validations, []); + } + assert.equal(result.overall_passed, true, 'unavailable forcing is a coverage skip'); + assert.equal(result.failed_count, 0); + assert.equal(result.passed_count, 0, 'skipped observations receive no passing-step credit'); + }); + } + + test('an independent phase still executes and its failure remains visible', async () => { + const { result, calls, steps } = await runForceDependency({ layout: 'independent' }); + assert.deepEqual(calls, ['comply_test_controller', 'get_products']); + assert.notEqual(steps[1].skipped, true); + assert.equal(steps[1].passed, false); + assert.equal(result.overall_passed, false); + }); + + test('non-stateful unsupported forcing does not create a state dependency', async () => { + const { calls, steps } = await runForceDependency({ statefulForce: false }); + assert.deepEqual(calls, ['comply_test_controller', 'get_products']); + assert.notEqual(steps[1].skipped, true); + }); + + test('successful forcing runs and validates the observation', async () => { + const { result, calls, steps } = await runForceDependency({ + controllerResponse: { success: true }, + observedStatus: 'rejected', + }); + assert.deepEqual(calls, ['comply_test_controller', 'get_products']); + assert.ok(steps.every(step => step.passed && !step.skipped)); + assert.equal(result.overall_passed, true); + }); + + test('a wrong outcome after successful forcing still fails', async () => { + const { result, calls, steps } = await runForceDependency({ controllerResponse: { success: true } }); + assert.deepEqual(calls, ['comply_test_controller', 'get_products']); + assert.equal(steps[1].passed, false); + assert.equal(result.overall_passed, false); + }); + + test('a real controller failure is not treated as unavailable forcing', async () => { + const { result, steps } = await runForceDependency({ + controllerResponse: { success: false, error: 'INVALID_PARAMS' }, + }); + assert.equal(steps[0].passed, false); + assert.equal(steps[1].skip_reason, 'prerequisite_failed'); + assert.equal(steps[1].passed, false); + assert.equal(result.overall_passed, false); + }); + + test('a real earlier failure takes precedence over unavailable forcing', async () => { + const { result, calls, steps } = await runForceDependency({ failBeforeForce: true }); + assert.deepEqual(calls, ['get_products', 'comply_test_controller']); + assert.equal(steps[0].passed, false); + assert.equal(steps[1].skip_reason, 'force_scenario_unsupported'); + assert.equal(steps[2].skip_reason, 'prerequisite_failed'); + assert.equal(steps[2].passed, false); + assert.equal(result.overall_passed, false); + }); + + for (const deferredSetup of ['missing_tool', 'not_applicable']) { + test(`unrescued ${deferredSetup} state takes precedence over unavailable forcing`, async () => { + const { result, calls, steps } = await runForceDependency({ deferredSetup }); + assert.deepEqual(calls, ['comply_test_controller']); + assert.equal(steps[0].skip_reason, deferredSetup); + assert.equal(steps[1].skip_reason, 'force_scenario_unsupported'); + const dependent = steps.at(-1); + assert.equal(dependent.skip_reason, 'prerequisite_failed'); + assert.match(dependent.skip.detail, /missing_setup/); + assert.equal(dependent.passed, false); + assert.equal(result.overall_passed, false); + }); + } +});