fix(storyboard): skip observations after unsupported forcing

.changeset/unsupported-force-scenario-cascade.md

@@ -0,0 +1,5 @@
+---
+'@adcp/sdk': patch
+---
+
+Skip dependent storyboard observations when a stateful test-controller force scenario is unsupported, including when forcing is the only stateful step in its phase. Preserve a not-applicable coverage result without dispatching unforced observations or awarding passing-step credit; independent phases and genuine failures retain their normal grading.

src/lib/testing/storyboard/runner.ts

@@ -4855,7 +4855,7 @@ async function executeStoryboardPass(
     stepId: string;
     reason: RunnerSkipReason | RunnerDetailedSkipReason;
     substitution_chain?: string;
-    /** The trigger is an explicit phase capability gate, not a failed setup. */
+    /** The trigger is unavailable coverage (capability or forcing), not a failed setup. */
     capabilityUnavailable?: boolean;
   };
   const phaseStatefulCascades = new Map<string, CascadeTrigger | null>();
@@ -6137,7 +6137,9 @@ async function executeStoryboardPass(
         }
         if (capabilityUnavailable) {
           recordUnavailableOutputs(phase.id, step);
-          if (routingContext) phaseInheritedCapabilityTrigger ??= trigger ?? undefined;
+          if (routingContext || trigger?.reason === 'force_scenario_unsupported') {
+            phaseInheritedCapabilityTrigger ??= trigger ?? undefined;
+          }
         } else if (routingContext) {
           recordUnavailableOutputs(phase.id, step, missingPrerequisiteContextKeysByPhase);
           recordHardPrerequisiteFailure(step, trigger === null);
@@ -6392,6 +6394,19 @@ async function executeStoryboardPass(
         if (step.stateful) {
           if (isHardMissingStateSkipReason(result.skip_reason)) {
             recordHardMissingState(step, result);
+          } else if (result.skip_reason === 'force_scenario_unsupported') {
+            // An unsupported force_* call never established the controlled state
+            // its observations require. Unlike ordinary implicit-account setup,
+            // it cannot use the sole-stateful-step exemption. Skip dependent
+            // coverage neutrally, while retaining any earlier hard failure.
+            recordUnavailableOutputs(phase.id, step);
+            if (!phaseStatefulCascades.has(phase.id)) {
+              phaseStatefulCascades.set(phase.id, {
+                stepId: step.id,
+                reason: result.skip_reason,
+                capabilityUnavailable: true,
+              });
+            }
           } else if (
             result.skip_reason === 'not_applicable' ||
             result.skip_reason === 'capability_prerequisite_unavailable'
@@ -6535,7 +6550,7 @@ async function executeStoryboardPass(
     if (
       phasePendingMissingTool.trigger &&
       !phaseRescuedTargets.has(phasePendingMissingTool.trigger.stepId) &&
-      !phaseStatefulCascades.has(phase.id)
+      (!phaseStatefulCascades.has(phase.id) || phaseStatefulCascades.get(phase.id)?.capabilityUnavailable === true)
     ) {
       const subs = phasePendingMissingTool.trigger.substitutes;
       const subsList = subs.length === 1 ? `"${subs[0]}"` : subs.map(s => `"${s}"`).join(', ');
@@ -6567,7 +6582,13 @@ async function executeStoryboardPass(
     // simply doesn't use this sync pathway — which is valid. Cascading on a
     // sole not_applicable would incorrectly penalise adapters that manage
     // state implicitly and have no list_accounts peer (adcp-client#1146).
-    if (phasePendingNotApplicable && !phaseEstablishedStatefulState && !phaseStatefulCascades.has(phase.id)) {
+    if (
+      phasePendingNotApplicable &&
+      !phaseEstablishedStatefulState &&
+      (!phaseStatefulCascades.has(phase.id) ||
+        (phasePendingNotApplicable.capabilityUnavailable !== true &&
+          phaseStatefulCascades.get(phase.id)?.capabilityUnavailable === true))
+    ) {
       const hadStatefulPeers = phaseStatefulStepIds.some(id => id !== phasePendingNotApplicable!.stepId);
       if (hadStatefulPeers) {
         phaseStatefulCascades.set(phase.id, phasePendingNotApplicable);

test/lib/storyboard-force-scenario-unsupported.test.js

@@ -18,7 +18,7 @@
 const { describe, test } = require('node:test');
 const assert = require('node:assert/strict');
 
-const { runStoryboardStep } = require('../../dist/lib/testing/storyboard/runner');
+const { runStoryboard, runStoryboardStep } = require('../../dist/lib/testing/storyboard/runner');
 const { DETAILED_SKIP_TO_CANONICAL } = require('../../dist/lib/testing/storyboard/types');
 
 function buildControllerStubClient(controllerResponse) {
@@ -148,3 +148,165 @@ describe('runStoryboardStep: force_scenario_unsupported (adcp#1805)', () => {
     assert.notEqual(result.skip_reason, 'force_scenario_unsupported');
   });
 });
+
+async function runForceDependency({
+  layout = 'downstream',
+  controllerResponse = { success: false, error: 'UNKNOWN_SCENARIO' },
+  observedStatus = 'completed',
+  failBeforeForce = false,
+  statefulForce = true,
+  deferredSetup = null,
+} = {}) {
+  const force = structuredClone(storyboard.phases[0].steps[0]);
+  force.stateful = statefulForce;
+  const observe = id => ({
+    id,
+    title: 'Observe forced outcome',
+    task: 'get_products',
+    stateful: true,
+    sample_request: { buying_mode: 'brief', brief: 'Display inventory' },
+    validations: [{ check: 'field_value', path: 'status', value: 'rejected', description: 'Forced rejection' }],
+  });
+  const setup = { id: 'setup', title: 'Set up forced state', steps: [force] };
+  if (deferredSetup) {
+    setup.steps.unshift({
+      id: 'missing_setup',
+      title: 'Prepare account state',
+      task: 'sync_accounts',
+      stateful: true,
+      sample_request: {},
+    });
+    setup.steps.push({ ...observe('substitute'), provides_state_for: 'missing_setup' });
+  }
+  const phases = [setup];
+  if (failBeforeForce) {
+    phases.unshift({ id: 'failure', title: 'Independent failure', depends_on: [], steps: [observe('real_failure')] });
+    setup.depends_on = [];
+  }
+  if (layout === 'same_phase') setup.steps.push(observe('observe'));
+  else {
+    phases.push({
+      id: 'observe',
+      title: 'Observe forced state',
+      depends_on: layout === 'independent' ? [] : failBeforeForce ? ['setup', 'failure'] : ['setup'],
+      steps: [observe('observe')],
+    });
+    if (layout === 'chain') {
+      phases.push({ id: 'readback', title: 'Read back', depends_on: ['observe'], steps: [observe('readback')] });
+    }
+  }
+  const calls = [];
+  const tools = ['comply_test_controller', 'get_products'];
+  const client = {
+    executeTask: async task => {
+      calls.push(task);
+      return {
+        success: true,
+        data: task === 'comply_test_controller' ? controllerResponse : { status: observedStatus },
+      };
+    },
+  };
+  const result = await runStoryboard(
+    'https://stub.example/mcp',
+    { ...storyboard, phases },
+    {
+      protocol: 'mcp',
+      agentTools: tools,
+      skip_controller_seeding: true,
+      _client: client,
+      _profile: {
+        name: 'stub',
+        tools,
+        ...(deferredSetup === 'not_applicable' && {
+          raw_capabilities: { account: { require_operator_auth: true } },
+        }),
+      },
+    }
+  );
+  return { result, calls, steps: result.phases.flatMap(phase => phase.steps) };
+}
+
+describe('runStoryboard: unsupported force setup cascades (adcp-client#3549)', () => {
+  for (const layout of ['same_phase', 'downstream', 'chain']) {
+    test(`${layout}: observations skip without dispatch or seller failures`, async () => {
+      const { result, calls, steps } = await runForceDependency({ layout });
+      assert.deepEqual(calls, ['comply_test_controller'], 'unforced observations must never dispatch');
+      assert.equal(steps[0].skip_reason, 'force_scenario_unsupported');
+      for (const step of steps.slice(1)) {
+        assert.equal(step.skipped, true);
+        assert.equal(step.passed, true);
+        assert.equal(step.skip.reason, 'not_applicable');
+        assert.match(step.skip.detail, /force_creative.*force_scenario_unsupported/);
+        assert.deepEqual(step.validations, []);
+      }
+      assert.equal(result.overall_passed, true, 'unavailable forcing is a coverage skip');
+      assert.equal(result.failed_count, 0);
+      assert.equal(result.passed_count, 0, 'skipped observations receive no passing-step credit');
+    });
+  }
+
+  test('an independent phase still executes and its failure remains visible', async () => {
+    const { result, calls, steps } = await runForceDependency({ layout: 'independent' });
+    assert.deepEqual(calls, ['comply_test_controller', 'get_products']);
+    assert.notEqual(steps[1].skipped, true);
+    assert.equal(steps[1].passed, false);
+    assert.equal(result.overall_passed, false);
+  });
+
+  test('non-stateful unsupported forcing does not create a state dependency', async () => {
+    const { calls, steps } = await runForceDependency({ statefulForce: false });
+    assert.deepEqual(calls, ['comply_test_controller', 'get_products']);
+    assert.notEqual(steps[1].skipped, true);
+  });
+
+  test('successful forcing runs and validates the observation', async () => {
+    const { result, calls, steps } = await runForceDependency({
+      controllerResponse: { success: true },
+      observedStatus: 'rejected',
+    });
+    assert.deepEqual(calls, ['comply_test_controller', 'get_products']);
+    assert.ok(steps.every(step => step.passed && !step.skipped));
+    assert.equal(result.overall_passed, true);
+  });
+
+  test('a wrong outcome after successful forcing still fails', async () => {
+    const { result, calls, steps } = await runForceDependency({ controllerResponse: { success: true } });
+    assert.deepEqual(calls, ['comply_test_controller', 'get_products']);
+    assert.equal(steps[1].passed, false);
+    assert.equal(result.overall_passed, false);
+  });
+
+  test('a real controller failure is not treated as unavailable forcing', async () => {
+    const { result, steps } = await runForceDependency({
+      controllerResponse: { success: false, error: 'INVALID_PARAMS' },
+    });
+    assert.equal(steps[0].passed, false);
+    assert.equal(steps[1].skip_reason, 'prerequisite_failed');
+    assert.equal(steps[1].passed, false);
+    assert.equal(result.overall_passed, false);
+  });
+
+  test('a real earlier failure takes precedence over unavailable forcing', async () => {
+    const { result, calls, steps } = await runForceDependency({ failBeforeForce: true });
+    assert.deepEqual(calls, ['get_products', 'comply_test_controller']);
+    assert.equal(steps[0].passed, false);
+    assert.equal(steps[1].skip_reason, 'force_scenario_unsupported');
+    assert.equal(steps[2].skip_reason, 'prerequisite_failed');
+    assert.equal(steps[2].passed, false);
+    assert.equal(result.overall_passed, false);
+  });
+
+  for (const deferredSetup of ['missing_tool', 'not_applicable']) {
+    test(`unrescued ${deferredSetup} state takes precedence over unavailable forcing`, async () => {
+      const { result, calls, steps } = await runForceDependency({ deferredSetup });
+      assert.deepEqual(calls, ['comply_test_controller']);
+      assert.equal(steps[0].skip_reason, deferredSetup);
+      assert.equal(steps[1].skip_reason, 'force_scenario_unsupported');
+      const dependent = steps.at(-1);
+      assert.equal(dependent.skip_reason, 'prerequisite_failed');
+      assert.match(dependent.skip.detail, /missing_setup/);
+      assert.equal(dependent.passed, false);
+      assert.equal(result.overall_passed, false);
+    });
+  }
+});