[rank1]: Traceback (most recent call last):
[rank1]: File "/root/Magma-main/train.py", line 625, in
[rank1]: train()
[rank1]: File "/root/Magma-main/train.py", line 601, in train
[rank1]: trainer.train()
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank1]: return inner_training_loop(
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank1]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank1]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank1]: outputs = model(**inputs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank1]: ret_val = func(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank1]: loss = self.module(*inputs, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank1]: result = forward_call(*args, **kwargs)
[rank1]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank1]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank1]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank1]: raise ValueError(
[rank1]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).
[rank3]: Traceback (most recent call last):
[rank3]: File "/root/Magma-main/train.py", line 625, in
[rank3]: train()
[rank3]: File "/root/Magma-main/train.py", line 601, in train
[rank3]: trainer.train()
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank3]: return inner_training_loop(
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank3]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank3]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank3]: outputs = model(**inputs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank3]: ret_val = func(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank3]: loss = self.module(*inputs, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank3]: result = forward_call(*args, **kwargs)
[rank3]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank3]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank3]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank3]: raise ValueError(
[rank3]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).
[rank0]: Traceback (most recent call last):
[rank0]: File "/root/Magma-main/train.py", line 625, in
[rank0]: train()
[rank0]: File "/root/Magma-main/train.py", line 601, in train
[rank0]: trainer.train()
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank0]: return inner_training_loop(
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank0]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank0]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank0]: outputs = model(**inputs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank0]: ret_val = func(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank0]: loss = self.module(*inputs, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank0]: result = forward_call(*args, **kwargs)
[rank0]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank0]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank0]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank0]: raise ValueError(
[rank0]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).
[rank2]: Traceback (most recent call last):
[rank2]: File "/root/Magma-main/train.py", line 625, in
[rank2]: train()
[rank2]: File "/root/Magma-main/train.py", line 601, in train
[rank2]: trainer.train()
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank2]: return inner_training_loop(
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank2]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank2]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank2]: outputs = model(**inputs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank2]: ret_val = func(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank2]: loss = self.module(*inputs, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank2]: result = forward_call(*args, **kwargs)
[rank2]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank2]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank2]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank2]: raise ValueError(
[rank2]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).z
这怎么解决
[rank1]: Traceback (most recent call last):
[rank1]: File "/root/Magma-main/train.py", line 625, in
[rank1]: train()
[rank1]: File "/root/Magma-main/train.py", line 601, in train
[rank1]: trainer.train()
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank1]: return inner_training_loop(
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank1]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank1]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank1]: outputs = model(**inputs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank1]: ret_val = func(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank1]: loss = self.module(*inputs, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank1]: result = forward_call(*args, **kwargs)
[rank1]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank1]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank1]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank1]: raise ValueError(
[rank1]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).
[rank3]: Traceback (most recent call last):
[rank3]: File "/root/Magma-main/train.py", line 625, in
[rank3]: train()
[rank3]: File "/root/Magma-main/train.py", line 601, in train
[rank3]: trainer.train()
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank3]: return inner_training_loop(
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank3]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank3]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank3]: outputs = model(**inputs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank3]: ret_val = func(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank3]: loss = self.module(*inputs, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank3]: result = forward_call(*args, **kwargs)
[rank3]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank3]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank3]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank3]: raise ValueError(
[rank3]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).
[rank0]: Traceback (most recent call last):
[rank0]: File "/root/Magma-main/train.py", line 625, in
[rank0]: train()
[rank0]: File "/root/Magma-main/train.py", line 601, in train
[rank0]: trainer.train()
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank0]: return inner_training_loop(
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank0]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank0]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank0]: outputs = model(**inputs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank0]: ret_val = func(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank0]: loss = self.module(*inputs, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank0]: result = forward_call(*args, **kwargs)
[rank0]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank0]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank0]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank0]: raise ValueError(
[rank0]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).
[rank2]: Traceback (most recent call last):
[rank2]: File "/root/Magma-main/train.py", line 625, in
[rank2]: train()
[rank2]: File "/root/Magma-main/train.py", line 601, in train
[rank2]: trainer.train()
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2245, in train
[rank2]: return inner_training_loop(
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 2560, in _inner_training_loop
[rank2]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3736, in training_step
[rank2]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/transformers/trainer.py", line 3801, in compute_loss
[rank2]: outputs = model(**inputs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
[rank2]: ret_val = func(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1833, in forward
[rank2]: loss = self.module(*inputs, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: File "/root/miniconda3/envs/magma/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1582, in _call_impl
[rank2]: result = forward_call(*args, **kwargs)
[rank2]: File "/root/Magma-main/magma/modeling_magma.py", line 721, in forward
[rank2]: inputs_embeds, attention_mask, position_ids, labels = self._merge_input_ids_with_image_features(
[rank2]: File "/root/Magma-main/magma/modeling_magma.py", line 500, in _merge_input_ids_with_image_features
[rank2]: raise ValueError(
[rank2]: ValueError: Number of image tokens in input_ids (0) different from num_images (4).z
这怎么解决