[Bugfix][Qwen3-Omni] Fix synchronous Thinker pipeline parallelism (#7345)
Signed-off-by: Deep Shah <deep@socratic.co>
This commit is contained in:
1 parent
909b121e0a
commit
2f1a18e69f
7 files changed
+279
-9
No files matched your search
@@ -34,6 +34,7 @@ from vllm.v1.worker.gpu_model_runner import GPUModelRunner, PerLayerAttnMetadata
|
||||
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
|
||||
|
||||
from vllm_omni.core.prefix_cache import OmniTensorPrefixCache
|
||||
from vllm_omni.data_entry_keys import OmniPayload
|
||||
from vllm_omni.engine.serialization import deserialize_additional_information
|
||||
from vllm_omni.model_executor.layers.rotary_embedding.mrope import OmniMRotaryEmbedding as MRotaryEmbedding
|
||||
from vllm_omni.model_executor.models.model_local_kv import collect_model_local_kv_specs
|
||||
@@ -892,8 +893,8 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
@torch.inference_mode()
|
||||
def extract_multimodal_outputs(
|
||||
self, hidden_states: torch.Tensor | list[torch.Tensor] | OmniOutput
|
||||
) -> tuple[Any, Any]:
|
||||
self, hidden_states: torch.Tensor | list[torch.Tensor] | OmniOutput | IntermediateTensors
|
||||
) -> tuple[torch.Tensor | IntermediateTensors, OmniPayload | None]:
|
||||
if (
|
||||
hasattr(self.model, "have_multimodal_outputs")
|
||||
and self.model.have_multimodal_outputs
|
||||
@@ -902,7 +903,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
text_hidden_states = hidden_states.text_hidden_states
|
||||
multimodal_outputs = hidden_states.multimodal_outputs
|
||||
|
||||
elif isinstance(hidden_states, torch.Tensor):
|
||||
elif isinstance(hidden_states, (torch.Tensor, IntermediateTensors)):
|
||||
text_hidden_states = hidden_states
|
||||
multimodal_outputs = {}
|
||||
elif isinstance(hidden_states, list) or isinstance(hidden_states, tuple):
|
||||
@@ -1225,6 +1226,9 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
else:
|
||||
hidden_states = outputs
|
||||
hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states)
|
||||
if isinstance(hidden_states, IntermediateTensors):
|
||||
# Profiling/graph capture needs a tensor to select dummy sampler rows.
|
||||
hidden_states = hidden_states["hidden_states"]
|
||||
if self.speculative_config and (
|
||||
self.speculative_config.use_eagle()
|
||||
or self.speculative_config.uses_draft_model()
|
||||
@@ -2072,7 +2076,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
**model_kwargs,
|
||||
**model_kwargs_extra,
|
||||
)
|
||||
if not isinstance(model_output, OmniOutput) and hasattr(self.model, "make_omni_output"):
|
||||
if not isinstance(model_output, (OmniOutput, IntermediateTensors)) and hasattr(self.model, "make_omni_output"):
|
||||
model_output = self.model.make_omni_output(model_output, **model_kwargs, **model_kwargs_extra)
|
||||
# Cache model output so later sample_tokens can consume multimodal results.
|
||||
self._omni_last_model_output = model_output
|
||||
|
||||
Reference in new issue
Block a user