[3/5][core]refactor communication layer: PR 3 of 5, all other models in non async mode (#3719)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
NATURE authored and GitHub committed 2026-05-28 14:06:41 +08:00
1 parent 0f3264a9c0
commit 3a347b10b0
38 files changed
+3021 -275

No files matched your search

+69 -6
View File
@@ -84,11 +84,23 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
self.inputs_embeds = self._make_buffer(self.max_num_tokens, self.hidden_size, dtype=self.dtype, numpy=False)
# Initialize KV cache manager (preserve vllm_config fallback behavior)
self.kv_transfer_manager = OmniKVTransferManager.from_vllm_config(self.vllm_config, self.model_config)
# Only Qwen3-Omni currently consumes the connector-based full-payload
# handoff added in this PR. Other model architectures (e.g. Bagel
# diffusion) retain their pre-existing runner behavior so this PR
# does not perturb them.
if getattr(self.model_config, "model_arch", None) == "Qwen3OmniMoeForConditionalGeneration":
# Worker-connector init is gated by a per-`model_arch` allowlist
# (covers both producer-side and consumer-side runners for the
# arches below). Consumer-wait stages must be registered
# separately as `(model_arch, model_stage)` tuples in
# `omni_scheduling_coordinator._FULL_PAYLOAD_INPUT_STAGES`;
# forgetting that produces a Stage-1 hang on the consumer.
_OMNI_CONNECTOR_INIT_ARCHS = {
"Qwen3OmniMoeForConditionalGeneration",
"Qwen2_5OmniForConditionalGeneration",
"CovoAudioForConditionalGeneration",
"MiMoAudioModel",
"Qwen3TTSTalkerForConditionalGeneration",
"Qwen3TTSCode2Wav",
"CosyVoice3Model",
"DyninOmniForConditionalGeneration",
}
if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS:
self.init_omni_connectors(
vllm_config=self.vllm_config,
model_config=self.model_config,
@@ -109,6 +121,57 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
with maybe_disable_pin_memory_for_ray(self, total_bytes):
return super()._make_buffer(*size, dtype=dtype, numpy=numpy)
def _build_model_sampler_output_token_ids(self) -> list[list[int]]:
"""Build decoded-token history for custom model samplers.
vLLM only populates sampling_metadata.output_token_ids when penalties or
logits processors require it. CosyVoice3's custom RAS sampler also
depends on this history, so we reconstruct it directly from the input
batch for prefer_model_sampler models.
"""
req_output_token_ids = getattr(self.input_batch, "req_output_token_ids", [])
req_ids = list(getattr(self.input_batch, "req_ids", []))
output_token_ids = [list(req_output_token_ids[idx] or []) for idx in range(len(req_ids))]
sampled_token_ids_cpu = getattr(self.input_batch, "sampled_token_ids_cpu", None)
async_copy_ready_event = getattr(self.input_batch, "async_copy_ready_event", None)
prev_req_id_to_index = getattr(self.input_batch, "prev_req_id_to_index", None)
if sampled_token_ids_cpu is None or not output_token_ids or prev_req_id_to_index is None:
return output_token_ids
sampled_token_ids: list[list[int]] | None = None
for index, req_id in enumerate(req_ids):
prev_index = prev_req_id_to_index.get(req_id)
if prev_index is None:
continue
req_history = output_token_ids[index]
if not req_history or req_history[-1] != -1:
continue
if sampled_token_ids is None:
assert async_copy_ready_event is not None
async_copy_ready_event.synchronize()
sampled_token_ids = sampled_token_ids_cpu.tolist()
new_ids = list(sampled_token_ids[prev_index])
if not new_ids:
continue
num_sampled_ids = len(new_ids) if new_ids[-1] != -1 else new_ids.index(-1)
first_placeholder = req_history.index(-1)
num_placeholders = len(req_history) - first_placeholder
num_to_replace = min(num_sampled_ids, num_placeholders)
req_history[first_placeholder : first_placeholder + num_to_replace] = new_ids[:num_to_replace]
for index, req_history in enumerate(output_token_ids):
if -1 in req_history:
output_token_ids[index] = req_history[: req_history.index(-1)]
return output_token_ids
def _sampling_metadata_for_model_sampler(self, sampling_metadata):
output_token_ids = self._build_model_sampler_output_token_ids()
if output_token_ids == sampling_metadata.output_token_ids:
return sampling_metadata
return replace(sampling_metadata, output_token_ids=output_token_ids)
def _request_final_stage_id(self, req_id: str) -> int | None:
info = self.model_intermediate_buffer.get(req_id)
if not isinstance(info, dict):
@@ -974,7 +1037,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
req_hidden_states_cpu[rid] = hidden_states[start:end].detach().to("cpu").contiguous()
pooler_output = []
for rid in req_ids_output_copy:
for out_idx, rid in enumerate(req_ids_output_copy):
if rid not in downstream_req_id_set:
pooler_output.append({})
continue