[3/5][core]refactor communication layer: PR 3 of 5, all other models in non async mode (#3719)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
0f3264a9c0
commit
3a347b10b0
38 files changed
+3021
-275
No files matched your search
@@ -84,11 +84,23 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
self.inputs_embeds = self._make_buffer(self.max_num_tokens, self.hidden_size, dtype=self.dtype, numpy=False)
|
||||
# Initialize KV cache manager (preserve vllm_config fallback behavior)
|
||||
self.kv_transfer_manager = OmniKVTransferManager.from_vllm_config(self.vllm_config, self.model_config)
|
||||
# Only Qwen3-Omni currently consumes the connector-based full-payload
|
||||
# handoff added in this PR. Other model architectures (e.g. Bagel
|
||||
# diffusion) retain their pre-existing runner behavior so this PR
|
||||
# does not perturb them.
|
||||
if getattr(self.model_config, "model_arch", None) == "Qwen3OmniMoeForConditionalGeneration":
|
||||
# Worker-connector init is gated by a per-`model_arch` allowlist
|
||||
# (covers both producer-side and consumer-side runners for the
|
||||
# arches below). Consumer-wait stages must be registered
|
||||
# separately as `(model_arch, model_stage)` tuples in
|
||||
# `omni_scheduling_coordinator._FULL_PAYLOAD_INPUT_STAGES`;
|
||||
# forgetting that produces a Stage-1 hang on the consumer.
|
||||
_OMNI_CONNECTOR_INIT_ARCHS = {
|
||||
"Qwen3OmniMoeForConditionalGeneration",
|
||||
"Qwen2_5OmniForConditionalGeneration",
|
||||
"CovoAudioForConditionalGeneration",
|
||||
"MiMoAudioModel",
|
||||
"Qwen3TTSTalkerForConditionalGeneration",
|
||||
"Qwen3TTSCode2Wav",
|
||||
"CosyVoice3Model",
|
||||
"DyninOmniForConditionalGeneration",
|
||||
}
|
||||
if getattr(self.model_config, "model_arch", None) in _OMNI_CONNECTOR_INIT_ARCHS:
|
||||
self.init_omni_connectors(
|
||||
vllm_config=self.vllm_config,
|
||||
model_config=self.model_config,
|
||||
@@ -109,6 +121,57 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
with maybe_disable_pin_memory_for_ray(self, total_bytes):
|
||||
return super()._make_buffer(*size, dtype=dtype, numpy=numpy)
|
||||
|
||||
def _build_model_sampler_output_token_ids(self) -> list[list[int]]:
|
||||
"""Build decoded-token history for custom model samplers.
|
||||
|
||||
vLLM only populates sampling_metadata.output_token_ids when penalties or
|
||||
logits processors require it. CosyVoice3's custom RAS sampler also
|
||||
depends on this history, so we reconstruct it directly from the input
|
||||
batch for prefer_model_sampler models.
|
||||
"""
|
||||
req_output_token_ids = getattr(self.input_batch, "req_output_token_ids", [])
|
||||
req_ids = list(getattr(self.input_batch, "req_ids", []))
|
||||
output_token_ids = [list(req_output_token_ids[idx] or []) for idx in range(len(req_ids))]
|
||||
|
||||
sampled_token_ids_cpu = getattr(self.input_batch, "sampled_token_ids_cpu", None)
|
||||
async_copy_ready_event = getattr(self.input_batch, "async_copy_ready_event", None)
|
||||
prev_req_id_to_index = getattr(self.input_batch, "prev_req_id_to_index", None)
|
||||
if sampled_token_ids_cpu is None or not output_token_ids or prev_req_id_to_index is None:
|
||||
return output_token_ids
|
||||
|
||||
sampled_token_ids: list[list[int]] | None = None
|
||||
for index, req_id in enumerate(req_ids):
|
||||
prev_index = prev_req_id_to_index.get(req_id)
|
||||
if prev_index is None:
|
||||
continue
|
||||
req_history = output_token_ids[index]
|
||||
if not req_history or req_history[-1] != -1:
|
||||
continue
|
||||
if sampled_token_ids is None:
|
||||
assert async_copy_ready_event is not None
|
||||
async_copy_ready_event.synchronize()
|
||||
sampled_token_ids = sampled_token_ids_cpu.tolist()
|
||||
new_ids = list(sampled_token_ids[prev_index])
|
||||
if not new_ids:
|
||||
continue
|
||||
num_sampled_ids = len(new_ids) if new_ids[-1] != -1 else new_ids.index(-1)
|
||||
first_placeholder = req_history.index(-1)
|
||||
num_placeholders = len(req_history) - first_placeholder
|
||||
num_to_replace = min(num_sampled_ids, num_placeholders)
|
||||
req_history[first_placeholder : first_placeholder + num_to_replace] = new_ids[:num_to_replace]
|
||||
|
||||
for index, req_history in enumerate(output_token_ids):
|
||||
if -1 in req_history:
|
||||
output_token_ids[index] = req_history[: req_history.index(-1)]
|
||||
|
||||
return output_token_ids
|
||||
|
||||
def _sampling_metadata_for_model_sampler(self, sampling_metadata):
|
||||
output_token_ids = self._build_model_sampler_output_token_ids()
|
||||
if output_token_ids == sampling_metadata.output_token_ids:
|
||||
return sampling_metadata
|
||||
return replace(sampling_metadata, output_token_ids=output_token_ids)
|
||||
|
||||
def _request_final_stage_id(self, req_id: str) -> int | None:
|
||||
info = self.model_intermediate_buffer.get(req_id)
|
||||
if not isinstance(info, dict):
|
||||
@@ -974,7 +1037,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
req_hidden_states_cpu[rid] = hidden_states[start:end].detach().to("cpu").contiguous()
|
||||
|
||||
pooler_output = []
|
||||
for rid in req_ids_output_copy:
|
||||
for out_idx, rid in enumerate(req_ids_output_copy):
|
||||
if rid not in downstream_req_id_set:
|
||||
pooler_output.append({})
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user