[3/5][core]refactor communication layer: PR 3 of 5, all other models in non async mode (#3719)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
NATURE authored and GitHub committed 2026-05-28 14:06:41 +08:00
1 parent 0f3264a9c0
commit 3a347b10b0
38 files changed
+3021 -275

No files matched your search

@@ -784,12 +784,14 @@ class CosyVoice3Model(
else:
self._stream_vocoder_cache_by_req[req_id] = new_cache_state
else:
token_offset = max(0, meta.talker_prefill_offset or 0) if meta else 0
tts_speech = self.code2wav.forward(
token=token.unsqueeze(0),
prompt_token=speech_token[:1],
prompt_feat=speech_feat[:1],
embedding=embedding[:1],
n_timesteps=10,
token_offset_tokens=token_offset,
)
audio = tts_speech.reshape(-1).to(dtype=torch.float32)
@@ -292,6 +292,7 @@ class CosyVoice3Code2Wav(nn.Module):
prompt_feat: torch.Tensor,
embedding: torch.Tensor,
n_timesteps: int = 10,
token_offset_tokens: int = 0,
) -> torch.Tensor:
"""Generate audio waveform from speech tokens."""
feat = self._forward_mel(
@@ -300,7 +301,7 @@ class CosyVoice3Code2Wav(nn.Module):
prompt_feat=prompt_feat,
embedding=embedding,
n_timesteps=n_timesteps,
token_offset_tokens=0,
token_offset_tokens=token_offset_tokens,
streaming=False,
finalize=True,
)
@@ -31,6 +31,7 @@ COSYVOICE3_PIPELINE = PipelineConfig(
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload",
sampling_constraints={
# merged speech stop token (logsumexp of all 200 stop logits)
"stop_token_ids": [6562],
@@ -44,7 +45,8 @@ COSYVOICE3_PIPELINE = PipelineConfig(
final_output=True,
final_output_type="audio",
engine_output_type="latent",
sync_process_input_func=f"{_PROC}.text2flow",
custom_process_input_func=f"{_PROC}.text2flow",
sync_process_input_func=f"{_PROC}.text2flow_token_only",
),
),
)
@@ -29,6 +29,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
owns_tokenizer=True,
requires_multimodal_data=True,
engine_output_type="latent",
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
"stop_token_ids": [151645],
@@ -44,6 +45,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
final_output_type="audio",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.llm2code2wav",
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
sampling_constraints={"detokenize": False},
),
),
@@ -39,6 +39,7 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
# Stop once the speech/text interleaved span ends. Code2Wav
@@ -55,7 +56,8 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sync_process_input_func=f"{_PROC}.llm2code2wav",
custom_process_input_func=f"{_PROC}.llm2code2wav",
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
sampling_constraints={"detokenize": False},
),
),
@@ -58,6 +58,7 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig(
engine_output_type="audio",
tokenizer_subdir="talker/llm",
custom_process_input_func=f"{_PROC}.thinker2talker",
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
),
),
)
@@ -30,6 +30,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
requires_multimodal_data=True,
engine_output_type="latent",
sampling_constraints={"detokenize": True},
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
),
StagePipelineConfig(
stage_id=1,
@@ -38,6 +39,8 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
input_sources=(0,),
engine_output_type="latent",
custom_process_input_func=f"{_PROC}.thinker2talker",
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
"stop_token_ids": [8294],
@@ -52,6 +55,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
final_output_type="audio",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.talker2code2wav",
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
),
@@ -74,6 +78,7 @@ QWEN2_5_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
requires_multimodal_data=True,
engine_output_type="latent",
sampling_constraints={"detokenize": True},
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
),
),
)
@@ -26,6 +26,7 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [2150],
@@ -40,7 +41,14 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
final_output_type="audio",
engine_output_type="audio",
model_arch="Qwen3TTSCode2Wav",
sync_process_input_func=f"{_PROC}.talker2code2wav",
# ``sync_process_input_func`` is the only input-proc override for
# this stage in sync (non-async-chunk) mode: a length-only
# ``_token_only`` placeholder. The bulk codec payload itself
# ships via the worker connector from stage 0's
# ``talker2code2wav_full_payload`` producer. Under async_chunk
# mode no pre-stage processing is needed -- chunks deliver
# directly to the consumer.
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
extras={"tts_args": {"max_instructions_length": 500}},
),
@@ -25,6 +25,27 @@ from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
logger = init_logger(__name__)
def _codec_ids_from_payload_or_input(
input_ids: torch.Tensor,
runtime_info: dict[str, Any] | None,
) -> torch.Tensor:
"""Prefer connector-delivered codec ids over token placeholders.
In non-async full-payload mode, the scheduler only needs placeholder
token ids for allocation. The real codec sequence is delivered through
model_intermediate_buffer as ``codes.audio``.
"""
if isinstance(runtime_info, dict):
codes = runtime_info.get("codes")
if isinstance(codes, dict):
audio = codes.get("audio")
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
return audio.reshape(-1).to(device=input_ids.device, dtype=torch.long)
if isinstance(audio, (list, tuple)) and audio:
return torch.as_tensor(audio, device=input_ids.device, dtype=torch.long).reshape(-1)
return input_ids.reshape(-1).to(dtype=torch.long)
class Qwen3TTSCode2Wav(nn.Module):
"""Stage-1 code2wav model for Qwen3-TTS (GenerationModelRunner).
Consumes frame-aligned codec tokens from input_ids and decodes waveform
@@ -239,6 +260,7 @@ class Qwen3TTSCode2Wav(nn.Module):
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
)
runtime_infos = runtime_additional_information or []
ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
@@ -246,14 +268,18 @@ class Qwen3TTSCode2Wav(nn.Module):
valid_codes_qf: list[torch.Tensor] = []
valid_indices: list[int] = []
left_context_size = [0] * len(request_ids_list)
if runtime_additional_information is not None:
for i, info in enumerate(runtime_additional_information):
if runtime_infos:
for i, info in enumerate(runtime_infos):
if i >= len(left_context_size):
break
if not isinstance(info, dict):
continue
meta = info.get("meta", {})
if "left_context_size" in meta:
left_context_size[i] = meta["left_context_size"]
for i, req_ids in enumerate(request_ids_list):
runtime_info = runtime_infos[i] if i < len(runtime_infos) else None
req_ids = _codec_ids_from_payload_or_input(req_ids, runtime_info)
if req_ids.numel() < 1:
parsed.append((0, 0))
continue
@@ -6,6 +6,7 @@ stage_args:
max_batch_size: 1
engine_args:
model_stage: token2text
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
model_arch: DyninOmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
@@ -6,6 +6,7 @@ stage_args:
max_batch_size: 1
engine_args:
model_stage: token2text
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
model_arch: DyninOmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
@@ -7,6 +7,7 @@ from typing import Any
import numpy as np
import torch
from vllm.inputs import TextPrompt
from vllm.logger import init_logger
from vllm_omni.data_entry_keys import (
CodesStruct,
@@ -16,6 +17,10 @@ from vllm_omni.data_entry_keys import (
)
from vllm_omni.inputs.data import OmniTokensPrompt
logger = init_logger(__name__)
_COSYVOICE3_SPEECH_TOKEN_SIZE = 6561
def _build_prompt_embed_struct(prompt_payload: dict[str, Any]) -> EmbeddingsStruct | None:
"""Wrap prompt_payload's flat speech_token/speech_feat/embedding tensors into EmbeddingsStruct."""
@@ -46,6 +51,52 @@ def _ensure_list(x: Any) -> list[Any]:
return [x]
def _to_token_id_list(value: Any) -> list[int]:
if value is None:
return []
if isinstance(value, torch.Tensor):
value = value.detach().to("cpu").reshape(-1).tolist()
token_ids: list[int] = []
for item in _ensure_list(value):
if isinstance(item, torch.Tensor):
token_ids.extend(_to_token_id_list(item))
continue
if isinstance(item, (list, tuple)):
token_ids.extend(_to_token_id_list(item))
continue
token_ids.append(int(item))
return token_ids
def _strip_prompt_prefix(output_ids: list[Any], prefix_ids: list[Any]) -> list[Any]:
if prefix_ids and len(output_ids) >= len(prefix_ids) and output_ids[: len(prefix_ids)] == prefix_ids:
return output_ids[len(prefix_ids) :]
return output_ids
def _prompt_speech_token_ids(multi_modal_data: dict[str, Any]) -> list[int]:
speech_token = multi_modal_data.get("speech_token")
if speech_token is None:
embed = multi_modal_data.get("embed")
if isinstance(embed, dict):
speech_token = embed.get("speech_token")
return _to_token_id_list(speech_token)
def _has_speech_stop_token(output_ids: list[Any]) -> bool:
return any(token_id >= _COSYVOICE3_SPEECH_TOKEN_SIZE for token_id in _to_token_id_list(output_ids))
def _set_non_stream_prompt_trim(additional_info: dict[str, Any], prompt_speech_len: int) -> None:
if prompt_speech_len <= 0:
return
meta = additional_info.get("meta")
if not isinstance(meta, dict):
meta = {}
additional_info["meta"] = meta
meta["talker_prefill_offset"] = prompt_speech_len
def _to_cpu_tensor(x: Any) -> torch.Tensor | None:
if isinstance(x, list):
if not x:
@@ -95,9 +146,14 @@ def text2flow(
if multi_modal_data is None:
raise RuntimeError(f"Missing multimodal_output for request {source_output.request_id}")
output_ids = _ensure_list(output.cumulative_token_ids)
prefix_ids = _ensure_list(source_output.prompt_token_ids)
raw_output_ids = _ensure_list(output.cumulative_token_ids)
prompt_speech_ids = _prompt_speech_token_ids(multi_modal_data)
output_ids = _strip_prompt_prefix(raw_output_ids, prefix_ids)
output_ids = _strip_prompt_prefix(output_ids, prompt_speech_ids)
additional_info = dict(multi_modal_data)
if _has_speech_stop_token(raw_output_ids):
_set_non_stream_prompt_trim(additional_info, len(prompt_speech_ids))
additional_info.setdefault("ids", {})["prompt"] = prefix_ids
engine_inputs.append(OmniTokensPrompt(prompt_token_ids=output_ids, additional_information=additional_info))
return engine_inputs
@@ -271,3 +327,107 @@ def talker2code2wav_async_chunk(
state["emitted_chunks"] = int(state.get("emitted_chunks", 0)) + 1
return payload
# ============================================================================
# Worker-connector data plane (non-async-chunk path).
# cosyvoice3 talker emits `multimodal_outputs={"embed": {"speech_token": t,
# "speech_feat": t, "embedding": t}}` ONLY at prefill (decode steps emit
# `{}`). After flatten_payload these become flat top-level keys
# `embed.speech_token` etc., persisted across decode steps by the
# full-payload accumulator (decode doesn't re-emit them). Shipping via the connector
# keeps the orchestrator off the heavy-tensor path.
# ============================================================================
# All three embed tensors are emitted once at prefill and must REPLACE-not-
# CONCAT across the (already trivial) per-request accumulator history so a
# regression where decode unexpectedly re-emits them does not silently
# duplicate the prefill tensor. See mixin._FULL_PAYLOAD_REPLACE_KEYS.
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset({"embed.speech_token", "embed.speech_feat", "embed.embedding"})
def text2flow_token_only(
source_outputs: list,
prompt: OmniTokensPrompt | TextPrompt = None,
_requires_multimodal_data: bool = True,
):
"""Sync-side builder for the non-async-chunk text→flow path.
CosyVoice3 sync keeps codec ids on the legacy token path. Some vLLM v1
histories include the source prompt prefix, so strip it only when it is an
exact leading match.
"""
del prompt
engine_inputs: list[OmniTokensPrompt] = []
for source_output in source_outputs:
if not source_output.finished:
continue
output = source_output.outputs[0]
prefix_ids = _ensure_list(source_output.prompt_token_ids)
raw_output_ids = _ensure_list(output.cumulative_token_ids)
output_ids = _strip_prompt_prefix(raw_output_ids, prefix_ids)
multi_modal_data = output.multimodal_output
if multi_modal_data is None:
raise RuntimeError(f"Missing multimodal_output for request {source_output.request_id}")
prompt_speech_ids = _prompt_speech_token_ids(multi_modal_data)
output_ids = _strip_prompt_prefix(output_ids, prompt_speech_ids)
additional_info: dict[str, Any] = dict(multi_modal_data)
if _has_speech_stop_token(raw_output_ids):
_set_non_stream_prompt_trim(additional_info, len(prompt_speech_ids))
additional_info.setdefault("ids", {})["prompt"] = prefix_ids
engine_inputs.append(
OmniTokensPrompt(
prompt_token_ids=output_ids,
additional_information=additional_info,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return engine_inputs
def text2flow_full_payload(
transfer_manager,
pooling_output,
request,
):
"""Producer-side payload builder.
Reads prefill-emitted `embed.{speech_token, speech_feat, embedding}` from
the accumulator and ships prompt conditioning as a connector payload.
The downstream flow stage reads these from `model_intermediate_buffer`
(see cosyvoice3.py:671 in the code2wav forward — runtime_info pickup).
"""
del transfer_manager
rid = getattr(request, "external_req_id", None) or getattr(request, "request_id", "?")
if not isinstance(pooling_output, dict):
logger.warning(
"cosyvoice3.text2flow_full_payload: pooling_output not a dict "
"(type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
rid,
)
return None
embed_out: dict[str, Any] = {}
for key in ("speech_token", "speech_feat", "embedding"):
v = pooling_output.get(f"embed.{key}")
if v is None:
nested = pooling_output.get("embed")
if isinstance(nested, dict):
v = nested.get(key)
if isinstance(v, torch.Tensor) and v.numel() > 0:
embed_out[key] = v
if not embed_out:
logger.warning(
"cosyvoice3.text2flow_full_payload: no embed.{speech_token,speech_feat,embedding} "
"found in pooling_output (keys=%s) for req=%s; consumer wait gate may hang.",
list(pooling_output.keys()),
rid,
)
return None
return {
"meta": {
"finished": torch.tensor(True, dtype=torch.bool),
},
"embed": embed_out,
}
@@ -1,27 +1,44 @@
# Copyright 2026 Tencent.
from typing import Any
import torch
from vllm.logger import init_logger
from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.model_executor.models.covo_audio.config_covo_audio import COVO_AUDIO_TOKEN_INDEX
logger = init_logger(__name__)
# Per-model REPLACE-keys for the full-payload accumulator (none for covo_audio:
# the producer side does not emit per-step hidden_states / model_outputs;
# llm2code2wav_full_payload reads token_ids directly from `request`).
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
def _filter_audio_codes(token_ids: list[int]) -> list[int]:
"""Filter codec-range token ids and rebase by COVO_AUDIO_TOKEN_INDEX."""
audio_codes = [t - COVO_AUDIO_TOKEN_INDEX for t in token_ids if t >= COVO_AUDIO_TOKEN_INDEX]
if not audio_codes:
audio_codes = [-1]
return audio_codes
def llm2code2wav(
source_outputs: list[Any],
prompt: Any = None,
requires_multimodal_data: bool = False,
) -> list[OmniTokensPrompt]:
"""Legacy orchestrator-path builder (retained for async_chunk + back-compat).
The non-async-chunk path now goes through ``llm2code2wav_token_only`` +
worker connector + ``llm2code2wav_full_payload``.
"""
talker_outputs = source_outputs
code2wav_inputs = []
for i, talker_output in enumerate(talker_outputs):
for talker_output in talker_outputs:
output = talker_output.outputs[0]
token_ids = output.token_ids
audio_codes = [t - COVO_AUDIO_TOKEN_INDEX for t in token_ids if t >= COVO_AUDIO_TOKEN_INDEX]
if not audio_codes:
audio_codes = [-1]
audio_codes = _filter_audio_codes(list(output.token_ids))
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=audio_codes,
@@ -31,3 +48,56 @@ def llm2code2wav(
)
return code2wav_inputs
def llm2code2wav_token_only(
source_outputs: list[Any],
prompt: Any = None,
requires_multimodal_data: bool = False,
) -> list[OmniTokensPrompt]:
"""Sync-side placeholder for the non-async-chunk Stage-1 input.
Returns an OmniTokensPrompt sized to the code2wav stage's expected
prefill length (one slot per audio code). The actual codec ids are
delivered via the worker connector payload built by
``llm2code2wav_full_payload``.
"""
code2wav_inputs: list[OmniTokensPrompt] = []
for output_wrapper in source_outputs:
output = output_wrapper.outputs[0]
audio_codes = _filter_audio_codes(list(output.token_ids))
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[0] * len(audio_codes),
additional_information=None,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return code2wav_inputs
def llm2code2wav_full_payload(
transfer_manager: Any,
pooling_output: dict[str, Any],
request: Any,
) -> dict[str, Any] | None:
"""Producer-side payload builder for the worker connector data plane.
covo_audio's fused_thinker_talker stage emits codec ids via
``request.output_token_ids`` (token-ids only -- no
hidden_states or embed tensors), so the connector payload is
just the filtered audio codes plus a finished marker.
"""
output_token_ids = list(getattr(request, "output_token_ids", None) or [])
if not output_token_ids:
logger.warning(
"covo_audio.llm2code2wav_full_payload: empty output_token_ids for req=%s; consumer wait gate may hang.",
getattr(request, "request_id", "?"),
)
return None
audio_codes = _filter_audio_codes(output_token_ids)
return {
"codes": {"audio": audio_codes},
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
}
@@ -5,9 +5,12 @@ from typing import Any
import torch
from vllm.inputs import TextPrompt
from vllm.logger import init_logger
from vllm_omni.inputs.data import OmniTokensPrompt
logger = init_logger(__name__)
def _to_prompt_dict(prompt_item: OmniTokensPrompt | TextPrompt | str | None) -> dict[str, Any]:
if isinstance(prompt_item, dict):
@@ -146,3 +149,128 @@ def token2image_to_token2audio(
requires_multimodal_data: bool = False,
):
return _bridge_tokens(source_outputs, prompt, requires_multimodal_data)
# ============================================================================
# Worker-connector data plane (non-async-chunk path).
# ============================================================================
# Per-model REPLACE-keys for the full-payload accumulator. dynin_omni's
# producer model emits new chunks per step (token_ids / runtime_info_json),
# all of which use the default CONCAT/replace semantics — no model_outputs
# entry needs explicit REPLACE.
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
def _build_full_payload(pooling_output: dict[str, Any] | None, request: Any) -> dict[str, Any] | None:
"""Producer-side payload builder: assemble dynin_omni connector payload.
Reads token_ids from ``pooling_output["token_ids"]`` (preferred) or
``request.output_token_ids`` (fallback). Reads structured non-tensor
metadata from ``pooling_output["runtime_info_json"]`` (JSON-in-uint8)
if present, falling back to ``pooling_output["runtime_info"]`` dict.
Carries forward ``request.additional_information`` so prompt-side
metadata (speaker / language / detok_id) survives the IPC boundary.
"""
if not isinstance(pooling_output, dict):
pooling_output = {}
token_ids = _to_token_id_list(pooling_output.get("token_ids"))
if not token_ids:
token_ids = _to_token_id_list(pooling_output.get("text_tokens"))
if not token_ids and request is not None:
token_ids = _to_token_id_list(getattr(request, "output_token_ids", None))
if not token_ids:
logger.warning(
"dynin_omni._build_full_payload: no token_ids found in pooling_output "
"(keys=%s) or request.output_token_ids for req=%s; consumer wait gate may hang.",
list(pooling_output.keys()),
getattr(request, "request_id", "?"),
)
return None
src_additional_info = getattr(request, "additional_information", {}) if request is not None else {}
if not isinstance(src_additional_info, dict):
src_additional_info = {}
runtime_bridge_info = _decode_runtime_bridge_info(pooling_output.get("runtime_info_json"))
if not runtime_bridge_info:
runtime_bridge_info = pooling_output.get("runtime_info", {}) or {}
payload = _normalize_additional_info(src_additional_info)
payload.update(_normalize_additional_info(runtime_bridge_info))
payload["detok_id"] = [_to_int(pooling_output.get("detok_id"), default=_to_int(payload.get("detok_id"), default=0))]
# Use nested OmniPayload shape so the scheduling-metadata extractor in
# OmniConnectorModelRunnerMixin reads codes.audio and meta.finished
# (flat keys at the top level are silently dropped with a warning).
payload["codes"] = {"audio": token_ids}
payload["meta"] = {"finished": torch.tensor(True, dtype=torch.bool)}
return payload
def token2text_to_token2image_full_payload(
transfer_manager: Any,
pooling_output: dict[str, Any],
request: Any,
) -> dict[str, Any] | None:
"""Producer-side payload builder for the Stage-0 → Stage-1 (text → image) transition."""
del transfer_manager
return _build_full_payload(pooling_output, request)
def token2image_to_token2audio_full_payload(
transfer_manager: Any,
pooling_output: dict[str, Any],
request: Any,
) -> dict[str, Any] | None:
"""Producer-side payload builder for the Stage-1 → Stage-2 (image → audio) transition."""
del transfer_manager
return _build_full_payload(pooling_output, request)
def _token_only_from_source(source_outputs: list[Any]) -> list[OmniTokensPrompt]:
"""Length-only placeholder list mirroring ``_bridge_tokens`` token counts."""
inputs: list[OmniTokensPrompt] = []
for source_output in source_outputs:
output = source_output.outputs[0]
mm_out = getattr(output, "multimodal_output", None) or {}
token_ids = _to_token_id_list(mm_out.get("token_ids"))
if not token_ids:
token_ids = _to_token_id_list(mm_out.get("text_tokens"))
if not token_ids:
token_ids = list(getattr(output, "token_ids", []) or [])
if not token_ids:
token_ids = [0]
inputs.append(
OmniTokensPrompt(
prompt_token_ids=[0] * len(token_ids),
additional_information=None,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return inputs
def token2text_to_token2image_token_only(
stage_list,
engine_input_source,
prompt: OmniTokensPrompt | TextPrompt = None,
requires_multimodal_data: bool = False,
) -> list[OmniTokensPrompt]:
"""Sync-side placeholder for Stage-1 input (token2image)."""
source_stage_id = engine_input_source[0] if engine_input_source else 0
source_outputs = stage_list[source_stage_id].engine_outputs
return _token_only_from_source(source_outputs)
def token2image_to_token2audio_token_only(
stage_list,
engine_input_source,
prompt: OmniTokensPrompt | TextPrompt = None,
requires_multimodal_data: bool = False,
) -> list[OmniTokensPrompt]:
"""Sync-side placeholder for Stage-2 input (token2audio)."""
source_stage_id = engine_input_source[0] if engine_input_source else 0
source_outputs = stage_list[source_stage_id].engine_outputs
return _token_only_from_source(source_outputs)
@@ -147,6 +147,18 @@ def llm2code2wav_async_chunk(
Accumulates codes in connector per request_id,
returns payload only when chunk_size is full or request is finished; returns None when waiting.
"""
# Null guard: chunk_transfer_adapter calls this every emit step
# including no-output steps where pooling_output is None.
if pooling_output is None or not isinstance(pooling_output, dict):
if is_finished:
connector = getattr(transfer_manager, "connector", None)
raw_cfg = getattr(connector, "config", {}) or {}
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
chunk_size = int(cfg.get("codec_chunk_frames", 3))
left_context_size = int(cfg.get("codec_left_context_frames", 3))
request_id = getattr(request, "external_req_id", None)
return _flush_remaining_codes(transfer_manager, request_id, chunk_size, left_context_size)
return None
connector = getattr(transfer_manager, "connector", None)
raw_cfg = getattr(connector, "config", {}) or {}
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
@@ -300,3 +312,137 @@ def llm2code2wav(
)
return code2wav_inputs
# ============================================================================
# Worker-connector data plane (non-async-chunk path).
# AR runner's `flatten_payload` converts the model emit
# `multimodal_outputs={"codes": {"audio": ...}}` to flat
# `pooling_output["codes.audio"]` before the full-payload accumulator runs, so default
# CONCAT semantics build the full codec tensor across all decode steps.
# ============================================================================
# Per-model REPLACE-keys for the full-payload accumulator. mimo_audio's
# producer side emits per-step codec frames that should be CONCAT'd across
# steps (not REPLACE'd), so this stays empty.
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
def _filter_zero_codec_rows(codec_codes: torch.Tensor) -> torch.Tensor:
"""Drop zero-padded codec rows from a 4-D `[N, 1, 8, 4]` tensor.
Mirrors the zero-row filter in the orchestrator-path `llm2code2wav`
body (see this file's ``llm2code2wav`` around line 224).
"""
if codec_codes.ndim != 4 or codec_codes.numel() == 0:
return codec_codes
is_all_zero = (codec_codes == 0).all(dim=(1, 2, 3))
nonzero_idx = (~is_all_zero).nonzero(as_tuple=True)[0]
if len(nonzero_idx) == 0:
# All rows are zero-padded; return an empty tensor so the caller
# can detect this via numel()==0 and skip the request.
return codec_codes[:0]
if len(nonzero_idx) < codec_codes.shape[0]:
return codec_codes[nonzero_idx]
return codec_codes
def llm2code2wav_token_only(
source_outputs: list,
_prompt=None,
_requires_multimodal_data: bool = False,
) -> list:
"""Sync-side placeholder for the non-async-chunk Stage-1 (code2wav) input.
Returns an ``OmniTokensPrompt`` sized to the orchestrator-shape codec
length so the consumer runtime allocates the right number of slots.
The actual codec ids are delivered via the worker connector payload
built by ``llm2code2wav_full_payload``.
"""
from vllm_omni.inputs.data import OmniTokensPrompt
code2wav_inputs: list = []
for output_wrapper in source_outputs:
out = output_wrapper.outputs[0]
mm = out.multimodal_output if hasattr(out, "multimodal_output") else None
mm = mm if isinstance(mm, dict) else {}
mm_codes = mm.get("codes", {}) if isinstance(mm, dict) else {}
prompt_len = 0
if isinstance(mm_codes, dict) and "audio" in mm_codes:
audio = mm_codes["audio"]
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
audio = audio.to(torch.long)
audio = _filter_zero_codec_rows(audio)
# +B*4 per batch row for the prepended pad_vec (see prepend_and_flatten_colmajor)
batch_size = int(audio.shape[0]) if audio.ndim >= 1 else 1
prompt_len = int(audio.numel()) + batch_size * 4
if prompt_len > MAX_CODE2WAV_TOKENS:
prompt_len = MAX_CODE2WAV_TOKENS
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[0] * prompt_len,
additional_information=None,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return code2wav_inputs
def llm2code2wav_full_payload(
transfer_manager,
pooling_output: dict,
request,
) -> dict | None:
"""Producer-side payload builder for the worker connector data plane.
AR runner's ``flatten_payload`` converts the per-step model emit
``{"codes": {"audio": ...}}`` to ``pooling_output["codes.audio"]``.
The accumulator CONCATs per-step tensors along dim 0, so by flush
time this holds the full ``[total_steps, 1, 8, 4]`` codec tensor.
A back-compat fallback to nested ``pooling_output["codes"]["audio"]``
is kept in case a future runtime path bypasses `flatten_payload`.
"""
del transfer_manager
rid = getattr(request, "request_id", "?")
if not isinstance(pooling_output, dict):
logger.warning(
"mimo_audio.llm2code2wav_full_payload: pooling_output not a dict "
"(type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
rid,
)
return None
codec_codes = pooling_output.get("codes.audio")
if codec_codes is None:
# Back-compat fallback for un-flattened pooler emits.
codes = pooling_output.get("codes")
if isinstance(codes, dict):
codec_codes = codes.get("audio")
if not isinstance(codec_codes, torch.Tensor) or codec_codes.numel() == 0:
logger.warning(
"mimo_audio.llm2code2wav_full_payload: missing/empty codes.audio "
"(keys=%s) for req=%s; consumer wait gate may hang.",
list(pooling_output.keys()),
rid,
)
return None
codec_codes = codec_codes.to(torch.long)
codec_codes = _filter_zero_codec_rows(codec_codes)
if codec_codes.numel() == 0:
logger.warning(
"mimo_audio.llm2code2wav_full_payload: codec_codes empty after _filter_zero_codec_rows for req=%s.",
rid,
)
return None
pad_vec = torch.tensor([TALKER_CODEC_PAD_TOKEN_ID] * 4)
code_final = prepend_and_flatten_colmajor(codec_codes, pad_vec).tolist()
if len(code_final) > MAX_CODE2WAV_TOKENS:
code_final = code_final[:MAX_CODE2WAV_TOKENS]
return {
"codes": {"audio": code_final},
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
}
@@ -535,9 +535,97 @@ def thinker2talker(
return talker_inputs
# ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` or
# ``_FULL_PAYLOAD_INPUT_STAGES``, so the worker connector is not
# initialised for this arch and the consumer never waits on a connector
# payload. Data flows through ``additional_information`` written by
# ``thinker2talker_token_only`` (wired as ``sync_process_input_func``
# in the pipeline) or the legacy ``thinker2talker`` (wired as
# ``custom_process_input_func``).
def thinker2talker_token_only(
source_outputs: list[Any],
prompt: OmniTokensPrompt | TextPrompt | None = None,
_requires_multimodal_data: bool = False,
) -> list[OmniTokensPrompt]:
"""Sync-side builder for the non-async-chunk thinker→talker path.
Ports the legacy ``thinker2talker`` body to the new stage-input-processor signature
(``source_outputs`` instead of ``stage_list, engine_input_source``).
Body is otherwise identical: extracts the
generated text from each thinker output and packages it with the
request's voice/speaker additional_information for the talker.
"""
if not isinstance(prompt, list):
prompt = [prompt]
talker_inputs: list[OmniTokensPrompt] = []
for i, source_output in enumerate(source_outputs):
output = source_output.outputs[0]
generated_text = output.text if hasattr(output, "text") and output.text else ""
original_prompt = prompt[i] if i < len(prompt) else None
additional_info: dict[str, Any] = {}
if original_prompt is not None and hasattr(original_prompt, "additional_information"):
additional_info = original_prompt.additional_information or {}
spk_emb = additional_info.get("spk_emb", None)
if isinstance(spk_emb, list) and spk_emb and not hasattr(spk_emb[0], "device"):
import torch
spk_emb = torch.tensor(spk_emb, dtype=torch.float32).unsqueeze(0)
talker_info = {
"ming_task": "omni",
"text": generated_text,
"spk_emb": spk_emb,
"voice_name": additional_info.get("voice_name", "DB30"),
"prompt_text": additional_info.get("prompt_text", None),
"prompt_wav_lat": additional_info.get("prompt_wav_lat", None),
"prompt_wav_emb": additional_info.get("prompt_wav_emb", None),
"max_text_length": additional_info.get("max_text_length", 50),
}
talker_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[0],
additional_information=talker_info,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return talker_inputs
thinker2talker_token_only._is_sync_input = True
def thinker2talker_full_payload(
transfer_manager,
pooling_output,
request,
):
"""Producer-side payload builder — no-op.
ming_flash_omni's thinker emits no heavy tensor to ship via the
worker connector (the bridge passes text only, and speaker metadata
arrives through the USER request's additional_information).
ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` so this
function is never invoked at runtime; it is retained for forward
compatibility with the connector path.
"""
del transfer_manager, pooling_output, request
return None
__all__ = [
"CFG_TEXT_SUFFIX",
"expand_cfg_prompts",
"thinker2imagegen",
"thinker2talker",
"thinker2talker_full_payload",
"thinker2talker_token_only",
]
@@ -1,3 +1,5 @@
import logging
import torch
from vllm.inputs import TextPrompt
@@ -11,6 +13,8 @@ from vllm_omni.data_entry_keys import (
)
from vllm_omni.inputs.data import OmniTokensPrompt
logger = logging.getLogger(__name__)
TALKER_CODEC_PAD_TOKEN_ID = 8292
TALKER_CODEC_START_TOKEN_ID = 8293
TALKER_CODEC_END_TOKEN_ID = 8294
@@ -86,3 +90,347 @@ def talker2code2wav(
)
)
return code2wav_inputs
# ============================================================================
# Worker-connector data plane (non-async-chunk path).
# Both transitions ship payloads via the worker connector
# (registered in ``_FULL_PAYLOAD_INPUT_STAGES`` in
# omni_scheduling_coordinator):
# - thinker->talker reads accumulated ``pooling_output["hidden"]`` and
# packs an OmniPayload-shaped dict (embed.prefill /
# hidden_states.output / ids.prompt / ids.output) for the talker, which
# the talker's ``talker_preprocess`` reads from
# ``model_intermediate_buffer``. The shape matches what legacy
# ``thinker2talker`` writes into ``additional_information`` as a debug
# fallback; ``thinker2talker_token_only`` only allocates prompt slots.
# - talker->code2wav strips TALKER_CODEC_{START,END} boundary tokens
# and ships the codec token ids.
# ============================================================================
# Per-model REPLACE-keys for the full-payload accumulator. qwen2_5_omni's
# producer side does not emit model_outputs through pooler_output (it ships
# token_ids on the request directly), so the empty set preserves correctness.
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
def _strip_codec_boundaries(token_ids: list[int]) -> list[int]:
"""Keep only real codec ids for the code2wav stage.
The talker stream can contain prompt/control ids (START/PAD/END/MASK) in
addition to sampled codec ids. Code2wav expects codec ids only; carrying
the prompt PAD span forward can inflate the sequence enough to OOM on L4.
Async scheduling may also leave trailing ``-1`` placeholders, so preserve
their length by repeating the last valid codec id.
"""
tids = list(token_ids)
trailing_placeholder_count = 0
while trailing_placeholder_count < len(tids) and tids[-1 - trailing_placeholder_count] == -1:
trailing_placeholder_count += 1
if tids and tids[-1] == TALKER_CODEC_END_TOKEN_ID:
tids = tids[:-1]
trailing_placeholder_count = 0
codec_ids = [tid for tid in tids if 0 <= tid < TALKER_CODEC_PAD_TOKEN_ID]
if trailing_placeholder_count > 0 and codec_ids:
codec_ids.extend([codec_ids[-1]] * trailing_placeholder_count)
return codec_ids
def talker2code2wav_token_only(
source_outputs,
_prompt: OmniTokensPrompt | TextPrompt = None,
_requires_multimodal_data: bool = False,
):
"""Sync-side placeholder for Stage-2 input (code2wav).
Returns OmniTokensPrompt sized to the stripped codec token count.
Actual codec ids are delivered via the worker connector payload built
by ``talker2code2wav_full_payload``.
"""
code2wav_inputs = []
for talker_output in source_outputs:
output = talker_output.outputs[0]
token_ids = _strip_codec_boundaries(list(output.cumulative_token_ids))
if not token_ids:
continue
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[0] * len(token_ids),
additional_information=None,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return code2wav_inputs
def talker2code2wav_full_payload(
transfer_manager,
pooling_output: dict,
request,
) -> dict | None:
"""Producer-side payload builder: ship the stripped codec ids via connector.
Token-ids-only shape. The talker stage's output already
carries the codec ids on ``request.output_token_ids``; we strip the
boundary tokens and pack a minimal payload.
"""
del transfer_manager
rid = getattr(request, "request_id", "?")
token_ids = list(getattr(request, "output_token_ids", None) or [])
if not token_ids:
logger.warning(
"qwen2_5_omni.talker2code2wav_full_payload: empty output_token_ids "
"for req=%s; consumer wait gate may hang.",
rid,
)
return None
token_ids = _strip_codec_boundaries(token_ids)
if not token_ids:
logger.warning(
"qwen2_5_omni.talker2code2wav_full_payload: codec ids empty after "
"stripping boundary tokens for req=%s; consumer wait gate may hang.",
rid,
)
return None
return {
"codes": {"audio": token_ids},
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
}
# ============================================================================
# Worker-connector data plane (non-async-chunk path) -- thinker->talker.
#
# qwen2_5_omni's talker consumes the thinker's last-layer hidden state
# via Linear(3584, 896). The AR runner publishes those hidden states
# per decode step on ``pooling_output["hidden"]`` (unpacked from
# ``OmniOutput.text_hidden_states``); the full-payload accumulator
# concatenates them so ``thinker2talker_full_payload`` sees the full
# prefill+decode trajectory and packs an OmniPayload-shaped dict that
# the talker's ``talker_preprocess`` reads from
# ``model_intermediate_buffer``. ``thinker2talker_token_only`` only
# allocates the talker's codec prompt slots; legacy
# ``thinker2talker`` above remains as a debug fallback that bundles the
# same shape into ``additional_information``.
# ============================================================================
def thinker2talker_token_only(
source_outputs,
prompt: OmniTokensPrompt | TextPrompt = None,
requires_multimodal_data: bool = False,
):
"""Placeholder builder for the connector-driven thinker->talker path.
Allocates the TALKER_CODEC_{START,PAD,END} prompt slots sized to the
thinker prompt length and forwards ``multi_modal_data``. The bulk
payload (hidden_states / embed / ids) ships exclusively through
``thinker2talker_full_payload`` via the worker connector and lands
in ``model_intermediate_buffer`` before the talker's forward() runs.
Consumer-wait gating is whitelist-driven via
``_FULL_PAYLOAD_INPUT_STAGES`` (see the mixin
``should_accumulate_full_payload_output`` docstring).
"""
thinker_outputs = source_outputs
talker_inputs = []
if not isinstance(prompt, list):
prompt = [prompt]
multi_modal_data = {
thinker_output.request_id: p.get("multi_modal_data", None) for thinker_output, p in zip(thinker_outputs, prompt)
}
for thinker_output in thinker_outputs:
prompt_token_ids = thinker_output.prompt_token_ids
talker_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[TALKER_CODEC_START_TOKEN_ID]
+ [TALKER_CODEC_PAD_TOKEN_ID] * (len(prompt_token_ids))
+ [TALKER_CODEC_END_TOKEN_ID],
additional_information=None,
multi_modal_data=(
multi_modal_data[thinker_output.request_id]
if requires_multimodal_data and multi_modal_data is not None
else None
),
mm_processor_kwargs=None,
)
)
return talker_inputs
def thinker2talker_full_payload(
transfer_manager,
pooling_output,
request,
):
"""Producer-side payload builder for the worker-connector data plane.
The AR runner emits per-step ``pooling_output["hidden"]`` (the
thinker's last-layer hidden states for the request span, unpacked
from ``OmniOutput.text_hidden_states``). The full-payload
accumulator concatenates those per-step rows across decode steps, so
by the time this builder fires the materialized
``pooling_output["hidden"]`` contains the full prefill+decode
hidden-state trajectory of size
``len(prompt_token_ids) + len(output_token_ids)``.
We split it at ``len(prompt_token_ids)`` into prefill embeddings and
decode hidden states, then pack the ``OmniPayload``-shaped dict that
the talker's ``thinker_to_talker_process`` reads from
``model_intermediate_buffer`` (keys ``embed.prefill`` /
``hidden_states.output`` / ``ids.prompt`` / ``ids.output``). Shape
matches what legacy ``thinker2talker`` writes into
``additional_information`` as a debug fallback, so the talker
consumes the same payload layout from either path.
Like ``qwen3_omni.thinker2talker_full_payload``, we apply a
finish-reason-aware stop-row trim: vLLM v1 appends the sampled
token to ``output_token_ids`` before ``check_stop``, so a request
that finished via ``FINISHED_STOPPED`` has one extra accumulated
hidden-state row that the talker must not consume. Max-token
finishes need no drop. Status is read from the request when
available; otherwise we fall back to a last-token-in-stop-set
heuristic.
"""
del transfer_manager
rid = getattr(request, "request_id", "?")
if not isinstance(pooling_output, dict):
logger.warning(
"qwen2_5_omni.thinker2talker_full_payload: pooling_output not a dict "
"(type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
rid,
)
return None
hidden = pooling_output.get("hidden")
if not isinstance(hidden, torch.Tensor):
logger.warning(
"qwen2_5_omni.thinker2talker_full_payload: missing 'hidden' tensor "
"(keys=%s) for req=%s; consumer wait gate may hang.",
list(pooling_output.keys()),
rid,
)
return None
def _ensure_list(x):
if x is None:
return []
if hasattr(x, "_x"):
# vLLM wraps cached token-id lists in ConstantList-like objects.
return list(x._x)
if isinstance(x, list):
return list(x)
return list(x)
prompt_token_ids = _ensure_list(getattr(request, "prompt_token_ids", None))
output_token_ids = _ensure_list(getattr(request, "output_token_ids", None))
all_token_ids = _ensure_list(getattr(request, "all_token_ids", None) or [])
if not all_token_ids:
all_token_ids = list(prompt_token_ids) + list(output_token_ids)
# Length-aware trim of accumulated thinker output, finish-reason-aware.
# Mirror qwen3_omni.thinker2talker_full_payload's logic so a stop-finish
# does not leak an extra hidden-state row to the talker.
status = getattr(request, "status", None)
status_name = getattr(status, "name", None) or ""
if not status_name and status is not None:
status_name = str(status).rsplit(".", 1)[-1]
stop_emission_drop = 1 if status_name == "FINISHED_STOPPED" else 0
if stop_emission_drop == 0 and not status_name and output_token_ids:
# Worker-side CachedRequestState has no `.status` field in vLLM v1;
# fall back to a last-token-in-stop-set heuristic.
sampling_params = getattr(request, "sampling_params", None)
if sampling_params is not None:
stop_ids: set[int] = set()
ignore_eos = bool(getattr(sampling_params, "ignore_eos", False))
for sid in getattr(sampling_params, "stop_token_ids", None) or ():
if isinstance(sid, int):
stop_ids.add(sid)
if not ignore_eos:
for eos in (
getattr(sampling_params, "eos_token_id", None),
getattr(sampling_params, "_eos_token_id", None),
):
if isinstance(eos, int):
stop_ids.add(eos)
for sid in (
getattr(sampling_params, "all_stop_token_ids", None)
or getattr(sampling_params, "_all_stop_token_ids", None)
or ()
):
if isinstance(sid, int):
stop_ids.add(sid)
if stop_ids and output_token_ids[-1] in stop_ids:
stop_emission_drop = 1
# Trim accumulated thinker output based on stop_emission_drop computed
# above. Mirror qwen3_omni.thinker2talker_full_payload's contract:
# target_rows = len(all_token_ids) - stop_emission_drop
# which excludes the stop-emission row for FINISHED_STOPPED but keeps
# all rows for FINISHED_LENGTH_CAPPED (max_tokens) finishes.
if stop_emission_drop > 0 and len(output_token_ids) >= stop_emission_drop:
output_token_ids = output_token_ids[:-stop_emission_drop]
h = hidden.detach().cpu().to(torch.float32)
target_rows = max(0, len(all_token_ids) - stop_emission_drop)
if target_rows <= 0:
logger.warning(
"qwen2_5_omni.thinker2talker_full_payload: target_rows<=0 "
"(all_token_ids=%d, stop_drop=%d) for req=%s; nothing to ship.",
len(all_token_ids),
stop_emission_drop,
getattr(request, "request_id", "?"),
)
return None
if h.dim() >= 1 and h.shape[0] > target_rows:
logger.warning(
"qwen2_5_omni.thinker2talker_full_payload: excess hidden rows "
"(got %d, target %d, stop_drop %d) for req=%s; trimming",
int(h.shape[0]),
target_rows,
stop_emission_drop,
getattr(request, "request_id", None),
)
h = h[:target_rows]
prompt_len = len(prompt_token_ids)
if h.shape[0] < prompt_len:
# Under-captured prefill -- defensively skip rather than ship a
# truncated payload that would confuse the talker's prefill path.
logger.warning(
"qwen2_5_omni.thinker2talker_full_payload: hidden rows=%d < prompt_len=%d "
"for req=%s; under-captured prefill, skipping payload.",
int(h.shape[0]),
prompt_len,
getattr(request, "request_id", "?"),
)
return None
prefill_hidden = h[:prompt_len]
decode_hidden = h[prompt_len:]
payload: OmniPayload = to_dict(
OmniPayloadStruct(
hidden_states=HiddenStatesStruct(
output=decode_hidden,
output_shape=list(decode_hidden.shape),
),
embed=EmbeddingsStruct(
prefill=prefill_hidden,
prefill_shape=list(prefill_hidden.shape),
),
ids=IdsStruct(
prompt=list(prompt_token_ids),
output=list(output_token_ids),
),
)
)
# Intentionally omit payload["meta"]: the thinker->talker transition
# carries no scheduler-relevant metadata (next_stage_prompt_len /
# left_context_size are not set on this edge).
return payload
@@ -35,6 +35,12 @@ logger = logging.getLogger(__name__)
# Pooling output layer keys: "0" = word embedding, "24" = accept_hidden_layer
_EMBED_LAYER_KEY = "0"
_HIDDEN_LAYER_KEY = "24"
# Per-model REPLACE-keys for the full-payload accumulator. Keys in this
# set use REPLACE semantics (subsequent emissions discard prior chunks)
# instead of CONCAT. qwen3-omni currently has none — model_outputs is
# not emitted by the thinker/talker forward.
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
_QWEN3_CODEC_CODEBOOK_SIZE = 2048
_QWEN3_CODEC_PAD_TOKEN_ID = 4196
_QWEN3_CODEC_BOS_TOKEN_ID = 4197
@@ -119,19 +125,6 @@ def _is_valid_qwen3_codec_token_id(token_id: Any) -> bool:
return 0 <= token_id < _QWEN3_CODEC_CODEBOOK_SIZE
def should_accumulate_qwen3_omni_full_payload_output(
model_config: Any,
custom_process_func: Any,
) -> bool:
"""Return whether Qwen3-Omni should accumulate full-payload outputs."""
return (
custom_process_func is not None
and not getattr(model_config, "async_chunk", False)
and getattr(model_config, "model_arch", None) == "Qwen3OmniMoeForConditionalGeneration"
and getattr(model_config, "model_stage", None) in {"thinker", "talker"}
)
def _extract_qwen3_full_payload_codec_rows(
code_predictor_codes: torch.Tensor,
output_token_ids: list[int],
@@ -462,7 +455,13 @@ def thinker2talker_full_payload(
request: OmniEngineCoreRequest,
) -> dict[str, Any] | None:
"""Pack complete thinker output for the non-async connector path."""
rid = getattr(request, "request_id", None)
if not isinstance(pooling_output, dict):
logger.warning(
"thinker2talker_full_payload: pooling_output not a dict (type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
rid,
)
return None
layers = {
@@ -475,11 +474,13 @@ def thinker2talker_full_payload(
hidden = pooling_output.get("hidden")
thinker_emb = hidden if isinstance(hidden, torch.Tensor) else None
if thinker_emb is None or thinker_hid is None:
logger.debug(
"thinker2talker_full_payload: missing thinker tensors for req=%s (embed=%s hidden=%s)",
getattr(request, "request_id", None),
logger.warning(
"thinker2talker_full_payload: missing thinker tensors for req=%s "
"(embed=%s hidden=%s keys=%s); consumer wait gate may hang.",
rid,
thinker_emb is not None,
thinker_hid is not None,
list(pooling_output.keys()),
)
return None
@@ -493,7 +494,7 @@ def thinker2talker_full_payload(
# The accumulator captures one hidden-state row per executed thinker
# forward (prefill + every decode step including the one that emitted
# the stop_token), so for a finished request thinker_emb has exactly one
# row more than the rows the talker should consume. async_chunk's
# row more than the rows the talker should consume. async_chunk's
# chunk-0 path naturally captures only the prefill / non-stop portion,
# which is why the [async_chunk] parametrization passes while [default]
# over-generates one codec frame on short outputs (e.g.
@@ -797,7 +798,14 @@ def talker2code2wav_full_payload(
request: OmniEngineCoreRequest,
) -> dict[str, Any] | None:
"""Pack complete talker codec output for the non-async connector path."""
rid = getattr(request, "request_id", None)
if not isinstance(pooling_output, dict):
logger.warning(
"talker2code2wav_full_payload: pooling_output not a dict "
"(type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
rid,
)
return None
code_predictor_codes = pooling_output.get("codes.audio")
if code_predictor_codes is None:
@@ -805,10 +813,19 @@ def talker2code2wav_full_payload(
if isinstance(codes, dict):
code_predictor_codes = codes.get("audio")
if code_predictor_codes is None:
logger.warning(
"talker2code2wav_full_payload: missing codes.audio (keys=%s) for req=%s; consumer wait gate may hang.",
list(pooling_output.keys()),
rid,
)
return None
if not isinstance(code_predictor_codes, torch.Tensor):
code_predictor_codes = torch.as_tensor(code_predictor_codes)
if code_predictor_codes.numel() == 0:
logger.warning(
"talker2code2wav_full_payload: empty codes.audio for req=%s; consumer wait gate may hang.",
rid,
)
return None
output_token_ids = _ensure_list(getattr(request, "output_token_ids", []) or [])
@@ -818,6 +835,16 @@ def talker2code2wav_full_payload(
list(output_token_ids),
)
if code_predictor_codes.numel() == 0:
logger.warning(
"talker2code2wav_full_payload: no valid codec rows after filtering "
"(raw_shape=%s output_ids_len=%d aligned_rows=%s valid_rows=%s) for req=%s; "
"consumer wait gate may hang.",
raw_shape,
len(output_token_ids),
codec_stats["aligned_rows"],
codec_stats["valid_rows"],
rid,
)
return None
codec_codes = code_predictor_codes.transpose(0, 1).cpu().reshape(-1).tolist()
@@ -53,10 +53,13 @@ def talker2code2wav(
# audio_codes may still contain zero-padded / invalid rows, so trim only
# after filtering valid frames instead of trying to align EOS indices.
seq_len = max(len(token_ids) - 1, 0)
# Filter invalid frames: zero-padded (EOS) and frames containing
# out-of-range values (e.g. stop_token_id=2150 exceeds codebook_size=2048).
# Filter invalid frames: zero-padded (EOS), out-of-range values (e.g.
# stop_token_id=2150 exceeds codebook_size=2048), and negative
# sentinels (e.g. -1 padding).
_CODEBOOK_SIZE = 2048
valid_mask = audio_codes.any(dim=1) & (audio_codes.max(dim=1).values < _CODEBOOK_SIZE)
valid_mask = (
(audio_codes >= 0).all(dim=1) & audio_codes.any(dim=1) & (audio_codes.max(dim=1).values < _CODEBOOK_SIZE)
)
audio_codes = audio_codes[valid_mask]
if seq_len > 0 and audio_codes.ndim == 2 and int(audio_codes.shape[0]) > seq_len:
audio_codes = audio_codes[-seq_len:]
@@ -280,3 +283,220 @@ def talker2code2wav_async_chunk(
speaker=extract_speaker_from_request(request),
language=extract_language_from_request(request),
)
# ============================================================================
# Worker-connector data plane (non-async-chunk path).
# AR runner's `flatten_payload` converts the model emit
# `multimodal_outputs={"codes": {"audio": ..., "ref": ...},
# "meta": {"ref_code_len": ..., "codec_streaming": ...}}` to flat dotted
# keys (`codes.audio`, `codes.ref`, `meta.ref_code_len`,
# `meta.codec_streaming`) before the full-payload accumulator runs.
# - codes.audio is 2-D so default CONCAT across steps builds the full sequence.
# - codes.ref is a list (not Tensor with dim>=2) so accumulator LATEST-wins
# keeps the prefill-emitted ref tensor across decode steps (which don't emit
# ref again).
# - meta.ref_code_len is 1-D so LATEST-wins; consumer reads [-1].
# ============================================================================
# Per-model REPLACE-keys for the full-payload accumulator. qwen3_tts's
# producer side emits codec frames that should CONCAT (codes.audio) plus
# scalars/lists that are correctly handled by default LATEST-wins, so this
# stays empty.
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
_CODEBOOK_SIZE = 2048
_NUM_QUANTIZERS_DEFAULT = 16
def _filter_audio_codes_qwen3_tts(audio_codes: torch.Tensor) -> torch.Tensor:
"""Filter zero-padded, out-of-range, and negative-padded codec frames.
Mirrors the orchestrator-path body in `talker2code2wav` above.
"""
if not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
return audio_codes
if audio_codes.ndim != 2:
return audio_codes
valid_mask = (
(audio_codes >= 0).all(dim=1) & audio_codes.any(dim=1) & (audio_codes.max(dim=1).values < _CODEBOOK_SIZE)
)
return audio_codes[valid_mask]
def _coerce_ref_code_len(raw) -> int:
"""Coerce mm["meta"]["ref_code_len"] / pooling_output["meta.ref_code_len"]
raw value (Tensor | int | None) into a non-negative int. Mirrors the
extraction inlined in the legacy ``talker2code2wav`` path; clamps any
negative input to 0 since downstream code treats this as a non-negative
frame count."""
if isinstance(raw, torch.Tensor):
value = int(raw.reshape(-1)[-1].item()) if raw.numel() > 0 else 0
elif raw is None:
value = 0
else:
value = int(raw)
return max(value, 0)
def _normalize_ref_code(ref_code, num_quantizers: int, ref_code_len: int):
"""Coerce ref_code into a [ref_len, Q] tensor or None. Mirrors orchestrator path."""
if isinstance(ref_code, list):
ref_code = ref_code[0] if ref_code else None
if not isinstance(ref_code, torch.Tensor) or ref_code.numel() == 0:
return None, 0
ref_code = ref_code.to(torch.long).cpu().contiguous()
if ref_code.ndim == 1:
if ref_code.numel() % num_quantizers != 0:
return None, 0
ref_code = ref_code.reshape(-1, num_quantizers)
elif ref_code.ndim != 2:
return None, 0
if ref_code_len > 0 and int(ref_code.shape[0]) > ref_code_len:
ref_code = ref_code[:ref_code_len]
return ref_code, int(ref_code.shape[0])
def talker2code2wav_token_only(
source_outputs: list,
prompt=None,
_requires_multimodal_data: bool = False,
) -> list:
"""Sync-side placeholder for the non-async-chunk Stage-1 (code2wav) input.
Sized to the expected codec token count (codebook-major flat:
Q * (ref_frames + audio_frames)). Speaker / language metadata are
extracted from `prompt` and threaded via `additional_information`
(orchestrator-style; same as the legacy `talker2code2wav` builder).
Actual codec ids are delivered via the worker connector payload built
by `talker2code2wav_full_payload`.
"""
from vllm_omni.inputs.data import OmniTokensPrompt
code2wav_inputs: list = []
for i, talker_output in enumerate(source_outputs):
if not talker_output.finished:
continue
output = talker_output.outputs[0]
mm = output.multimodal_output if hasattr(output, "multimodal_output") else None
mm = mm if isinstance(mm, dict) else {}
mm_codes = mm.get("codes", {}) if isinstance(mm, dict) else {}
token_ids = getattr(output, "cumulative_token_ids", []) or []
seq_len = max(len(token_ids) - 1, 0)
audio = mm_codes.get("audio") if isinstance(mm_codes, dict) else None
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
audio = audio.to(torch.long)
audio = _filter_audio_codes_qwen3_tts(audio)
if seq_len > 0 and audio.ndim == 2 and int(audio.shape[0]) > seq_len:
audio = audio[-seq_len:]
num_audio_frames = int(audio.shape[0]) if audio.ndim == 2 else 0
num_quantizers = int(audio.shape[1]) if audio.ndim == 2 and audio.shape[1] > 0 else _NUM_QUANTIZERS_DEFAULT
else:
num_audio_frames = 0
num_quantizers = _NUM_QUANTIZERS_DEFAULT
ref_code_raw = mm_codes.get("ref") if isinstance(mm_codes, dict) else None
ref_code_len_raw = mm.get("meta", {}).get("ref_code_len") if isinstance(mm.get("meta"), dict) else None
ref_code_len = _coerce_ref_code_len(ref_code_len_raw)
_, ref_frames = _normalize_ref_code(ref_code_raw, num_quantizers, ref_code_len)
# Codebook-major flat: Q * (ref_frames + audio_frames)
prompt_len = num_quantizers * (ref_frames + num_audio_frames)
additional_info = to_dict(
OmniPayloadStruct(
meta=MetaStruct(left_context_size=ref_frames) if ref_frames > 0 else None,
speaker=extract_speaker_from_prompt(prompt, index=i),
language=extract_language_from_prompt(prompt, index=i),
)
)
code2wav_inputs.append(
OmniTokensPrompt(
prompt_token_ids=[0] * prompt_len,
additional_information=additional_info if additional_info else None,
multi_modal_data=None,
mm_processor_kwargs=None,
)
)
return code2wav_inputs
def talker2code2wav_full_payload(
transfer_manager,
pooling_output,
request,
):
"""Producer-side payload builder.
Reads accumulated codec from `pooling_output["codes.audio"]` (CONCAT
across steps via flatten_payload), latest `pooling_output["codes.ref"]`
(prefill-emitted), and latest `pooling_output["meta.ref_code_len"]`.
Replicates the orchestrator-path body of `talker2code2wav` (filter,
crop to seq_len, prepend ref, codebook-major flatten).
"""
del transfer_manager
rid = getattr(request, "request_id", "?")
if not isinstance(pooling_output, dict):
logger.warning(
"qwen3_tts.talker2code2wav_full_payload: pooling_output not a dict "
"(type=%s) for req=%s; consumer wait gate may hang.",
type(pooling_output).__name__,
rid,
)
return None
# codes.audio — try flat dotted first (flatten_payload), then nested fallback.
audio = pooling_output.get("codes.audio")
if audio is None:
codes_nested = pooling_output.get("codes")
if isinstance(codes_nested, dict):
audio = codes_nested.get("audio")
if not isinstance(audio, torch.Tensor) or audio.numel() == 0:
logger.warning(
"qwen3_tts.talker2code2wav_full_payload: missing/empty codes.audio "
"(keys=%s) for req=%s; consumer wait gate may hang.",
list(pooling_output.keys()),
rid,
)
return None
audio = audio.to(torch.long)
audio = _filter_audio_codes_qwen3_tts(audio)
if audio.numel() == 0:
logger.warning(
"qwen3_tts.talker2code2wav_full_payload: audio empty after codec "
"filter (negative/all-zero/out-of-range rows dropped) for req=%s.",
rid,
)
return None
output_token_ids = list(getattr(request, "output_token_ids", None) or [])
seq_len = max(len(output_token_ids) - 1, 0)
if seq_len > 0 and audio.ndim == 2 and int(audio.shape[0]) > seq_len:
audio = audio[-seq_len:]
num_quantizers = int(audio.shape[1]) if audio.ndim == 2 and audio.shape[1] > 0 else _NUM_QUANTIZERS_DEFAULT
# meta.ref_code_len — flat dotted then nested fallback.
ref_code_len_raw = pooling_output.get("meta.ref_code_len")
if ref_code_len_raw is None:
meta_nested = pooling_output.get("meta")
if isinstance(meta_nested, dict):
ref_code_len_raw = meta_nested.get("ref_code_len")
ref_code_len = _coerce_ref_code_len(ref_code_len_raw)
# codes.ref — flat dotted then nested fallback.
ref_code_raw = pooling_output.get("codes.ref")
if ref_code_raw is None:
codes_nested = pooling_output.get("codes")
if isinstance(codes_nested, dict):
ref_code_raw = codes_nested.get("ref")
ref_code, ref_frames = _normalize_ref_code(ref_code_raw, num_quantizers, ref_code_len)
if ref_code is not None:
audio = torch.cat([ref_code.to(audio.device), audio], dim=0)
codec_codes = audio.transpose(0, 1).to(device="cpu", dtype=torch.long).reshape(-1).contiguous()
return {
"codes": {"audio": codec_codes},
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
}