[3/5][core]refactor communication layer: PR 3 of 5, all other models in non async mode (#3719)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
0f3264a9c0
commit
3a347b10b0
38 files changed
+3021
-275
No files matched your search
@@ -784,12 +784,14 @@ class CosyVoice3Model(
|
||||
else:
|
||||
self._stream_vocoder_cache_by_req[req_id] = new_cache_state
|
||||
else:
|
||||
token_offset = max(0, meta.talker_prefill_offset or 0) if meta else 0
|
||||
tts_speech = self.code2wav.forward(
|
||||
token=token.unsqueeze(0),
|
||||
prompt_token=speech_token[:1],
|
||||
prompt_feat=speech_feat[:1],
|
||||
embedding=embedding[:1],
|
||||
n_timesteps=10,
|
||||
token_offset_tokens=token_offset,
|
||||
)
|
||||
|
||||
audio = tts_speech.reshape(-1).to(dtype=torch.float32)
|
||||
|
||||
@@ -292,6 +292,7 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
prompt_feat: torch.Tensor,
|
||||
embedding: torch.Tensor,
|
||||
n_timesteps: int = 10,
|
||||
token_offset_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Generate audio waveform from speech tokens."""
|
||||
feat = self._forward_mel(
|
||||
@@ -300,7 +301,7 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
prompt_feat=prompt_feat,
|
||||
embedding=embedding,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=0,
|
||||
token_offset_tokens=token_offset_tokens,
|
||||
streaming=False,
|
||||
finalize=True,
|
||||
)
|
||||
|
||||
@@ -31,6 +31,7 @@ COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload",
|
||||
sampling_constraints={
|
||||
# merged speech stop token (logsumexp of all 200 stop logits)
|
||||
"stop_token_ids": [6562],
|
||||
@@ -44,7 +45,8 @@ COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="latent",
|
||||
sync_process_input_func=f"{_PROC}.text2flow",
|
||||
custom_process_input_func=f"{_PROC}.text2flow",
|
||||
sync_process_input_func=f"{_PROC}.text2flow_token_only",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -29,6 +29,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [151645],
|
||||
@@ -44,6 +45,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
|
||||
@@ -39,6 +39,7 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
# Stop once the speech/text interleaved span ends. Code2Wav
|
||||
@@ -55,7 +56,8 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
custom_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
|
||||
@@ -58,6 +58,7 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig(
|
||||
engine_output_type="audio",
|
||||
tokenizer_subdir="talker/llm",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
@@ -38,6 +39,8 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
input_sources=(0,),
|
||||
engine_output_type="latent",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [8294],
|
||||
@@ -52,6 +55,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
@@ -74,6 +78,7 @@ QWEN2_5_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -26,6 +26,7 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [2150],
|
||||
@@ -40,7 +41,14 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="Qwen3TTSCode2Wav",
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
# ``sync_process_input_func`` is the only input-proc override for
|
||||
# this stage in sync (non-async-chunk) mode: a length-only
|
||||
# ``_token_only`` placeholder. The bulk codec payload itself
|
||||
# ships via the worker connector from stage 0's
|
||||
# ``talker2code2wav_full_payload`` producer. Under async_chunk
|
||||
# mode no pre-stage processing is needed -- chunks deliver
|
||||
# directly to the consumer.
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
extras={"tts_args": {"max_instructions_length": 500}},
|
||||
),
|
||||
|
||||
@@ -25,6 +25,27 @@ from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _codec_ids_from_payload_or_input(
|
||||
input_ids: torch.Tensor,
|
||||
runtime_info: dict[str, Any] | None,
|
||||
) -> torch.Tensor:
|
||||
"""Prefer connector-delivered codec ids over token placeholders.
|
||||
|
||||
In non-async full-payload mode, the scheduler only needs placeholder
|
||||
token ids for allocation. The real codec sequence is delivered through
|
||||
model_intermediate_buffer as ``codes.audio``.
|
||||
"""
|
||||
if isinstance(runtime_info, dict):
|
||||
codes = runtime_info.get("codes")
|
||||
if isinstance(codes, dict):
|
||||
audio = codes.get("audio")
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
return audio.reshape(-1).to(device=input_ids.device, dtype=torch.long)
|
||||
if isinstance(audio, (list, tuple)) and audio:
|
||||
return torch.as_tensor(audio, device=input_ids.device, dtype=torch.long).reshape(-1)
|
||||
return input_ids.reshape(-1).to(dtype=torch.long)
|
||||
|
||||
|
||||
class Qwen3TTSCode2Wav(nn.Module):
|
||||
"""Stage-1 code2wav model for Qwen3-TTS (GenerationModelRunner).
|
||||
Consumes frame-aligned codec tokens from input_ids and decodes waveform
|
||||
@@ -239,6 +260,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
|
||||
)
|
||||
|
||||
runtime_infos = runtime_additional_information or []
|
||||
ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
|
||||
|
||||
@@ -246,14 +268,18 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
valid_codes_qf: list[torch.Tensor] = []
|
||||
valid_indices: list[int] = []
|
||||
left_context_size = [0] * len(request_ids_list)
|
||||
if runtime_additional_information is not None:
|
||||
for i, info in enumerate(runtime_additional_information):
|
||||
if runtime_infos:
|
||||
for i, info in enumerate(runtime_infos):
|
||||
if i >= len(left_context_size):
|
||||
break
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
meta = info.get("meta", {})
|
||||
if "left_context_size" in meta:
|
||||
left_context_size[i] = meta["left_context_size"]
|
||||
for i, req_ids in enumerate(request_ids_list):
|
||||
runtime_info = runtime_infos[i] if i < len(runtime_infos) else None
|
||||
req_ids = _codec_ids_from_payload_or_input(req_ids, runtime_info)
|
||||
if req_ids.numel() < 1:
|
||||
parsed.append((0, 0))
|
||||
continue
|
||||
|
||||
@@ -6,6 +6,7 @@ stage_args:
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2text
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
|
||||
@@ -6,6 +6,7 @@ stage_args:
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2text
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image_full_payload
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
import numpy as np
|
||||
import torch
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.data_entry_keys import (
|
||||
CodesStruct,
|
||||
@@ -16,6 +17,10 @@ from vllm_omni.data_entry_keys import (
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_COSYVOICE3_SPEECH_TOKEN_SIZE = 6561
|
||||
|
||||
|
||||
def _build_prompt_embed_struct(prompt_payload: dict[str, Any]) -> EmbeddingsStruct | None:
|
||||
"""Wrap prompt_payload's flat speech_token/speech_feat/embedding tensors into EmbeddingsStruct."""
|
||||
@@ -46,6 +51,52 @@ def _ensure_list(x: Any) -> list[Any]:
|
||||
return [x]
|
||||
|
||||
|
||||
def _to_token_id_list(value: Any) -> list[int]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = value.detach().to("cpu").reshape(-1).tolist()
|
||||
token_ids: list[int] = []
|
||||
for item in _ensure_list(value):
|
||||
if isinstance(item, torch.Tensor):
|
||||
token_ids.extend(_to_token_id_list(item))
|
||||
continue
|
||||
if isinstance(item, (list, tuple)):
|
||||
token_ids.extend(_to_token_id_list(item))
|
||||
continue
|
||||
token_ids.append(int(item))
|
||||
return token_ids
|
||||
|
||||
|
||||
def _strip_prompt_prefix(output_ids: list[Any], prefix_ids: list[Any]) -> list[Any]:
|
||||
if prefix_ids and len(output_ids) >= len(prefix_ids) and output_ids[: len(prefix_ids)] == prefix_ids:
|
||||
return output_ids[len(prefix_ids) :]
|
||||
return output_ids
|
||||
|
||||
|
||||
def _prompt_speech_token_ids(multi_modal_data: dict[str, Any]) -> list[int]:
|
||||
speech_token = multi_modal_data.get("speech_token")
|
||||
if speech_token is None:
|
||||
embed = multi_modal_data.get("embed")
|
||||
if isinstance(embed, dict):
|
||||
speech_token = embed.get("speech_token")
|
||||
return _to_token_id_list(speech_token)
|
||||
|
||||
|
||||
def _has_speech_stop_token(output_ids: list[Any]) -> bool:
|
||||
return any(token_id >= _COSYVOICE3_SPEECH_TOKEN_SIZE for token_id in _to_token_id_list(output_ids))
|
||||
|
||||
|
||||
def _set_non_stream_prompt_trim(additional_info: dict[str, Any], prompt_speech_len: int) -> None:
|
||||
if prompt_speech_len <= 0:
|
||||
return
|
||||
meta = additional_info.get("meta")
|
||||
if not isinstance(meta, dict):
|
||||
meta = {}
|
||||
additional_info["meta"] = meta
|
||||
meta["talker_prefill_offset"] = prompt_speech_len
|
||||
|
||||
|
||||
def _to_cpu_tensor(x: Any) -> torch.Tensor | None:
|
||||
if isinstance(x, list):
|
||||
if not x:
|
||||
@@ -95,9 +146,14 @@ def text2flow(
|
||||
if multi_modal_data is None:
|
||||
raise RuntimeError(f"Missing multimodal_output for request {source_output.request_id}")
|
||||
|
||||
output_ids = _ensure_list(output.cumulative_token_ids)
|
||||
prefix_ids = _ensure_list(source_output.prompt_token_ids)
|
||||
raw_output_ids = _ensure_list(output.cumulative_token_ids)
|
||||
prompt_speech_ids = _prompt_speech_token_ids(multi_modal_data)
|
||||
output_ids = _strip_prompt_prefix(raw_output_ids, prefix_ids)
|
||||
output_ids = _strip_prompt_prefix(output_ids, prompt_speech_ids)
|
||||
additional_info = dict(multi_modal_data)
|
||||
if _has_speech_stop_token(raw_output_ids):
|
||||
_set_non_stream_prompt_trim(additional_info, len(prompt_speech_ids))
|
||||
additional_info.setdefault("ids", {})["prompt"] = prefix_ids
|
||||
engine_inputs.append(OmniTokensPrompt(prompt_token_ids=output_ids, additional_information=additional_info))
|
||||
return engine_inputs
|
||||
@@ -271,3 +327,107 @@ def talker2code2wav_async_chunk(
|
||||
|
||||
state["emitted_chunks"] = int(state.get("emitted_chunks", 0)) + 1
|
||||
return payload
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path).
|
||||
# cosyvoice3 talker emits `multimodal_outputs={"embed": {"speech_token": t,
|
||||
# "speech_feat": t, "embedding": t}}` ONLY at prefill (decode steps emit
|
||||
# `{}`). After flatten_payload these become flat top-level keys
|
||||
# `embed.speech_token` etc., persisted across decode steps by the
|
||||
# full-payload accumulator (decode doesn't re-emit them). Shipping via the connector
|
||||
# keeps the orchestrator off the heavy-tensor path.
|
||||
# ============================================================================
|
||||
|
||||
# All three embed tensors are emitted once at prefill and must REPLACE-not-
|
||||
# CONCAT across the (already trivial) per-request accumulator history so a
|
||||
# regression where decode unexpectedly re-emits them does not silently
|
||||
# duplicate the prefill tensor. See mixin._FULL_PAYLOAD_REPLACE_KEYS.
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset({"embed.speech_token", "embed.speech_feat", "embed.embedding"})
|
||||
|
||||
|
||||
def text2flow_token_only(
|
||||
source_outputs: list,
|
||||
prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
_requires_multimodal_data: bool = True,
|
||||
):
|
||||
"""Sync-side builder for the non-async-chunk text→flow path.
|
||||
|
||||
CosyVoice3 sync keeps codec ids on the legacy token path. Some vLLM v1
|
||||
histories include the source prompt prefix, so strip it only when it is an
|
||||
exact leading match.
|
||||
"""
|
||||
del prompt
|
||||
engine_inputs: list[OmniTokensPrompt] = []
|
||||
for source_output in source_outputs:
|
||||
if not source_output.finished:
|
||||
continue
|
||||
output = source_output.outputs[0]
|
||||
prefix_ids = _ensure_list(source_output.prompt_token_ids)
|
||||
raw_output_ids = _ensure_list(output.cumulative_token_ids)
|
||||
output_ids = _strip_prompt_prefix(raw_output_ids, prefix_ids)
|
||||
multi_modal_data = output.multimodal_output
|
||||
if multi_modal_data is None:
|
||||
raise RuntimeError(f"Missing multimodal_output for request {source_output.request_id}")
|
||||
prompt_speech_ids = _prompt_speech_token_ids(multi_modal_data)
|
||||
output_ids = _strip_prompt_prefix(output_ids, prompt_speech_ids)
|
||||
additional_info: dict[str, Any] = dict(multi_modal_data)
|
||||
if _has_speech_stop_token(raw_output_ids):
|
||||
_set_non_stream_prompt_trim(additional_info, len(prompt_speech_ids))
|
||||
additional_info.setdefault("ids", {})["prompt"] = prefix_ids
|
||||
engine_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=output_ids,
|
||||
additional_information=additional_info,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return engine_inputs
|
||||
|
||||
|
||||
def text2flow_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output,
|
||||
request,
|
||||
):
|
||||
"""Producer-side payload builder.
|
||||
|
||||
Reads prefill-emitted `embed.{speech_token, speech_feat, embedding}` from
|
||||
the accumulator and ships prompt conditioning as a connector payload.
|
||||
The downstream flow stage reads these from `model_intermediate_buffer`
|
||||
(see cosyvoice3.py:671 in the code2wav forward — runtime_info pickup).
|
||||
"""
|
||||
del transfer_manager
|
||||
rid = getattr(request, "external_req_id", None) or getattr(request, "request_id", "?")
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.warning(
|
||||
"cosyvoice3.text2flow_full_payload: pooling_output not a dict "
|
||||
"(type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
embed_out: dict[str, Any] = {}
|
||||
for key in ("speech_token", "speech_feat", "embedding"):
|
||||
v = pooling_output.get(f"embed.{key}")
|
||||
if v is None:
|
||||
nested = pooling_output.get("embed")
|
||||
if isinstance(nested, dict):
|
||||
v = nested.get(key)
|
||||
if isinstance(v, torch.Tensor) and v.numel() > 0:
|
||||
embed_out[key] = v
|
||||
if not embed_out:
|
||||
logger.warning(
|
||||
"cosyvoice3.text2flow_full_payload: no embed.{speech_token,speech_feat,embedding} "
|
||||
"found in pooling_output (keys=%s) for req=%s; consumer wait gate may hang.",
|
||||
list(pooling_output.keys()),
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
return {
|
||||
"meta": {
|
||||
"finished": torch.tensor(True, dtype=torch.bool),
|
||||
},
|
||||
"embed": embed_out,
|
||||
}
|
||||
@@ -1,27 +1,44 @@
|
||||
# Copyright 2026 Tencent.
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
from vllm_omni.model_executor.models.covo_audio.config_covo_audio import COVO_AUDIO_TOKEN_INDEX
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Per-model REPLACE-keys for the full-payload accumulator (none for covo_audio:
|
||||
# the producer side does not emit per-step hidden_states / model_outputs;
|
||||
# llm2code2wav_full_payload reads token_ids directly from `request`).
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _filter_audio_codes(token_ids: list[int]) -> list[int]:
|
||||
"""Filter codec-range token ids and rebase by COVO_AUDIO_TOKEN_INDEX."""
|
||||
audio_codes = [t - COVO_AUDIO_TOKEN_INDEX for t in token_ids if t >= COVO_AUDIO_TOKEN_INDEX]
|
||||
if not audio_codes:
|
||||
audio_codes = [-1]
|
||||
return audio_codes
|
||||
|
||||
|
||||
def llm2code2wav(
|
||||
source_outputs: list[Any],
|
||||
prompt: Any = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Legacy orchestrator-path builder (retained for async_chunk + back-compat).
|
||||
|
||||
The non-async-chunk path now goes through ``llm2code2wav_token_only`` +
|
||||
worker connector + ``llm2code2wav_full_payload``.
|
||||
"""
|
||||
talker_outputs = source_outputs
|
||||
code2wav_inputs = []
|
||||
|
||||
for i, talker_output in enumerate(talker_outputs):
|
||||
for talker_output in talker_outputs:
|
||||
output = talker_output.outputs[0]
|
||||
token_ids = output.token_ids
|
||||
|
||||
audio_codes = [t - COVO_AUDIO_TOKEN_INDEX for t in token_ids if t >= COVO_AUDIO_TOKEN_INDEX]
|
||||
|
||||
if not audio_codes:
|
||||
audio_codes = [-1]
|
||||
|
||||
audio_codes = _filter_audio_codes(list(output.token_ids))
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=audio_codes,
|
||||
@@ -31,3 +48,56 @@ def llm2code2wav(
|
||||
)
|
||||
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def llm2code2wav_token_only(
|
||||
source_outputs: list[Any],
|
||||
prompt: Any = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Sync-side placeholder for the non-async-chunk Stage-1 input.
|
||||
|
||||
Returns an OmniTokensPrompt sized to the code2wav stage's expected
|
||||
prefill length (one slot per audio code). The actual codec ids are
|
||||
delivered via the worker connector payload built by
|
||||
``llm2code2wav_full_payload``.
|
||||
"""
|
||||
code2wav_inputs: list[OmniTokensPrompt] = []
|
||||
for output_wrapper in source_outputs:
|
||||
output = output_wrapper.outputs[0]
|
||||
audio_codes = _filter_audio_codes(list(output.token_ids))
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0] * len(audio_codes),
|
||||
additional_information=None,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def llm2code2wav_full_payload(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any],
|
||||
request: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Producer-side payload builder for the worker connector data plane.
|
||||
|
||||
covo_audio's fused_thinker_talker stage emits codec ids via
|
||||
``request.output_token_ids`` (token-ids only -- no
|
||||
hidden_states or embed tensors), so the connector payload is
|
||||
just the filtered audio codes plus a finished marker.
|
||||
"""
|
||||
output_token_ids = list(getattr(request, "output_token_ids", None) or [])
|
||||
if not output_token_ids:
|
||||
logger.warning(
|
||||
"covo_audio.llm2code2wav_full_payload: empty output_token_ids for req=%s; consumer wait gate may hang.",
|
||||
getattr(request, "request_id", "?"),
|
||||
)
|
||||
return None
|
||||
audio_codes = _filter_audio_codes(output_token_ids)
|
||||
return {
|
||||
"codes": {"audio": audio_codes},
|
||||
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
|
||||
}
|
||||
@@ -5,9 +5,12 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _to_prompt_dict(prompt_item: OmniTokensPrompt | TextPrompt | str | None) -> dict[str, Any]:
|
||||
if isinstance(prompt_item, dict):
|
||||
@@ -146,3 +149,128 @@ def token2image_to_token2audio(
|
||||
requires_multimodal_data: bool = False,
|
||||
):
|
||||
return _bridge_tokens(source_outputs, prompt, requires_multimodal_data)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path).
|
||||
# ============================================================================
|
||||
|
||||
# Per-model REPLACE-keys for the full-payload accumulator. dynin_omni's
|
||||
# producer model emits new chunks per step (token_ids / runtime_info_json),
|
||||
# all of which use the default CONCAT/replace semantics — no model_outputs
|
||||
# entry needs explicit REPLACE.
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _build_full_payload(pooling_output: dict[str, Any] | None, request: Any) -> dict[str, Any] | None:
|
||||
"""Producer-side payload builder: assemble dynin_omni connector payload.
|
||||
|
||||
Reads token_ids from ``pooling_output["token_ids"]`` (preferred) or
|
||||
``request.output_token_ids`` (fallback). Reads structured non-tensor
|
||||
metadata from ``pooling_output["runtime_info_json"]`` (JSON-in-uint8)
|
||||
if present, falling back to ``pooling_output["runtime_info"]`` dict.
|
||||
Carries forward ``request.additional_information`` so prompt-side
|
||||
metadata (speaker / language / detok_id) survives the IPC boundary.
|
||||
"""
|
||||
if not isinstance(pooling_output, dict):
|
||||
pooling_output = {}
|
||||
|
||||
token_ids = _to_token_id_list(pooling_output.get("token_ids"))
|
||||
if not token_ids:
|
||||
token_ids = _to_token_id_list(pooling_output.get("text_tokens"))
|
||||
if not token_ids and request is not None:
|
||||
token_ids = _to_token_id_list(getattr(request, "output_token_ids", None))
|
||||
if not token_ids:
|
||||
logger.warning(
|
||||
"dynin_omni._build_full_payload: no token_ids found in pooling_output "
|
||||
"(keys=%s) or request.output_token_ids for req=%s; consumer wait gate may hang.",
|
||||
list(pooling_output.keys()),
|
||||
getattr(request, "request_id", "?"),
|
||||
)
|
||||
return None
|
||||
|
||||
src_additional_info = getattr(request, "additional_information", {}) if request is not None else {}
|
||||
if not isinstance(src_additional_info, dict):
|
||||
src_additional_info = {}
|
||||
|
||||
runtime_bridge_info = _decode_runtime_bridge_info(pooling_output.get("runtime_info_json"))
|
||||
if not runtime_bridge_info:
|
||||
runtime_bridge_info = pooling_output.get("runtime_info", {}) or {}
|
||||
|
||||
payload = _normalize_additional_info(src_additional_info)
|
||||
payload.update(_normalize_additional_info(runtime_bridge_info))
|
||||
payload["detok_id"] = [_to_int(pooling_output.get("detok_id"), default=_to_int(payload.get("detok_id"), default=0))]
|
||||
# Use nested OmniPayload shape so the scheduling-metadata extractor in
|
||||
# OmniConnectorModelRunnerMixin reads codes.audio and meta.finished
|
||||
# (flat keys at the top level are silently dropped with a warning).
|
||||
payload["codes"] = {"audio": token_ids}
|
||||
payload["meta"] = {"finished": torch.tensor(True, dtype=torch.bool)}
|
||||
return payload
|
||||
|
||||
|
||||
def token2text_to_token2image_full_payload(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any],
|
||||
request: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Producer-side payload builder for the Stage-0 → Stage-1 (text → image) transition."""
|
||||
del transfer_manager
|
||||
return _build_full_payload(pooling_output, request)
|
||||
|
||||
|
||||
def token2image_to_token2audio_full_payload(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any],
|
||||
request: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Producer-side payload builder for the Stage-1 → Stage-2 (image → audio) transition."""
|
||||
del transfer_manager
|
||||
return _build_full_payload(pooling_output, request)
|
||||
|
||||
|
||||
def _token_only_from_source(source_outputs: list[Any]) -> list[OmniTokensPrompt]:
|
||||
"""Length-only placeholder list mirroring ``_bridge_tokens`` token counts."""
|
||||
inputs: list[OmniTokensPrompt] = []
|
||||
for source_output in source_outputs:
|
||||
output = source_output.outputs[0]
|
||||
mm_out = getattr(output, "multimodal_output", None) or {}
|
||||
token_ids = _to_token_id_list(mm_out.get("token_ids"))
|
||||
if not token_ids:
|
||||
token_ids = _to_token_id_list(mm_out.get("text_tokens"))
|
||||
if not token_ids:
|
||||
token_ids = list(getattr(output, "token_ids", []) or [])
|
||||
if not token_ids:
|
||||
token_ids = [0]
|
||||
inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0] * len(token_ids),
|
||||
additional_information=None,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
def token2text_to_token2image_token_only(
|
||||
stage_list,
|
||||
engine_input_source,
|
||||
prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Sync-side placeholder for Stage-1 input (token2image)."""
|
||||
source_stage_id = engine_input_source[0] if engine_input_source else 0
|
||||
source_outputs = stage_list[source_stage_id].engine_outputs
|
||||
return _token_only_from_source(source_outputs)
|
||||
|
||||
|
||||
def token2image_to_token2audio_token_only(
|
||||
stage_list,
|
||||
engine_input_source,
|
||||
prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Sync-side placeholder for Stage-2 input (token2audio)."""
|
||||
source_stage_id = engine_input_source[0] if engine_input_source else 0
|
||||
source_outputs = stage_list[source_stage_id].engine_outputs
|
||||
return _token_only_from_source(source_outputs)
|
||||
@@ -147,6 +147,18 @@ def llm2code2wav_async_chunk(
|
||||
Accumulates codes in connector per request_id,
|
||||
returns payload only when chunk_size is full or request is finished; returns None when waiting.
|
||||
"""
|
||||
# Null guard: chunk_transfer_adapter calls this every emit step
|
||||
# including no-output steps where pooling_output is None.
|
||||
if pooling_output is None or not isinstance(pooling_output, dict):
|
||||
if is_finished:
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
raw_cfg = getattr(connector, "config", {}) or {}
|
||||
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
|
||||
chunk_size = int(cfg.get("codec_chunk_frames", 3))
|
||||
left_context_size = int(cfg.get("codec_left_context_frames", 3))
|
||||
request_id = getattr(request, "external_req_id", None)
|
||||
return _flush_remaining_codes(transfer_manager, request_id, chunk_size, left_context_size)
|
||||
return None
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
raw_cfg = getattr(connector, "config", {}) or {}
|
||||
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
|
||||
@@ -300,3 +312,137 @@ def llm2code2wav(
|
||||
)
|
||||
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path).
|
||||
# AR runner's `flatten_payload` converts the model emit
|
||||
# `multimodal_outputs={"codes": {"audio": ...}}` to flat
|
||||
# `pooling_output["codes.audio"]` before the full-payload accumulator runs, so default
|
||||
# CONCAT semantics build the full codec tensor across all decode steps.
|
||||
# ============================================================================
|
||||
|
||||
# Per-model REPLACE-keys for the full-payload accumulator. mimo_audio's
|
||||
# producer side emits per-step codec frames that should be CONCAT'd across
|
||||
# steps (not REPLACE'd), so this stays empty.
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _filter_zero_codec_rows(codec_codes: torch.Tensor) -> torch.Tensor:
|
||||
"""Drop zero-padded codec rows from a 4-D `[N, 1, 8, 4]` tensor.
|
||||
|
||||
Mirrors the zero-row filter in the orchestrator-path `llm2code2wav`
|
||||
body (see this file's ``llm2code2wav`` around line 224).
|
||||
"""
|
||||
if codec_codes.ndim != 4 or codec_codes.numel() == 0:
|
||||
return codec_codes
|
||||
is_all_zero = (codec_codes == 0).all(dim=(1, 2, 3))
|
||||
nonzero_idx = (~is_all_zero).nonzero(as_tuple=True)[0]
|
||||
if len(nonzero_idx) == 0:
|
||||
# All rows are zero-padded; return an empty tensor so the caller
|
||||
# can detect this via numel()==0 and skip the request.
|
||||
return codec_codes[:0]
|
||||
if len(nonzero_idx) < codec_codes.shape[0]:
|
||||
return codec_codes[nonzero_idx]
|
||||
return codec_codes
|
||||
|
||||
|
||||
def llm2code2wav_token_only(
|
||||
source_outputs: list,
|
||||
_prompt=None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
) -> list:
|
||||
"""Sync-side placeholder for the non-async-chunk Stage-1 (code2wav) input.
|
||||
|
||||
Returns an ``OmniTokensPrompt`` sized to the orchestrator-shape codec
|
||||
length so the consumer runtime allocates the right number of slots.
|
||||
The actual codec ids are delivered via the worker connector payload
|
||||
built by ``llm2code2wav_full_payload``.
|
||||
"""
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
code2wav_inputs: list = []
|
||||
for output_wrapper in source_outputs:
|
||||
out = output_wrapper.outputs[0]
|
||||
mm = out.multimodal_output if hasattr(out, "multimodal_output") else None
|
||||
mm = mm if isinstance(mm, dict) else {}
|
||||
mm_codes = mm.get("codes", {}) if isinstance(mm, dict) else {}
|
||||
prompt_len = 0
|
||||
if isinstance(mm_codes, dict) and "audio" in mm_codes:
|
||||
audio = mm_codes["audio"]
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
audio = audio.to(torch.long)
|
||||
audio = _filter_zero_codec_rows(audio)
|
||||
# +B*4 per batch row for the prepended pad_vec (see prepend_and_flatten_colmajor)
|
||||
batch_size = int(audio.shape[0]) if audio.ndim >= 1 else 1
|
||||
prompt_len = int(audio.numel()) + batch_size * 4
|
||||
if prompt_len > MAX_CODE2WAV_TOKENS:
|
||||
prompt_len = MAX_CODE2WAV_TOKENS
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0] * prompt_len,
|
||||
additional_information=None,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def llm2code2wav_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output: dict,
|
||||
request,
|
||||
) -> dict | None:
|
||||
"""Producer-side payload builder for the worker connector data plane.
|
||||
|
||||
AR runner's ``flatten_payload`` converts the per-step model emit
|
||||
``{"codes": {"audio": ...}}`` to ``pooling_output["codes.audio"]``.
|
||||
The accumulator CONCATs per-step tensors along dim 0, so by flush
|
||||
time this holds the full ``[total_steps, 1, 8, 4]`` codec tensor.
|
||||
|
||||
A back-compat fallback to nested ``pooling_output["codes"]["audio"]``
|
||||
is kept in case a future runtime path bypasses `flatten_payload`.
|
||||
"""
|
||||
del transfer_manager
|
||||
rid = getattr(request, "request_id", "?")
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.warning(
|
||||
"mimo_audio.llm2code2wav_full_payload: pooling_output not a dict "
|
||||
"(type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
codec_codes = pooling_output.get("codes.audio")
|
||||
if codec_codes is None:
|
||||
# Back-compat fallback for un-flattened pooler emits.
|
||||
codes = pooling_output.get("codes")
|
||||
if isinstance(codes, dict):
|
||||
codec_codes = codes.get("audio")
|
||||
if not isinstance(codec_codes, torch.Tensor) or codec_codes.numel() == 0:
|
||||
logger.warning(
|
||||
"mimo_audio.llm2code2wav_full_payload: missing/empty codes.audio "
|
||||
"(keys=%s) for req=%s; consumer wait gate may hang.",
|
||||
list(pooling_output.keys()),
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
codec_codes = codec_codes.to(torch.long)
|
||||
codec_codes = _filter_zero_codec_rows(codec_codes)
|
||||
if codec_codes.numel() == 0:
|
||||
logger.warning(
|
||||
"mimo_audio.llm2code2wav_full_payload: codec_codes empty after _filter_zero_codec_rows for req=%s.",
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
pad_vec = torch.tensor([TALKER_CODEC_PAD_TOKEN_ID] * 4)
|
||||
code_final = prepend_and_flatten_colmajor(codec_codes, pad_vec).tolist()
|
||||
if len(code_final) > MAX_CODE2WAV_TOKENS:
|
||||
code_final = code_final[:MAX_CODE2WAV_TOKENS]
|
||||
|
||||
return {
|
||||
"codes": {"audio": code_final},
|
||||
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
|
||||
}
|
||||
@@ -535,9 +535,97 @@ def thinker2talker(
|
||||
return talker_inputs
|
||||
|
||||
|
||||
# ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` or
|
||||
# ``_FULL_PAYLOAD_INPUT_STAGES``, so the worker connector is not
|
||||
# initialised for this arch and the consumer never waits on a connector
|
||||
# payload. Data flows through ``additional_information`` written by
|
||||
# ``thinker2talker_token_only`` (wired as ``sync_process_input_func``
|
||||
# in the pipeline) or the legacy ``thinker2talker`` (wired as
|
||||
# ``custom_process_input_func``).
|
||||
|
||||
|
||||
def thinker2talker_token_only(
|
||||
source_outputs: list[Any],
|
||||
prompt: OmniTokensPrompt | TextPrompt | None = None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Sync-side builder for the non-async-chunk thinker→talker path.
|
||||
|
||||
Ports the legacy ``thinker2talker`` body to the new stage-input-processor signature
|
||||
(``source_outputs`` instead of ``stage_list, engine_input_source``).
|
||||
Body is otherwise identical: extracts the
|
||||
generated text from each thinker output and packages it with the
|
||||
request's voice/speaker additional_information for the talker.
|
||||
"""
|
||||
if not isinstance(prompt, list):
|
||||
prompt = [prompt]
|
||||
|
||||
talker_inputs: list[OmniTokensPrompt] = []
|
||||
for i, source_output in enumerate(source_outputs):
|
||||
output = source_output.outputs[0]
|
||||
|
||||
generated_text = output.text if hasattr(output, "text") and output.text else ""
|
||||
|
||||
original_prompt = prompt[i] if i < len(prompt) else None
|
||||
additional_info: dict[str, Any] = {}
|
||||
if original_prompt is not None and hasattr(original_prompt, "additional_information"):
|
||||
additional_info = original_prompt.additional_information or {}
|
||||
|
||||
spk_emb = additional_info.get("spk_emb", None)
|
||||
if isinstance(spk_emb, list) and spk_emb and not hasattr(spk_emb[0], "device"):
|
||||
import torch
|
||||
|
||||
spk_emb = torch.tensor(spk_emb, dtype=torch.float32).unsqueeze(0)
|
||||
|
||||
talker_info = {
|
||||
"ming_task": "omni",
|
||||
"text": generated_text,
|
||||
"spk_emb": spk_emb,
|
||||
"voice_name": additional_info.get("voice_name", "DB30"),
|
||||
"prompt_text": additional_info.get("prompt_text", None),
|
||||
"prompt_wav_lat": additional_info.get("prompt_wav_lat", None),
|
||||
"prompt_wav_emb": additional_info.get("prompt_wav_emb", None),
|
||||
"max_text_length": additional_info.get("max_text_length", 50),
|
||||
}
|
||||
|
||||
talker_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0],
|
||||
additional_information=talker_info,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
|
||||
return talker_inputs
|
||||
|
||||
|
||||
thinker2talker_token_only._is_sync_input = True
|
||||
|
||||
|
||||
def thinker2talker_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output,
|
||||
request,
|
||||
):
|
||||
"""Producer-side payload builder — no-op.
|
||||
|
||||
ming_flash_omni's thinker emits no heavy tensor to ship via the
|
||||
worker connector (the bridge passes text only, and speaker metadata
|
||||
arrives through the USER request's additional_information).
|
||||
ming_flash_omni is not in ``_OMNI_CONNECTOR_INIT_ARCHS`` so this
|
||||
function is never invoked at runtime; it is retained for forward
|
||||
compatibility with the connector path.
|
||||
"""
|
||||
del transfer_manager, pooling_output, request
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CFG_TEXT_SUFFIX",
|
||||
"expand_cfg_prompts",
|
||||
"thinker2imagegen",
|
||||
"thinker2talker",
|
||||
"thinker2talker_full_payload",
|
||||
"thinker2talker_token_only",
|
||||
]
|
||||
@@ -1,3 +1,5 @@
|
||||
import logging
|
||||
|
||||
import torch
|
||||
from vllm.inputs import TextPrompt
|
||||
|
||||
@@ -11,6 +13,8 @@ from vllm_omni.data_entry_keys import (
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TALKER_CODEC_PAD_TOKEN_ID = 8292
|
||||
TALKER_CODEC_START_TOKEN_ID = 8293
|
||||
TALKER_CODEC_END_TOKEN_ID = 8294
|
||||
@@ -86,3 +90,347 @@ def talker2code2wav(
|
||||
)
|
||||
)
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path).
|
||||
# Both transitions ship payloads via the worker connector
|
||||
# (registered in ``_FULL_PAYLOAD_INPUT_STAGES`` in
|
||||
# omni_scheduling_coordinator):
|
||||
# - thinker->talker reads accumulated ``pooling_output["hidden"]`` and
|
||||
# packs an OmniPayload-shaped dict (embed.prefill /
|
||||
# hidden_states.output / ids.prompt / ids.output) for the talker, which
|
||||
# the talker's ``talker_preprocess`` reads from
|
||||
# ``model_intermediate_buffer``. The shape matches what legacy
|
||||
# ``thinker2talker`` writes into ``additional_information`` as a debug
|
||||
# fallback; ``thinker2talker_token_only`` only allocates prompt slots.
|
||||
# - talker->code2wav strips TALKER_CODEC_{START,END} boundary tokens
|
||||
# and ships the codec token ids.
|
||||
# ============================================================================
|
||||
|
||||
# Per-model REPLACE-keys for the full-payload accumulator. qwen2_5_omni's
|
||||
# producer side does not emit model_outputs through pooler_output (it ships
|
||||
# token_ids on the request directly), so the empty set preserves correctness.
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _strip_codec_boundaries(token_ids: list[int]) -> list[int]:
|
||||
"""Keep only real codec ids for the code2wav stage.
|
||||
|
||||
The talker stream can contain prompt/control ids (START/PAD/END/MASK) in
|
||||
addition to sampled codec ids. Code2wav expects codec ids only; carrying
|
||||
the prompt PAD span forward can inflate the sequence enough to OOM on L4.
|
||||
Async scheduling may also leave trailing ``-1`` placeholders, so preserve
|
||||
their length by repeating the last valid codec id.
|
||||
"""
|
||||
tids = list(token_ids)
|
||||
trailing_placeholder_count = 0
|
||||
while trailing_placeholder_count < len(tids) and tids[-1 - trailing_placeholder_count] == -1:
|
||||
trailing_placeholder_count += 1
|
||||
|
||||
if tids and tids[-1] == TALKER_CODEC_END_TOKEN_ID:
|
||||
tids = tids[:-1]
|
||||
trailing_placeholder_count = 0
|
||||
|
||||
codec_ids = [tid for tid in tids if 0 <= tid < TALKER_CODEC_PAD_TOKEN_ID]
|
||||
if trailing_placeholder_count > 0 and codec_ids:
|
||||
codec_ids.extend([codec_ids[-1]] * trailing_placeholder_count)
|
||||
return codec_ids
|
||||
|
||||
|
||||
def talker2code2wav_token_only(
|
||||
source_outputs,
|
||||
_prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
):
|
||||
"""Sync-side placeholder for Stage-2 input (code2wav).
|
||||
|
||||
Returns OmniTokensPrompt sized to the stripped codec token count.
|
||||
Actual codec ids are delivered via the worker connector payload built
|
||||
by ``talker2code2wav_full_payload``.
|
||||
"""
|
||||
code2wav_inputs = []
|
||||
for talker_output in source_outputs:
|
||||
output = talker_output.outputs[0]
|
||||
token_ids = _strip_codec_boundaries(list(output.cumulative_token_ids))
|
||||
if not token_ids:
|
||||
continue
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0] * len(token_ids),
|
||||
additional_information=None,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def talker2code2wav_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output: dict,
|
||||
request,
|
||||
) -> dict | None:
|
||||
"""Producer-side payload builder: ship the stripped codec ids via connector.
|
||||
|
||||
Token-ids-only shape. The talker stage's output already
|
||||
carries the codec ids on ``request.output_token_ids``; we strip the
|
||||
boundary tokens and pack a minimal payload.
|
||||
"""
|
||||
del transfer_manager
|
||||
rid = getattr(request, "request_id", "?")
|
||||
token_ids = list(getattr(request, "output_token_ids", None) or [])
|
||||
if not token_ids:
|
||||
logger.warning(
|
||||
"qwen2_5_omni.talker2code2wav_full_payload: empty output_token_ids "
|
||||
"for req=%s; consumer wait gate may hang.",
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
token_ids = _strip_codec_boundaries(token_ids)
|
||||
if not token_ids:
|
||||
logger.warning(
|
||||
"qwen2_5_omni.talker2code2wav_full_payload: codec ids empty after "
|
||||
"stripping boundary tokens for req=%s; consumer wait gate may hang.",
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
return {
|
||||
"codes": {"audio": token_ids},
|
||||
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path) -- thinker->talker.
|
||||
#
|
||||
# qwen2_5_omni's talker consumes the thinker's last-layer hidden state
|
||||
# via Linear(3584, 896). The AR runner publishes those hidden states
|
||||
# per decode step on ``pooling_output["hidden"]`` (unpacked from
|
||||
# ``OmniOutput.text_hidden_states``); the full-payload accumulator
|
||||
# concatenates them so ``thinker2talker_full_payload`` sees the full
|
||||
# prefill+decode trajectory and packs an OmniPayload-shaped dict that
|
||||
# the talker's ``talker_preprocess`` reads from
|
||||
# ``model_intermediate_buffer``. ``thinker2talker_token_only`` only
|
||||
# allocates the talker's codec prompt slots; legacy
|
||||
# ``thinker2talker`` above remains as a debug fallback that bundles the
|
||||
# same shape into ``additional_information``.
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def thinker2talker_token_only(
|
||||
source_outputs,
|
||||
prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
):
|
||||
"""Placeholder builder for the connector-driven thinker->talker path.
|
||||
|
||||
Allocates the TALKER_CODEC_{START,PAD,END} prompt slots sized to the
|
||||
thinker prompt length and forwards ``multi_modal_data``. The bulk
|
||||
payload (hidden_states / embed / ids) ships exclusively through
|
||||
``thinker2talker_full_payload`` via the worker connector and lands
|
||||
in ``model_intermediate_buffer`` before the talker's forward() runs.
|
||||
|
||||
Consumer-wait gating is whitelist-driven via
|
||||
``_FULL_PAYLOAD_INPUT_STAGES`` (see the mixin
|
||||
``should_accumulate_full_payload_output`` docstring).
|
||||
"""
|
||||
thinker_outputs = source_outputs
|
||||
talker_inputs = []
|
||||
if not isinstance(prompt, list):
|
||||
prompt = [prompt]
|
||||
multi_modal_data = {
|
||||
thinker_output.request_id: p.get("multi_modal_data", None) for thinker_output, p in zip(thinker_outputs, prompt)
|
||||
}
|
||||
|
||||
for thinker_output in thinker_outputs:
|
||||
prompt_token_ids = thinker_output.prompt_token_ids
|
||||
talker_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[TALKER_CODEC_START_TOKEN_ID]
|
||||
+ [TALKER_CODEC_PAD_TOKEN_ID] * (len(prompt_token_ids))
|
||||
+ [TALKER_CODEC_END_TOKEN_ID],
|
||||
additional_information=None,
|
||||
multi_modal_data=(
|
||||
multi_modal_data[thinker_output.request_id]
|
||||
if requires_multimodal_data and multi_modal_data is not None
|
||||
else None
|
||||
),
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
|
||||
return talker_inputs
|
||||
|
||||
|
||||
def thinker2talker_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output,
|
||||
request,
|
||||
):
|
||||
"""Producer-side payload builder for the worker-connector data plane.
|
||||
|
||||
The AR runner emits per-step ``pooling_output["hidden"]`` (the
|
||||
thinker's last-layer hidden states for the request span, unpacked
|
||||
from ``OmniOutput.text_hidden_states``). The full-payload
|
||||
accumulator concatenates those per-step rows across decode steps, so
|
||||
by the time this builder fires the materialized
|
||||
``pooling_output["hidden"]`` contains the full prefill+decode
|
||||
hidden-state trajectory of size
|
||||
``len(prompt_token_ids) + len(output_token_ids)``.
|
||||
|
||||
We split it at ``len(prompt_token_ids)`` into prefill embeddings and
|
||||
decode hidden states, then pack the ``OmniPayload``-shaped dict that
|
||||
the talker's ``thinker_to_talker_process`` reads from
|
||||
``model_intermediate_buffer`` (keys ``embed.prefill`` /
|
||||
``hidden_states.output`` / ``ids.prompt`` / ``ids.output``). Shape
|
||||
matches what legacy ``thinker2talker`` writes into
|
||||
``additional_information`` as a debug fallback, so the talker
|
||||
consumes the same payload layout from either path.
|
||||
|
||||
Like ``qwen3_omni.thinker2talker_full_payload``, we apply a
|
||||
finish-reason-aware stop-row trim: vLLM v1 appends the sampled
|
||||
token to ``output_token_ids`` before ``check_stop``, so a request
|
||||
that finished via ``FINISHED_STOPPED`` has one extra accumulated
|
||||
hidden-state row that the talker must not consume. Max-token
|
||||
finishes need no drop. Status is read from the request when
|
||||
available; otherwise we fall back to a last-token-in-stop-set
|
||||
heuristic.
|
||||
"""
|
||||
del transfer_manager
|
||||
rid = getattr(request, "request_id", "?")
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.warning(
|
||||
"qwen2_5_omni.thinker2talker_full_payload: pooling_output not a dict "
|
||||
"(type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
hidden = pooling_output.get("hidden")
|
||||
if not isinstance(hidden, torch.Tensor):
|
||||
logger.warning(
|
||||
"qwen2_5_omni.thinker2talker_full_payload: missing 'hidden' tensor "
|
||||
"(keys=%s) for req=%s; consumer wait gate may hang.",
|
||||
list(pooling_output.keys()),
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
def _ensure_list(x):
|
||||
if x is None:
|
||||
return []
|
||||
if hasattr(x, "_x"):
|
||||
# vLLM wraps cached token-id lists in ConstantList-like objects.
|
||||
return list(x._x)
|
||||
if isinstance(x, list):
|
||||
return list(x)
|
||||
return list(x)
|
||||
|
||||
prompt_token_ids = _ensure_list(getattr(request, "prompt_token_ids", None))
|
||||
output_token_ids = _ensure_list(getattr(request, "output_token_ids", None))
|
||||
all_token_ids = _ensure_list(getattr(request, "all_token_ids", None) or [])
|
||||
if not all_token_ids:
|
||||
all_token_ids = list(prompt_token_ids) + list(output_token_ids)
|
||||
|
||||
# Length-aware trim of accumulated thinker output, finish-reason-aware.
|
||||
# Mirror qwen3_omni.thinker2talker_full_payload's logic so a stop-finish
|
||||
# does not leak an extra hidden-state row to the talker.
|
||||
status = getattr(request, "status", None)
|
||||
status_name = getattr(status, "name", None) or ""
|
||||
if not status_name and status is not None:
|
||||
status_name = str(status).rsplit(".", 1)[-1]
|
||||
stop_emission_drop = 1 if status_name == "FINISHED_STOPPED" else 0
|
||||
if stop_emission_drop == 0 and not status_name and output_token_ids:
|
||||
# Worker-side CachedRequestState has no `.status` field in vLLM v1;
|
||||
# fall back to a last-token-in-stop-set heuristic.
|
||||
sampling_params = getattr(request, "sampling_params", None)
|
||||
if sampling_params is not None:
|
||||
stop_ids: set[int] = set()
|
||||
ignore_eos = bool(getattr(sampling_params, "ignore_eos", False))
|
||||
for sid in getattr(sampling_params, "stop_token_ids", None) or ():
|
||||
if isinstance(sid, int):
|
||||
stop_ids.add(sid)
|
||||
if not ignore_eos:
|
||||
for eos in (
|
||||
getattr(sampling_params, "eos_token_id", None),
|
||||
getattr(sampling_params, "_eos_token_id", None),
|
||||
):
|
||||
if isinstance(eos, int):
|
||||
stop_ids.add(eos)
|
||||
for sid in (
|
||||
getattr(sampling_params, "all_stop_token_ids", None)
|
||||
or getattr(sampling_params, "_all_stop_token_ids", None)
|
||||
or ()
|
||||
):
|
||||
if isinstance(sid, int):
|
||||
stop_ids.add(sid)
|
||||
if stop_ids and output_token_ids[-1] in stop_ids:
|
||||
stop_emission_drop = 1
|
||||
|
||||
# Trim accumulated thinker output based on stop_emission_drop computed
|
||||
# above. Mirror qwen3_omni.thinker2talker_full_payload's contract:
|
||||
# target_rows = len(all_token_ids) - stop_emission_drop
|
||||
# which excludes the stop-emission row for FINISHED_STOPPED but keeps
|
||||
# all rows for FINISHED_LENGTH_CAPPED (max_tokens) finishes.
|
||||
if stop_emission_drop > 0 and len(output_token_ids) >= stop_emission_drop:
|
||||
output_token_ids = output_token_ids[:-stop_emission_drop]
|
||||
h = hidden.detach().cpu().to(torch.float32)
|
||||
target_rows = max(0, len(all_token_ids) - stop_emission_drop)
|
||||
if target_rows <= 0:
|
||||
logger.warning(
|
||||
"qwen2_5_omni.thinker2talker_full_payload: target_rows<=0 "
|
||||
"(all_token_ids=%d, stop_drop=%d) for req=%s; nothing to ship.",
|
||||
len(all_token_ids),
|
||||
stop_emission_drop,
|
||||
getattr(request, "request_id", "?"),
|
||||
)
|
||||
return None
|
||||
if h.dim() >= 1 and h.shape[0] > target_rows:
|
||||
logger.warning(
|
||||
"qwen2_5_omni.thinker2talker_full_payload: excess hidden rows "
|
||||
"(got %d, target %d, stop_drop %d) for req=%s; trimming",
|
||||
int(h.shape[0]),
|
||||
target_rows,
|
||||
stop_emission_drop,
|
||||
getattr(request, "request_id", None),
|
||||
)
|
||||
h = h[:target_rows]
|
||||
|
||||
prompt_len = len(prompt_token_ids)
|
||||
if h.shape[0] < prompt_len:
|
||||
# Under-captured prefill -- defensively skip rather than ship a
|
||||
# truncated payload that would confuse the talker's prefill path.
|
||||
logger.warning(
|
||||
"qwen2_5_omni.thinker2talker_full_payload: hidden rows=%d < prompt_len=%d "
|
||||
"for req=%s; under-captured prefill, skipping payload.",
|
||||
int(h.shape[0]),
|
||||
prompt_len,
|
||||
getattr(request, "request_id", "?"),
|
||||
)
|
||||
return None
|
||||
|
||||
prefill_hidden = h[:prompt_len]
|
||||
decode_hidden = h[prompt_len:]
|
||||
|
||||
payload: OmniPayload = to_dict(
|
||||
OmniPayloadStruct(
|
||||
hidden_states=HiddenStatesStruct(
|
||||
output=decode_hidden,
|
||||
output_shape=list(decode_hidden.shape),
|
||||
),
|
||||
embed=EmbeddingsStruct(
|
||||
prefill=prefill_hidden,
|
||||
prefill_shape=list(prefill_hidden.shape),
|
||||
),
|
||||
ids=IdsStruct(
|
||||
prompt=list(prompt_token_ids),
|
||||
output=list(output_token_ids),
|
||||
),
|
||||
)
|
||||
)
|
||||
# Intentionally omit payload["meta"]: the thinker->talker transition
|
||||
# carries no scheduler-relevant metadata (next_stage_prompt_len /
|
||||
# left_context_size are not set on this edge).
|
||||
return payload
|
||||
@@ -35,6 +35,12 @@ logger = logging.getLogger(__name__)
|
||||
# Pooling output layer keys: "0" = word embedding, "24" = accept_hidden_layer
|
||||
_EMBED_LAYER_KEY = "0"
|
||||
_HIDDEN_LAYER_KEY = "24"
|
||||
# Per-model REPLACE-keys for the full-payload accumulator. Keys in this
|
||||
# set use REPLACE semantics (subsequent emissions discard prior chunks)
|
||||
# instead of CONCAT. qwen3-omni currently has none — model_outputs is
|
||||
# not emitted by the thinker/talker forward.
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
|
||||
|
||||
_QWEN3_CODEC_CODEBOOK_SIZE = 2048
|
||||
_QWEN3_CODEC_PAD_TOKEN_ID = 4196
|
||||
_QWEN3_CODEC_BOS_TOKEN_ID = 4197
|
||||
@@ -119,19 +125,6 @@ def _is_valid_qwen3_codec_token_id(token_id: Any) -> bool:
|
||||
return 0 <= token_id < _QWEN3_CODEC_CODEBOOK_SIZE
|
||||
|
||||
|
||||
def should_accumulate_qwen3_omni_full_payload_output(
|
||||
model_config: Any,
|
||||
custom_process_func: Any,
|
||||
) -> bool:
|
||||
"""Return whether Qwen3-Omni should accumulate full-payload outputs."""
|
||||
return (
|
||||
custom_process_func is not None
|
||||
and not getattr(model_config, "async_chunk", False)
|
||||
and getattr(model_config, "model_arch", None) == "Qwen3OmniMoeForConditionalGeneration"
|
||||
and getattr(model_config, "model_stage", None) in {"thinker", "talker"}
|
||||
)
|
||||
|
||||
|
||||
def _extract_qwen3_full_payload_codec_rows(
|
||||
code_predictor_codes: torch.Tensor,
|
||||
output_token_ids: list[int],
|
||||
@@ -462,7 +455,13 @@ def thinker2talker_full_payload(
|
||||
request: OmniEngineCoreRequest,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pack complete thinker output for the non-async connector path."""
|
||||
rid = getattr(request, "request_id", None)
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.warning(
|
||||
"thinker2talker_full_payload: pooling_output not a dict (type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
layers = {
|
||||
@@ -475,11 +474,13 @@ def thinker2talker_full_payload(
|
||||
hidden = pooling_output.get("hidden")
|
||||
thinker_emb = hidden if isinstance(hidden, torch.Tensor) else None
|
||||
if thinker_emb is None or thinker_hid is None:
|
||||
logger.debug(
|
||||
"thinker2talker_full_payload: missing thinker tensors for req=%s (embed=%s hidden=%s)",
|
||||
getattr(request, "request_id", None),
|
||||
logger.warning(
|
||||
"thinker2talker_full_payload: missing thinker tensors for req=%s "
|
||||
"(embed=%s hidden=%s keys=%s); consumer wait gate may hang.",
|
||||
rid,
|
||||
thinker_emb is not None,
|
||||
thinker_hid is not None,
|
||||
list(pooling_output.keys()),
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -493,7 +494,7 @@ def thinker2talker_full_payload(
|
||||
# The accumulator captures one hidden-state row per executed thinker
|
||||
# forward (prefill + every decode step including the one that emitted
|
||||
# the stop_token), so for a finished request thinker_emb has exactly one
|
||||
# row more than the rows the talker should consume. async_chunk's
|
||||
# row more than the rows the talker should consume. async_chunk's
|
||||
# chunk-0 path naturally captures only the prefill / non-stop portion,
|
||||
# which is why the [async_chunk] parametrization passes while [default]
|
||||
# over-generates one codec frame on short outputs (e.g.
|
||||
@@ -797,7 +798,14 @@ def talker2code2wav_full_payload(
|
||||
request: OmniEngineCoreRequest,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pack complete talker codec output for the non-async connector path."""
|
||||
rid = getattr(request, "request_id", None)
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.warning(
|
||||
"talker2code2wav_full_payload: pooling_output not a dict "
|
||||
"(type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
code_predictor_codes = pooling_output.get("codes.audio")
|
||||
if code_predictor_codes is None:
|
||||
@@ -805,10 +813,19 @@ def talker2code2wav_full_payload(
|
||||
if isinstance(codes, dict):
|
||||
code_predictor_codes = codes.get("audio")
|
||||
if code_predictor_codes is None:
|
||||
logger.warning(
|
||||
"talker2code2wav_full_payload: missing codes.audio (keys=%s) for req=%s; consumer wait gate may hang.",
|
||||
list(pooling_output.keys()),
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
if not isinstance(code_predictor_codes, torch.Tensor):
|
||||
code_predictor_codes = torch.as_tensor(code_predictor_codes)
|
||||
if code_predictor_codes.numel() == 0:
|
||||
logger.warning(
|
||||
"talker2code2wav_full_payload: empty codes.audio for req=%s; consumer wait gate may hang.",
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
output_token_ids = _ensure_list(getattr(request, "output_token_ids", []) or [])
|
||||
@@ -818,6 +835,16 @@ def talker2code2wav_full_payload(
|
||||
list(output_token_ids),
|
||||
)
|
||||
if code_predictor_codes.numel() == 0:
|
||||
logger.warning(
|
||||
"talker2code2wav_full_payload: no valid codec rows after filtering "
|
||||
"(raw_shape=%s output_ids_len=%d aligned_rows=%s valid_rows=%s) for req=%s; "
|
||||
"consumer wait gate may hang.",
|
||||
raw_shape,
|
||||
len(output_token_ids),
|
||||
codec_stats["aligned_rows"],
|
||||
codec_stats["valid_rows"],
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
codec_codes = code_predictor_codes.transpose(0, 1).cpu().reshape(-1).tolist()
|
||||
|
||||
@@ -53,10 +53,13 @@ def talker2code2wav(
|
||||
# audio_codes may still contain zero-padded / invalid rows, so trim only
|
||||
# after filtering valid frames instead of trying to align EOS indices.
|
||||
seq_len = max(len(token_ids) - 1, 0)
|
||||
# Filter invalid frames: zero-padded (EOS) and frames containing
|
||||
# out-of-range values (e.g. stop_token_id=2150 exceeds codebook_size=2048).
|
||||
# Filter invalid frames: zero-padded (EOS), out-of-range values (e.g.
|
||||
# stop_token_id=2150 exceeds codebook_size=2048), and negative
|
||||
# sentinels (e.g. -1 padding).
|
||||
_CODEBOOK_SIZE = 2048
|
||||
valid_mask = audio_codes.any(dim=1) & (audio_codes.max(dim=1).values < _CODEBOOK_SIZE)
|
||||
valid_mask = (
|
||||
(audio_codes >= 0).all(dim=1) & audio_codes.any(dim=1) & (audio_codes.max(dim=1).values < _CODEBOOK_SIZE)
|
||||
)
|
||||
audio_codes = audio_codes[valid_mask]
|
||||
if seq_len > 0 and audio_codes.ndim == 2 and int(audio_codes.shape[0]) > seq_len:
|
||||
audio_codes = audio_codes[-seq_len:]
|
||||
@@ -280,3 +283,220 @@ def talker2code2wav_async_chunk(
|
||||
speaker=extract_speaker_from_request(request),
|
||||
language=extract_language_from_request(request),
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Worker-connector data plane (non-async-chunk path).
|
||||
# AR runner's `flatten_payload` converts the model emit
|
||||
# `multimodal_outputs={"codes": {"audio": ..., "ref": ...},
|
||||
# "meta": {"ref_code_len": ..., "codec_streaming": ...}}` to flat dotted
|
||||
# keys (`codes.audio`, `codes.ref`, `meta.ref_code_len`,
|
||||
# `meta.codec_streaming`) before the full-payload accumulator runs.
|
||||
# - codes.audio is 2-D so default CONCAT across steps builds the full sequence.
|
||||
# - codes.ref is a list (not Tensor with dim>=2) so accumulator LATEST-wins
|
||||
# keeps the prefill-emitted ref tensor across decode steps (which don't emit
|
||||
# ref again).
|
||||
# - meta.ref_code_len is 1-D so LATEST-wins; consumer reads [-1].
|
||||
# ============================================================================
|
||||
|
||||
# Per-model REPLACE-keys for the full-payload accumulator. qwen3_tts's
|
||||
# producer side emits codec frames that should CONCAT (codes.audio) plus
|
||||
# scalars/lists that are correctly handled by default LATEST-wins, so this
|
||||
# stays empty.
|
||||
_FULL_PAYLOAD_REPLACE_KEYS: frozenset[str] = frozenset()
|
||||
|
||||
_CODEBOOK_SIZE = 2048
|
||||
_NUM_QUANTIZERS_DEFAULT = 16
|
||||
|
||||
|
||||
def _filter_audio_codes_qwen3_tts(audio_codes: torch.Tensor) -> torch.Tensor:
|
||||
"""Filter zero-padded, out-of-range, and negative-padded codec frames.
|
||||
|
||||
Mirrors the orchestrator-path body in `talker2code2wav` above.
|
||||
"""
|
||||
if not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
|
||||
return audio_codes
|
||||
if audio_codes.ndim != 2:
|
||||
return audio_codes
|
||||
valid_mask = (
|
||||
(audio_codes >= 0).all(dim=1) & audio_codes.any(dim=1) & (audio_codes.max(dim=1).values < _CODEBOOK_SIZE)
|
||||
)
|
||||
return audio_codes[valid_mask]
|
||||
|
||||
|
||||
def _coerce_ref_code_len(raw) -> int:
|
||||
"""Coerce mm["meta"]["ref_code_len"] / pooling_output["meta.ref_code_len"]
|
||||
raw value (Tensor | int | None) into a non-negative int. Mirrors the
|
||||
extraction inlined in the legacy ``talker2code2wav`` path; clamps any
|
||||
negative input to 0 since downstream code treats this as a non-negative
|
||||
frame count."""
|
||||
if isinstance(raw, torch.Tensor):
|
||||
value = int(raw.reshape(-1)[-1].item()) if raw.numel() > 0 else 0
|
||||
elif raw is None:
|
||||
value = 0
|
||||
else:
|
||||
value = int(raw)
|
||||
return max(value, 0)
|
||||
|
||||
|
||||
def _normalize_ref_code(ref_code, num_quantizers: int, ref_code_len: int):
|
||||
"""Coerce ref_code into a [ref_len, Q] tensor or None. Mirrors orchestrator path."""
|
||||
if isinstance(ref_code, list):
|
||||
ref_code = ref_code[0] if ref_code else None
|
||||
if not isinstance(ref_code, torch.Tensor) or ref_code.numel() == 0:
|
||||
return None, 0
|
||||
ref_code = ref_code.to(torch.long).cpu().contiguous()
|
||||
if ref_code.ndim == 1:
|
||||
if ref_code.numel() % num_quantizers != 0:
|
||||
return None, 0
|
||||
ref_code = ref_code.reshape(-1, num_quantizers)
|
||||
elif ref_code.ndim != 2:
|
||||
return None, 0
|
||||
if ref_code_len > 0 and int(ref_code.shape[0]) > ref_code_len:
|
||||
ref_code = ref_code[:ref_code_len]
|
||||
return ref_code, int(ref_code.shape[0])
|
||||
|
||||
|
||||
def talker2code2wav_token_only(
|
||||
source_outputs: list,
|
||||
prompt=None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
) -> list:
|
||||
"""Sync-side placeholder for the non-async-chunk Stage-1 (code2wav) input.
|
||||
|
||||
Sized to the expected codec token count (codebook-major flat:
|
||||
Q * (ref_frames + audio_frames)). Speaker / language metadata are
|
||||
extracted from `prompt` and threaded via `additional_information`
|
||||
(orchestrator-style; same as the legacy `talker2code2wav` builder).
|
||||
Actual codec ids are delivered via the worker connector payload built
|
||||
by `talker2code2wav_full_payload`.
|
||||
"""
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
code2wav_inputs: list = []
|
||||
for i, talker_output in enumerate(source_outputs):
|
||||
if not talker_output.finished:
|
||||
continue
|
||||
output = talker_output.outputs[0]
|
||||
mm = output.multimodal_output if hasattr(output, "multimodal_output") else None
|
||||
mm = mm if isinstance(mm, dict) else {}
|
||||
mm_codes = mm.get("codes", {}) if isinstance(mm, dict) else {}
|
||||
token_ids = getattr(output, "cumulative_token_ids", []) or []
|
||||
seq_len = max(len(token_ids) - 1, 0)
|
||||
|
||||
audio = mm_codes.get("audio") if isinstance(mm_codes, dict) else None
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
audio = audio.to(torch.long)
|
||||
audio = _filter_audio_codes_qwen3_tts(audio)
|
||||
if seq_len > 0 and audio.ndim == 2 and int(audio.shape[0]) > seq_len:
|
||||
audio = audio[-seq_len:]
|
||||
num_audio_frames = int(audio.shape[0]) if audio.ndim == 2 else 0
|
||||
num_quantizers = int(audio.shape[1]) if audio.ndim == 2 and audio.shape[1] > 0 else _NUM_QUANTIZERS_DEFAULT
|
||||
else:
|
||||
num_audio_frames = 0
|
||||
num_quantizers = _NUM_QUANTIZERS_DEFAULT
|
||||
|
||||
ref_code_raw = mm_codes.get("ref") if isinstance(mm_codes, dict) else None
|
||||
ref_code_len_raw = mm.get("meta", {}).get("ref_code_len") if isinstance(mm.get("meta"), dict) else None
|
||||
ref_code_len = _coerce_ref_code_len(ref_code_len_raw)
|
||||
_, ref_frames = _normalize_ref_code(ref_code_raw, num_quantizers, ref_code_len)
|
||||
|
||||
# Codebook-major flat: Q * (ref_frames + audio_frames)
|
||||
prompt_len = num_quantizers * (ref_frames + num_audio_frames)
|
||||
|
||||
additional_info = to_dict(
|
||||
OmniPayloadStruct(
|
||||
meta=MetaStruct(left_context_size=ref_frames) if ref_frames > 0 else None,
|
||||
speaker=extract_speaker_from_prompt(prompt, index=i),
|
||||
language=extract_language_from_prompt(prompt, index=i),
|
||||
)
|
||||
)
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0] * prompt_len,
|
||||
additional_information=additional_info if additional_info else None,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
return code2wav_inputs
|
||||
|
||||
|
||||
def talker2code2wav_full_payload(
|
||||
transfer_manager,
|
||||
pooling_output,
|
||||
request,
|
||||
):
|
||||
"""Producer-side payload builder.
|
||||
|
||||
Reads accumulated codec from `pooling_output["codes.audio"]` (CONCAT
|
||||
across steps via flatten_payload), latest `pooling_output["codes.ref"]`
|
||||
(prefill-emitted), and latest `pooling_output["meta.ref_code_len"]`.
|
||||
Replicates the orchestrator-path body of `talker2code2wav` (filter,
|
||||
crop to seq_len, prepend ref, codebook-major flatten).
|
||||
"""
|
||||
del transfer_manager
|
||||
rid = getattr(request, "request_id", "?")
|
||||
if not isinstance(pooling_output, dict):
|
||||
logger.warning(
|
||||
"qwen3_tts.talker2code2wav_full_payload: pooling_output not a dict "
|
||||
"(type=%s) for req=%s; consumer wait gate may hang.",
|
||||
type(pooling_output).__name__,
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
# codes.audio — try flat dotted first (flatten_payload), then nested fallback.
|
||||
audio = pooling_output.get("codes.audio")
|
||||
if audio is None:
|
||||
codes_nested = pooling_output.get("codes")
|
||||
if isinstance(codes_nested, dict):
|
||||
audio = codes_nested.get("audio")
|
||||
if not isinstance(audio, torch.Tensor) or audio.numel() == 0:
|
||||
logger.warning(
|
||||
"qwen3_tts.talker2code2wav_full_payload: missing/empty codes.audio "
|
||||
"(keys=%s) for req=%s; consumer wait gate may hang.",
|
||||
list(pooling_output.keys()),
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
audio = audio.to(torch.long)
|
||||
audio = _filter_audio_codes_qwen3_tts(audio)
|
||||
if audio.numel() == 0:
|
||||
logger.warning(
|
||||
"qwen3_tts.talker2code2wav_full_payload: audio empty after codec "
|
||||
"filter (negative/all-zero/out-of-range rows dropped) for req=%s.",
|
||||
rid,
|
||||
)
|
||||
return None
|
||||
|
||||
output_token_ids = list(getattr(request, "output_token_ids", None) or [])
|
||||
seq_len = max(len(output_token_ids) - 1, 0)
|
||||
if seq_len > 0 and audio.ndim == 2 and int(audio.shape[0]) > seq_len:
|
||||
audio = audio[-seq_len:]
|
||||
|
||||
num_quantizers = int(audio.shape[1]) if audio.ndim == 2 and audio.shape[1] > 0 else _NUM_QUANTIZERS_DEFAULT
|
||||
|
||||
# meta.ref_code_len — flat dotted then nested fallback.
|
||||
ref_code_len_raw = pooling_output.get("meta.ref_code_len")
|
||||
if ref_code_len_raw is None:
|
||||
meta_nested = pooling_output.get("meta")
|
||||
if isinstance(meta_nested, dict):
|
||||
ref_code_len_raw = meta_nested.get("ref_code_len")
|
||||
ref_code_len = _coerce_ref_code_len(ref_code_len_raw)
|
||||
|
||||
# codes.ref — flat dotted then nested fallback.
|
||||
ref_code_raw = pooling_output.get("codes.ref")
|
||||
if ref_code_raw is None:
|
||||
codes_nested = pooling_output.get("codes")
|
||||
if isinstance(codes_nested, dict):
|
||||
ref_code_raw = codes_nested.get("ref")
|
||||
ref_code, ref_frames = _normalize_ref_code(ref_code_raw, num_quantizers, ref_code_len)
|
||||
if ref_code is not None:
|
||||
audio = torch.cat([ref_code.to(audio.device), audio], dim=0)
|
||||
|
||||
codec_codes = audio.transpose(0, 1).to(device="cpu", dtype=torch.long).reshape(-1).contiguous()
|
||||
return {
|
||||
"codes": {"audio": codec_codes},
|
||||
"meta": {"finished": torch.tensor(True, dtype=torch.bool)},
|
||||
}
|
||||
Reference in new issue
Block a user