[3/5][core]refactor communication layer: PR 3 of 5, all other models in non async mode (#3719)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
NATURE authored and GitHub committed 2026-05-28 14:06:41 +08:00
1 parent 0f3264a9c0
commit 3a347b10b0
38 files changed
+3021 -275

No files matched your search

@@ -784,12 +784,14 @@ class CosyVoice3Model(
else:
self._stream_vocoder_cache_by_req[req_id] = new_cache_state
else:
token_offset = max(0, meta.talker_prefill_offset or 0) if meta else 0
tts_speech = self.code2wav.forward(
token=token.unsqueeze(0),
prompt_token=speech_token[:1],
prompt_feat=speech_feat[:1],
embedding=embedding[:1],
n_timesteps=10,
token_offset_tokens=token_offset,
)
audio = tts_speech.reshape(-1).to(dtype=torch.float32)
@@ -292,6 +292,7 @@ class CosyVoice3Code2Wav(nn.Module):
prompt_feat: torch.Tensor,
embedding: torch.Tensor,
n_timesteps: int = 10,
token_offset_tokens: int = 0,
) -> torch.Tensor:
"""Generate audio waveform from speech tokens."""
feat = self._forward_mel(
@@ -300,7 +301,7 @@ class CosyVoice3Code2Wav(nn.Module):
prompt_feat=prompt_feat,
embedding=embedding,
n_timesteps=n_timesteps,
token_offset_tokens=0,
token_offset_tokens=token_offset_tokens,
streaming=False,
finalize=True,
)
@@ -31,6 +31,7 @@ COSYVOICE3_PIPELINE = PipelineConfig(
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload",
sampling_constraints={
# merged speech stop token (logsumexp of all 200 stop logits)
"stop_token_ids": [6562],
@@ -44,7 +45,8 @@ COSYVOICE3_PIPELINE = PipelineConfig(
final_output=True,
final_output_type="audio",
engine_output_type="latent",
sync_process_input_func=f"{_PROC}.text2flow",
custom_process_input_func=f"{_PROC}.text2flow",
sync_process_input_func=f"{_PROC}.text2flow_token_only",
),
),
)
@@ -29,6 +29,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
owns_tokenizer=True,
requires_multimodal_data=True,
engine_output_type="latent",
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
"stop_token_ids": [151645],
@@ -44,6 +45,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
final_output_type="audio",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.llm2code2wav",
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
sampling_constraints={"detokenize": False},
),
),
@@ -39,6 +39,7 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
# Stop once the speech/text interleaved span ends. Code2Wav
@@ -55,7 +56,8 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sync_process_input_func=f"{_PROC}.llm2code2wav",
custom_process_input_func=f"{_PROC}.llm2code2wav",
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
sampling_constraints={"detokenize": False},
),
),
@@ -58,6 +58,7 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig(
engine_output_type="audio",
tokenizer_subdir="talker/llm",
custom_process_input_func=f"{_PROC}.thinker2talker",
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
),
),
)
@@ -30,6 +30,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
requires_multimodal_data=True,
engine_output_type="latent",
sampling_constraints={"detokenize": True},
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
),
StagePipelineConfig(
stage_id=1,
@@ -38,6 +39,8 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
input_sources=(0,),
engine_output_type="latent",
custom_process_input_func=f"{_PROC}.thinker2talker",
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
"stop_token_ids": [8294],
@@ -52,6 +55,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
final_output_type="audio",
engine_output_type="audio",
custom_process_input_func=f"{_PROC}.talker2code2wav",
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
),
@@ -74,6 +78,7 @@ QWEN2_5_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
requires_multimodal_data=True,
engine_output_type="latent",
sampling_constraints={"detokenize": True},
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
),
),
)
@@ -26,6 +26,7 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [2150],
@@ -40,7 +41,14 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
final_output_type="audio",
engine_output_type="audio",
model_arch="Qwen3TTSCode2Wav",
sync_process_input_func=f"{_PROC}.talker2code2wav",
# ``sync_process_input_func`` is the only input-proc override for
# this stage in sync (non-async-chunk) mode: a length-only
# ``_token_only`` placeholder. The bulk codec payload itself
# ships via the worker connector from stage 0's
# ``talker2code2wav_full_payload`` producer. Under async_chunk
# mode no pre-stage processing is needed -- chunks deliver
# directly to the consumer.
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
extras={"tts_args": {"max_instructions_length": 500}},
),
@@ -25,6 +25,27 @@ from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
logger = init_logger(__name__)
def _codec_ids_from_payload_or_input(
input_ids: torch.Tensor,
runtime_info: dict[str, Any] | None,
) -> torch.Tensor:
"""Prefer connector-delivered codec ids over token placeholders.
In non-async full-payload mode, the scheduler only needs placeholder
token ids for allocation. The real codec sequence is delivered through
model_intermediate_buffer as ``codes.audio``.
"""
if isinstance(runtime_info, dict):
codes = runtime_info.get("codes")
if isinstance(codes, dict):
audio = codes.get("audio")
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
return audio.reshape(-1).to(device=input_ids.device, dtype=torch.long)
if isinstance(audio, (list, tuple)) and audio:
return torch.as_tensor(audio, device=input_ids.device, dtype=torch.long).reshape(-1)
return input_ids.reshape(-1).to(dtype=torch.long)
class Qwen3TTSCode2Wav(nn.Module):
"""Stage-1 code2wav model for Qwen3-TTS (GenerationModelRunner).
Consumes frame-aligned codec tokens from input_ids and decodes waveform
@@ -239,6 +260,7 @@ class Qwen3TTSCode2Wav(nn.Module):
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
)
runtime_infos = runtime_additional_information or []
ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
@@ -246,14 +268,18 @@ class Qwen3TTSCode2Wav(nn.Module):
valid_codes_qf: list[torch.Tensor] = []
valid_indices: list[int] = []
left_context_size = [0] * len(request_ids_list)
if runtime_additional_information is not None:
for i, info in enumerate(runtime_additional_information):
if runtime_infos:
for i, info in enumerate(runtime_infos):
if i >= len(left_context_size):
break
if not isinstance(info, dict):
continue
meta = info.get("meta", {})
if "left_context_size" in meta:
left_context_size[i] = meta["left_context_size"]
for i, req_ids in enumerate(request_ids_list):
runtime_info = runtime_infos[i] if i < len(runtime_infos) else None
req_ids = _codec_ids_from_payload_or_input(req_ids, runtime_info)
if req_ids.numel() < 1:
parsed.append((0, 0))
continue