[3/5][core]refactor communication layer: PR 3 of 5, all other models in non async mode (#3719)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
0f3264a9c0
commit
3a347b10b0
38 files changed
+3021
-275
No files matched your search
@@ -784,12 +784,14 @@ class CosyVoice3Model(
|
||||
else:
|
||||
self._stream_vocoder_cache_by_req[req_id] = new_cache_state
|
||||
else:
|
||||
token_offset = max(0, meta.talker_prefill_offset or 0) if meta else 0
|
||||
tts_speech = self.code2wav.forward(
|
||||
token=token.unsqueeze(0),
|
||||
prompt_token=speech_token[:1],
|
||||
prompt_feat=speech_feat[:1],
|
||||
embedding=embedding[:1],
|
||||
n_timesteps=10,
|
||||
token_offset_tokens=token_offset,
|
||||
)
|
||||
|
||||
audio = tts_speech.reshape(-1).to(dtype=torch.float32)
|
||||
|
||||
@@ -292,6 +292,7 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
prompt_feat: torch.Tensor,
|
||||
embedding: torch.Tensor,
|
||||
n_timesteps: int = 10,
|
||||
token_offset_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Generate audio waveform from speech tokens."""
|
||||
feat = self._forward_mel(
|
||||
@@ -300,7 +301,7 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
prompt_feat=prompt_feat,
|
||||
embedding=embedding,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=0,
|
||||
token_offset_tokens=token_offset_tokens,
|
||||
streaming=False,
|
||||
finalize=True,
|
||||
)
|
||||
|
||||
@@ -31,6 +31,7 @@ COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload",
|
||||
sampling_constraints={
|
||||
# merged speech stop token (logsumexp of all 200 stop logits)
|
||||
"stop_token_ids": [6562],
|
||||
@@ -44,7 +45,8 @@ COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="latent",
|
||||
sync_process_input_func=f"{_PROC}.text2flow",
|
||||
custom_process_input_func=f"{_PROC}.text2flow",
|
||||
sync_process_input_func=f"{_PROC}.text2flow_token_only",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -29,6 +29,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [151645],
|
||||
@@ -44,6 +45,7 @@ COVO_AUDIO_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
|
||||
@@ -39,6 +39,7 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.llm2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
# Stop once the speech/text interleaved span ends. Code2Wav
|
||||
@@ -55,7 +56,8 @@ MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
custom_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
|
||||
@@ -58,6 +58,7 @@ MING_FLASH_OMNI_PIPELINE = PipelineConfig(
|
||||
engine_output_type="audio",
|
||||
tokenizer_subdir="talker/llm",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -30,6 +30,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
@@ -38,6 +39,8 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
input_sources=(0,),
|
||||
engine_output_type="latent",
|
||||
custom_process_input_func=f"{_PROC}.thinker2talker",
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [8294],
|
||||
@@ -52,6 +55,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
custom_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
@@ -74,6 +78,7 @@ QWEN2_5_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -26,6 +26,7 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [2150],
|
||||
@@ -40,7 +41,14 @@ QWEN3_TTS_PIPELINE = PipelineConfig(
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="Qwen3TTSCode2Wav",
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav",
|
||||
# ``sync_process_input_func`` is the only input-proc override for
|
||||
# this stage in sync (non-async-chunk) mode: a length-only
|
||||
# ``_token_only`` placeholder. The bulk codec payload itself
|
||||
# ships via the worker connector from stage 0's
|
||||
# ``talker2code2wav_full_payload`` producer. Under async_chunk
|
||||
# mode no pre-stage processing is needed -- chunks deliver
|
||||
# directly to the consumer.
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
extras={"tts_args": {"max_instructions_length": 500}},
|
||||
),
|
||||
|
||||
@@ -25,6 +25,27 @@ from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _codec_ids_from_payload_or_input(
|
||||
input_ids: torch.Tensor,
|
||||
runtime_info: dict[str, Any] | None,
|
||||
) -> torch.Tensor:
|
||||
"""Prefer connector-delivered codec ids over token placeholders.
|
||||
|
||||
In non-async full-payload mode, the scheduler only needs placeholder
|
||||
token ids for allocation. The real codec sequence is delivered through
|
||||
model_intermediate_buffer as ``codes.audio``.
|
||||
"""
|
||||
if isinstance(runtime_info, dict):
|
||||
codes = runtime_info.get("codes")
|
||||
if isinstance(codes, dict):
|
||||
audio = codes.get("audio")
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
return audio.reshape(-1).to(device=input_ids.device, dtype=torch.long)
|
||||
if isinstance(audio, (list, tuple)) and audio:
|
||||
return torch.as_tensor(audio, device=input_ids.device, dtype=torch.long).reshape(-1)
|
||||
return input_ids.reshape(-1).to(dtype=torch.long)
|
||||
|
||||
|
||||
class Qwen3TTSCode2Wav(nn.Module):
|
||||
"""Stage-1 code2wav model for Qwen3-TTS (GenerationModelRunner).
|
||||
Consumes frame-aligned codec tokens from input_ids and decodes waveform
|
||||
@@ -239,6 +260,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
|
||||
)
|
||||
|
||||
runtime_infos = runtime_additional_information or []
|
||||
ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
|
||||
|
||||
@@ -246,14 +268,18 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
valid_codes_qf: list[torch.Tensor] = []
|
||||
valid_indices: list[int] = []
|
||||
left_context_size = [0] * len(request_ids_list)
|
||||
if runtime_additional_information is not None:
|
||||
for i, info in enumerate(runtime_additional_information):
|
||||
if runtime_infos:
|
||||
for i, info in enumerate(runtime_infos):
|
||||
if i >= len(left_context_size):
|
||||
break
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
meta = info.get("meta", {})
|
||||
if "left_context_size" in meta:
|
||||
left_context_size[i] = meta["left_context_size"]
|
||||
for i, req_ids in enumerate(request_ids_list):
|
||||
runtime_info = runtime_infos[i] if i < len(runtime_infos) else None
|
||||
req_ids = _codec_ids_from_payload_or_input(req_ids, runtime_info)
|
||||
if req_ids.numel() < 1:
|
||||
parsed.append((0, 0))
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user