Merge current Omni main and retain conditioning regression coverage

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-10-04 13:50:06 +08:00
commit 6f5f9b97e3
21 files changed
+787 -134

No files matched your search

@@ -73,6 +73,27 @@ logger = init_logger(__name__)
_RUNTIME_COMPONENTS_CACHE: dict[str, dict] = {}
def _normalize_request_conditioning(payload: dict) -> dict:
"""Unwrap singleton conditioning containers at the per-request boundary.
A processor/prefix-cache passthrough can retain a one-item tensor list.
The code2wav schema consumes one request, so that wrapper carries no
batch dimension. A multi-item list here is an unsplit batch and must
fail instead of silently choosing another request's voice.
"""
embed = payload.get("embed")
if not isinstance(embed, Mapping):
return payload
normalized = dict(embed)
for key in ("speech_token", "speech_feat", "embedding", "speech_token_len"):
value = normalized.get(key)
if isinstance(value, (list, tuple)):
if len(value) != 1:
raise ValueError(f"CosyVoice3 per-request {key} contains an unsplit batch of {len(value)} items")
normalized[key] = value[0]
return {**payload, "embed": normalized}
def _cosyvoice3_trt_enabled() -> bool:
"""COSYVOICE3_TRT env toggle (default on) for the optional TensorRT paths.
@@ -1500,9 +1521,10 @@ class CosyVoice3Model(
# [total_tokens, hidden]
hidden_states = self.model.llm(inputs_embeds, positions)
if getattr(self, "_mrv2_tensor_output", False) or cosyvoice3_packed_inference_enabled():
return hidden_states
return self.make_omni_output(hidden_states, **kwargs)
# Both runners attach the live step's conditioning after replay.
# Capturing an OmniOutput here would retain the dummy prefill lists
# and re-emit them during decode, replacing the real reference.
return hidden_states
elif self.model_stage == "cosyvoice3_code2wav":
# Lazily swap the flow-decoder estimator to a TensorRT engine on the
# first code2wav step (after weights are loaded), gated by the same
@@ -1538,7 +1560,7 @@ class CosyVoice3Model(
for idx, req_ids in enumerate(request_ids_list):
raw = runtime_info[idx] if idx < len(runtime_info) and isinstance(runtime_info[idx], dict) else {}
payload = to_struct(raw)
payload = to_struct(_normalize_request_conditioning(raw))
meta = payload.meta
embed = payload.embed
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CosyVoice3 pipeline topology (frozen).
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
@@ -12,11 +12,17 @@ Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
through the shared-memory connector.
"""
import shutil
from dataclasses import replace
from transformers import PretrainedConfig
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
@@ -35,8 +41,9 @@ COSYVOICE3_PIPELINE = PipelineConfig(
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload",
sampling_constraints={
# merged speech stop token (logsumexp of all 200 stop logits)
"stop_token_ids": [6562],
# Standard sampling can emit any of the 200 control tokens.
# RAS merges their logits into 6562, which is also in this range.
"stop_token_ids": list(range(6561, 6761)),
},
),
StagePipelineConfig(
@@ -52,3 +59,28 @@ COSYVOICE3_PIPELINE = PipelineConfig(
),
),
)
def resolve_cosyvoice3_pipeline(hf_config: PretrainedConfig | None = None) -> PipelineConfig | None:
"""Select a streaming default compatible with the device and runtime.
Query the platform through NVML without initializing CUDA in the parent.
Explicit deploy configs still take precedence over this pipeline default.
"""
from vllm.platforms import current_platform
if hf_config is not None and not isinstance(hf_config, CosyVoice3Config):
return None
if current_platform.is_cuda():
capability = current_platform.get_device_capability()
if (
capability is not None
and capability.major == 9
and current_platform.get_device_total_memory() >= 140 * 1024**3
and shutil.which("nvidia-cuda-mps-control") is not None
):
return replace(
COSYVOICE3_PIPELINE,
default_deploy_config_name="cosyvoice3_packed_streaming_optimized_standard.yaml",
)
return COSYVOICE3_PIPELINE
@@ -65,6 +65,24 @@ def _cached_reference_rope(self, q: torch.Tensor, k: torch.Tensor, cache: dict):
return qo.view(batch, heads, seqlen, dim), ko.view(batch, heads, seqlen, dim)
def _reference_flash_attn_varlen_func():
"""Resolve the platform's variable-length FlashAttention entrypoint.
vLLM's bundled extension is NVIDIA-only. ROCm exposes the compatible
unpacked-QKV API through AITER, with upstream flash-attn as its fallback.
"""
if torch.version.hip is not None:
try:
from aiter import flash_attn_varlen_func
return flash_attn_varlen_func, True
except ImportError:
from flash_attn import flash_attn_varlen_func
else:
from vllm.vllm_flash_attn import flash_attn_varlen_func
return flash_attn_varlen_func, False
def _windowed_attention(
self,
x: torch.Tensor,
@@ -86,7 +104,7 @@ def _windowed_attention(
q, k = self._apply_dense_rope(q, k) if rope_cache is None else _cached_reference_rope(self, q, k, rope_cache)
if q.dtype not in (torch.bfloat16, torch.float16):
return sdpa(x, input_lengths)
from vllm.vllm_flash_attn import flash_attn_varlen_func
flash_attn_varlen_func, is_aiter = _reference_flash_attn_varlen_func()
heads, dim = q.shape[1], q.shape[-1]
@@ -95,7 +113,16 @@ def _windowed_attention(
starts = torch.arange(0, (batch + 1) * seqlen, seqlen, device=x.device, dtype=torch.int32)
window = [self.context - 1, 0] if self.causal and self.context is not None else [-1, -1]
out = flash_attn_varlen_func(
kwargs = {}
if torch.version.hip is None:
# Only vLLM's NVIDIA wrapper accepts its backend-selection argument.
kwargs["fa_version"] = fa_version
elif is_aiter and torch.is_grad_enabled() and any(t.requires_grad for t in (q, k, v)):
# AITER's autograd wrapper requires LSE to be retained for backward.
# The inference path avoids this allocation, and upstream flash-attn
# manages its own autograd state without this AITER-specific keyword.
kwargs["return_lse"] = True
result = flash_attn_varlen_func(
packed(q),
packed(k),
packed(v),
@@ -105,8 +132,10 @@ def _windowed_attention(
cu_seqlens_k=starts,
causal=self.causal,
window_size=window,
fa_version=fa_version,
).view(batch, seqlen, heads, dim)
**kwargs,
)
out = result[0] if isinstance(result, tuple) else result
out = out.view(batch, seqlen, heads, dim)
if not skip_padded_query_mask:
valid = (torch.arange(seqlen, device=x.device).view(1, seqlen) < input_lengths.view(-1, 1)).view(
batch, seqlen, 1, 1
@@ -370,6 +370,11 @@ class Qwen3TTSPromptEmbedsBuilder:
# CustomVoice prompt pieces that do not depend on the text, per
# (device, speaker, language): bounded by the model's voices.
self._custom_voice_prompt_cache: dict[tuple, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
# Speaker codec id and embedding per (device, speaker).
self._speaker_embed_cache: dict[tuple[str, str], tuple[int, torch.Tensor]] = {}
# Text-conditioning embeddings projected by ``preprocess_infos_batch``
# for the current step's new requests, keyed by request id.
self._batched_text_embeds: dict[str, torch.Tensor] = {}
self._ref_audio_artifact_cache_max_entries = int(ref_audio_artifact_cache_max_entries)
self._ref_audio_artifact_cache: OrderedDict[str, dict[str, torch.Tensor | bool]] = OrderedDict()
@@ -752,6 +757,18 @@ class Qwen3TTSPromptEmbedsBuilder:
return bool(value)
return False
@staticmethod
def _non_streaming_mode(info_dict: dict[str, Any], task_type: str) -> bool:
value = info_dict.get("non_streaming_mode")
if isinstance(value, list):
value = value[0] if value else None
if isinstance(value, bool):
return value
# Match official inference defaults:
# - CustomVoice/VoiceDesign: non_streaming_mode=True
# - Base: non_streaming_mode=False
return task_type in ("CustomVoice", "VoiceDesign")
@staticmethod
def _needs_initial_prompt_preprocess(info_dict: dict[str, Any]) -> bool:
embed = info_dict.get("embed")
@@ -806,15 +823,16 @@ class Qwen3TTSPromptEmbedsBuilder:
req_infos: list[dict[str, Any]],
device: torch.device,
) -> None:
"""Batch Base voice-clone ref-audio codec extraction for current prefill requests.
"""Batch per-request prompt work for current prefill requests.
Tokenizes assistant / ref text and runs the talker-supplied
``encode_ref_audio_batch`` callable once per sample-rate group,
stashing the precomputed artifacts under
For Base voice cloning, tokenizes assistant / ref text and runs the
talker-supplied ``encode_ref_audio_batch`` callable once per
sample-rate group, stashing the precomputed artifacts under
:data:`PRECOMPUTED_TEXT_IDS_KEY`, :data:`PRECOMPUTED_REF_IDS_KEY`,
:data:`PRECOMPUTED_REF_CODE_KEY`, and :data:`NORMALIZED_REF_AUDIO_KEY`
so the per-request :meth:`build_prompt_embeds` call can skip the
equivalent serial work.
:data:`PRECOMPUTED_REF_CODE_KEY`, and :data:`NORMALIZED_REF_AUDIO_KEY`.
For non-streaming CustomVoice / VoiceDesign, projects every request's
text at once (:meth:`_project_texts_batch`). The per-request
:meth:`build_prompt_embeds` call then skips the equivalent serial work.
"""
pending_text: list[tuple[dict[str, Any], str]] = []
pending_ref_text: list[tuple[dict[str, Any], str]] = []
@@ -919,6 +937,54 @@ class Qwen3TTSPromptEmbedsBuilder:
if isinstance(cache_key, str) and cache_key:
self.put_ref_audio_artifacts(cache_key, ref_code=ref_code_t)
self._batched_text_embeds = self._project_texts_batch(req_infos)
def _project_texts_batch(self, req_infos: list[dict[str, Any]]) -> dict[str, torch.Tensor]:
"""Embed and project the text of new non-streaming CustomVoice / VoiceDesign requests at once.
Each request otherwise uploads its text ids and runs the text
embedding + projection MLP on its own. Here every request's ids go up
in one pinned host-to-device copy and the MLP runs once over the
concatenated text tokens; the rows are independent, so each request's
slice equals its own per-request result. The device ids replace
:data:`PRECOMPUTED_TEXT_IDS_KEY` so ``build_prompt_embeds`` skips the
per-request copy too.
"""
items: list[tuple[dict[str, Any], str, np.ndarray]] = []
for info_dict in req_infos:
req_id = info_dict.get("req_id")
if not isinstance(req_id, str) or not self._needs_initial_prompt_preprocess(info_dict):
continue
task_type = first_value(info_dict.get("task_type"), "CustomVoice")
if task_type not in ("CustomVoice", "VoiceDesign") or not self._non_streaming_mode(info_dict, task_type):
continue
ids = first_value(info_dict.get(PRECOMPUTED_TEXT_IDS_KEY))
if isinstance(ids, torch.Tensor) and ids.device.type == "cpu":
ids = ids.reshape(-1).numpy()
elif isinstance(ids, list) and ids and all(isinstance(v, (int, np.integer)) for v in ids):
ids = np.asarray(ids)
elif not (isinstance(ids, np.ndarray) and ids.ndim == 1):
continue
# The assistant template wraps the text in 3 leading and 5 trailing tokens.
if ids.size <= 8:
continue
items.append((info_dict, req_id, ids.astype(np.int64, copy=False)))
if not items:
return {}
full_ids = [ids for _info, _req_id, ids in items]
text_ids = [ids[3:-5] for ids in full_ids]
packed = to_device_nonblocking(torch.from_numpy(np.concatenate(full_ids + text_ids)), self._device())
num_full = sum(ids.size for ids in full_ids)
projected = self._text_projection(self._text_embedding(packed[num_full:].view(1, -1)))
embeds: dict[str, torch.Tensor] = {}
full_offset = text_offset = 0
for (info_dict, req_id, ids), text in zip(items, text_ids, strict=True):
info_dict[PRECOMPUTED_TEXT_IDS_KEY] = packed[full_offset : full_offset + ids.size].view(1, -1)
embeds[req_id] = projected[:, text_offset : text_offset + text.size]
full_offset += ids.size
text_offset += text.size
return embeds
# -------------------- ICL prompt assembly --------------------
def _generate_icl_prompt(
@@ -1010,18 +1076,7 @@ class Qwen3TTSPromptEmbedsBuilder:
text = (info_dict.get("text") or [""])[0]
language = (info_dict.get("language") or ["Auto"])[0]
non_streaming_mode_val = info_dict.get("non_streaming_mode")
if isinstance(non_streaming_mode_val, list):
non_streaming_mode_raw = non_streaming_mode_val[0] if non_streaming_mode_val else None
else:
non_streaming_mode_raw = non_streaming_mode_val
if isinstance(non_streaming_mode_raw, bool):
non_streaming_mode = non_streaming_mode_raw
else:
# Match official inference defaults:
# - CustomVoice/VoiceDesign: non_streaming_mode=True
# - Base: non_streaming_mode=False
non_streaming_mode = task_type in ("CustomVoice", "VoiceDesign")
non_streaming_mode = self._non_streaming_mode(info_dict, task_type)
# Text ids for assistant template (always).
tok = self.get_text_tokenizer()
@@ -1412,15 +1467,19 @@ class Qwen3TTSPromptEmbedsBuilder:
)
if not speaker:
raise ValueError("CustomVoice requires additional_information.speaker.")
spk_id_map = {k.lower(): v for k, v in (getattr(talker_config, "spk_id", None) or {}).items()}
if speaker not in spk_id_map:
raise ValueError(f"Unsupported speaker: {speaker}")
spk_id = spk_id_map[speaker]
# Keep it at least 1D; embedding on a 0-d tensor can return 1D.
spk_tensor = index_to_device([spk_id], input_ids.device)
spk_embed = codec_embed(spk_tensor)
if spk_embed.ndim in (1, 2):
spk_embed = spk_embed.view(1, 1, -1)
speaker_key = (str(input_ids.device), speaker)
cached_speaker = self._speaker_embed_cache.get(speaker_key)
if cached_speaker is None:
spk_id_map = {k.lower(): v for k, v in (getattr(talker_config, "spk_id", None) or {}).items()}
if speaker not in spk_id_map:
raise ValueError(f"Unsupported speaker: {speaker}")
spk_id = spk_id_map[speaker]
# Keep it at least 1D; embedding on a 0-d tensor can return 1D.
spk_embed = codec_embed(index_to_device([spk_id], input_ids.device))
if spk_embed.ndim in (1, 2):
spk_embed = spk_embed.view(1, 1, -1)
cached_speaker = self._speaker_embed_cache[speaker_key] = (spk_id, spk_embed)
spk_id, spk_embed = cached_speaker
speaker_embed = spk_embed
codec_input = torch.cat([codec_input_0, speaker_embed, codec_input_1], dim=1)
@@ -1452,7 +1511,9 @@ class Qwen3TTSPromptEmbedsBuilder:
talker_prompt = prefix
if non_streaming_mode:
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
text_all = self._batched_text_embeds.pop(str(info_dict.get("req_id")), None)
if text_all is None:
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
text_all = torch.cat([text_all, tts_eos_embed], dim=1)
talker_prompt = torch.cat([talker_prompt, text_all + pad_row, tail], dim=1)
trailing_text_hidden = tts_pad_embed
@@ -1477,7 +1538,9 @@ class Qwen3TTSPromptEmbedsBuilder:
talker_prompt = torch.cat((role_embed, codec_prefix), dim=1)
if non_streaming_mode:
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
text_all = self._batched_text_embeds.pop(str(info_dict.get("req_id")), None)
if text_all is None:
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
text_all = torch.cat([text_all, tts_eos_embed], dim=1)
pad_ids = torch.full(
(1, int(text_all.shape[1])),