Merge current Omni main and retain conditioning regression coverage
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
6f5f9b97e3
21 files changed
+787
-134
No files matched your search
@@ -73,6 +73,27 @@ logger = init_logger(__name__)
|
||||
_RUNTIME_COMPONENTS_CACHE: dict[str, dict] = {}
|
||||
|
||||
|
||||
def _normalize_request_conditioning(payload: dict) -> dict:
|
||||
"""Unwrap singleton conditioning containers at the per-request boundary.
|
||||
|
||||
A processor/prefix-cache passthrough can retain a one-item tensor list.
|
||||
The code2wav schema consumes one request, so that wrapper carries no
|
||||
batch dimension. A multi-item list here is an unsplit batch and must
|
||||
fail instead of silently choosing another request's voice.
|
||||
"""
|
||||
embed = payload.get("embed")
|
||||
if not isinstance(embed, Mapping):
|
||||
return payload
|
||||
normalized = dict(embed)
|
||||
for key in ("speech_token", "speech_feat", "embedding", "speech_token_len"):
|
||||
value = normalized.get(key)
|
||||
if isinstance(value, (list, tuple)):
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"CosyVoice3 per-request {key} contains an unsplit batch of {len(value)} items")
|
||||
normalized[key] = value[0]
|
||||
return {**payload, "embed": normalized}
|
||||
|
||||
|
||||
def _cosyvoice3_trt_enabled() -> bool:
|
||||
"""COSYVOICE3_TRT env toggle (default on) for the optional TensorRT paths.
|
||||
|
||||
@@ -1500,9 +1521,10 @@ class CosyVoice3Model(
|
||||
# [total_tokens, hidden]
|
||||
hidden_states = self.model.llm(inputs_embeds, positions)
|
||||
|
||||
if getattr(self, "_mrv2_tensor_output", False) or cosyvoice3_packed_inference_enabled():
|
||||
return hidden_states
|
||||
return self.make_omni_output(hidden_states, **kwargs)
|
||||
# Both runners attach the live step's conditioning after replay.
|
||||
# Capturing an OmniOutput here would retain the dummy prefill lists
|
||||
# and re-emit them during decode, replacing the real reference.
|
||||
return hidden_states
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
# Lazily swap the flow-decoder estimator to a TensorRT engine on the
|
||||
# first code2wav step (after weights are loaded), gated by the same
|
||||
@@ -1538,7 +1560,7 @@ class CosyVoice3Model(
|
||||
|
||||
for idx, req_ids in enumerate(request_ids_list):
|
||||
raw = runtime_info[idx] if idx < len(runtime_info) and isinstance(runtime_info[idx], dict) else {}
|
||||
payload = to_struct(raw)
|
||||
payload = to_struct(_normalize_request_conditioning(raw))
|
||||
meta = payload.meta
|
||||
embed = payload.embed
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CosyVoice3 pipeline topology (frozen).
|
||||
|
||||
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
|
||||
@@ -12,11 +12,17 @@ Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
|
||||
through the shared-memory connector.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
from dataclasses import replace
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
|
||||
|
||||
@@ -35,8 +41,9 @@ COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload",
|
||||
sampling_constraints={
|
||||
# merged speech stop token (logsumexp of all 200 stop logits)
|
||||
"stop_token_ids": [6562],
|
||||
# Standard sampling can emit any of the 200 control tokens.
|
||||
# RAS merges their logits into 6562, which is also in this range.
|
||||
"stop_token_ids": list(range(6561, 6761)),
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
@@ -52,3 +59,28 @@ COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def resolve_cosyvoice3_pipeline(hf_config: PretrainedConfig | None = None) -> PipelineConfig | None:
|
||||
"""Select a streaming default compatible with the device and runtime.
|
||||
|
||||
Query the platform through NVML without initializing CUDA in the parent.
|
||||
Explicit deploy configs still take precedence over this pipeline default.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if hf_config is not None and not isinstance(hf_config, CosyVoice3Config):
|
||||
return None
|
||||
if current_platform.is_cuda():
|
||||
capability = current_platform.get_device_capability()
|
||||
if (
|
||||
capability is not None
|
||||
and capability.major == 9
|
||||
and current_platform.get_device_total_memory() >= 140 * 1024**3
|
||||
and shutil.which("nvidia-cuda-mps-control") is not None
|
||||
):
|
||||
return replace(
|
||||
COSYVOICE3_PIPELINE,
|
||||
default_deploy_config_name="cosyvoice3_packed_streaming_optimized_standard.yaml",
|
||||
)
|
||||
return COSYVOICE3_PIPELINE
|
||||
@@ -65,6 +65,24 @@ def _cached_reference_rope(self, q: torch.Tensor, k: torch.Tensor, cache: dict):
|
||||
return qo.view(batch, heads, seqlen, dim), ko.view(batch, heads, seqlen, dim)
|
||||
|
||||
|
||||
def _reference_flash_attn_varlen_func():
|
||||
"""Resolve the platform's variable-length FlashAttention entrypoint.
|
||||
|
||||
vLLM's bundled extension is NVIDIA-only. ROCm exposes the compatible
|
||||
unpacked-QKV API through AITER, with upstream flash-attn as its fallback.
|
||||
"""
|
||||
if torch.version.hip is not None:
|
||||
try:
|
||||
from aiter import flash_attn_varlen_func
|
||||
|
||||
return flash_attn_varlen_func, True
|
||||
except ImportError:
|
||||
from flash_attn import flash_attn_varlen_func
|
||||
else:
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
return flash_attn_varlen_func, False
|
||||
|
||||
|
||||
def _windowed_attention(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
@@ -86,7 +104,7 @@ def _windowed_attention(
|
||||
q, k = self._apply_dense_rope(q, k) if rope_cache is None else _cached_reference_rope(self, q, k, rope_cache)
|
||||
if q.dtype not in (torch.bfloat16, torch.float16):
|
||||
return sdpa(x, input_lengths)
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
flash_attn_varlen_func, is_aiter = _reference_flash_attn_varlen_func()
|
||||
|
||||
heads, dim = q.shape[1], q.shape[-1]
|
||||
|
||||
@@ -95,7 +113,16 @@ def _windowed_attention(
|
||||
|
||||
starts = torch.arange(0, (batch + 1) * seqlen, seqlen, device=x.device, dtype=torch.int32)
|
||||
window = [self.context - 1, 0] if self.causal and self.context is not None else [-1, -1]
|
||||
out = flash_attn_varlen_func(
|
||||
kwargs = {}
|
||||
if torch.version.hip is None:
|
||||
# Only vLLM's NVIDIA wrapper accepts its backend-selection argument.
|
||||
kwargs["fa_version"] = fa_version
|
||||
elif is_aiter and torch.is_grad_enabled() and any(t.requires_grad for t in (q, k, v)):
|
||||
# AITER's autograd wrapper requires LSE to be retained for backward.
|
||||
# The inference path avoids this allocation, and upstream flash-attn
|
||||
# manages its own autograd state without this AITER-specific keyword.
|
||||
kwargs["return_lse"] = True
|
||||
result = flash_attn_varlen_func(
|
||||
packed(q),
|
||||
packed(k),
|
||||
packed(v),
|
||||
@@ -105,8 +132,10 @@ def _windowed_attention(
|
||||
cu_seqlens_k=starts,
|
||||
causal=self.causal,
|
||||
window_size=window,
|
||||
fa_version=fa_version,
|
||||
).view(batch, seqlen, heads, dim)
|
||||
**kwargs,
|
||||
)
|
||||
out = result[0] if isinstance(result, tuple) else result
|
||||
out = out.view(batch, seqlen, heads, dim)
|
||||
if not skip_padded_query_mask:
|
||||
valid = (torch.arange(seqlen, device=x.device).view(1, seqlen) < input_lengths.view(-1, 1)).view(
|
||||
batch, seqlen, 1, 1
|
||||
|
||||
@@ -370,6 +370,11 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
# CustomVoice prompt pieces that do not depend on the text, per
|
||||
# (device, speaker, language): bounded by the model's voices.
|
||||
self._custom_voice_prompt_cache: dict[tuple, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
|
||||
# Speaker codec id and embedding per (device, speaker).
|
||||
self._speaker_embed_cache: dict[tuple[str, str], tuple[int, torch.Tensor]] = {}
|
||||
# Text-conditioning embeddings projected by ``preprocess_infos_batch``
|
||||
# for the current step's new requests, keyed by request id.
|
||||
self._batched_text_embeds: dict[str, torch.Tensor] = {}
|
||||
|
||||
self._ref_audio_artifact_cache_max_entries = int(ref_audio_artifact_cache_max_entries)
|
||||
self._ref_audio_artifact_cache: OrderedDict[str, dict[str, torch.Tensor | bool]] = OrderedDict()
|
||||
@@ -752,6 +757,18 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
return bool(value)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _non_streaming_mode(info_dict: dict[str, Any], task_type: str) -> bool:
|
||||
value = info_dict.get("non_streaming_mode")
|
||||
if isinstance(value, list):
|
||||
value = value[0] if value else None
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
# Match official inference defaults:
|
||||
# - CustomVoice/VoiceDesign: non_streaming_mode=True
|
||||
# - Base: non_streaming_mode=False
|
||||
return task_type in ("CustomVoice", "VoiceDesign")
|
||||
|
||||
@staticmethod
|
||||
def _needs_initial_prompt_preprocess(info_dict: dict[str, Any]) -> bool:
|
||||
embed = info_dict.get("embed")
|
||||
@@ -806,15 +823,16 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
req_infos: list[dict[str, Any]],
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""Batch Base voice-clone ref-audio codec extraction for current prefill requests.
|
||||
"""Batch per-request prompt work for current prefill requests.
|
||||
|
||||
Tokenizes assistant / ref text and runs the talker-supplied
|
||||
``encode_ref_audio_batch`` callable once per sample-rate group,
|
||||
stashing the precomputed artifacts under
|
||||
For Base voice cloning, tokenizes assistant / ref text and runs the
|
||||
talker-supplied ``encode_ref_audio_batch`` callable once per
|
||||
sample-rate group, stashing the precomputed artifacts under
|
||||
:data:`PRECOMPUTED_TEXT_IDS_KEY`, :data:`PRECOMPUTED_REF_IDS_KEY`,
|
||||
:data:`PRECOMPUTED_REF_CODE_KEY`, and :data:`NORMALIZED_REF_AUDIO_KEY`
|
||||
so the per-request :meth:`build_prompt_embeds` call can skip the
|
||||
equivalent serial work.
|
||||
:data:`PRECOMPUTED_REF_CODE_KEY`, and :data:`NORMALIZED_REF_AUDIO_KEY`.
|
||||
For non-streaming CustomVoice / VoiceDesign, projects every request's
|
||||
text at once (:meth:`_project_texts_batch`). The per-request
|
||||
:meth:`build_prompt_embeds` call then skips the equivalent serial work.
|
||||
"""
|
||||
pending_text: list[tuple[dict[str, Any], str]] = []
|
||||
pending_ref_text: list[tuple[dict[str, Any], str]] = []
|
||||
@@ -919,6 +937,54 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
if isinstance(cache_key, str) and cache_key:
|
||||
self.put_ref_audio_artifacts(cache_key, ref_code=ref_code_t)
|
||||
|
||||
self._batched_text_embeds = self._project_texts_batch(req_infos)
|
||||
|
||||
def _project_texts_batch(self, req_infos: list[dict[str, Any]]) -> dict[str, torch.Tensor]:
|
||||
"""Embed and project the text of new non-streaming CustomVoice / VoiceDesign requests at once.
|
||||
|
||||
Each request otherwise uploads its text ids and runs the text
|
||||
embedding + projection MLP on its own. Here every request's ids go up
|
||||
in one pinned host-to-device copy and the MLP runs once over the
|
||||
concatenated text tokens; the rows are independent, so each request's
|
||||
slice equals its own per-request result. The device ids replace
|
||||
:data:`PRECOMPUTED_TEXT_IDS_KEY` so ``build_prompt_embeds`` skips the
|
||||
per-request copy too.
|
||||
"""
|
||||
items: list[tuple[dict[str, Any], str, np.ndarray]] = []
|
||||
for info_dict in req_infos:
|
||||
req_id = info_dict.get("req_id")
|
||||
if not isinstance(req_id, str) or not self._needs_initial_prompt_preprocess(info_dict):
|
||||
continue
|
||||
task_type = first_value(info_dict.get("task_type"), "CustomVoice")
|
||||
if task_type not in ("CustomVoice", "VoiceDesign") or not self._non_streaming_mode(info_dict, task_type):
|
||||
continue
|
||||
ids = first_value(info_dict.get(PRECOMPUTED_TEXT_IDS_KEY))
|
||||
if isinstance(ids, torch.Tensor) and ids.device.type == "cpu":
|
||||
ids = ids.reshape(-1).numpy()
|
||||
elif isinstance(ids, list) and ids and all(isinstance(v, (int, np.integer)) for v in ids):
|
||||
ids = np.asarray(ids)
|
||||
elif not (isinstance(ids, np.ndarray) and ids.ndim == 1):
|
||||
continue
|
||||
# The assistant template wraps the text in 3 leading and 5 trailing tokens.
|
||||
if ids.size <= 8:
|
||||
continue
|
||||
items.append((info_dict, req_id, ids.astype(np.int64, copy=False)))
|
||||
if not items:
|
||||
return {}
|
||||
full_ids = [ids for _info, _req_id, ids in items]
|
||||
text_ids = [ids[3:-5] for ids in full_ids]
|
||||
packed = to_device_nonblocking(torch.from_numpy(np.concatenate(full_ids + text_ids)), self._device())
|
||||
num_full = sum(ids.size for ids in full_ids)
|
||||
projected = self._text_projection(self._text_embedding(packed[num_full:].view(1, -1)))
|
||||
embeds: dict[str, torch.Tensor] = {}
|
||||
full_offset = text_offset = 0
|
||||
for (info_dict, req_id, ids), text in zip(items, text_ids, strict=True):
|
||||
info_dict[PRECOMPUTED_TEXT_IDS_KEY] = packed[full_offset : full_offset + ids.size].view(1, -1)
|
||||
embeds[req_id] = projected[:, text_offset : text_offset + text.size]
|
||||
full_offset += ids.size
|
||||
text_offset += text.size
|
||||
return embeds
|
||||
|
||||
# -------------------- ICL prompt assembly --------------------
|
||||
|
||||
def _generate_icl_prompt(
|
||||
@@ -1010,18 +1076,7 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
|
||||
text = (info_dict.get("text") or [""])[0]
|
||||
language = (info_dict.get("language") or ["Auto"])[0]
|
||||
non_streaming_mode_val = info_dict.get("non_streaming_mode")
|
||||
if isinstance(non_streaming_mode_val, list):
|
||||
non_streaming_mode_raw = non_streaming_mode_val[0] if non_streaming_mode_val else None
|
||||
else:
|
||||
non_streaming_mode_raw = non_streaming_mode_val
|
||||
if isinstance(non_streaming_mode_raw, bool):
|
||||
non_streaming_mode = non_streaming_mode_raw
|
||||
else:
|
||||
# Match official inference defaults:
|
||||
# - CustomVoice/VoiceDesign: non_streaming_mode=True
|
||||
# - Base: non_streaming_mode=False
|
||||
non_streaming_mode = task_type in ("CustomVoice", "VoiceDesign")
|
||||
non_streaming_mode = self._non_streaming_mode(info_dict, task_type)
|
||||
|
||||
# Text ids for assistant template (always).
|
||||
tok = self.get_text_tokenizer()
|
||||
@@ -1412,15 +1467,19 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
)
|
||||
if not speaker:
|
||||
raise ValueError("CustomVoice requires additional_information.speaker.")
|
||||
spk_id_map = {k.lower(): v for k, v in (getattr(talker_config, "spk_id", None) or {}).items()}
|
||||
if speaker not in spk_id_map:
|
||||
raise ValueError(f"Unsupported speaker: {speaker}")
|
||||
spk_id = spk_id_map[speaker]
|
||||
# Keep it at least 1D; embedding on a 0-d tensor can return 1D.
|
||||
spk_tensor = index_to_device([spk_id], input_ids.device)
|
||||
spk_embed = codec_embed(spk_tensor)
|
||||
if spk_embed.ndim in (1, 2):
|
||||
spk_embed = spk_embed.view(1, 1, -1)
|
||||
speaker_key = (str(input_ids.device), speaker)
|
||||
cached_speaker = self._speaker_embed_cache.get(speaker_key)
|
||||
if cached_speaker is None:
|
||||
spk_id_map = {k.lower(): v for k, v in (getattr(talker_config, "spk_id", None) or {}).items()}
|
||||
if speaker not in spk_id_map:
|
||||
raise ValueError(f"Unsupported speaker: {speaker}")
|
||||
spk_id = spk_id_map[speaker]
|
||||
# Keep it at least 1D; embedding on a 0-d tensor can return 1D.
|
||||
spk_embed = codec_embed(index_to_device([spk_id], input_ids.device))
|
||||
if spk_embed.ndim in (1, 2):
|
||||
spk_embed = spk_embed.view(1, 1, -1)
|
||||
cached_speaker = self._speaker_embed_cache[speaker_key] = (spk_id, spk_embed)
|
||||
spk_id, spk_embed = cached_speaker
|
||||
speaker_embed = spk_embed
|
||||
codec_input = torch.cat([codec_input_0, speaker_embed, codec_input_1], dim=1)
|
||||
|
||||
@@ -1452,7 +1511,9 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
talker_prompt = prefix
|
||||
|
||||
if non_streaming_mode:
|
||||
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
|
||||
text_all = self._batched_text_embeds.pop(str(info_dict.get("req_id")), None)
|
||||
if text_all is None:
|
||||
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
|
||||
text_all = torch.cat([text_all, tts_eos_embed], dim=1)
|
||||
talker_prompt = torch.cat([talker_prompt, text_all + pad_row, tail], dim=1)
|
||||
trailing_text_hidden = tts_pad_embed
|
||||
@@ -1477,7 +1538,9 @@ class Qwen3TTSPromptEmbedsBuilder:
|
||||
talker_prompt = torch.cat((role_embed, codec_prefix), dim=1)
|
||||
|
||||
if non_streaming_mode:
|
||||
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
|
||||
text_all = self._batched_text_embeds.pop(str(info_dict.get("req_id")), None)
|
||||
if text_all is None:
|
||||
text_all = text_projection(text_embedding(input_ids[:, 3:-5]))
|
||||
text_all = torch.cat([text_all, tts_eos_embed], dim=1)
|
||||
pad_ids = torch.full(
|
||||
(1, int(text_all.shape[1])),
|
||||
|
||||
Reference in new issue
Block a user