[Perf] Optimize Higgs Audio v3 serving (#4204)
Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
21 files changed
+1976
-734
No files matched your search
@@ -162,6 +162,54 @@ def _build_delay_pattern(codes: torch.Tensor) -> torch.Tensor:
|
||||
_ENCODER_CACHE: Any | None = None
|
||||
_K2_OMNIVOICE_REPO = "k2-fsa/OmniVoice"
|
||||
_K2_OMNIVOICE_SUBDIR = "audio_tokenizer"
|
||||
_AUDIO_TOKENIZER_PATH_ENVS = (
|
||||
"HIGGS_AUDIO_TOKENIZER_PATH",
|
||||
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_audio_tokenizer_dir(path: str) -> str | None:
|
||||
if not path:
|
||||
return None
|
||||
expanded = os.path.abspath(os.path.expanduser(path))
|
||||
if os.path.isfile(os.path.join(expanded, "config.json")):
|
||||
return expanded
|
||||
nested = os.path.join(expanded, _K2_OMNIVOICE_SUBDIR)
|
||||
if os.path.isfile(os.path.join(nested, "config.json")):
|
||||
return nested
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_audio_tokenizer_dir() -> str | None:
|
||||
"""Resolve the Higgs codec encoder dir without requiring online HF access.
|
||||
|
||||
Voice cloning runs in the API server process. Production H20 jobs are often
|
||||
offline, so prefer explicit local directories and already-populated HF cache
|
||||
before falling back to ``snapshot_download``.
|
||||
"""
|
||||
for env_name in _AUDIO_TOKENIZER_PATH_ENVS:
|
||||
candidate = _normalize_audio_tokenizer_dir(os.getenv(env_name, ""))
|
||||
if candidate is not None:
|
||||
return candidate
|
||||
|
||||
from huggingface_hub import try_to_load_from_cache
|
||||
|
||||
cached_config = try_to_load_from_cache(
|
||||
repo_id=_K2_OMNIVOICE_REPO,
|
||||
filename=f"{_K2_OMNIVOICE_SUBDIR}/config.json",
|
||||
)
|
||||
if isinstance(cached_config, str) and os.path.isfile(cached_config):
|
||||
return os.path.dirname(cached_config)
|
||||
|
||||
from huggingface_hub.constants import HF_HUB_CACHE
|
||||
|
||||
safe = _K2_OMNIVOICE_REPO.replace("/", "--")
|
||||
snapshots_dir = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
|
||||
if os.path.isdir(snapshots_dir):
|
||||
for rev in os.listdir(snapshots_dir):
|
||||
candidate = _normalize_audio_tokenizer_dir(os.path.join(snapshots_dir, rev))
|
||||
if candidate is not None:
|
||||
return candidate
|
||||
|
||||
|
||||
def _load_audio_tokenizer():
|
||||
@@ -180,14 +228,17 @@ def _load_audio_tokenizer():
|
||||
|
||||
Only the ``audio_tokenizer/`` subdirectory (~806 MB) is downloaded.
|
||||
"""
|
||||
from huggingface_hub import snapshot_download
|
||||
from transformers import HiggsAudioV2TokenizerModel
|
||||
|
||||
repo_path = snapshot_download(
|
||||
_K2_OMNIVOICE_REPO,
|
||||
allow_patterns=[f"{_K2_OMNIVOICE_SUBDIR}/*"],
|
||||
)
|
||||
audio_tokenizer_dir = os.path.join(repo_path, _K2_OMNIVOICE_SUBDIR)
|
||||
audio_tokenizer_dir = _resolve_audio_tokenizer_dir()
|
||||
if audio_tokenizer_dir is None:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
repo_path = snapshot_download(
|
||||
_K2_OMNIVOICE_REPO,
|
||||
allow_patterns=[f"{_K2_OMNIVOICE_SUBDIR}/*"],
|
||||
)
|
||||
audio_tokenizer_dir = os.path.join(repo_path, _K2_OMNIVOICE_SUBDIR)
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model = HiggsAudioV2TokenizerModel.from_pretrained(audio_tokenizer_dir).to(device)
|
||||
return model.eval()
|
||||
|
||||
@@ -93,8 +93,13 @@ class HiggsAudioV3Config(PretrainedConfig):
|
||||
tts_token_id: int | None = None,
|
||||
text_token_id: int | None = None,
|
||||
audio_continuation_id: int | None = None,
|
||||
enable_flashinfer_api_unwrap: bool = True,
|
||||
enable_mlp_cudagraph: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# Legacy perf knob removed: Higgs v3 scheduler tokens now come from
|
||||
# SamplerOutput.sampled_token_ids only.
|
||||
kwargs.pop("enable_cpu_token_override", None)
|
||||
self.audio_token_id = audio_token_id
|
||||
self.mel_per_sample = mel_per_sample
|
||||
|
||||
@@ -124,6 +129,8 @@ class HiggsAudioV3Config(PretrainedConfig):
|
||||
self.tts_token_id = tts_token_id
|
||||
self.text_token_id = text_token_id
|
||||
self.audio_continuation_id = audio_continuation_id
|
||||
self.enable_flashinfer_api_unwrap = bool(enable_flashinfer_api_unwrap)
|
||||
self.enable_mlp_cudagraph = bool(enable_mlp_cudagraph)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ __all__ = [
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Prefix under which codec weights are stored in the v3 checkpoint.
|
||||
_CODEC_PREFIX = "tied.embedding.modality_embeddings.0.model."
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -32,6 +32,7 @@ from vllm_omni.data_entry_keys import (
|
||||
OmniPayload,
|
||||
OmniPayloadStruct,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
__all__ = ["talker2code2wav", "talker2code2wav_async_chunk"]
|
||||
|
||||
@@ -48,6 +49,15 @@ _DEFAULT_CODEC_LEFT_CONTEXT_FRAMES = 25
|
||||
_DEFAULT_CODEC_RIGHT_HOLDBACK_FRAMES = 4
|
||||
|
||||
|
||||
def _empty_code2wav_prompt() -> Any:
|
||||
return OmniTokensPrompt(
|
||||
prompt_token_ids=[],
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
additional_information=None,
|
||||
)
|
||||
|
||||
|
||||
def _revert_delay_pattern(audio_codes_qt: torch.Tensor) -> torch.Tensor:
|
||||
"""Reverse the MusicGen-style delay pattern.
|
||||
|
||||
@@ -91,9 +101,8 @@ def talker2code2wav(
|
||||
_requires_multimodal_data: bool = False,
|
||||
) -> list[Any]:
|
||||
"""Sync: collect all talker codes, revert delay pattern, filter, pass to code2wav."""
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
code2wav_inputs: list[OmniTokensPrompt] = []
|
||||
code2wav_inputs: list[Any] = []
|
||||
for talker_output in source_outputs:
|
||||
if not talker_output.finished:
|
||||
continue
|
||||
@@ -103,14 +112,7 @@ def talker2code2wav(
|
||||
|
||||
audio_codes = mm_codes.get("audio")
|
||||
if audio_codes is None or not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[],
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
additional_information=None,
|
||||
)
|
||||
)
|
||||
code2wav_inputs.append(_empty_code2wav_prompt())
|
||||
continue
|
||||
|
||||
audio_codes = audio_codes.to(torch.long)
|
||||
@@ -137,22 +139,9 @@ def talker2code2wav(
|
||||
# Step 1: Revert delay pattern
|
||||
try:
|
||||
codes_qt = _revert_delay_pattern(codes_qt)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"Skipping request: insufficient frames for delay pattern "
|
||||
"reversal (T=%d < Q=%d). This is a request-level error, "
|
||||
"not a server-fatal condition.",
|
||||
codes_qt.shape[1],
|
||||
codes_qt.shape[0],
|
||||
)
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[],
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
additional_information=None,
|
||||
)
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.warning("Skipping invalid Higgs Audio v3 code sequence for Stage 1: %s", exc)
|
||||
code2wav_inputs.append(_empty_code2wav_prompt())
|
||||
continue
|
||||
|
||||
# Step 2: Replace out-of-range codes (BOC=1024, EOC=1025, -1) with 0.
|
||||
@@ -172,14 +161,7 @@ def talker2code2wav(
|
||||
codes_qt = codes_qt[:, :-1]
|
||||
|
||||
if codes_qt.numel() == 0:
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[],
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
additional_information=None,
|
||||
)
|
||||
)
|
||||
code2wav_inputs.append(_empty_code2wav_prompt())
|
||||
continue
|
||||
|
||||
# Code2Wav expects codebook-major flat: [Q * num_frames]
|
||||
|
||||
Reference in new issue
Block a user