[Perf] Optimize Higgs Audio v3 serving (#4204)

Signed-off-by: Sy03 <1370724210@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-06-11 13:29:31 -07:00
1 parent eb964dcec5
commit e27fd46ab9
21 files changed
+1976 -734

No files matched your search

@@ -162,6 +162,54 @@ def _build_delay_pattern(codes: torch.Tensor) -> torch.Tensor:
_ENCODER_CACHE: Any | None = None
_K2_OMNIVOICE_REPO = "k2-fsa/OmniVoice"
_K2_OMNIVOICE_SUBDIR = "audio_tokenizer"
_AUDIO_TOKENIZER_PATH_ENVS = (
"HIGGS_AUDIO_TOKENIZER_PATH",
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
)
def _normalize_audio_tokenizer_dir(path: str) -> str | None:
if not path:
return None
expanded = os.path.abspath(os.path.expanduser(path))
if os.path.isfile(os.path.join(expanded, "config.json")):
return expanded
nested = os.path.join(expanded, _K2_OMNIVOICE_SUBDIR)
if os.path.isfile(os.path.join(nested, "config.json")):
return nested
return None
def _resolve_audio_tokenizer_dir() -> str | None:
"""Resolve the Higgs codec encoder dir without requiring online HF access.
Voice cloning runs in the API server process. Production H20 jobs are often
offline, so prefer explicit local directories and already-populated HF cache
before falling back to ``snapshot_download``.
"""
for env_name in _AUDIO_TOKENIZER_PATH_ENVS:
candidate = _normalize_audio_tokenizer_dir(os.getenv(env_name, ""))
if candidate is not None:
return candidate
from huggingface_hub import try_to_load_from_cache
cached_config = try_to_load_from_cache(
repo_id=_K2_OMNIVOICE_REPO,
filename=f"{_K2_OMNIVOICE_SUBDIR}/config.json",
)
if isinstance(cached_config, str) and os.path.isfile(cached_config):
return os.path.dirname(cached_config)
from huggingface_hub.constants import HF_HUB_CACHE
safe = _K2_OMNIVOICE_REPO.replace("/", "--")
snapshots_dir = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
if os.path.isdir(snapshots_dir):
for rev in os.listdir(snapshots_dir):
candidate = _normalize_audio_tokenizer_dir(os.path.join(snapshots_dir, rev))
if candidate is not None:
return candidate
def _load_audio_tokenizer():
@@ -180,14 +228,17 @@ def _load_audio_tokenizer():
Only the ``audio_tokenizer/`` subdirectory (~806 MB) is downloaded.
"""
from huggingface_hub import snapshot_download
from transformers import HiggsAudioV2TokenizerModel
repo_path = snapshot_download(
_K2_OMNIVOICE_REPO,
allow_patterns=[f"{_K2_OMNIVOICE_SUBDIR}/*"],
)
audio_tokenizer_dir = os.path.join(repo_path, _K2_OMNIVOICE_SUBDIR)
audio_tokenizer_dir = _resolve_audio_tokenizer_dir()
if audio_tokenizer_dir is None:
from huggingface_hub import snapshot_download
repo_path = snapshot_download(
_K2_OMNIVOICE_REPO,
allow_patterns=[f"{_K2_OMNIVOICE_SUBDIR}/*"],
)
audio_tokenizer_dir = os.path.join(repo_path, _K2_OMNIVOICE_SUBDIR)
device = "cuda" if torch.cuda.is_available() else "cpu"
model = HiggsAudioV2TokenizerModel.from_pretrained(audio_tokenizer_dir).to(device)
return model.eval()
@@ -93,8 +93,13 @@ class HiggsAudioV3Config(PretrainedConfig):
tts_token_id: int | None = None,
text_token_id: int | None = None,
audio_continuation_id: int | None = None,
enable_flashinfer_api_unwrap: bool = True,
enable_mlp_cudagraph: bool = True,
**kwargs: Any,
) -> None:
# Legacy perf knob removed: Higgs v3 scheduler tokens now come from
# SamplerOutput.sampled_token_ids only.
kwargs.pop("enable_cpu_token_override", None)
self.audio_token_id = audio_token_id
self.mel_per_sample = mel_per_sample
@@ -124,6 +129,8 @@ class HiggsAudioV3Config(PretrainedConfig):
self.tts_token_id = tts_token_id
self.text_token_id = text_token_id
self.audio_continuation_id = audio_continuation_id
self.enable_flashinfer_api_unwrap = bool(enable_flashinfer_api_unwrap)
self.enable_mlp_cudagraph = bool(enable_mlp_cudagraph)
super().__init__(**kwargs)
@@ -35,6 +35,7 @@ __all__ = [
logger = init_logger(__name__)
# Prefix under which codec weights are stored in the v3 checkpoint.
_CODEC_PREFIX = "tied.embedding.modality_embeddings.0.model."
File diff suppressed because it is too large. Load diff