[Model] Support NVIDIA-NemotronLabs-VoiceChat-11B offline speech-to-speech (#5842)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yuekai ZhangandClaude Fable 5 authored and GitHub committed 2026-08-12 16:25:48 +08:00
1 parent 9b0df3909a
commit 5be7541d01
46 files changed
+13034

No files matched your search

+22
View File
@@ -30,6 +30,11 @@ _ARCH_TO_MODEL_TYPE: dict[str, str] = {
# the HF config without manual overrides.
"PersonaPlexTalkerForConditionalGeneration": "personaplex",
"PersonaPlexCode2Wav": "personaplex",
# NemotronVoiceChat ships a NeMo-style config.json without model_type;
# create_model_config() patches model_type=nemotron_voicechat from the arch.
"NemotronVoiceChatThinkerForConditionalGeneration": "nemotron_voicechat",
"NemotronVoiceChatTalkerForConditionalGeneration": "nemotron_voicechat",
"NemotronVoiceChatCode2Wav": "nemotron_voicechat",
"VoxCPM2TalkerForConditionalGeneration": "voxcpm2",
}
@@ -56,6 +61,9 @@ def _register_omni_hf_configs() -> None:
MossTTSLocalConfig,
MossTTSRealtimeConfig,
)
from vllm_omni.model_executor.models.nemotron_voicechat.configuration_nemotron_voicechat import (
NemotronVoiceChatConfig,
)
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
PersonaPlexConfig,
)
@@ -87,6 +95,7 @@ def _register_omni_hf_configs() -> None:
("moss_tts_realtime", MossTTSRealtimeConfig),
("qwen3_tts", Qwen3TTSConfig),
("personaplex", PersonaPlexConfig),
("nemotron_voicechat", NemotronVoiceChatConfig),
("cosyvoice3", CosyVoice3Config),
("glm_tts", GLMTTSConfig),
("omnivoice", OmniVoiceConfig),
@@ -324,6 +333,19 @@ class OmniEngineArgs(EngineArgs):
if model_type is not None:
self._patch_empty_hf_config(model_type)
# NemotronVoiceChat: the checkpoint's only tokenizer subfolder is the
# auxiliary rnnt_tokenizer/ (ASR loss vocab), which the generic
# subfolder auto-detect below would wrongly pick up. The text tokenizer
# is the Nemotron backbone's, resolved from a local override or its
# HF id (cache-friendly).
if not self.tokenizer and self.model_arch in (
"NemotronVoiceChatThinkerForConditionalGeneration",
"NemotronVoiceChatTalkerForConditionalGeneration",
"NemotronVoiceChatCode2Wav",
):
self.tokenizer = os.environ.get("NEMOTRON_VOICECHAT_LLM_PATH") or "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
logger.info("NemotronVoiceChat: using text tokenizer from %s", self.tokenizer)
# Auto-detect tokenizer for models that store it in a subdirectory
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
if not self.tokenizer and self.model: