[Model] Support NVIDIA-NemotronLabs-VoiceChat-11B offline speech-to-speech (#5842)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9b0df3909a
commit
5be7541d01
46 files changed
+13034
No files matched your search
@@ -30,6 +30,11 @@ _ARCH_TO_MODEL_TYPE: dict[str, str] = {
|
||||
# the HF config without manual overrides.
|
||||
"PersonaPlexTalkerForConditionalGeneration": "personaplex",
|
||||
"PersonaPlexCode2Wav": "personaplex",
|
||||
# NemotronVoiceChat ships a NeMo-style config.json without model_type;
|
||||
# create_model_config() patches model_type=nemotron_voicechat from the arch.
|
||||
"NemotronVoiceChatThinkerForConditionalGeneration": "nemotron_voicechat",
|
||||
"NemotronVoiceChatTalkerForConditionalGeneration": "nemotron_voicechat",
|
||||
"NemotronVoiceChatCode2Wav": "nemotron_voicechat",
|
||||
"VoxCPM2TalkerForConditionalGeneration": "voxcpm2",
|
||||
}
|
||||
|
||||
@@ -56,6 +61,9 @@ def _register_omni_hf_configs() -> None:
|
||||
MossTTSLocalConfig,
|
||||
MossTTSRealtimeConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.configuration_nemotron_voicechat import (
|
||||
NemotronVoiceChatConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
|
||||
PersonaPlexConfig,
|
||||
)
|
||||
@@ -87,6 +95,7 @@ def _register_omni_hf_configs() -> None:
|
||||
("moss_tts_realtime", MossTTSRealtimeConfig),
|
||||
("qwen3_tts", Qwen3TTSConfig),
|
||||
("personaplex", PersonaPlexConfig),
|
||||
("nemotron_voicechat", NemotronVoiceChatConfig),
|
||||
("cosyvoice3", CosyVoice3Config),
|
||||
("glm_tts", GLMTTSConfig),
|
||||
("omnivoice", OmniVoiceConfig),
|
||||
@@ -324,6 +333,19 @@ class OmniEngineArgs(EngineArgs):
|
||||
if model_type is not None:
|
||||
self._patch_empty_hf_config(model_type)
|
||||
|
||||
# NemotronVoiceChat: the checkpoint's only tokenizer subfolder is the
|
||||
# auxiliary rnnt_tokenizer/ (ASR loss vocab), which the generic
|
||||
# subfolder auto-detect below would wrongly pick up. The text tokenizer
|
||||
# is the Nemotron backbone's, resolved from a local override or its
|
||||
# HF id (cache-friendly).
|
||||
if not self.tokenizer and self.model_arch in (
|
||||
"NemotronVoiceChatThinkerForConditionalGeneration",
|
||||
"NemotronVoiceChatTalkerForConditionalGeneration",
|
||||
"NemotronVoiceChatCode2Wav",
|
||||
):
|
||||
self.tokenizer = os.environ.get("NEMOTRON_VOICECHAT_LLM_PATH") or "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
|
||||
logger.info("NemotronVoiceChat: using text tokenizer from %s", self.tokenizer)
|
||||
|
||||
# Auto-detect tokenizer for models that store it in a subdirectory
|
||||
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
|
||||
if not self.tokenizer and self.model:
|
||||
|
||||
Reference in new issue
Block a user