From 3f504b45d2c097deddf84e03b35c51fa3777b016 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Tue, 21 Apr 2026 09:17:53 -0400 Subject: [PATCH] [Config Refactor] Migrate 5 TTS models (VoxCPM2 / CosyVoice3 / MiMo Audio / Voxtral TTS / Fish Speech S2 Pro) to Pipeline + Deploy schema (#2958) Signed-off-by: Yueqian Lin Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> --- docs/serving/speech_api.md | 15 +-- .../examples/offline_inference/cosyvoice3.md | 11 +- .../examples/offline_inference/mimo_audio.md | 12 -- .../examples/offline_inference/voxtral_tts.md | 7 +- .../examples/online_serving/fish_speech.md | 10 +- .../examples/online_serving/mimo_audio.md | 8 +- .../offline_inference/cosyvoice3/README.md | 11 +- .../cosyvoice3/verify_e2e_cosyvoice.py | 18 +-- .../offline_inference/mimo_audio/README.md | 12 -- .../offline_inference/mimo_audio/end2end.py | 9 +- .../offline_inference/voxtral_tts/README.md | 7 +- .../offline_inference/voxtral_tts/end2end.py | 9 +- examples/online_serving/fish_speech/README.md | 11 +- .../fish_speech/run_gradio_demo.sh | 11 +- .../online_serving/fish_speech/run_server.sh | 10 +- examples/online_serving/mimo_audio/README.md | 8 +- examples/online_serving/voxcpm2/README.md | 9 +- .../online_serving/voxcpm2/gradio_demo.py | 5 +- .../voxcpm2/openai_speech_client.py | 5 +- .../e2e/offline_inference/test_cosyvoice3.py | 114 ++++++++---------- tests/e2e/offline_inference/test_voxcpm2.py | 14 +-- .../e2e/offline_inference/test_voxtral_tts.py | 12 +- .../e2e/online_serving/test_cosyvoice3_tts.py | 13 +- tests/e2e/online_serving/test_mimo_audio.py | 30 +---- tests/e2e/online_serving/test_voxtral_tts.py | 7 +- tests/e2e/stage_configs/mimo_audio_ci.yaml | 71 ----------- vllm_omni/config/pipeline_registry.py | 20 +++ vllm_omni/config/stage_config.py | 25 +++- vllm_omni/deploy/cosyvoice3.yaml | 58 +++++++++ vllm_omni/deploy/fish_qwen3_omni.yaml | 61 ++++++++++ vllm_omni/deploy/mimo_audio.yaml | 56 +++++++++ vllm_omni/deploy/voxcpm2.yaml | 29 +++++ vllm_omni/deploy/voxtral_tts.yaml | 65 ++++++++++ vllm_omni/entrypoints/utils.py | 27 +++-- .../models/cosyvoice3/pipeline.py | 50 ++++++++ .../models/fish_speech/pipeline.py | 51 ++++++++ .../models/mimo_audio/pipeline.py | 54 +++++++++ .../model_executor/models/voxcpm2/pipeline.py | 35 ++++++ .../models/voxtral_tts/__init__.py | 0 .../models/voxtral_tts/pipeline.py | 45 +++++++ .../stage_configs/cosyvoice3.yaml | 53 -------- .../stage_configs/cosyvoice3_async_chunk.yaml | 81 ------------- .../stage_configs/fish_speech_s2_pro.yaml | 91 -------------- .../stage_configs/mimo_audio.yaml | 69 ----------- .../stage_configs/mimo_audio_async_chunk.yaml | 91 -------------- .../model_executor/stage_configs/voxcpm2.yaml | 36 ------ .../stage_configs/voxtral_tts.yaml | 100 --------------- 47 files changed, 702 insertions(+), 844 deletions(-) delete mode 100644 tests/e2e/stage_configs/mimo_audio_ci.yaml create mode 100644 vllm_omni/deploy/cosyvoice3.yaml create mode 100644 vllm_omni/deploy/fish_qwen3_omni.yaml create mode 100644 vllm_omni/deploy/mimo_audio.yaml create mode 100644 vllm_omni/deploy/voxcpm2.yaml create mode 100644 vllm_omni/deploy/voxtral_tts.yaml create mode 100644 vllm_omni/model_executor/models/cosyvoice3/pipeline.py create mode 100644 vllm_omni/model_executor/models/fish_speech/pipeline.py create mode 100644 vllm_omni/model_executor/models/mimo_audio/pipeline.py create mode 100644 vllm_omni/model_executor/models/voxcpm2/pipeline.py create mode 100644 vllm_omni/model_executor/models/voxtral_tts/__init__.py create mode 100644 vllm_omni/model_executor/models/voxtral_tts/pipeline.py delete mode 100644 vllm_omni/model_executor/stage_configs/cosyvoice3.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/cosyvoice3_async_chunk.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/mimo_audio.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/mimo_audio_async_chunk.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/voxcpm2.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/voxtral_tts.yaml diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 733811081..8f78d6a20 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -22,21 +22,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ --enforce-eager # Fish Speech S2 Pro -vllm-omni serve fishaudio/s2-pro \ - --stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \ - --omni \ - --port 8091 \ - --trust-remote-code \ - --enforce-eager \ - --gpu-memory-utilization 0.9 +vllm serve fishaudio/s2-pro --omni --port 8091 # Voxtral TTS -vllm serve mistralai/Voxtral-4B-TTS-2603 \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ - --omni \ - --port 8091 \ - --trust-remote-code \ - --enforce-eager +vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091 ``` ### Generate Speech diff --git a/docs/user_guide/examples/offline_inference/cosyvoice3.md b/docs/user_guide/examples/offline_inference/cosyvoice3.md index ebb7c02ef..d0638f414 100644 --- a/docs/user_guide/examples/offline_inference/cosyvoice3.md +++ b/docs/user_guide/examples/offline_inference/cosyvoice3.md @@ -61,10 +61,17 @@ Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features. - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform. -Stage wiring is configured in `vllm_omni/model_executor/stage_configs/cosyvoice3.yaml`. +Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`; +runtime tunables (batch size, memory limits, sampling) live in +`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by +HF `model_type` and defaults to `async_chunk: true` (shared-memory +streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the +legacy sync path where stage 1 runs `text2flow` over the full +speech-token sequence. - Stage 0 emits latent speech tokens. -- Stage 1 consumes them via `custom_process_input_func` and outputs audio. +- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the + shared-memory connector (async-chunk mode) and outputs audio. ## Example materials diff --git a/docs/user_guide/examples/offline_inference/mimo_audio.md b/docs/user_guide/examples/offline_inference/mimo_audio.md index 4e8052697..1cba2f77d 100644 --- a/docs/user_guide/examples/offline_inference/mimo_audio.md +++ b/docs/user_guide/examples/offline_inference/mimo_audio.md @@ -38,7 +38,6 @@ Run a single sample for basic TTS: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft ``` @@ -47,7 +46,6 @@ Run batch samples for basic TTS: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft \ --num-prompts {batch_size} @@ -65,7 +63,6 @@ Generate speech from text input: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft \ --text "The weather is so nice today." @@ -77,7 +74,6 @@ Generate speech with explicit voice style instructions: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft_with_instruct \ --text "The weather is so nice today." \ @@ -90,7 +86,6 @@ Generate speech using an audio reference for voice cloning: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft_with_audio \ --text "The weather is so nice today." \ @@ -103,7 +98,6 @@ Generate speech from text containing natural voice descriptions: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft_with_natural_instruction \ --text "In a panting young male voice, he said: I can't run anymore, wait for me!" @@ -115,7 +109,6 @@ Transcribe audio to text: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type audio_trancribing_sft \ --audio-path "./spoken_dialogue_assistant_turn_1.wav" @@ -127,7 +120,6 @@ Understand and analyze audio content with text queries: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type audio_understanding_sft \ --text "Summarize the audio." \ @@ -140,7 +132,6 @@ Audio understanding with reasoning chain: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type audio_understanding_sft_with_thinking \ --text "Summarize the audio." \ @@ -153,7 +144,6 @@ Multi-turn dialogue with audio input and output: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type spoken_dialogue_sft_multiturn \ --audio-path "./prompt_speech_zh_m.wav" @@ -167,7 +157,6 @@ Multi-turn dialogue converting speech to text: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type speech2text_dialogue_sft_multiturn ``` @@ -180,7 +169,6 @@ Multi-turn text-only dialogue: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type text_dialogue_sft_multiturn ``` diff --git a/docs/user_guide/examples/offline_inference/voxtral_tts.md b/docs/user_guide/examples/offline_inference/voxtral_tts.md index db4c2e26b..1a4f2c21b 100644 --- a/docs/user_guide/examples/offline_inference/voxtral_tts.md +++ b/docs/user_guide/examples/offline_inference/voxtral_tts.md @@ -13,28 +13,24 @@ When `mistral_common` has `SpeechRequest` support, prompt token IDs are built vi ```bash # Basic single-prompt with cheerful_female voice preset python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --write-audio --voice cheerful_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" # 32 replicate prompts with cheerful_female voice preset python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --num-prompts 32 --write-audio --voice cheerful_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" # Streaming with neutral_female voice preset python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --streaming --write-audio --voice neutral_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" # 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" @@ -42,7 +38,6 @@ python3 examples/offline_inference/voxtral_tts/end2end.py \ # Short debug prompt with reference audio # Note: Reference audio capability is not yet released. python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --write-audio \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "This is a test message." \ @@ -57,7 +52,7 @@ python3 examples/offline_inference/voxtral_tts/end2end.py \ | `--text TEXT` | Text to synthesize (default: `"This is a test message."`) | | `--audio-path PATH` | Path to reference audio file for voice cloning | | `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) | -| `--stage-configs-path PATH` | Path to stage configs YAML (currently it must be set for VoxtralTTS) | +| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. | | `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) | | `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) | | `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) | diff --git a/docs/user_guide/examples/online_serving/fish_speech.md b/docs/user_guide/examples/online_serving/fish_speech.md index 7322d06aa..2a15ef44a 100644 --- a/docs/user_guide/examples/online_serving/fish_speech.md +++ b/docs/user_guide/examples/online_serving/fish_speech.md @@ -41,15 +41,11 @@ Features: ## Launch the Server ```bash -vllm-omni serve fishaudio/s2-pro \ - --stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \ - --omni \ - --port 8091 \ - --trust-remote-code \ - --enforce-eager \ - --gpu-memory-utilization 0.9 +vllm serve fishaudio/s2-pro --omni --port 8091 ``` +The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`. + Or use the convenience script: ```bash diff --git a/docs/user_guide/examples/online_serving/mimo_audio.md b/docs/user_guide/examples/online_serving/mimo_audio.md index 4737eca36..c8752f578 100644 --- a/docs/user_guide/examples/online_serving/mimo_audio.md +++ b/docs/user_guide/examples/online_serving/mimo_audio.md @@ -13,10 +13,10 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/ ```bash export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer" -vllm-omni serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \ ---served-model-name "MiMo-Audio-7B-Instruct" \ ---port 18091 --stage-configs-path ./vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ ---chat-template ./examples/online_serving/mimo_audio/chat_template.jinja +vllm serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \ + --served-model-name "MiMo-Audio-7B-Instruct" \ + --port 18091 \ + --chat-template ./examples/online_serving/mimo_audio/chat_template.jinja ``` > ⚠️ **Important** > **MiMo-Audio is not compatible with the default chat template.** diff --git a/examples/offline_inference/cosyvoice3/README.md b/examples/offline_inference/cosyvoice3/README.md index e16134e6e..704b49614 100644 --- a/examples/offline_inference/cosyvoice3/README.md +++ b/examples/offline_inference/cosyvoice3/README.md @@ -58,7 +58,14 @@ Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features. - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform. -Stage wiring is configured in `vllm_omni/model_executor/stage_configs/cosyvoice3.yaml`. +Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`; +runtime tunables (batch size, memory limits, sampling) live in +`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by +HF `model_type` and defaults to `async_chunk: true` (shared-memory +streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the +legacy sync path where stage 1 runs `text2flow` over the full +speech-token sequence. - Stage 0 emits latent speech tokens. -- Stage 1 consumes them via `custom_process_input_func` and outputs audio. +- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the + shared-memory connector (async-chunk mode) and outputs audio. diff --git a/examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py b/examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py index 6311bbc90..de2712891 100644 --- a/examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py +++ b/examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py @@ -24,7 +24,13 @@ def run_e2e(): required=True, help="Path to CosyVoice3 model directory (e.g., pretrained_models/Fun-CosyVoice3-0.5B/).", ) - parser.add_argument("--stage-config", type=str, default="vllm_omni/model_executor/stage_configs/cosyvoice3.yaml") + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Override the deploy config path. If unset, auto-loads " + "vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.", + ) parser.add_argument("--prompt", type=str, default="Hello, this is a test of the CosyVoice system capability.") parser.add_argument( "--prompt-text", @@ -43,18 +49,14 @@ def run_e2e(): if not os.path.exists(args.tokenizer): raise FileNotFoundError(f"{args.tokenizer} does not exist!") - # Ensure stage config exists - if not os.path.exists(args.stage_config): - raise FileNotFoundError(f"{args.stage_config} does not exist!") + if args.deploy_config is not None and not os.path.exists(args.deploy_config): + raise FileNotFoundError(f"{args.deploy_config} does not exist!") print(f"Initializing cosyvoice E2E with model={args.model}") - # Initialize Omni - # This spins up the engine(s) based on the stage config - # We pass trust_remote_code=True same as Qwen examples omni = Omni( model=args.model, - stage_configs_path=args.stage_config, + deploy_config=args.deploy_config, trust_remote_code=True, tokenizer=args.tokenizer, log_stats=True, diff --git a/examples/offline_inference/mimo_audio/README.md b/examples/offline_inference/mimo_audio/README.md index 596afabee..5615dea51 100644 --- a/examples/offline_inference/mimo_audio/README.md +++ b/examples/offline_inference/mimo_audio/README.md @@ -39,7 +39,6 @@ Run a single sample for basic TTS: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft ``` @@ -48,7 +47,6 @@ Run batch samples for basic TTS: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft \ --num-prompts {batch_size} @@ -66,7 +64,6 @@ Generate speech from text input: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft \ --text "The weather is so nice today." @@ -78,7 +75,6 @@ Generate speech with explicit voice style instructions: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft_with_instruct \ --text "The weather is so nice today." \ @@ -91,7 +87,6 @@ Generate speech using an audio reference for voice cloning: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft_with_audio \ --text "The weather is so nice today." \ @@ -104,7 +99,6 @@ Generate speech from text containing natural voice descriptions: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type tts_sft_with_natural_instruction \ --text "In a panting young male voice, he said: I can't run anymore, wait for me!" @@ -116,7 +110,6 @@ Transcribe audio to text: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type audio_trancribing_sft \ --audio-path "./spoken_dialogue_assistant_turn_1.wav" @@ -128,7 +121,6 @@ Understand and analyze audio content with text queries: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type audio_understanding_sft \ --text "Summarize the audio." \ @@ -141,7 +133,6 @@ Audio understanding with reasoning chain: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type audio_understanding_sft_with_thinking \ --text "Summarize the audio." \ @@ -154,7 +145,6 @@ Multi-turn dialogue with audio input and output: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type spoken_dialogue_sft_multiturn \ --audio-path "./prompt_speech_zh_m.wav" @@ -168,7 +158,6 @@ Multi-turn dialogue converting speech to text: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type speech2text_dialogue_sft_multiturn ``` @@ -181,7 +170,6 @@ Multi-turn text-only dialogue: ```bash python3 -u end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ --query-type text_dialogue_sft_multiturn ``` diff --git a/examples/offline_inference/mimo_audio/end2end.py b/examples/offline_inference/mimo_audio/end2end.py index ae044d2e8..2c67d66fa 100644 --- a/examples/offline_inference/mimo_audio/end2end.py +++ b/examples/offline_inference/mimo_audio/end2end.py @@ -182,7 +182,7 @@ def main(args): omni = Omni( model=model_name, - stage_configs_path=args.stage_configs_path, + deploy_config=args.deploy_config, log_stats=args.enable_stats, log_file=("omni_pipeline.log" if args.enable_stats else None), init_sleep_seconds=args.init_sleep_seconds, @@ -428,10 +428,11 @@ def parse_args(): help="Sampling rate for audio.", ) parser.add_argument( - "--stage-configs-path", + "--deploy-config", type=str, - default="../../../model_executor/stage_configs/mimo_audio.yaml", - help="Path to a stage configs file.", + default=None, + help="Override the deploy config path. If unset, auto-loads " + "vllm_omni/deploy/mimo_audio.yaml based on the HF model_type.", ) return parser.parse_args() diff --git a/examples/offline_inference/voxtral_tts/README.md b/examples/offline_inference/voxtral_tts/README.md index 5f3d5413b..a55ce8830 100644 --- a/examples/offline_inference/voxtral_tts/README.md +++ b/examples/offline_inference/voxtral_tts/README.md @@ -10,28 +10,24 @@ When `mistral_common` has `SpeechRequest` support, prompt token IDs are built vi ```bash # Basic single-prompt with cheerful_female voice preset python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --write-audio --voice cheerful_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" # 32 replicate prompts with cheerful_female voice preset python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --num-prompts 32 --write-audio --voice cheerful_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" # Streaming with neutral_female voice preset python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --streaming --write-audio --voice neutral_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" # 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" @@ -39,7 +35,6 @@ python3 examples/offline_inference/voxtral_tts/end2end.py \ # Short debug prompt with reference audio # Note: Reference audio capability is not yet released. python3 examples/offline_inference/voxtral_tts/end2end.py \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ --write-audio \ --model mistralai/Voxtral-4B-TTS-2603 \ --text "This is a test message." \ @@ -54,7 +49,7 @@ python3 examples/offline_inference/voxtral_tts/end2end.py \ | `--text TEXT` | Text to synthesize (default: `"This is a test message."`) | | `--audio-path PATH` | Path to reference audio file for voice cloning | | `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) | -| `--stage-configs-path PATH` | Path to stage configs YAML (currently it must be set for VoxtralTTS) | +| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. | | `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) | | `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) | | `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) | diff --git a/examples/offline_inference/voxtral_tts/end2end.py b/examples/offline_inference/voxtral_tts/end2end.py index 075024645..cf28f917e 100644 --- a/examples/offline_inference/voxtral_tts/end2end.py +++ b/examples/offline_inference/voxtral_tts/end2end.py @@ -39,7 +39,7 @@ logger = logging.getLogger(__name__) async def run_streaming(inputs, sampling_params_list, model_name, args, output_dir): async_omni = AsyncOmni( model=model_name, - stage_configs_path=args.stage_configs_path, + deploy_config=args.deploy_config, log_stats=args.log_stats, ) @@ -192,7 +192,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir llm = Omni( model=model_name, log_stats=args.log_stats, - stage_configs_path=args.stage_configs_path, + deploy_config=args.deploy_config, ) if args.profiling_mode: @@ -253,10 +253,11 @@ def parse_args() -> Namespace: help="Directory to write output wav files.", ) parser.add_argument( - "--stage-configs-path", + "--deploy-config", type=str, default=None, - help="Path to stage configs YAML. Auto-resolved from model if not set.", + help="Override the deploy config path. If unset, auto-loads " + "vllm_omni/deploy/voxtral_tts.yaml based on the HF model_type.", ) parser.add_argument( "--num-prompts", type=int, default=1, help="Number of replicate prompts to run for measuring performance" diff --git a/examples/online_serving/fish_speech/README.md b/examples/online_serving/fish_speech/README.md index ae968d3ba..9b4e3cc40 100644 --- a/examples/online_serving/fish_speech/README.md +++ b/examples/online_serving/fish_speech/README.md @@ -29,15 +29,12 @@ Features: ## Launch the Server ```bash -vllm-omni serve fishaudio/s2-pro \ - --stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \ - --omni \ - --port 8091 \ - --trust-remote-code \ - --enforce-eager \ - --gpu-memory-utilization 0.9 +vllm serve fishaudio/s2-pro --omni --port 8091 ``` +The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml` +(the HF `model_type` on the fishaudio checkpoints is `fish_qwen3_omni`). + Or use the convenience script: ```bash diff --git a/examples/online_serving/fish_speech/run_gradio_demo.sh b/examples/online_serving/fish_speech/run_gradio_demo.sh index 98a696644..a0370b9cc 100755 --- a/examples/online_serving/fish_speech/run_gradio_demo.sh +++ b/examples/online_serving/fish_speech/run_gradio_demo.sh @@ -11,18 +11,13 @@ MODEL="${MODEL:-fishaudio/s2-pro}" PORT="${PORT:-8091}" GRADIO_PORT="${GRADIO_PORT:-7860}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" echo "Starting Fish Speech S2 Pro server (port $PORT)..." FLASHINFER_DISABLE_VERSION_CHECK=1 \ -vllm-omni serve "$MODEL" \ - --stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml" \ +vllm serve "$MODEL" \ + --omni \ --host 0.0.0.0 \ - --port "$PORT" \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni & + --port "$PORT" & SERVER_PID=$! cleanup() { diff --git a/examples/online_serving/fish_speech/run_server.sh b/examples/online_serving/fish_speech/run_server.sh index 59c09c7fe..a865daf93 100755 --- a/examples/online_serving/fish_speech/run_server.sh +++ b/examples/online_serving/fish_speech/run_server.sh @@ -13,11 +13,7 @@ PORT="${PORT:-8091}" echo "Starting Fish Speech S2 Pro server with model: $MODEL" FLASHINFER_DISABLE_VERSION_CHECK=1 \ -vllm-omni serve "$MODEL" \ - --stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \ +vllm serve "$MODEL" \ + --omni \ --host 0.0.0.0 \ - --port "$PORT" \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni + --port "$PORT" diff --git a/examples/online_serving/mimo_audio/README.md b/examples/online_serving/mimo_audio/README.md index 9f70d59cb..9c1be7f21 100644 --- a/examples/online_serving/mimo_audio/README.md +++ b/examples/online_serving/mimo_audio/README.md @@ -13,10 +13,10 @@ Please refer to [README.md](../../../README.md) ```bash export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer" -vllm-omni serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \ ---served-model-name "MiMo-Audio-7B-Instruct" \ ---port 18091 --stage-configs-path ./vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ ---chat-template ./examples/online_serving/mimo_audio/chat_template.jinja +vllm serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \ + --served-model-name "MiMo-Audio-7B-Instruct" \ + --port 18091 \ + --chat-template ./examples/online_serving/mimo_audio/chat_template.jinja ``` > ⚠️ **Important** > **MiMo-Audio is not compatible with the default chat template.** diff --git a/examples/online_serving/voxcpm2/README.md b/examples/online_serving/voxcpm2/README.md index 8735180f0..9ca2ae708 100644 --- a/examples/online_serving/voxcpm2/README.md +++ b/examples/online_serving/voxcpm2/README.md @@ -5,12 +5,13 @@ Serve VoxCPM2 TTS via the OpenAI-compatible `/v1/audio/speech` endpoint. ## Start the Server ```bash -python -m vllm_omni.entrypoints.openai.api_server \ - --model openbmb/VoxCPM2 \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \ - --host 0.0.0.0 --port 8000 +vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000 ``` +The deploy config is auto-loaded from `vllm_omni/deploy/voxcpm2.yaml`. Pass +`--deploy-config ` to override, or `--stage-N- ` (e.g. +`--stage-0-max-num-seqs 8`) for per-stage runtime overrides. + ## Zero-shot Synthesis ```bash diff --git a/examples/online_serving/voxcpm2/gradio_demo.py b/examples/online_serving/voxcpm2/gradio_demo.py index a33a2d924..c6706198a 100644 --- a/examples/online_serving/voxcpm2/gradio_demo.py +++ b/examples/online_serving/voxcpm2/gradio_demo.py @@ -8,10 +8,7 @@ the audio clock rate. Usage: # Start the vLLM server first: - python -m vllm_omni.entrypoints.openai.api_server \ - --model openbmb/VoxCPM2 \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \ - --host 0.0.0.0 --port 8000 + vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000 # Then launch the demo: python gradio_demo.py --api-base http://localhost:8000 diff --git a/examples/online_serving/voxcpm2/openai_speech_client.py b/examples/online_serving/voxcpm2/openai_speech_client.py index a117d24fd..127b8cebb 100644 --- a/examples/online_serving/voxcpm2/openai_speech_client.py +++ b/examples/online_serving/voxcpm2/openai_speech_client.py @@ -13,10 +13,7 @@ Examples: --ref-audio "https://example.com/reference.wav" Server setup: - python -m vllm_omni.entrypoints.openai.api_server \ - --model openbmb/VoxCPM2 \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \ - --host 0.0.0.0 --port 8000 + vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000 """ from __future__ import annotations diff --git a/tests/e2e/offline_inference/test_cosyvoice3.py b/tests/e2e/offline_inference/test_cosyvoice3.py index db5debac8..7206f1e7b 100644 --- a/tests/e2e/offline_inference/test_cosyvoice3.py +++ b/tests/e2e/offline_inference/test_cosyvoice3.py @@ -15,19 +15,18 @@ from __future__ import annotations import functools import io import os -import tempfile from pathlib import Path from urllib.request import urlopen import numpy as np import pytest import soundfile as sf -import yaml from huggingface_hub import snapshot_download from vllm.sampling_params import SamplingParams from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner +from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer @@ -44,13 +43,9 @@ REFERENCE_STAGE0_TEMPERATURE = 1.0 REFERENCE_STAGE0_REPETITION_PENALTY = 2.0 -def _stage_config(name: str) -> str: - return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name) - - -STAGE_CONFIGS = [ - _stage_config("cosyvoice3.yaml"), - _stage_config("cosyvoice3_async_chunk.yaml"), +ASYNC_CHUNK_MODES = [ + pytest.param(False, id="sync"), + pytest.param(True, id="async_chunk"), ] @@ -138,19 +133,6 @@ def _get_stage_engine_outputs(omni_runner: OmniRunner, stage_id: int): raise AttributeError("Unable to locate stage outputs on Omni runner") -def _patched_stage_config(base_stage_config: str, model_dir: Path, tmp_dir: Path) -> str: - cfg = yaml.safe_load(Path(base_stage_config).read_text(encoding="utf-8")) - tokenizer_path = str(model_dir / "CosyVoice-BlankEN") - for stage in cfg.get("stage_args", []): - engine_args = stage.setdefault("engine_args", {}) - engine_args["tokenizer"] = tokenizer_path - engine_args["enforce_eager"] = True - engine_args["hf_overrides"] = {"architectures": ["CosyVoice3Model"]} - out_path = tmp_dir / Path(base_stage_config).name - out_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") - return str(out_path) - - def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[str, object]]: return [ { @@ -165,54 +147,54 @@ def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[s @pytest.mark.core_model @pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.parametrize("base_stage_config", STAGE_CONFIGS) -def test_cosyvoice3_offline_reference_zero_shot(base_stage_config: str) -> None: +@pytest.mark.parametrize("async_chunk", ASYNC_CHUNK_MODES) +def test_cosyvoice3_offline_reference_zero_shot(async_chunk: bool) -> None: """CosyVoice3 zero-shot reference inference should stop cleanly and produce sane audio.""" prompt_audio, prompt_sr = _load_reference_prompt_wav() model_dir = _resolve_model_dir() expected_stop_token = int(CosyVoice3Config().llm["eos_token_id"]) - with tempfile.TemporaryDirectory(prefix="cv3-e2e-") as tmp: - stage_config = _patched_stage_config(base_stage_config, model_dir, Path(tmp)) - with OmniRunner( - str(model_dir), seed=42, stage_configs_path=stage_config, stage_init_timeout=300 - ) as omni_runner: - sampling_params_list = omni_runner.get_default_sampling_params_list() - sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT) + with OmniRunner( + str(model_dir), + seed=42, + stage_configs_path=get_deploy_config_path("cosyvoice3.yaml"), + async_chunk=async_chunk, + stage_init_timeout=300, + ) as omni_runner: + sampling_params_list = omni_runner.get_default_sampling_params_list() + sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT) - outputs = omni_runner.omni.generate( - _build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list + outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list) + + assert outputs, "No outputs returned" + audio_mm = outputs[0].multimodal_output + assert "audio" in audio_mm, "No audio output found" + + audio = _concat_audio(audio_mm["audio"]) + assert audio.size > 0, "Generated audio is empty" + + sr_val = audio_mm.get("sr", 24000) + if isinstance(sr_val, list) and sr_val: + sr_val = sr_val[-1] + if hasattr(sr_val, "item"): + sr_val = sr_val.item() + sr = int(sr_val) + assert sr == 24000, f"Unexpected sample_rate={sr}" + + duration_s = audio.size / sr + assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})" + + stage0_outputs = _get_stage_engine_outputs(omni_runner, 0) + if stage0_outputs: + completion = stage0_outputs[0].outputs[0] + finish_reason = getattr(completion, "finish_reason", None) + stop_reason = getattr(completion, "stop_reason", None) + num_tokens = len(getattr(completion, "token_ids", []) or []) + + assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'" + assert int(stop_reason) == expected_stop_token, ( + f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}" ) - - assert outputs, "No outputs returned" - audio_mm = outputs[0].multimodal_output - assert "audio" in audio_mm, "No audio output found" - - audio = _concat_audio(audio_mm["audio"]) - assert audio.size > 0, "Generated audio is empty" - - sr_val = audio_mm.get("sr", 24000) - if isinstance(sr_val, list) and sr_val: - sr_val = sr_val[-1] - if hasattr(sr_val, "item"): - sr_val = sr_val.item() - sr = int(sr_val) - assert sr == 24000, f"Unexpected sample_rate={sr}" - - duration_s = audio.size / sr - assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})" - - stage0_outputs = _get_stage_engine_outputs(omni_runner, 0) - if stage0_outputs: - completion = stage0_outputs[0].outputs[0] - finish_reason = getattr(completion, "finish_reason", None) - stop_reason = getattr(completion, "stop_reason", None) - num_tokens = len(getattr(completion, "token_ids", []) or []) - - assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'" - assert int(stop_reason) == expected_stop_token, ( - f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}" - ) - assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range" - else: - assert "async_chunk" in Path(base_stage_config).name, "Stage-0 produced no engine outputs" + assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range" + else: + assert async_chunk, "Stage-0 produced no engine outputs" diff --git a/tests/e2e/offline_inference/test_voxcpm2.py b/tests/e2e/offline_inference/test_voxcpm2.py index cffdee800..913a6969d 100644 --- a/tests/e2e/offline_inference/test_voxcpm2.py +++ b/tests/e2e/offline_inference/test_voxcpm2.py @@ -7,25 +7,17 @@ import torch from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner +from tests.helpers.stage_config import get_deploy_config_path VOXCPM2_MODEL = "openbmb/VoxCPM2" -STAGE_CONFIG = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "vllm_omni", - "model_executor", - "stage_configs", - "voxcpm2.yaml", -) +DEPLOY_CONFIG = get_deploy_config_path("voxcpm2.yaml") SAMPLE_RATE = 48000 @pytest.fixture(scope="module") def voxcpm2_engine(): """Create VoxCPM2 engine for testing.""" - with OmniRunner(VOXCPM2_MODEL, stage_configs_path=STAGE_CONFIG) as runner: + with OmniRunner(VOXCPM2_MODEL, stage_configs_path=DEPLOY_CONFIG) as runner: yield runner.omni diff --git a/tests/e2e/offline_inference/test_voxtral_tts.py b/tests/e2e/offline_inference/test_voxtral_tts.py index 6bf95fce1..5386e2db4 100644 --- a/tests/e2e/offline_inference/test_voxtral_tts.py +++ b/tests/e2e/offline_inference/test_voxtral_tts.py @@ -31,13 +31,11 @@ from vllm import SamplingParams from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner -from tests.helpers.stage_config import modify_stage_config +from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config from vllm_omni.entrypoints.async_omni import AsyncOmni MODEL = "mistralai/Voxtral-4B-TTS-2603" -STAGE_CONFIG = str( - Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxtral_tts.yaml" -) +STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml") SAMPLE_RATE = 24000 # Minimum expected audio samples for a short sentence (~0.04s of 24kHz audio) MIN_AUDIO_SAMPLES = 1000 @@ -66,9 +64,9 @@ def _resolve_stage_config(run_level: str) -> str: return modify_stage_config( STAGE_CONFIG, deletes={ - "stage_args": { - 0: ["engine_args.load_format"], - 1: ["engine_args.load_format"], + "stages": { + 0: ["load_format"], + 1: ["load_format"], } }, ) diff --git a/tests/e2e/online_serving/test_cosyvoice3_tts.py b/tests/e2e/online_serving/test_cosyvoice3_tts.py index e05b5e34f..76e5e4f49 100644 --- a/tests/e2e/online_serving/test_cosyvoice3_tts.py +++ b/tests/e2e/online_serving/test_cosyvoice3_tts.py @@ -12,12 +12,11 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" -from pathlib import Path - import pytest from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512" @@ -27,8 +26,8 @@ REF_TEXT = "希望你以后能够做的比我还好呦。" def get_stage_config(name: str = "cosyvoice3.yaml"): - """Get the stage config path from vllm_omni model_executor stage_configs.""" - return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name) + """Get the deploy config path for CosyVoice3.""" + return get_deploy_config_path(name) def get_prompt(prompt_type="zh"): @@ -44,7 +43,7 @@ tts_server_params = [ OmniServerParams( model=MODEL, stage_config_path=get_stage_config(), - server_args=["--trust-remote-code", "--disable-log-stats"], + server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"], ), id="cosyvoice3", ) @@ -54,7 +53,7 @@ tts_async_chunk_server_params = [ pytest.param( OmniServerParams( model=MODEL, - stage_config_path=get_stage_config("cosyvoice3_async_chunk.yaml"), + stage_config_path=get_stage_config(), server_args=["--trust-remote-code", "--disable-log-stats"], ), id="cosyvoice3_async_chunk", @@ -93,7 +92,7 @@ def test_voice_clone_zh_001(omni_server, openai_client) -> None: def test_voice_clone_zh_002(omni_server, openai_client) -> None: """ Test voice cloning TTS with Chinese text via async_chunk streaming. - Deploy Setting: cosyvoice3_async_chunk.yaml + Deploy Setting: cosyvoice3.yaml with default ``async_chunk: true`` Input Modal: text + ref_audio + ref_text Output Modal: audio (streamed) Input Setting: stream=True diff --git a/tests/e2e/online_serving/test_mimo_audio.py b/tests/e2e/online_serving/test_mimo_audio.py index 331baf18a..df00c6416 100644 --- a/tests/e2e/online_serving/test_mimo_audio.py +++ b/tests/e2e/online_serving/test_mimo_audio.py @@ -12,7 +12,7 @@ import pytest from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data -from tests.helpers.stage_config import modify_stage_config +from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) @@ -26,26 +26,6 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" models = ["XiaomiMiMo/MiMo-Audio-7B-Instruct"] -def get_chunk_config(): - path = modify_stage_config( - str(Path(__file__).parent.parent / "stage_configs" / "mimo_audio_ci.yaml"), - updates={ - "async_chunk": True, - "stage_args": { - 0: { - "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav_async_chunk" - }, - 1: { - "engine_args.max_model_len": 8192, - "engine_args.max_num_batched_tokens": 8192, - }, - }, - }, - deletes={"stage_args": {1: ["custom_process_input_func"]}}, - ) - return path - - def download_tokenizer(): tokenizer_path = os.environ.get("MIMO_AUDIO_TOKENIZER_PATH", MIMO_AUDIO_TOKENIZER_REPO) if os.path.exists(tokenizer_path): @@ -59,19 +39,21 @@ def download_tokenizer(): return local_path -# CI stage config for H100 / MI325 # Guard module-level setup so test collection doesn't fail in environments # where the model cache is read-only or models aren't available. try: - stage_configs = [get_chunk_config()] + stage_configs = [get_deploy_config_path("mimo_audio.yaml")] tokenizer_path = download_tokenizer() os.environ["MIMO_AUDIO_TOKENIZER_PATH"] = tokenizer_path + # --load-format dummy applies to every stage pipeline-wide, avoiding a + # per-stage yaml rewrite (the old approach wrote a tempfile + atexit-unlink + # which raced with CI's process lifecycle). test_params = [ OmniServerParams( model=model, stage_config_path=stage_config, - server_args=["--chat-template", CHAT_TEMPLATE_PATH], + server_args=["--chat-template", CHAT_TEMPLATE_PATH, "--load-format", "dummy"], ) for model in models for stage_config in stage_configs diff --git a/tests/e2e/online_serving/test_voxtral_tts.py b/tests/e2e/online_serving/test_voxtral_tts.py index 2dd46fcfa..649f822ba 100644 --- a/tests/e2e/online_serving/test_voxtral_tts.py +++ b/tests/e2e/online_serving/test_voxtral_tts.py @@ -12,19 +12,16 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" -from pathlib import Path - import httpx import pytest from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path MODEL = "mistralai/Voxtral-4B-TTS-2603" -STAGE_CONFIG = str( - Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxtral_tts.yaml" -) +STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml") EXTRA_ARGS = ["--trust-remote-code", "--enforce-eager", "--disable-log-stats"] TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, server_args=EXTRA_ARGS)] diff --git a/tests/e2e/stage_configs/mimo_audio_ci.yaml b/tests/e2e/stage_configs/mimo_audio_ci.yaml deleted file mode 100644 index 7127a71b4..000000000 --- a/tests/e2e/stage_configs/mimo_audio_ci.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# CI stage config for running MiMo-Audio in multi-stage omni runtime. -# Based on mimo_audio.yaml with load_format: dummy for CI testing. - -async_chunk: false -stage_args: - - stage_id: 0 - stage_type: llm - runtime: - process: true - devices: "0" - engine_args: - dtype: bfloat16 - max_num_seqs: 1 - model_stage: fused_thinker_talker - model_arch: MiMoAudioForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - tensor_parallel_size: 1 - gpu_memory_utilization: 0.5 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: latent - max_model_len: 8192 - max_num_batched_tokens: 8192 - load_format: dummy - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.6 - top_p: 0.95 - top_k: 50 - max_tokens: 18192 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - - - stage_id: 1 - stage_type: llm - runtime: - process: true - devices: "0" - engine_args: - model_stage: code2wav - max_num_seqs: 1 - model_arch: MiMoAudioForConditionalGeneration - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - tensor_parallel_size: 1 - gpu_memory_utilization: 0.2 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: audio - max_model_len: 18192 - max_num_batched_tokens: 18192 - async_scheduling: false - load_format: dummy - engine_input_source: [ 0 ] - is_comprehension: false - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 18192 - seed: 42 - detokenize: false diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index c07bc2610..69a004015 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -43,6 +43,26 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = { "vllm_omni.model_executor.models.qwen3_tts.pipeline", "QWEN3_TTS_PIPELINE", ), + "voxcpm2": ( + "vllm_omni.model_executor.models.voxcpm2.pipeline", + "VOXCPM2_PIPELINE", + ), + "cosyvoice3": ( + "vllm_omni.model_executor.models.cosyvoice3.pipeline", + "COSYVOICE3_PIPELINE", + ), + "mimo_audio": ( + "vllm_omni.model_executor.models.mimo_audio.pipeline", + "MIMO_AUDIO_PIPELINE", + ), + "voxtral_tts": ( + "vllm_omni.model_executor.models.voxtral_tts.pipeline", + "VOXTRAL_TTS_PIPELINE", + ), + "fish_qwen3_omni": ( + "vllm_omni.model_executor.models.fish_speech.pipeline", + "FISH_SPEECH_PIPELINE", + ), } # --- Single-stage diffusion pipelines (populated in PR 3/N) --- diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 9b41660a5..9839a5389 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -1367,11 +1367,32 @@ class StageConfigFactory: from vllm.transformers_utils.config import get_hf_file_to_dict config_dict = get_hf_file_to_dict("config.json", model, revision=None) - if config_dict and "model_type" in config_dict: - return config_dict["model_type"], None + if config_dict: + if "model_type" in config_dict: + return config_dict["model_type"], None + # VoxCPM2-style configs use singular ``architecture`` rather + # than HF's standard ``model_type`` / ``architectures``. Accept + # it as a fallback so the pipeline registry can still match. + if "architecture" in config_dict and isinstance(config_dict["architecture"], str): + return config_dict["architecture"], None except Exception as e: logger.debug(f"Failed to auto-detect model type for {model}: {e}") + # Final fallback: some models (e.g. CosyVoice3) ship an empty + # config.json and rely on naming conventions. Match the model path + # basename against registered pipeline keys — longest match wins + # so "cosyvoice3" (length 10) beats "cosyvoice" (length 9). + model_lower = model.lower().replace("-", "").replace("_", "") + best: str | None = None + best_len = 0 + for registered_key in _PIPELINE_REGISTRY.keys(): + candidate = registered_key.lower().replace("-", "").replace("_", "") + if candidate and candidate in model_lower and len(candidate) > best_len: + best = registered_key + best_len = len(candidate) + if best is not None: + return best, None + return None, None @classmethod diff --git a/vllm_omni/deploy/cosyvoice3.yaml b/vllm_omni/deploy/cosyvoice3.yaml new file mode 100644 index 000000000..53e3eb3f3 --- /dev/null +++ b/vllm_omni/deploy/cosyvoice3.yaml @@ -0,0 +1,58 @@ +# CosyVoice3 deploy: 2-stage talker → flow-matching code2wav. +# +# Default mode is async-chunk streaming through SharedMemoryConnector. Pass +# ``--no-async-chunk`` to the serve CLI for the legacy sync path (stage 1 +# builds flow input from the full speech-token sequence via ``text2flow``). +# The shared-memory connector definition and input/output_connectors +# references stay in the yaml unconditionally; the runtime only activates +# them when ``async_chunk: true``. +# +# enforce_eager=true everywhere: +# * Stage 0 (talker) — cudagraph not verified on this checkpoint. +# * Stage 1 (code2wav) — CUDA graphs don't work with dynamic conv shapes. +async_chunk: true +dtype: float32 + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + codec_streaming: true + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 25 + codec_vocab_size: 6561 + +stages: + - stage_id: 0 + max_num_seqs: 1 + gpu_memory_utilization: 0.4 + enforce_eager: true + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + max_tokens: 2048 + top_k: 25 + top_p: 0.8 + # near-identity repetition penalty forces vLLM to track + # output_token_ids for RAS (stop-token logit logsumexp). + repetition_penalty: 1.0001 + disable_hybrid_kv_cache_manager: true + mm_processor_cache_gb: 0 + skip_mm_profiling: true + + - stage_id: 1 + max_num_seqs: 1 + gpu_memory_utilization: 0.2 + enforce_eager: true + max_model_len: 32768 + devices: "0" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + max_tokens: 2048 + disable_hybrid_kv_cache_manager: true + skip_mm_profiling: true diff --git a/vllm_omni/deploy/fish_qwen3_omni.yaml b/vllm_omni/deploy/fish_qwen3_omni.yaml new file mode 100644 index 000000000..a5bee925b --- /dev/null +++ b/vllm_omni/deploy/fish_qwen3_omni.yaml @@ -0,0 +1,61 @@ +# Fish Speech S2 Pro deploy: async-chunk streaming slow_ar → dac_decoder. +# Verified on 1× H20. +# Registry key and filename use the HF ``model_type=fish_qwen3_omni`` for +# auto-detection; human-readable "fish_speech" stays as the source directory. +async_chunk: true +enable_chunked_prefill: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 + codec_streaming: true + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + # ~21 Hz codec; 25 frames ≈ 1.16s of audio. + codec_chunk_frames: 25 + codec_left_context_frames: 25 + initial_codec_chunk_frames: 4 + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.6 + enforce_eager: false + async_scheduling: false + # vLLM >=0.19 requires max_num_batched_tokens >= max_model_len when + # enable_chunked_prefill=false. Bumped from legacy 3072 to match + # max_model_len; chunked prefill stays off because the SlowAR decode + # loop isn't chunked-prefill-safe. + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.8 + top_k: 30 + top_p: 0.9 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.0 + + - stage_id: 1 + max_num_seqs: 1 + gpu_memory_utilization: 0.1 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + repetition_penalty: 1.0 diff --git a/vllm_omni/deploy/mimo_audio.yaml b/vllm_omni/deploy/mimo_audio.yaml new file mode 100644 index 000000000..f5e704f9b --- /dev/null +++ b/vllm_omni/deploy/mimo_audio.yaml @@ -0,0 +1,56 @@ +# MiMo Audio deploy: 2-stage thinker+talker → code2wav. +# +# Default mode is async-chunk streaming on a single GPU (both stages on +# device 0) through SharedMemoryConnector. For the legacy 2-GPU sync +# pipeline (stage 1 on a second card), pass ``--no-async-chunk +# --stage-1-devices 1 --stage-1-max-model-len 18192 +# --stage-1-max-num-batched-tokens 18192`` to the serve CLI. +async_chunk: true +dtype: bfloat16 + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 365536 + codec_streaming: true + connector_get_sleep_s: 0.001 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 3 + codec_left_context_frames: 3 + +stages: + - stage_id: 0 + max_num_seqs: 1 + gpu_memory_utilization: 0.3 + enforce_eager: true + max_num_batched_tokens: 8192 + max_model_len: 8192 + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.6 + top_p: 0.95 + top_k: 50 + max_tokens: 18192 + seed: 42 + repetition_penalty: 1.1 + + - stage_id: 1 + max_num_seqs: 1 + gpu_memory_utilization: 0.2 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 8192 + max_model_len: 8192 + devices: "0" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 18192 + seed: 42 diff --git a/vllm_omni/deploy/voxcpm2.yaml b/vllm_omni/deploy/voxcpm2.yaml new file mode 100644 index 000000000..b49906710 --- /dev/null +++ b/vllm_omni/deploy/voxcpm2.yaml @@ -0,0 +1,29 @@ +# VoxCPM2 deploy: single-stage AR pipeline with per-request state batching. +# Verified on 1x H20 141GB. +# +# Fields omitted from a stage fall back to StageDeployConfig defaults (see +# vllm_omni/config/stage_config.py). Notable choices: +# * enforce_eager=true because the talker's KV-cache save/restore loop is +# not cudagraph-compatible (captured separately via voxcpm2_talker's own +# _CapturedGraph path). +# * max_num_seqs=4 matches the legacy max_batch_size=4 contract: concurrent +# requests are supported via per-request StaticKVCache save/restore. +async_chunk: false +dtype: bfloat16 + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 + enforce_eager: true + async_scheduling: true + max_num_batched_tokens: 4096 + max_model_len: 4096 + devices: "0" + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 4096 + seed: 42 + repetition_penalty: 1.0 diff --git a/vllm_omni/deploy/voxtral_tts.yaml b/vllm_omni/deploy/voxtral_tts.yaml new file mode 100644 index 000000000..b0899a399 --- /dev/null +++ b/vllm_omni/deploy/voxtral_tts.yaml @@ -0,0 +1,65 @@ +# Voxtral TTS deploy: async-chunk streaming generator → audio tokenizer. +# Verified on 1× H20. +# +# Mistral tokenizer/config/load flags are declared per-stage here because +# DeployConfig does not yet expose them as pipeline-wide fields. They must +# match on every stage so the engine args stay consistent across workers. +async_chunk: true + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 + codec_streaming: true + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_chunk_frames_at_begin: 5 + codec_left_context_frames: 25 + +stages: + - stage_id: 0 + max_num_seqs: 32 + gpu_memory_utilization: 0.8 + enforce_eager: false + async_scheduling: true + max_model_len: 4096 + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.1 + tokenizer_mode: mistral + config_format: mistral + load_format: mistral + skip_mm_profiling: true + enable_chunked_prefill: false + + - stage_id: 1 + max_num_seqs: 32 + gpu_memory_utilization: 0.1 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "0" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.9 + top_p: 0.8 + top_k: 40 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.05 + tokenizer_mode: mistral + config_format: mistral + load_format: mistral + skip_mm_profiling: true diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index 5757d3899..04c0ffe47 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -247,22 +247,25 @@ def _convert_dataclasses_to_dict(obj: Any) -> Any: def _try_resolve_omni_model_type(model: str) -> str | None: """Try to resolve model_type for omni models with empty config.json. - Checks if any registered omni stage config file name matches a substring - in the model name (e.g. 'cosyvoice3' in 'FunAudioLLM/Fun-CosyVoice3-0.5B-2512'). - When multiple configs match, the longest stem wins to avoid ambiguity - (e.g. 'bagel_single_stage' over 'bagel'). + Searches both the legacy ``stage_configs/*.yaml`` directory and the + migrated ``deploy/*.yaml`` directory for a stem that substring-matches + the model path (e.g. ``cosyvoice3`` in + ``FunAudioLLM/Fun-CosyVoice3-0.5B-2512``). The longest match wins so + ``cosyvoice3`` beats ``cosyvoice`` and ``bagel_single_stage`` beats + ``bagel``. """ - stage_configs_dir = PROJECT_ROOT / "vllm_omni" / "model_executor" / "stage_configs" - if not stage_configs_dir.exists(): - return None model_lower = model.lower().replace("-", "").replace("_", "") best_match: str | None = None best_len = 0 - for config_file in sorted(stage_configs_dir.glob("*.yaml")): - candidate = config_file.stem.replace("-", "").replace("_", "") - if candidate in model_lower and len(candidate) > best_len: - best_match = config_file.stem - best_len = len(candidate) + for subdir in ("model_executor/stage_configs", "deploy"): + config_dir = PROJECT_ROOT / "vllm_omni" / subdir + if not config_dir.exists(): + continue + for config_file in sorted(config_dir.glob("*.yaml")): + candidate = config_file.stem.replace("-", "").replace("_", "") + if candidate and candidate in model_lower and len(candidate) > best_len: + best_match = config_file.stem + best_len = len(candidate) return best_match diff --git a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py new file mode 100644 index 000000000..4480a0dd8 --- /dev/null +++ b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CosyVoice3 pipeline topology (frozen). + +Stage 0: Talker — text prompt → speech tokens (LLM autoregressive). +Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform. + * ``sync_process_input_func`` runs when ``deploy.async_chunk=false``: + stage 1 builds full-sequence flow input via ``text2flow``. + * ``async_chunk_process_next_stage_input_func`` runs when + ``deploy.async_chunk=true``: stage 0 streams codec chunks to stage 1 + through the shared-memory connector. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3" + +COSYVOICE3_PIPELINE = PipelineConfig( + model_type="cosyvoice3", + model_arch="CosyVoice3Model", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="cosyvoice3_talker", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + owns_tokenizer=True, + engine_output_type="latent", + async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"), + sampling_constraints={ + # merged speech stop token (logsumexp of all 200 stop logits) + "stop_token_ids": [6562], + }, + ), + StagePipelineConfig( + stage_id=1, + model_stage="cosyvoice3_code2wav", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + final_output=True, + final_output_type="audio", + engine_output_type="latent", + sync_process_input_func=f"{_PROC}.text2flow", + ), + ), +) diff --git a/vllm_omni/model_executor/models/fish_speech/pipeline.py b/vllm_omni/model_executor/models/fish_speech/pipeline.py new file mode 100644 index 000000000..baee1b5e7 --- /dev/null +++ b/vllm_omni/model_executor/models/fish_speech/pipeline.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fish Speech S2 Pro pipeline topology (frozen). + +Stage 0: slow_ar — text → RVQ codec tokens (LLM autoregressive). +Stage 1: dac_decoder — RVQ tokens → audio waveform (LLM_GENERATION). + +The HF config top-level reports ``model_type = "fish_qwen3_omni"`` (the +OmniConfig that bundles slow-AR and fast-AR sub-configs), which is why the +registry key follows the HF name rather than the human-readable "fish_speech". +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +_PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech" + +FISH_SPEECH_PIPELINE = PipelineConfig( + model_type="fish_qwen3_omni", + model_arch="FishSpeechSlowARForConditionalGeneration", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="fish_speech_slow_ar", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + owns_tokenizer=True, + engine_output_type="latent", + async_chunk_process_next_stage_input_func=(f"{_PROC}.slow_ar_to_dac_decoder_async_chunk"), + sampling_constraints={ + "detokenize": False, + # <|im_end|> — stop when the model emits end-of-turn. + "stop_token_ids": [151645], + }, + ), + StagePipelineConfig( + stage_id=1, + model_stage="dac_decoder", + model_arch="FishSpeechDACDecoder", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + final_output=True, + final_output_type="audio", + engine_output_type="audio", + sampling_constraints={"detokenize": True}, + ), + ), +) diff --git a/vllm_omni/model_executor/models/mimo_audio/pipeline.py b/vllm_omni/model_executor/models/mimo_audio/pipeline.py new file mode 100644 index 000000000..33e78a172 --- /dev/null +++ b/vllm_omni/model_executor/models/mimo_audio/pipeline.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiMo Audio pipeline topology (frozen). + +Stage 0: fused thinker+talker — multimodal understanding + text + RVQ codes. +Stage 1: Code2Wav — RVQ codes → waveform. + +MiMoAudioConfig inherits from Qwen2Config, so the HF ``model_type`` field +reports ``qwen2`` — the registry's model_type-based auto-detect can't +disambiguate. ``hf_architectures`` lets ``StageConfigFactory`` fall back to +matching ``hf_config.architectures`` instead. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +_PROC = "vllm_omni.model_executor.stage_input_processors.mimo_audio" + +MIMO_AUDIO_PIPELINE = PipelineConfig( + model_type="mimo_audio", + # HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in + # ``model_executor/models/registry.py``; it resolves to the internal + # class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``. + model_arch="MiMoAudioModel", + hf_architectures=("MiMoAudioModel",), + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="fused_thinker_talker", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="text", + owns_tokenizer=True, + engine_output_type="latent", + async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"), + sampling_constraints={"detokenize": True}, + ), + StagePipelineConfig( + stage_id=1, + model_stage="code2wav", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + final_output=True, + final_output_type="audio", + engine_output_type="audio", + sync_process_input_func=f"{_PROC}.llm2code2wav", + sampling_constraints={"detokenize": False}, + ), + ), +) diff --git a/vllm_omni/model_executor/models/voxcpm2/pipeline.py b/vllm_omni/model_executor/models/voxcpm2/pipeline.py new file mode 100644 index 000000000..347fce177 --- /dev/null +++ b/vllm_omni/model_executor/models/voxcpm2/pipeline.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""VoxCPM2 pipeline topology (frozen). + +Single-stage AR TTS: text → speech waveform in one pass. +Uses the native MiniCPM4 base_lm with a per-request StaticKVCache that the +talker restores into the paged attention layer at step boundaries. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +VOXCPM2_PIPELINE = PipelineConfig( + model_type="voxcpm2", + model_arch="VoxCPM2TalkerForConditionalGeneration", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="latent_generator", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="audio", + owns_tokenizer=True, + engine_output_type="audio", + sampling_constraints={ + "detokenize": False, + "stop_token_ids": [1], + }, + ), + ), +) diff --git a/vllm_omni/model_executor/models/voxtral_tts/__init__.py b/vllm_omni/model_executor/models/voxtral_tts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/model_executor/models/voxtral_tts/pipeline.py b/vllm_omni/model_executor/models/voxtral_tts/pipeline.py new file mode 100644 index 000000000..3bc2743c6 --- /dev/null +++ b/vllm_omni/model_executor/models/voxtral_tts/pipeline.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Voxtral TTS pipeline topology (frozen). + +Stage 0: audio_generation — text → acoustic latents (LLM_AR, tokenizer owner). +Stage 1: audio_tokenizer — acoustic latents → waveform (LLM_GENERATION). +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +_PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts" + +VOXTRAL_TTS_PIPELINE = PipelineConfig( + model_type="voxtral_tts", + model_arch="VoxtralTTSForConditionalGeneration", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="audio_generation", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=False, + final_output_type="text", + owns_tokenizer=True, + engine_output_type="latent", + async_chunk_process_next_stage_input_func=(f"{_PROC}.generator2tokenizer_async_chunk"), + sampling_constraints={"detokenize": True}, + ), + StagePipelineConfig( + stage_id=1, + model_stage="audio_tokenizer", + execution_type=StageExecutionType.LLM_GENERATION, + input_sources=(0,), + final_output=True, + final_output_type="audio", + engine_output_type="audio", + sampling_constraints={"detokenize": True}, + extras={"tts_args": {"max_instructions_length": 500}}, + ), + ), +) diff --git a/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml b/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml deleted file mode 100644 index 8e0582723..000000000 --- a/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Stage config for running CosyVoice3 with 2 stage architecture -# Stage 0: Talker (text prompt → speech tokens) -# Stage 1: Code2Wav (flow matching → acoustic features → waveform) -# Right now I have coded up stage 2 in stage 1 only; can be split in future - -stage_args: - - stage_id: 0 - is_comprehension: true - runtime: - devices: 0 - engine_args: - model_stage: cosyvoice3_talker - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - model_arch: CosyVoice3Model - trust_remote_code: true - gpu_memory_utilization: 0.2 - engine_output_type: latent # Output speech tokens for chunk aware flow matching - disable_hybrid_kv_cache_manager: true - enable_prefix_caching: false - enforce_eager: true - mm_processor_cache_gb: 0 - skip_mm_profiling: true - dtype: "float32" - default_sampling_params: - max_tokens: 2048 - stop_token_ids: [6562] # merged speech stop token (logsumexp of all 200 stop logits) - top_k: 25 - top_p: 0.8 - repetition_penalty: 1.0001 # near-identity; forces vLLM to track output_token_ids for RAS - - - stage_id: 1 - runtime: - devices: 0 - engine_args: - model_stage: cosyvoice3_code2wav - model_arch: CosyVoice3Model - trust_remote_code: true - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - engine_output_type: latent - gpu_memory_utilization: 0.1 - enforce_eager: true # CUDA graphs don't work with dynamic conv shapes in code2wav - disable_hybrid_kv_cache_manager: true - enable_prefix_caching: false - skip_mm_profiling: true - dtype: "float32" - default_sampling_params: - max_tokens: 2048 - engine_input_source: [0] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow - final_output: true - final_output_type: audio diff --git a/vllm_omni/model_executor/stage_configs/cosyvoice3_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/cosyvoice3_async_chunk.yaml deleted file mode 100644 index 13419ef10..000000000 --- a/vllm_omni/model_executor/stage_configs/cosyvoice3_async_chunk.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Stage config for running CosyVoice3 with async_chunk architecture -# Stage 0: Talker (text prompt -> speech tokens streamed by chunks) -# Stage 1: Code2Wav (flow matching -> acoustic features -> waveform) -async_chunk: true - -stage_args: - - stage_id: 0 - is_comprehension: true - runtime: - devices: 0 - max_batch_size: 1 - engine_args: - model_stage: cosyvoice3_talker - model_arch: CosyVoice3Model - worker_type: ar - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.cosyvoice3.talker2code2wav_async_chunk - trust_remote_code: true - gpu_memory_utilization: 0.4 - engine_output_type: latent - disable_hybrid_kv_cache_manager: true - enable_prefix_caching: false - enforce_eager: true - mm_processor_cache_gb: 0 - skip_mm_profiling: true - dtype: "float32" - default_sampling_params: - max_tokens: 2048 - stop_token_ids: [6562] # merged speech stop token (logsumexp of all 200 stop logits) - top_k: 25 - top_p: 0.8 - repetition_penalty: 1.0001 # near-identity; forces vLLM to track output_token_ids for RAS - output_connectors: - to_stage_1: connector_of_shared_memory - - - stage_id: 1 - runtime: - devices: 0 - max_batch_size: 1 - engine_args: - model_stage: cosyvoice3_code2wav - model_arch: CosyVoice3Model - worker_type: generation - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - trust_remote_code: true - engine_output_type: latent - gpu_memory_utilization: 0.2 - enforce_eager: true - disable_hybrid_kv_cache_manager: true - enable_prefix_caching: false - skip_mm_profiling: true - max_model_len: 32768 - dtype: "float32" - default_sampling_params: - max_tokens: 2048 - engine_input_source: [0] - final_output: true - final_output_type: audio - input_connectors: - from_stage_0: connector_of_shared_memory - -runtime: - enabled: true - - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - codec_streaming: true - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 25 - codec_left_context_frames: 25 - codec_vocab_size: 6561 - - edges: - - from: 0 - to: 1 diff --git a/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml b/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml deleted file mode 100644 index 90f80c22d..000000000 --- a/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml +++ /dev/null @@ -1,91 +0,0 @@ -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - max_batch_size: 16 - engine_args: - max_num_seqs: 4 - model_stage: fish_speech_slow_ar - model_arch: FishSpeechSlowARForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: false - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: latent - gpu_memory_utilization: 0.6 - distributed_executor_backend: "mp" - max_num_batched_tokens: 3072 - max_model_len: 16384 - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.fish_speech.slow_ar_to_dac_decoder_async_chunk - output_connectors: - to_stage_1: connector_of_shared_memory - default_sampling_params: - temperature: 0.8 - top_k: 30 - top_p: 0.9 - max_tokens: 2048 - seed: 42 - detokenize: false - repetition_penalty: 1.0 - # <|im_end|> token -- stop when model emits end-of-turn. - stop_token_ids: [151645] - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - max_batch_size: 16 - engine_args: - max_num_seqs: 1 - model_stage: dac_decoder - model_arch: FishSpeechDACDecoder - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.1 - distributed_executor_backend: "mp" - max_num_batched_tokens: 8192 - max_model_len: 16384 - engine_input_source: [0] - final_output: true - final_output_type: audio - input_connectors: - from_stage_0: connector_of_shared_memory - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 65536 - seed: 42 - detokenize: true - repetition_penalty: 1.0 - -runtime: - enabled: true - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 - codec_streaming: true - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - # Chunk sizes for streaming -- ~21 Hz codec. - # 25 frames ≈ 1.16s of audio at 21.5 Hz. - codec_chunk_frames: 25 - codec_left_context_frames: 25 - initial_codec_chunk_frames: 4 - - edges: - - from: 0 - to: 1 diff --git a/vllm_omni/model_executor/stage_configs/mimo_audio.yaml b/vllm_omni/model_executor/stage_configs/mimo_audio.yaml deleted file mode 100644 index 123e5f0cf..000000000 --- a/vllm_omni/model_executor/stage_configs/mimo_audio.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# stage config for running mimo-audio for multi-stage omni runtime. - -# The following config has been verified on 1x H20-96G GPU. -async_chunk: false -stage_args: - - stage_id: 0 - stage_type: llm - runtime: - process: true # Run this stage in a separate process - devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device) - engine_args: - dtype: bfloat16 - max_num_seqs: 1 - model_stage: fused_thinker_talker - model_arch: MiMoAudioForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs) - gpu_memory_utilization: 0.3 - enforce_eager: true # need to discuss - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: latent # change the param name,such as pooling_output - max_model_len: 8192 - max_num_batched_tokens: 8192 - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.6 - top_p: 0.95 - top_k: 50 - max_tokens: 18192 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - - - stage_id: 1 - stage_type: llm - runtime: - process: true # Run this stage in a separate process - devices: "1" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device) - engine_args: - model_stage: code2wav - max_num_seqs: 1 - model_arch: MiMoAudioForConditionalGeneration - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs) - gpu_memory_utilization: 0.2 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: audio - max_model_len: 18192 - max_num_batched_tokens: 18192 - async_scheduling: false - engine_input_source: [ 0 ] - is_comprehension: false - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 18192 - seed: 42 - detokenize: false diff --git a/vllm_omni/model_executor/stage_configs/mimo_audio_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/mimo_audio_async_chunk.yaml deleted file mode 100644 index 2fa1b982a..000000000 --- a/vllm_omni/model_executor/stage_configs/mimo_audio_async_chunk.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# stage config for running mimo-audio for multi-stage omni runtime. - -# The following config has been verified on 1x H20-96G GPU. -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm - runtime: - process: true # Run this stage in a separate process - devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device) - engine_args: - dtype: bfloat16 - max_num_seqs: 1 - model_stage: fused_thinker_talker - model_arch: MiMoAudioForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs) - gpu_memory_utilization: 0.3 - enforce_eager: true # need to discuss - trust_remote_code: true - enable_prefix_caching: false - engine_output_type: latent # change the param name,such as pooling_output - max_model_len: 8192 - max_num_batched_tokens: 8192 - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav_async_chunk - output_connectors: - to_stage_1: connector_of_shared_memory - is_comprehension: true - final_output: true - final_output_type: text - default_sampling_params: - temperature: 0.6 - top_p: 0.95 - top_k: 50 - max_tokens: 18192 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - - - stage_id: 1 - stage_type: llm - runtime: - process: true # Run this stage in a separate process - devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device) - engine_args: - model_stage: code2wav - max_num_seqs: 1 - model_arch: MiMoAudioForConditionalGeneration - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs) - gpu_memory_utilization: 0.2 - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: audio - max_model_len: 8192 - max_num_batched_tokens: 8192 - engine_input_source: [ 0 ] - input_connectors: - from_stage_0: connector_of_shared_memory - is_comprehension: false - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 18192 - seed: 42 - detokenize: false - -runtime: - enabled: true - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 365536 - codec_streaming: true - connector_get_sleep_s: 0.001 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 3 - codec_left_context_frames: 3 - - edges: - - from: 0 - to: 1 diff --git a/vllm_omni/model_executor/stage_configs/voxcpm2.yaml b/vllm_omni/model_executor/stage_configs/voxcpm2.yaml deleted file mode 100644 index 7cc93d6b2..000000000 --- a/vllm_omni/model_executor/stage_configs/voxcpm2.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# VoxCPM2 AR pipeline with per-request state batching. -# Uses native MiniCPM4 base_lm + per-request StaticKVCache. -# max_batch_size > 1 supported via KV cache save/restore. -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - max_batch_size: 4 - engine_args: - dtype: bfloat16 - model_stage: latent_generator - model_arch: VoxCPM2TalkerForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: true - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.9 - distributed_executor_backend: "mp" - max_num_batched_tokens: 4096 - max_model_len: 4096 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 4096 - seed: 42 - detokenize: false - repetition_penalty: 1.0 - stop_token_ids: [1] - final_output: true - final_output_type: audio diff --git a/vllm_omni/model_executor/stage_configs/voxtral_tts.yaml b/vllm_omni/model_executor/stage_configs/voxtral_tts.yaml deleted file mode 100644 index b0d9a81e7..000000000 --- a/vllm_omni/model_executor/stage_configs/voxtral_tts.yaml +++ /dev/null @@ -1,100 +0,0 @@ -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm # Use llm stage type to launch OmniLLM - runtime: - process: true # Run this stage in a separate process - devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device) - engine_args: - max_num_seqs: 32 - model_stage: audio_generation - model_arch: VoxtralTTSForConditionalGeneration - worker_type: ar - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.8 - enforce_eager: false - trust_remote_code: true - async_scheduling: true - engine_output_type: latent - enable_prefix_caching: false - tokenizer_mode: mistral - config_format: mistral - load_format: mistral - skip_mm_profiling: true - enable_chunked_prefill: false - max_model_len: 4096 - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxtral_tts.generator2tokenizer_async_chunk - output_connectors: - to_stage_1: connector_of_shared_memory - is_comprehension: true - final_output: false - final_output_type: text - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.1 - - stage_id: 1 - stage_type: llm # Use llm stage type to launch OmniLLM - runtime: - process: true - devices: "0" - engine_args: - max_num_seqs: 32 - model_stage: audio_tokenizer - model_arch: VoxtralTTSForConditionalGeneration - worker_type: generation - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - async_scheduling: false - gpu_memory_utilization: 0.1 - enforce_eager: true - trust_remote_code: true - enable_prefix_caching: false - skip_mm_profiling: true - engine_output_type: audio - tokenizer_mode: mistral - config_format: mistral - load_format: mistral - max_num_batched_tokens: 65536 - max_model_len: 65536 - engine_input_source: [0] - is_comprehension: false - final_output: true - final_output_type: audio - input_connectors: - from_stage_0: connector_of_shared_memory - tts_args: - max_instructions_length: 500 - default_sampling_params: - temperature: 0.9 - top_p: 0.8 - top_k: 40 - max_tokens: 2048 - seed: 42 - detokenize: True - repetition_penalty: 1.05 - -# Top-level runtime config (concise): default windows and stage edges -runtime: - enabled: true - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 - codec_streaming: true - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 25 - codec_chunk_frames_at_begin: 5 - codec_left_context_frames: 25 - - edges: - - from: 0 # language_model → acoustic_transformer: trigger only after receiving full input (-1) - to: 1