diff --git a/recipes/fishaudio/Fish-Speech-S2-Pro.md b/recipes/fishaudio/Fish-Speech-S2-Pro.md index 91ac3fe03..2a88bd169 100644 --- a/recipes/fishaudio/Fish-Speech-S2-Pro.md +++ b/recipes/fishaudio/Fish-Speech-S2-Pro.md @@ -102,3 +102,30 @@ curl -X POST http://localhost:8091/v1/audio/speech \ both `ref_audio` and `ref_text` parameters. - Memory usage: Model loads at ~48.3 GiB, peaks at ~48.9 GiB during inference headroom for video frames and audio caches. + +### 2x H100 80GB + +#### Environment + +- OS: Linux +- Python: 3.10+ +- Required package: `fish-speech` (for DAC codec) +- CUDA 12.8 +- vLLM version: 0.19.0+ + +#### Command + +```bash +CUDA_VISIBLE_DEVICES=0,1 vllm serve fishaudio/s2-pro --omni --port 8091 \ + --deploy-config vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml +``` + +This profile pins Stage0 (Slow/Fast AR) to GPU 0 and Stage1 (DAC decoder) +to GPU 1 to remove AR/DAC contention, sets `max_num_seqs=64` on both +stages so concurrencies above 8 don't queue, and reduces +`connector_get_sleep_s` from 10 ms to 1 ms so DAC chunks aren't held in +the connector once they're ready. + +For benchmark numbers see the discussion in +vllm-project/vllm-omni#2515 (initial H100x2 sweep by @Sy0307) and +vllm-project/vllm-omni#3323 (H20x2 sweep with the shipped config). diff --git a/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml new file mode 100644 index 000000000..9fec34d69 --- /dev/null +++ b/vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml @@ -0,0 +1,72 @@ +# Fish Speech S2 Pro 2-GPU deploy: Stage0 AR on GPU 0, Stage1 DAC on GPU 1. +# +# Opt-in profile validated on H100x2. Compared with the single-GPU default +# (``fish_qwen3_omni.yaml``), it splits AR and DAC across devices to remove +# AR/DAC contention, enlarges Stage0 AR batching, and lets Stage1 consume +# multiple DAC chunks in parallel. +# +# Select via: +# CUDA_VISIBLE_DEVICES=0,1 vllm serve fishaudio/s2-pro --omni \ +# --deploy-config vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml +# +# Or programmatically: +# Omni(model="fishaudio/s2-pro", +# deploy_config_path="vllm_omni/deploy/fish_qwen3_omni_2gpu.yaml") + +async_chunk: true +enable_chunked_prefill: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 + codec_streaming: true + # Tightened from 10ms to 1ms: chunks become available sooner under + # cross-GPU pipelining, so connector idle waiting dominates if left high. + connector_get_sleep_s: 0.001 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + # ~21 Hz codec; 25 frames ≈ 1.16s of audio. + codec_chunk_frames: 25 + codec_left_context_frames: 25 + initial_codec_chunk_frames: 4 + +stages: + - stage_id: 0 + max_num_seqs: 64 + gpu_memory_utilization: 0.75 + enforce_eager: false + async_scheduling: false + # vLLM scheduler requires max_num_batched_tokens >= max_model_len when + # chunked prefill is off (SlowAR decode loop isn't chunked-prefill-safe). + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.8 + top_k: 30 + top_p: 0.9 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.0 + + - stage_id: 1 + max_num_seqs: 64 + gpu_memory_utilization: 0.2 + enforce_eager: true + async_scheduling: false + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "1" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + repetition_penalty: 1.0