diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 1e25cf2e0..88cb8dbb5 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -305,6 +305,51 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Omni · Perf Test · Multi-Replica" + key: nightly-omni-performance-multi-replicas + timeout_in_minutes: 180 + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 3 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" timeout_in_minutes: 45 @@ -1188,6 +1233,7 @@ steps: - nightly-omni-performance-no-async-chunk - nightly-omni-performance-async-chunk - nightly-omni-performance-vllm-text + - nightly-omni-performance-multi-replicas - nightly-tts-performance - nightly-diffusion-x2iat-performance-qwen-image - nightly-diffusion-x2iat-performance-qwen-image-edit @@ -1204,6 +1250,7 @@ steps: - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509 diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json new file mode 100644 index 000000000..0e7cb63f7 --- /dev/null +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -0,0 +1,115 @@ +[ + { + "test_name": "test_qwen3_omni_3gpu_replica2", + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "stage_overrides": { + "1": {"num_replicas": 2, "devices": "1,2"}, + "2": {"num_replicas": 2, "devices": "1,2"} + }, + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [128, 128, 128, 128], + "max_concurrency": [8, 16, 24, 32], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [1000, 2000, 3000, 4000], + "mean_tpot_ms": [200, 200, 200, 200], + "mean_audio_ttfp_ms": [10000, 15000, 20000, 25000], + "mean_audio_rtf": [1.5, 1.5, 1.5, 1.5] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [10], + "request_rate": [0.1], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [2000], + "mean_audio_ttfp_ms": [2000], + "mean_audio_rtf": [0.25] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [40], + "request_rate": [0.5], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [6000], + "mean_audio_ttfp_ms": [6000], + "mean_audio_rtf": [0.7] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [100], + "request_rate": [1.0], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [12000], + "mean_audio_ttfp_ms": [12000], + "mean_audio_rtf": [1.0] + } + } + ] + } +] diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index c459ecabe..7c17cf7ce 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -942,7 +942,7 @@ class StageConfig: # CLI overrides take precedence over YAML defaults for key, value in self.runtime_overrides.items(): - if value is not None and key not in ("devices", "max_batch_size"): + if value is not None and key not in ("devices", "max_batch_size", "num_replicas"): engine_args[key] = value # Build runtime config from YAML defaults + CLI overrides @@ -950,6 +950,8 @@ class StageConfig: runtime.setdefault("process", True) if self.runtime_overrides.get("devices") is not None: runtime["devices"] = self.runtime_overrides["devices"] + if self.runtime_overrides.get("num_replicas") is not None: + runtime["num_replicas"] = self.runtime_overrides["num_replicas"] # Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs legacy_mbs = runtime.pop("max_batch_size", None) diff --git a/vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml b/vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml index 791e29a8d..900c9e4bb 100644 --- a/vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml +++ b/vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml @@ -23,7 +23,11 @@ connectors: stages: - stage_id: 0 + max_num_batched_tokens: 32768 + max_num_seqs: 64 gpu_memory_utilization: 0.9 + trust_remote_code: true + enable_prefix_caching: false devices: "0" default_sampling_params: temperature: 0.4 @@ -34,7 +38,11 @@ stages: repetition_penalty: 1.05 - stage_id: 1 + max_num_batched_tokens: 32768 + max_num_seqs: 64 gpu_memory_utilization: 0.6 + trust_remote_code: true + enable_prefix_caching: false devices: "1,2" num_replicas: 2 input_connectors: @@ -47,11 +55,13 @@ stages: repetition_penalty: 1.05 - stage_id: 2 - gpu_memory_utilization: 0.1 - max_num_seqs: 1 - enforce_eager: true - async_scheduling: false max_num_batched_tokens: 51200 + max_num_seqs: 64 + gpu_memory_utilization: 0.1 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: false devices: "1,2" num_replicas: 2 input_connectors: @@ -70,20 +80,16 @@ platforms: - stage_id: 0 gpu_memory_utilization: 0.6 tensor_parallel_size: 2 + max_num_batched_tokens: 8192 devices: "0,1" - stage_id: 1 gpu_memory_utilization: 0.6 - enforce_eager: true + max_num_batched_tokens: 8192 devices: "2" - stage_id: 2 gpu_memory_utilization: 0.3 devices: "2" - rocm: - stages: - - stage_id: 0 - enforce_eager: true - xpu: stages: - stage_id: 0