anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI] Add nightly-ci for multi-stage deployment (#3610)

Browse Source 
Signed-off-by: ZhengWG <zwg0606@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Zheng WengangandHongsheng Liu authored and GitHub committed 2026-05-26 11:16:43 +08:00
1 parent 11a951b962
commit cad25fa20c
4 files changed
+181 -11
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/test-nightly.yml
+47
View File
@@ -305,6 +305,51 @@ steps:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Omni · Perf Test · Multi-Replica"
key: nightly-omni-performance-multi-replicas
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
agents:
queue: "mithril-h100-pool"
plugins:
- kubernetes:
podSpec:
containers:
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
resources:
limits:
nvidia.com/gpu: 3
volumeMounts:
- name: devshm
mountPath: /dev/shm
- name: hf-cache
mountPath: /root/.cache/huggingface
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
key: token
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
- name: devshm
emptyDir:
medium: Memory
- name: hf-cache
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
timeout_in_minutes: 45
@@ -1188,6 +1233,7 @@ steps:
- nightly-omni-performance-no-async-chunk
- nightly-omni-performance-async-chunk
- nightly-omni-performance-vllm-text
- nightly-omni-performance-multi-replicas
- nightly-tts-performance
- nightly-diffusion-x2iat-performance-qwen-image
- nightly-diffusion-x2iat-performance-qwen-image-edit
@@ -1204,6 +1250,7 @@ steps:
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+115
View File
@@ -0,0 +1,115 @@
[
{
"test_name": "test_qwen3_omni_3gpu_replica2",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"stage_overrides": {
"1": {"num_replicas": 2, "devices": "1,2"},
"2": {"num_replicas": 2, "devices": "1,2"}
},
"extra_cli_args": [
"--async-chunk"
]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [128, 128, 128, 128],
"max_concurrency": [8, 16, 24, 32],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 2000, 3000, 4000],
"mean_tpot_ms": [200, 200, 200, 200],
"mean_audio_ttfp_ms": [10000, 15000, 20000, 25000],
"mean_audio_rtf": [1.5, 1.5, 1.5, 1.5]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [2000],
"mean_audio_rtf": [0.25]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [6000],
"mean_audio_rtf": [0.7]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [12000],
"mean_audio_rtf": [1.0]
}
}
]
}
]
vllm_omni/config/stage_config.py
+3 -1
View File
@@ -942,7 +942,7 @@ class StageConfig:
# CLI overrides take precedence over YAML defaults
for key, value in self.runtime_overrides.items():
if value is not None and key not in ("devices", "max_batch_size"):
if value is not None and key not in ("devices", "max_batch_size", "num_replicas"):
engine_args[key] = value
# Build runtime config from YAML defaults + CLI overrides
@@ -950,6 +950,8 @@ class StageConfig:
runtime.setdefault("process", True)
if self.runtime_overrides.get("devices") is not None:
runtime["devices"] = self.runtime_overrides["devices"]
if self.runtime_overrides.get("num_replicas") is not None:
runtime["num_replicas"] = self.runtime_overrides["num_replicas"]
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
legacy_mbs = runtime.pop("max_batch_size", None)
vllm_omni/deploy/qwen3_omni_moe_multi_replicas.yaml
+16 -10
View File
@@ -23,7 +23,11 @@ connectors:
stages:
- stage_id: 0
max_num_batched_tokens: 32768
max_num_seqs: 64
gpu_memory_utilization: 0.9
trust_remote_code: true
enable_prefix_caching: false
devices: "0"
default_sampling_params:
temperature: 0.4
@@ -34,7 +38,11 @@ stages:
repetition_penalty: 1.05
- stage_id: 1
max_num_batched_tokens: 32768
max_num_seqs: 64
gpu_memory_utilization: 0.6
trust_remote_code: true
enable_prefix_caching: false
devices: "1,2"
num_replicas: 2
input_connectors:
@@ -47,11 +55,13 @@ stages:
repetition_penalty: 1.05
- stage_id: 2
gpu_memory_utilization: 0.1
max_num_seqs: 1
enforce_eager: true
async_scheduling: false
max_num_batched_tokens: 51200
max_num_seqs: 64
gpu_memory_utilization: 0.1
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
async_scheduling: false
devices: "1,2"
num_replicas: 2
input_connectors:
@@ -70,20 +80,16 @@ platforms:
- stage_id: 0
gpu_memory_utilization: 0.6
tensor_parallel_size: 2
max_num_batched_tokens: 8192
devices: "0,1"
- stage_id: 1
gpu_memory_utilization: 0.6
enforce_eager: true
max_num_batched_tokens: 8192
devices: "2"
- stage_id: 2
gpu_memory_utilization: 0.3
devices: "2"
rocm:
stages:
- stage_id: 0
enforce_eager: true
xpu:
stages:
- stage_id: 0