anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Perf] Bagel Performance Nightly CI test (#2175)

Browse Source 
Signed-off-by: NumberWan <wantszkin2003@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
NumberWanandHongsheng Liu authored and GitHub committed 2026-05-28 09:41:46 +08:00
1 parent 6ea66f8ed0
commit 8766c0c59f
2 files changed
+202
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/test-nightly.yml
+50
View File
@@ -938,6 +938,54 @@ steps:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
key: nightly-diffusion-x2iat-performance-bagel
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- export CACHE_DIT_VERSION=1.3.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
agents:
queue: "mithril-h100-pool"
plugins:
- kubernetes:
podSpec:
containers:
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
resources:
limits:
nvidia.com/gpu: 1
volumeMounts:
- name: devshm
mountPath: /dev/shm
- name: hf-cache
mountPath: /root/.cache/huggingface
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
key: token
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
- name: devshm
emptyDir:
medium: Memory
- name: hf-cache
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test"
key: nightly-hunyuan-image3-performance
soft_fail: true
@@ -1239,6 +1287,7 @@ steps:
- nightly-diffusion-x2iat-performance-qwen-image-edit
- nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- nightly-diffusion-x2iat-performance-qwen-image-layered
- nightly-diffusion-x2iat-performance-bagel
- nightly-diffusion-x2v-performance
- nightly-testcase-statistics
if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
@@ -1255,6 +1304,7 @@ steps:
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
- python tools/nightly/generate_nightly_perf_excel.py
tests/dfx/perf/tests/test_bagel_vllm_omni.json
+152
View File
@@ -0,0 +1,152 @@
[
{
"test_name": "test_bagel_single_device_single_stage_t2i",
"description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task",
"server_type": "vllm-omni",
"server_params": {
"model": "ByteDance-Seed/BAGEL-7B-MoT",
"serve_args": {
"enable-diffusion-pipeline-profiler": true,
"deploy-config": "../vllm_omni/deploy/bagel_single_stage.yaml",
"no-async-chunk": true,
"tensor-parallel-size": 1,
"cfg-parallel-size": 1,
"gpu-memory-utilization": 0.90,
"max-model-len": 8192,
"max-num-batched-tokens": 9216
}
},
"benchmark_params": [
{
"name": "512x512_t2i_steps20",
"dataset": "random",
"task": "t2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 20,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.3678,
"latency_mean": 2.691,
"peak_memory_mb_max": 33083.6,
"peak_memory_mb_mean": 33082.5
}
}
]
},
{
"test_name": "test_bagel_single_device_single_stage_i2i",
"description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task",
"server_type": "vllm-omni",
"server_params": {
"model": "ByteDance-Seed/BAGEL-7B-MoT",
"serve_args": {
"enable-diffusion-pipeline-profiler": true,
"deploy-config": "../vllm_omni/deploy/bagel_single_stage.yaml",
"no-async-chunk": true,
"tensor-parallel-size": 1,
"cfg-parallel-size": 1,
"gpu-memory-utilization": 0.90,
"max-model-len": 8192,
"max-num-batched-tokens": 9216
}
},
"benchmark_params": [
{
"name": "512x512_i2i_steps20",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 20,
"max-concurrency": 1,
"enable-negative-prompt": true,
"num-input-images": 1,
"baseline": {
"throughput_qps": 0.3124,
"latency_mean": 3.1686,
"peak_memory_mb_max": 33448.8,
"peak_memory_mb_mean": 33447.7
}
}
]
},
{
"test_name": "test_bagel_single_device_multi_stage_t2i",
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task",
"server_type": "vllm-omni",
"server_params": {
"model": "ByteDance-Seed/BAGEL-7B-MoT",
"serve_args": {
"enable-diffusion-pipeline-profiler": true,
"deploy-config": "../vllm_omni/deploy/bagel.yaml",
"no-async-chunk": true,
"tensor-parallel-size": 1,
"cfg-parallel-size": 1,
"gpu-memory-utilization": 0.45,
"max-model-len": 8192,
"max-num-batched-tokens": 9216
}
},
"benchmark_params": [
{
"name": "512x512_t2i_steps20",
"dataset": "random",
"task": "t2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 20,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.3746,
"latency_mean": 2.6425,
"peak_memory_mb_max": 33077.0,
"peak_memory_mb_mean": 33077.0
}
}
]
},
{
"test_name": "test_bagel_single_device_multi_stage_i2i",
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task",
"server_type": "vllm-omni",
"server_params": {
"model": "ByteDance-Seed/BAGEL-7B-MoT",
"serve_args": {
"enable-diffusion-pipeline-profiler": true,
"deploy-config": "../vllm_omni/deploy/bagel.yaml",
"no-async-chunk": true,
"tensor-parallel-size": 1,
"cfg-parallel-size": 1,
"gpu-memory-utilization": 0.45,
"max-model-len": 8192,
"max-num-batched-tokens": 9216
}
},
"benchmark_params": [
{
"name": "512x512_i2i_steps20",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 20,
"max-concurrency": 1,
"enable-negative-prompt": true,
"num-input-images": 1,
"baseline": {
"throughput_qps": 0.1349,
"latency_mean": 7.339,
"peak_memory_mb_max": 33321.2,
"peak_memory_mb_mean": 33321.2
}
}
]
}
]