diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 88cb8dbb5..3b3d986aa 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -938,6 +938,54 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" + key: nightly-diffusion-x2iat-performance-bagel + timeout_in_minutes: 180 + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + - export CACHE_DIT_VERSION=1.3.0 + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test" key: nightly-hunyuan-image3-performance soft_fail: true @@ -1239,6 +1287,7 @@ steps: - nightly-diffusion-x2iat-performance-qwen-image-edit - nightly-diffusion-x2iat-performance-qwen-image-edit-2509 - nightly-diffusion-x2iat-performance-qwen-image-layered + - nightly-diffusion-x2iat-performance-bagel - nightly-diffusion-x2v-performance - nightly-testcase-statistics if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" @@ -1255,6 +1304,7 @@ steps: - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509 - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics - python tools/nightly/generate_nightly_perf_excel.py diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json new file mode 100644 index 000000000..09fc96030 --- /dev/null +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -0,0 +1,152 @@ +[ + { + "test_name": "test_bagel_single_device_single_stage_t2i", + "description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task", + "server_type": "vllm-omni", + "server_params": { + "model": "ByteDance-Seed/BAGEL-7B-MoT", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "deploy-config": "../vllm_omni/deploy/bagel_single_stage.yaml", + "no-async-chunk": true, + "tensor-parallel-size": 1, + "cfg-parallel-size": 1, + "gpu-memory-utilization": 0.90, + "max-model-len": 8192, + "max-num-batched-tokens": 9216 + } + }, + "benchmark_params": [ + { + "name": "512x512_t2i_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 20, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.3678, + "latency_mean": 2.691, + "peak_memory_mb_max": 33083.6, + "peak_memory_mb_mean": 33082.5 + } + } + ] + }, + { + "test_name": "test_bagel_single_device_single_stage_i2i", + "description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task", + "server_type": "vllm-omni", + "server_params": { + "model": "ByteDance-Seed/BAGEL-7B-MoT", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "deploy-config": "../vllm_omni/deploy/bagel_single_stage.yaml", + "no-async-chunk": true, + "tensor-parallel-size": 1, + "cfg-parallel-size": 1, + "gpu-memory-utilization": 0.90, + "max-model-len": 8192, + "max-num-batched-tokens": 9216 + } + }, + "benchmark_params": [ + { + "name": "512x512_i2i_steps20", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 20, + "max-concurrency": 1, + "enable-negative-prompt": true, + "num-input-images": 1, + "baseline": { + "throughput_qps": 0.3124, + "latency_mean": 3.1686, + "peak_memory_mb_max": 33448.8, + "peak_memory_mb_mean": 33447.7 + } + } + ] + }, + { + "test_name": "test_bagel_single_device_multi_stage_t2i", + "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task", + "server_type": "vllm-omni", + "server_params": { + "model": "ByteDance-Seed/BAGEL-7B-MoT", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "deploy-config": "../vllm_omni/deploy/bagel.yaml", + "no-async-chunk": true, + "tensor-parallel-size": 1, + "cfg-parallel-size": 1, + "gpu-memory-utilization": 0.45, + "max-model-len": 8192, + "max-num-batched-tokens": 9216 + } + }, + "benchmark_params": [ + { + "name": "512x512_t2i_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 20, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.3746, + "latency_mean": 2.6425, + "peak_memory_mb_max": 33077.0, + "peak_memory_mb_mean": 33077.0 + } + } + ] + }, + { + "test_name": "test_bagel_single_device_multi_stage_i2i", + "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task", + "server_type": "vllm-omni", + "server_params": { + "model": "ByteDance-Seed/BAGEL-7B-MoT", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "deploy-config": "../vllm_omni/deploy/bagel.yaml", + "no-async-chunk": true, + "tensor-parallel-size": 1, + "cfg-parallel-size": 1, + "gpu-memory-utilization": 0.45, + "max-model-len": 8192, + "max-num-batched-tokens": 9216 + } + }, + "benchmark_params": [ + { + "name": "512x512_i2i_steps20", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 20, + "max-concurrency": 1, + "enable-negative-prompt": true, + "num-input-images": 1, + "baseline": { + "throughput_qps": 0.1349, + "latency_mean": 7.339, + "peak_memory_mb_max": 33321.2, + "peak_memory_mb_mean": 33321.2 + } + } + ] + } +]