From 64903b8e30b6595e4018c66232abbbd806aa9dfd Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:27:10 +0800 Subject: [PATCH] [CI/Build] Share nightly CUDA YAML between H100/L4 and B200 (#7028) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] [Your Email] Signed-off-by: [Your Name] Co-authored-by: Cursor Co-authored-by: amy-why-3459 --- .../common/ci_source_file_dependencies.yml | 6 + .buildkite/cuda/test-nightly.yml | 167 ++++++------------ .buildkite/cuda/test-weekly.yml | 14 +- tests/buildkite/test_upload_pipeline.py | 47 +++++ .../perf/tests/test_cosmos3_vllm_omni.json | 8 +- .../test_hunyuanvideo15_i2v_vllm_omni.json | 4 +- .../test_hunyuanvideo15_t2v_vllm_omni.json | 4 +- tests/dfx/perf/tests/test_minicpmo_4_5.json | 2 +- .../test_minicpmo_4_5_duplex_seed_tts.json | 2 +- .../perf/tests/test_minimax_h3_vllm_omni.json | 8 +- .../tests/test_qwen3_omni_async_chunk.json | 4 +- .../tests/test_qwen3_omni_no_async_chunk.json | 2 +- .../perf/tests/test_qwen_image_vllm_omni.json | 8 +- tests/dfx/perf/tests/test_tts.json | 8 +- .../perf/tests/test_wan22_i2v_vllm_omni.json | 4 +- .../distributed/test_cfg_parallel.py | 8 +- tests/diffusion/distributed/test_comm.py | 6 +- .../distributed/test_expert_parallel.py | 2 +- .../test_flux2_vae_patch_parallel.py | 2 +- .../distributed/test_pipeline_parallel.py | 6 +- .../distributed/test_sana_video_sp_numeric.py | 12 +- .../distributed/test_sequence_parallel.py | 2 +- .../test_vae_decode_parallelism.py | 4 +- .../distributed/test_wan_spatial_shard.py | 5 +- .../test_minimax_h3_quantization_quality.py | 2 +- .../offloader/test_diffusion_cpu_offload.py | 7 +- .../test_diffusion_layerwise_offload.py | 13 +- .../quantization/test_gguf_diffusion.py | 2 +- .../quantization/test_quantization_fp8.py | 24 +-- .../quantization/test_quantization_quality.py | 17 +- tests/e2e/accuracy/helpers.py | 26 +++ ...est_hunyuanvideo15_i2v_video_similarity.py | 6 +- ...est_hunyuanvideo15_t2v_video_similarity.py | 6 +- .../minicpmo_4_5/test_minicpmo_4_5.py | 6 +- .../test_minimax_h3_i2va_ref2va_similarity.py | 6 +- .../accuracy/qwen3_omni/test_qwen3_omni.py | 4 +- .../sana_video/test_sana_video_golden.py | 5 +- .../test_sana_video_i2v_alignment.py | 4 +- .../test_sana_video_pipeline_alignment.py | 24 ++- tests/e2e/accuracy/test_cosmos3_similarity.py | 7 +- .../test_diffusers_backend_similarity.py | 39 ++-- tests/e2e/accuracy/test_hunyuan_image3.py | 13 +- .../test_hunyuan_image3_pixel_accuracy.py | 4 +- tests/e2e/accuracy/test_qwen_image.py | 4 +- .../test_wan22_i2v_video_similarity.py | 6 +- .../test_async_omni_collective_rpc.py | 2 +- .../test_async_omni_qwen_image_generate.py | 2 +- .../custom_pipeline/test_worker_extension.py | 2 +- .../test_device_postprocess_equivalence.py | 2 +- .../test_dots_tts_expansion.py | 2 +- .../e2e/offline_inference/test_gepard_tts.py | 4 +- .../test_hunyuanimage3_i2t_expansion.py | 2 +- .../test_lingbot_world_v2.py | 2 +- .../test_longcat_video_avatar.py | 15 +- ...st_qwen_image_autoround_w4a16_expansion.py | 8 +- .../e2e/offline_inference/test_voxcpm2_tts.py | 3 + .../test_wan22_autoround_w4a16_expansion.py | 16 +- .../test_hunyuan_video_15_expansion.py | 26 +-- .../e2e/online_serving/test_lingbot_video.py | 2 +- .../test_lingbot_video_expansion.py | 4 +- .../online_serving/test_lingbot_video_moe.py | 2 +- .../test_minicpmo_4_5_duplex_expansion.py | 6 +- .../e2e/online_serving/test_pi0_expansion.py | 4 +- .../test_qwen3_omni_multi_replicas.py | 6 +- .../test_qwen3_tts_base_expansion.py | 6 +- .../test_qwen3_tts_customvoice_expansion.py | 10 +- .../test_qwen_image_expansion.py | 4 +- tests/e2e/online_serving/test_sana_wm.py | 2 +- .../online_serving/test_wan22_expansion.py | 4 +- .../online_serving/test_zimage_expansion.py | 2 +- .../offline_inference/test_image_to_video.py | 5 +- .../offline_inference/test_internvla_a1.py | 3 +- .../offline_inference/test_text_to_image.py | 2 +- .../test_vace_video_generation.py | 5 +- .../online_serving/test_qwen2_5_omni.py | 17 +- .../online_serving/test_qwen3_omni.py | 17 +- .../online_serving/test_text_to_image.py | 5 +- tests/model_tests/diffusion/case_filtering.py | 6 +- 78 files changed, 437 insertions(+), 321 deletions(-) diff --git a/.buildkite/common/ci_source_file_dependencies.yml b/.buildkite/common/ci_source_file_dependencies.yml index 724794c95..33def4de4 100644 --- a/.buildkite/common/ci_source_file_dependencies.yml +++ b/.buildkite/common/ci_source_file_dependencies.yml @@ -329,6 +329,7 @@ source_file_dependencies: - *qwen_image - tests/e2e/online_serving/test_qwen_image.py - tests/e2e/online_serving/test_qwen_image_expansion.py + - tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py diffusion_qwen_image_accuracy: - *qwen_image @@ -417,6 +418,11 @@ source_file_dependencies: - tests/e2e/online_serving/test_z_image.py - tests/e2e/online_serving/test_zimage_expansion.py + diffusion_sana_video_function: + - *sana_video + - tests/e2e/online_serving/test_sana_video_expansion.py + - tests/e2e/accuracy/sana_video/ + diffusion_stable_audio_function: - *stable_audio - tests/e2e/offline_inference/test_stable_audio_expansion.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index cfaca90fe..1fdb1ef4a 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -35,23 +35,20 @@ steps: source_file_dependencies: omni_minicpmo_4_5_duplex_function timeout_in_minutes: 50 commands: - - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model" - mirror_hardwares: h100_1 + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model" - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU" source_file_dependencies: omni_qwen2_5_omni_doc timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model" - mirror_hardwares: l4_4 + - pytest -s -v tests/examples/ -m "full_model and omni and L4 and B200 and cards_4" --run-level "full_model" - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU" source_file_dependencies: omni_qwen3_omni_doc timeout_in_minutes: 90 commands: - - pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model" - label: ":full_moon: Omni · Accuracy Test" source_file_dependencies: omni_qwen3_omni_accuracy @@ -61,11 +58,10 @@ steps: - export SEED_TTS_EVAL_DEVICE=cuda:1 - | set +e - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model + pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model EXIT=$$? buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json" exit $$EXIT - mirror_hardwares: h100_2 - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test" source_file_dependencies: omni_minicpmo_4_5_accuracy @@ -75,11 +71,10 @@ steps: - export SEED_TTS_EVAL_DEVICE=cuda:1 - | set +e - pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model + pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model EXIT=$$? buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json" exit $$EXIT - mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · No Async Chunk" source_file_dependencies: omni_qwen3_omni_perf @@ -89,11 +84,10 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Async Chunk" source_file_dependencies: omni_qwen3_omni_perf @@ -103,11 +97,10 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - mirror_hardwares: h100_2 - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test" source_file_dependencies: omni_minicpmo_4_5_perf @@ -117,11 +110,10 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and cards_1" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - mirror_hardwares: h100_1 - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test" source_file_dependencies: omni_minicpmo_4_5_duplex_perf @@ -131,18 +123,16 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - mirror_hardwares: h100_1 - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" source_file_dependencies: omni_qwen3_omni_function timeout_in_minutes: 45 commands: - - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model" - mirror_hardwares: h100_4 + - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model" - group: ":card_index_dividers: TTS Model Test" key: nightly-tts-test-group @@ -157,8 +147,7 @@ steps: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy - mirror_hardwares: l4_1 + - pytest -s -v tests/e2e/ -m "full_model and L4 and B200 and tts and cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy - label: ":full_moon: TTS · Perf Test · Single-GPU" source_file_dependencies: tts_qwen3_tts_perf @@ -169,11 +158,10 @@ steps: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - mirror_hardwares: h100_1 - label: ":full_moon: TTS · Perf Test · 2-GPU" source_file_dependencies: tts_qwen3_tts_perf @@ -184,11 +172,10 @@ steps: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2" + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - mirror_hardwares: h100_2 - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" key: nightly-tiny-model-test-group @@ -209,16 +196,14 @@ steps: # for the CI, since these tests should pass irrespective of whether or not we use # the full weights or not. commands: - - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model" - mirror_hardwares: l4_2 + - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level "core_model" - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU" source_file_dependencies: diffusion_tiny_model timeout_in_minutes: 30 # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count). commands: - - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model" - mirror_hardwares: l4_4 + - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level "core_model" - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" key: nightly-diffusion-x2iat-group @@ -235,9 +220,8 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py - -m "full_model and diffusion and H100 and cards_1" + -m "full_model and diffusion and H100 and B200 and cards_1" --run-level "full_model" - mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU" source_file_dependencies: diffusion_qwen_image_function @@ -246,36 +230,29 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py - -m "full_model and diffusion and H100 and cards_2" + -m "full_model and diffusion and H100 and B200 and cards_2" --run-level "full_model" - mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Z-Image Function Test · L4 · 4-GPU" - source_file_dependencies: diffusion_z_image_function + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU" + source_file_dependencies: diffusion_qwen_image_function timeout_in_minutes: 120 commands: - - >- - pytest -sv - tests/e2e/online_serving/test_zimage_expansion.py - -m "full_model and diffusion and L4 and cards_4" - --run-level "full_model" - mirror_hardwares: l4_4 + - pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m "full_model and diffusion and L4 and B200 and cards_1" --run-level "full_model" + - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" source_file_dependencies: diffusion_text_to_image_doc timeout_in_minutes: 60 commands: - - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100 and B200 and (cards_1 or cards_2)" --run-level "full_model" - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" source_file_dependencies: diffusion_qwen_image_accuracy timeout_in_minutes: 180 commands: - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 - - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model - mirror_hardwares: h100_1 + - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test" source_file_dependencies: diffusion_hunyuan_image3_accuracy @@ -283,8 +260,7 @@ steps: commands: - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct" - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3" - - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model - mirror_hardwares: h100_4 + - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and H100 and B200 and cards_4" --run-level full_model - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU" source_file_dependencies: diffusion_qwen_image_perf @@ -295,12 +271,11 @@ steps: - export CACHE_DIT_VERSION=1.5.0 - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1" + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU" source_file_dependencies: diffusion_qwen_image_perf @@ -308,16 +283,16 @@ steps: timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.5.0 + # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN; + # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM. - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4" + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - mirror_hardwares: h100_4 - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group @@ -331,36 +306,31 @@ steps: source_file_dependencies: diffusion_wan22_function timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model" - mirror_hardwares: h100_1 + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model" - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU" source_file_dependencies: diffusion_wan22_function timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model" - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU" source_file_dependencies: diffusion_wan22_function timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model" - mirror_hardwares: h100_1 + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model" - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU" source_file_dependencies: diffusion_wan22_function timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model" - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test" source_file_dependencies: diffusion_hunyuan_video_function timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model" - label: ":full_moon: Diffusion X2V · LingBot Function Test" source_file_dependencies: diffusion_lingbot_function @@ -371,23 +341,20 @@ steps: tests/e2e/online_serving/test_lingbot_video.py tests/e2e/online_serving/test_lingbot_video_moe.py tests/e2e/offline_inference/test_lingbot_world_v2.py - -m "full_model and diffusion and H100 and cards_1" + -m "full_model and diffusion and H100 and B200 and cards_1" --run-level "full_model" - mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2V · Doc Test" source_file_dependencies: diffusion_image_to_video_doc timeout_in_minutes: 60 commands: - - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model" - mirror_hardwares: h100_1 + - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100 and B200 and cards_1" --run-level "full_model" - label: ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test" source_file_dependencies: diffusion_wan22_function timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model" - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test" source_file_dependencies: diffusion_wan22_function @@ -395,31 +362,27 @@ steps: commands: - | VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \ - pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_1 + pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and H100 and B200 and cuda and cards_1" --run-level "full_model" - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" source_file_dependencies: diffusion_wan22_accuracy timeout_in_minutes: 180 commands: - - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model - mirror_hardwares: h100_2 + - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test" source_file_dependencies: diffusion_minimax_h3_accuracy timeout_in_minutes: 180 commands: - - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model - mirror_hardwares: h100_4 + - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m "full_model and H100 and B200 and cards_4" --run-level full_model - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test" source_file_dependencies: diffusion_hunyuan_video_accuracy timeout_in_minutes: 180 commands: - - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - mirror_hardwares: h100_2 + - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model + - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU" source_file_dependencies: diffusion_wan22_perf @@ -429,12 +392,11 @@ steps: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1" + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU" source_file_dependencies: @@ -444,18 +406,16 @@ steps: timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2" + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2" EXIT1=$$? - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2" + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2" EXIT2=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi exit $$EXIT2 - mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5" source_file_dependencies: diffusion_hunyuan_video_perf @@ -465,15 +425,14 @@ steps: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" EXIT1=$$? - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" EXIT2=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi exit $$EXIT2 - mirror_hardwares: h100_2 - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3" source_file_dependencies: diffusion_minimax_h3_perf @@ -483,12 +442,11 @@ steps: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT - mirror_hardwares: h100_4 - group: ":card_index_dividers: Diffusion Test" key: nightly-diffusion-test-group @@ -499,65 +457,56 @@ steps: source_file_dependencies: diffusion_distributed_attention timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model" - mirror_hardwares: l4_2 + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level "full_model" - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU" source_file_dependencies: diffusion_distributed_attention timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model" - mirror_hardwares: l4_3 + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level "full_model" - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU" source_file_dependencies: diffusion_distributed_attention timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model" - mirror_hardwares: l4_4 + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level "full_model" - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU" source_file_dependencies: diffusion_distributed_attention timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model" - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU" source_file_dependencies: diffusion_distributed_attention timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model" - mirror_hardwares: h100_4 + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level "full_model" - label: ":full_moon: Diffusion · Offloader Test" source_file_dependencies: diffusion_offloader timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/offloader -m "full_model and cuda" - mirror_hardwares: l4_1 + - pytest -sv tests/diffusion/offloader -m "full_model and L4 and B200 and cuda and cards_1" - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU" source_file_dependencies: diffusion_quantization timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" - mirror_hardwares: h100_1 + - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level "full_model" - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU" source_file_dependencies: diffusion_minimax_h3_function timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model" - mirror_hardwares: h100_2 + - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model" - label: ":full_moon: Diffusion · Quantization Test with L4" source_file_dependencies: diffusion_quantization timeout_in_minutes: 60 commands: - pip install "vllm-gguf-plugin==0.0.4" - - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" - mirror_hardwares: l4_1 + - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level "full_model" # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. # Do not add new depends_on entries or extend this step. diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index d53bc56e4..7809fcbc7 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -339,11 +339,14 @@ steps: mirror_hardwares: h100_1 - label: "Diffusion · SANA-Video · H100 · Single-GPU" - timeout_in_minutes: 90 + source_file_dependencies: diffusion_sana_video_function + timeout_in_minutes: 180 commands: + - export SANA_VIDEO_GOLDEN_BASE_URL=https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/sana-video/v2 - >- pytest -sv tests/e2e/online_serving/test_sana_video_expansion.py + tests/e2e/accuracy/sana_video/ -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 @@ -488,6 +491,15 @@ steps: --run-level "full_model" mirror_hardwares: h100_2 + - label: "Diffusion · H100 · 3-GPU" + timeout_in_minutes: 120 + commands: + - >- + pytest -sv + tests/e2e/ -m "slow and diffusion and H100 and cards_3" + --run-level "full_model" + mirror_hardwares: h100_3 + - label: "Diffusion · H100 · 4-GPU" timeout_in_minutes: 90 commands: diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index a3192dada..b39dba2cd 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -27,6 +27,7 @@ from upload_pipeline import ( # noqa: E402 pytestmark = [pytest.mark.core_model, pytest.mark.cpu] CUDA_BOOTSTRAP_STEPS = Path(".buildkite/cuda/bootstrap-upload-steps.yml") +NIGHTLY_YAML = Path(".buildkite/cuda/test-nightly.yml") BOOTSTRAP_STEPS_TEMPLATE = """steps: - key: image-build - key: upload-ready-pipeline @@ -368,6 +369,52 @@ def test_cpu_step_without_mirror_hardwares_is_unchanged() -> None: assert _expand_mirror_hardwares(step) is step +def _leaf_steps(steps: list) -> list[dict]: + leaves: list[dict] = [] + for step in steps: + if not isinstance(step, dict): + continue + nested = step.get("steps") + if nested is not None: + leaves.extend(_leaf_steps(nested)) + else: + leaves.append(step) + return leaves + + +def test_nightly_yaml_infers_h100_l4_and_b200(monkeypatch: pytest.MonkeyPatch) -> None: + """Nightly inferred jobs share card counts across H100/L4 and B200 mirrors.""" + src = yaml.safe_load(NIGHTLY_YAML.read_text(encoding="utf-8")) + pytest_leaves = [step for step in _leaf_steps(src["steps"]) if "pytest" in str(step.get("commands"))] + inferred_leaves = [step for step in pytest_leaves if "mirror_hardwares" not in step] + pinned_leaves = [step for step in pytest_leaves if "mirror_hardwares" in step] + assert inferred_leaves + + monkeypatch.setattr("upload_pipeline._get_mirror_hw_selector", lambda: "") + default_by_label = { + step["label"]: step for step in _leaf_steps(_render_test_pipeline(src, changed_files=None)["steps"]) + } + + monkeypatch.setattr("upload_pipeline._get_mirror_hw_selector", lambda: "b200") + b200_src = yaml.safe_load(NIGHTLY_YAML.read_text(encoding="utf-8")) + b200_by_label = { + step["label"]: step for step in _leaf_steps(_render_test_pipeline(b200_src, changed_files=None)["steps"]) + } + + pytest_labels = {step["label"] for step in inferred_leaves} + pinned_labels = {step["label"] for step in pinned_leaves} + assert pinned_labels <= set(default_by_label) + assert pinned_labels.isdisjoint(b200_by_label) + assert pytest_labels <= set(default_by_label) + assert pytest_labels <= set(b200_by_label) + for label in pytest_labels: + default_step = default_by_label[label] + b200_step = b200_by_label[label] + assert default_step["agents"]["queue"] in {"mithril-h100-pool", "l4-k8s"} + assert b200_step["agents"]["queue"] == "b200-k8s" + assert _gpu_limit(default_step) == _gpu_limit(b200_step) + + @pytest.mark.parametrize(("raw", "expected"), [("", ""), (" ", ""), ("b200", "b200"), ("B200", "b200")]) def test_mirror_hw_selector_empty_or_b200(monkeypatch: pytest.MonkeyPatch, raw: str, expected: str) -> None: monkeypatch.setenv("MIRROR_HW", raw) diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index bcd649de1..5e2b9d3dc 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } @@ -63,7 +63,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } @@ -132,7 +132,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } @@ -200,7 +200,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json index 8186edf2c..25d91c44f 100644 --- a/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 1 } @@ -55,7 +55,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json index 3771c1c81..ffaaba436 100644 --- a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 1 } @@ -54,7 +54,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index cac1880c5..7d7ca08ad 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100", + "cuda": ["H100", "B200"], "npu": "A3" }, "num_cards": 1 diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json index 801743134..bb0599419 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100", + "cuda": ["H100", "B200"], "npu": "A3" }, "num_cards": 1 diff --git a/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json b/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json index e16b40e91..7dfbe6f3d 100644 --- a/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 4 } @@ -76,7 +76,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 4 } @@ -148,7 +148,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 4 } @@ -219,7 +219,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 4 } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 6b0247aaf..1292a79e5 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100", + "cuda": ["H100", "B200"], "npu": "A3" }, "num_cards": 2 @@ -242,7 +242,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index d73629666..186bac452 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100", + "cuda": ["H100", "B200"], "npu": "A3" }, "num_cards": 2 diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 4f64d6ec1..61cb334eb 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 1 } @@ -66,7 +66,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 1 } @@ -150,7 +150,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 4 } @@ -197,7 +197,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 4 } diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 592f61ca2..c627c01e8 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100", + "cuda": ["H100", "B200"], "npu": "A3" }, "num_cards": 2 @@ -121,7 +121,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } @@ -285,7 +285,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 2 } @@ -416,7 +416,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 1 } diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 234bbf1db..3244d1c41 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -5,7 +5,7 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": ["H100", "B200"] }, "num_cards": 1 } @@ -57,7 +57,7 @@ { "hardware_marks": { "res": { - "cuda": "H100", + "cuda": ["H100", "B200"], "npu": "A3" }, "num_cards": 2 diff --git a/tests/diffusion/distributed/test_cfg_parallel.py b/tests/diffusion/distributed/test_cfg_parallel.py index 2ade13549..20679cf76 100644 --- a/tests/diffusion/distributed/test_cfg_parallel.py +++ b/tests/diffusion/distributed/test_cfg_parallel.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Tests for CFG (Classifier-Free Guidance) parallel functionality. CPU tests mock the CFG process group and validate mixin logic without GPUs. @@ -32,8 +32,8 @@ from vllm_omni.diffusion.distributed.parallel_state import ( ) from vllm_omni.platforms import current_omni_platform -_L4_TWO_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=2) -_L4_THREE_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=3) +_L4_TWO_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=2) +_L4_THREE_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=3) def _set_random_seeds(seed: int) -> None: @@ -1009,7 +1009,7 @@ def _run_multi_branch_gpu_parity( @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "L4"}, num_cards=2) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=2) @pytest.mark.parametrize("batch_size", [2]) @pytest.mark.parametrize("cfg_normalize", [False, True]) @pytest.mark.parametrize("extra_kwargs", [None, {"step_i": 1}]) diff --git a/tests/diffusion/distributed/test_comm.py b/tests/diffusion/distributed/test_comm.py index 647389048..effe2b541 100644 --- a/tests/diffusion/distributed/test_comm.py +++ b/tests/diffusion/distributed/test_comm.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Tests for SeqAllToAll4D, SeqAllToAll5D, and RingComm communication primitives. CPU tests use gloo on CPU tensors (no GPU). Nightly parity tests run the same @@ -24,8 +24,8 @@ from vllm_omni.diffusion.distributed.parallel_state import ( ) from vllm_omni.platforms import current_omni_platform -_L4_TWO_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=2) -_L4_FOUR_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=4) +_L4_TWO_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=2) +_L4_FOUR_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=4) DeviceKind = Literal["cpu", "cuda"] diff --git a/tests/diffusion/distributed/test_expert_parallel.py b/tests/diffusion/distributed/test_expert_parallel.py index 8e9be1367..053fed9d1 100644 --- a/tests/diffusion/distributed/test_expert_parallel.py +++ b/tests/diffusion/distributed/test_expert_parallel.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ System test for Expert Parallel (EP). diff --git a/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py b/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py index 8de3a42f8..204406ec1 100644 --- a/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py +++ b/tests/diffusion/distributed/test_flux2_vae_patch_parallel.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """FLUX.2-dev VAE patch-parallel correctness tests. diff --git a/tests/diffusion/distributed/test_pipeline_parallel.py b/tests/diffusion/distributed/test_pipeline_parallel.py index cccf116c3..08c3cc775 100644 --- a/tests/diffusion/distributed/test_pipeline_parallel.py +++ b/tests/diffusion/distributed/test_pipeline_parallel.py @@ -31,9 +31,9 @@ from vllm_omni.diffusion.distributed.parallel_state import ( from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, PipelineParallelMixin from vllm_omni.platforms import current_omni_platform -_L4_TWO_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=2) -_L4_THREE_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=3) -_L4_FOUR_GPU = hardware_marks(res={"cuda": "L4"}, num_cards=4) +_L4_TWO_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=2) +_L4_THREE_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=3) +_L4_FOUR_GPU = hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=4) DeviceKind = Literal["cpu", "cuda"] _UNIT_MARKS = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] diff --git a/tests/diffusion/distributed/test_sana_video_sp_numeric.py b/tests/diffusion/distributed/test_sana_video_sp_numeric.py index b18cafe4c..872dcbcbb 100644 --- a/tests/diffusion/distributed/test_sana_video_sp_numeric.py +++ b/tests/diffusion/distributed/test_sana_video_sp_numeric.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Real multi-process numeric equivalence gate for SANA-Video sequence parallel. Runs the native transformer directly (no pipeline, no topology validation) with @@ -273,7 +273,7 @@ def _baseline_worker( @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "L4"}, num_cards=2) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=2) @pytest.mark.parametrize("task", ["t2v", "i2v"]) @pytest.mark.parametrize("num_frames", [21, 5]) def test_sp2_matches_sp1_fp32(task: str, num_frames: int) -> None: @@ -290,7 +290,7 @@ def test_sp2_matches_sp1_fp32(task: str, num_frames: int) -> None: @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "L4"}, num_cards=4) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=4) @pytest.mark.parametrize("task", ["t2v", "i2v"]) @pytest.mark.parametrize("num_frames", [21, 5]) def test_sp4_matches_sp1_fp32(task: str, num_frames: int) -> None: @@ -307,7 +307,7 @@ def test_sp4_matches_sp1_fp32(task: str, num_frames: int) -> None: @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "L4"}, num_cards=4) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=4) def test_sp2_cfg2_matches_sp1_fp32() -> None: """Four processes: CFG branches with distinct inputs each run their own SP group; catches CFG/SP group crossover and deadlocks.""" @@ -324,7 +324,7 @@ def test_sp2_cfg2_matches_sp1_fp32() -> None: @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "L4"}, num_cards=4) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=4) def test_tp2_sp2_matches_sp1_fp32() -> None: """Four processes: TP shards heads and channels while SP shards frames; the combined mesh must still reproduce the serial output.""" @@ -342,7 +342,7 @@ def test_tp2_sp2_matches_sp1_fp32() -> None: @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "L4"}, num_cards=2) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=2) def test_sp2_bf16_drift_informational() -> None: """Records the bf16 reduction-reorder drift magnitude; the load-bearing numeric gate is the fp32 suite above.""" diff --git a/tests/diffusion/distributed/test_sequence_parallel.py b/tests/diffusion/distributed/test_sequence_parallel.py index f083e1d26..ba68669e9 100644 --- a/tests/diffusion/distributed/test_sequence_parallel.py +++ b/tests/diffusion/distributed/test_sequence_parallel.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ System test for Sequence Parallel (SP) backends: Ulysses and Ring attention. diff --git a/tests/diffusion/distributed/test_vae_decode_parallelism.py b/tests/diffusion/distributed/test_vae_decode_parallelism.py index 28ded4639..acd916806 100644 --- a/tests/diffusion/distributed/test_vae_decode_parallelism.py +++ b/tests/diffusion/distributed/test_vae_decode_parallelism.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """VAE patch-parallel decode parity tests for diffusion models. @@ -145,7 +145,7 @@ def _run_generate( @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 4}) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards={"cuda": 4}) @pytest.mark.parametrize("model_case", MODEL_CASES) def test_vae_patch_parallel_tp2(model_case: dict[str, Any], tmp_path: Path): if current_omni_platform.is_npu(): diff --git a/tests/diffusion/distributed/test_wan_spatial_shard.py b/tests/diffusion/distributed/test_wan_spatial_shard.py index beb5ac2db..d8842ee3f 100644 --- a/tests/diffusion/distributed/test_wan_spatial_shard.py +++ b/tests/diffusion/distributed/test_wan_spatial_shard.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import os from types import SimpleNamespace @@ -538,7 +541,7 @@ def _spatial_shard_decode_worker(rank: int, split_dim: str, return_dict, master_ @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel -@hardware_test(res={"cuda": "H100"}, num_cards=_SPATIAL_SHARD_WORLD_SIZE) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=_SPATIAL_SHARD_WORLD_SIZE) @pytest.mark.parametrize("split_dim", ["height", "width"]) def test_spatial_shard_decode_matches_reference(split_dim: str): manager = mp.get_context("spawn").Manager() diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py index a45631f7d..c7e53ca62 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py @@ -175,7 +175,7 @@ def _generate_joint_output(omni, config: QualityTestConfig): pytest.param( _QUALITY_CONFIG, id=_QUALITY_CONFIG.id, - marks=hardware_marks(res={"cuda": _QUALITY_CONFIG.gpu}, num_cards=2), + marks=hardware_marks(res={"cuda": ["H100", "B200"]}, num_cards=2), ) ], ) diff --git a/tests/diffusion/offloader/test_diffusion_cpu_offload.py b/tests/diffusion/offloader/test_diffusion_cpu_offload.py index 80a03180f..205fad8fd 100644 --- a/tests/diffusion/offloader/test_diffusion_cpu_offload.py +++ b/tests/diffusion/offloader/test_diffusion_cpu_offload.py @@ -9,7 +9,7 @@ import pytest import torch from vllm.distributed.parallel_state import cleanup_dist_env_and_memory -from tests.helpers.mark import hardware_test +from tests.helpers.mark import hardware_marks, hardware_test from tests.helpers.monitor import DeviceMemoryMonitor from tests.helpers.runtime import OmniRunner from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -27,7 +27,10 @@ MODELS: dict[str, dict[str, int | None]] = {**AUDIO_MODEL, **IMAGE_MODELS} MODEL_MARKS = { "riverclouds/qwen_image_random": pytest.mark.core_model, - "stabilityai/stable-audio-open-1.0": pytest.mark.full_model, + "stabilityai/stable-audio-open-1.0": [ + pytest.mark.full_model, + *hardware_marks(res={"cuda": ["L4", "B200"], "rocm": "MI325"}), + ], } _GATED_MODELS = {"stabilityai/stable-audio-open-1.0"} diff --git a/tests/diffusion/offloader/test_diffusion_layerwise_offload.py b/tests/diffusion/offloader/test_diffusion_layerwise_offload.py index c386871d3..c579affe9 100644 --- a/tests/diffusion/offloader/test_diffusion_layerwise_offload.py +++ b/tests/diffusion/offloader/test_diffusion_layerwise_offload.py @@ -12,7 +12,7 @@ import pytest import torch from vllm.distributed.parallel_state import cleanup_dist_env_and_memory -from tests.helpers.mark import hardware_test +from tests.helpers.mark import hardware_marks from tests.helpers.monitor import DeviceMemoryMonitor from tests.helpers.runtime import OmniRunner from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -34,8 +34,14 @@ IMAGE_VIDEO_MODELS: dict[str, dict[str, int | None]] = { MODELS: dict[str, dict[str, int | None]] = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS} MODEL_MARKS = { - "riverclouds/qwen_image_random": pytest.mark.core_model, - "stabilityai/stable-audio-open-1.0": pytest.mark.full_model, + "riverclouds/qwen_image_random": [ + pytest.mark.core_model, + *hardware_marks(res={"cuda": "L4", "rocm": "MI325"}), + ], + "stabilityai/stable-audio-open-1.0": [ + pytest.mark.full_model, + *hardware_marks(res={"cuda": ["L4", "B200"], "rocm": "MI325"}), + ], } AUDIO_MODEL_PARAMS: dict[str, dict[str, Any]] = { @@ -309,7 +315,6 @@ def test_stable_audio_requires_both_outputs(audio_no_offload, audio_offload, mes @pytest.mark.diffusion -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}) @pytest.mark.parametrize( "model_name", [pytest.param(name, marks=MODEL_MARKS[name]) for name in MODELS], diff --git a/tests/diffusion/quantization/test_gguf_diffusion.py b/tests/diffusion/quantization/test_gguf_diffusion.py index ad2e72edc..dc91a1e40 100644 --- a/tests/diffusion/quantization/test_gguf_diffusion.py +++ b/tests/diffusion/quantization/test_gguf_diffusion.py @@ -109,7 +109,7 @@ def _generate_single_stage_image( return images, peak_mem -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parametrize("model", [Z_IMAGE_CONFIG, FLUX_CONFIG], ids=["Z-Image-Turbo", "FLUX.2-klein"]) diff --git a/tests/diffusion/quantization/test_quantization_fp8.py b/tests/diffusion/quantization/test_quantization_fp8.py index 5793ad95c..5aa7faaa6 100644 --- a/tests/diffusion/quantization/test_quantization_fp8.py +++ b/tests/diffusion/quantization/test_quantization_fp8.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ End-to-end tests for the unified quantization framework (PR #1764). @@ -270,7 +270,7 @@ def _generate_bagel_image( # ─── Single-stage diffusion model tests ────────────────────────────────────── -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_single_stage_zimage_fp8(): """Z-Image-Turbo with FP8 generates valid images.""" images, _ = _generate_single_stage_image( @@ -281,7 +281,7 @@ def test_single_stage_zimage_fp8(): images[0].save("test_zimage_fp8.png") -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_single_stage_zimage_fp8_uses_less_memory(): """FP8 should use less peak memory than BF16 for Z-Image-Turbo.""" _, mem_bf16 = _generate_single_stage_image( @@ -300,7 +300,7 @@ def test_single_stage_zimage_fp8_uses_less_memory(): assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)" -@hardware_test(res={"cuda": "L4", "xpu": "B60"}) +@hardware_test(res={"cuda": ["L4", "B200"], "xpu": "B60"}) @pytest.mark.advanced_model def test_single_stage_qwen_image_fp8(): """Qwen-Image (random weights) with FP8 generates valid images.""" @@ -316,7 +316,7 @@ def test_single_stage_qwen_image_fp8(): images[0].save("test_qwen_image_fp8.png") -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.skip(reason="This model is not authorized on Hugging Face Hub yet") def test_single_stage_flux_fp8(): """FLUX.1-dev with FP8 generates valid images.""" @@ -331,7 +331,7 @@ def test_single_stage_flux_fp8(): images[0].save("test_flux_fp8.png") -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.skip(reason="This model is not authorized on Hugging Face Hub yet") def test_single_stage_flux_fp8_uses_less_memory(): """FP8 should use less peak memory than BF16 for FLUX.1-dev.""" @@ -357,7 +357,7 @@ def test_single_stage_flux_fp8_uses_less_memory(): assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)" -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) def test_single_stage_ltx2_fp8_uses_less_memory(): """FP8 should use less peak memory than BF16 for LTX-2.""" _, mem_bf16 = _generate_single_stage_video( @@ -379,7 +379,7 @@ def test_single_stage_ltx2_fp8_uses_less_memory(): # ─── Multi-stage model tests (BAGEL) ───────────────────────────────────────── -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) def test_bagel_fp8_generates_image(): """BAGEL with diffusion-stage FP8 generates a valid image. @@ -392,7 +392,7 @@ def test_bagel_fp8_generates_image(): image.save("test_bagel_fp8.png") -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) def test_bagel_bf16_generates_image(): """BAGEL without quantization generates a valid image (baseline).""" image, _ = _generate_bagel_image(diffusion_quantization_config=None) @@ -402,7 +402,7 @@ def test_bagel_bf16_generates_image(): # ─── Quantization config routing tests ──────────────────────────────────────── -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_quantization_key_maps_to_quantization_config(): """The old 'quantization' kwarg should map to 'quantization_config' in OmniDiffusionConfig.from_kwargs for backwards compatibility.""" @@ -413,7 +413,7 @@ def test_quantization_key_maps_to_quantization_config(): assert config.quantization_config.get_name() == "fp8" -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_quantization_config_key_takes_priority(): """When both 'quantization' and 'quantization_config' are set, 'quantization_config' takes priority.""" @@ -428,7 +428,7 @@ def test_quantization_config_key_takes_priority(): assert config.quantization_config.activation_scheme == "static" -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_single_stage_quantization_config_key(): """Single-stage model using quantization_config (dict) key generates images.""" images, _ = _generate_single_stage_image( diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index 47908daa0..4bace0da5 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -45,6 +45,7 @@ import pytest import torch from PIL import Image +from tests.e2e.accuracy.helpers import resolve_device_threshold from tests.helpers.mark import hardware_marks _REPO_ROOT = Path(__file__).resolve().parents[3] @@ -70,7 +71,7 @@ class QualityTestConfig: id: str # pytest ID, e.g. "fp8_z_image" task: str # "t2i" or "t2v" prompt: str # generation prompt - max_lpips: float # fail threshold — higher = more lenient + max_lpips: float | dict[str, float] # threshold, or {"H100": 0.15, "B200": 0.17} model: str | None = None # HF model name quantization: str | dict[str, object] | None = None # quantization method/config, e.g. "fp8" baseline_model: str | None = None # explicit BF16/local baseline path @@ -80,7 +81,6 @@ class QualityTestConfig: num_inference_steps: int = 20 # keep low for CI speed num_frames: int = 5 # only for t2v seed: int = 42 - gpu: str = "H100" # minimum GPU requirement negative_prompt: str | None = "" guidance_scale: float | None = None sigmas: list[float] | None = None @@ -190,7 +190,7 @@ QUALITY_CONFIGS = [ quantization={"text_encoder": "fp8", "transformer": None, "vae": None}, task="t2i", prompt="a cup of coffee on a wooden table, morning light", - max_lpips=0.15, + max_lpips={"H100": 0.15, "B200": 0.17}, num_inference_steps=10, enable_cpu_offload=True, height=1024, @@ -528,7 +528,7 @@ def test_ltx_quality_gate_uses_official_eager_defaults(monkeypatch): assert captured.sampling.guidance_scale is None -_marks = hardware_marks(res={"cuda": "H100"}) +_marks = hardware_marks(res={"cuda": ["H100", "B200"]}) _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUALITY_OUTPUT_DIR" in os.environ else None @@ -588,8 +588,9 @@ def test_quantization_quality(config: QualityTestConfig): # --- Similarity metrics --- lpips_score = _compute_lpips(baseline_out, quant_out, config.task) psnr_score, mae_score = _compute_psnr_and_mae(baseline_out, quant_out, config.task) - assert lpips_score <= config.max_lpips, ( - f"LPIPS {lpips_score:.4f} exceeds threshold {config.max_lpips} " + gpu_key, max_lpips = resolve_device_threshold(config.max_lpips, label=f"{config.id} max_lpips") + assert lpips_score <= max_lpips, ( + f"LPIPS {lpips_score:.4f} exceeds threshold {max_lpips} ({gpu_key}) " f"for {config.quantization_ref() or 'pre-quantized checkpoint'} on {config.quantized_ref()}" ) @@ -601,12 +602,12 @@ def test_quantization_quality(config: QualityTestConfig): print(f" Baseline: {config.baseline_ref()}") print(f" Quantized: {config.quantized_ref()}") print(f" Method: {config.quantization_ref() or 'pre-quantized checkpoint'}") - print(f" LPIPS: {lpips_score:.4f} (threshold: {config.max_lpips})") + print(f" LPIPS: {lpips_score:.4f} (threshold: {max_lpips}, gpu: {gpu_key})") print(f" PSNR: {psnr_score:.4f} dB (higher is better)") print(f" MAE: {mae_score:.6f} (lower is better)") print(f" BF16 memory: {bl_mem:.2f} GiB") print(f" Quant memory: {qt_mem:.2f} GiB ({mem_reduction:.0f}% reduction)") - print(f" Result: {'PASS' if lpips_score <= config.max_lpips else 'FAIL'}") + print(f" Result: {'PASS' if lpips_score <= max_lpips else 'FAIL'}") print(f"{'=' * 60}\n") assert np.isfinite(psnr_score) or np.isinf(psnr_score), f"PSNR is invalid for {config.id}: {psnr_score}" diff --git a/tests/e2e/accuracy/helpers.py b/tests/e2e/accuracy/helpers.py index cc1b7c589..0ea9074b8 100644 --- a/tests/e2e/accuracy/helpers.py +++ b/tests/e2e/accuracy/helpers.py @@ -135,6 +135,32 @@ def resolve_similarity_thresholds( return table.get(profile, table[DEFAULT_DEVICE_PROFILE]) +def resolve_device_threshold( + thresholds: float | int | Mapping[str, float], + *, + device_name: str | None = None, + label: str = "threshold", +) -> tuple[str, float]: + """Resolve a scalar or per-GPU threshold for the current CUDA device. + + Returns ``(profile_key, threshold)``. A bare number uses profile ``default``. + A mapping matches keys as substrings of the device name via + ``resolve_device_profile`` (for example ``{"H100": 0.3, "B200": 0.36}``). + Prefer an explicit ``default`` entry when unmatched GPUs should fall back; + otherwise unmatched devices raise. + """ + if isinstance(thresholds, float | int): + return DEFAULT_DEVICE_PROFILE, float(thresholds) + + profile = resolve_device_profile(device_name, profiles=thresholds) + if profile in thresholds: + return profile, float(thresholds[profile]) + if DEFAULT_DEVICE_PROFILE in thresholds: + return DEFAULT_DEVICE_PROFILE, float(thresholds[DEFAULT_DEVICE_PROFILE]) + known = ", ".join(str(key) for key in thresholds) + raise RuntimeError(f"no {label} entry for device profile {profile!r}; known: {known}") + + _SSIM_RE = re.compile(r"All:(?P[0-9.]+)") _PSNR_RE = re.compile(r"average:(?P[0-9.]+)") diff --git a/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py b/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py index eb8f2d769..f5df4004f 100644 --- a/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py +++ b/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py @@ -183,7 +183,7 @@ def _generate_online_video( @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_hunyuanvideo15_i2v_diffusers_offline_generates_video( hunyuanvideo15_i2v_image_source: str | None, ) -> None: @@ -203,7 +203,7 @@ def test_hunyuanvideo15_i2v_diffusers_offline_generates_video( @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) def test_hunyuanvideo15_i2v_online_serving_generates_video( omni_server, @@ -229,7 +229,7 @@ def test_hunyuanvideo15_i2v_online_serving_generates_video( @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_hunyuanvideo15_i2v_serving_matches_offline_video_similarity( hunyuanvideo15_i2v_image_source: str | None, ) -> None: diff --git a/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py b/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py index f5b90822d..f16813f12 100644 --- a/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py +++ b/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py @@ -144,7 +144,7 @@ def _generate_online_video(*, omni_server, online_client, timeout_seconds: int) @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None: if not torch.cuda.is_available(): pytest.skip("HunyuanVideo-1.5 T2V offline accuracy test requires CUDA.") @@ -160,7 +160,7 @@ def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None: @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) def test_hunyuanvideo15_t2v_online_serving_generates_video( omni_server, @@ -182,7 +182,7 @@ def test_hunyuanvideo15_t2v_online_serving_generates_video( @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_hunyuanvideo15_t2v_serving_matches_offline_video_similarity() -> None: if not torch.cuda.is_available(): pytest.skip("HunyuanVideo-1.5 T2V video similarity test requires CUDA.") diff --git a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py index e688b9360..ad659541d 100644 --- a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py +++ b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -141,7 +141,7 @@ def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None: @requires_daily_omni_deps -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", daily_test_params, indirect=True) def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: _require_vllm_cli() @@ -174,7 +174,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", seed_test_params, indirect=True) def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: _require_vllm_cli() @@ -200,7 +200,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", duplex_accuracy_test_params, indirect=True) def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None: """Gate Seed-TTS WER through the explicit Realtime TTS contract.""" diff --git a/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py b/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py index 60f829bee..4ae45e4e1 100644 --- a/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py +++ b/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py @@ -138,8 +138,6 @@ def _server_args() -> list[str]: "--vae-parallel-mode", "tile", "--vae-use-tiling", - "--diffusion-attention-backend", - "FLASH_ATTN", "--stage-init-timeout", "1800", "--init-timeout", @@ -228,7 +226,7 @@ def _assert_official_video( ) -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4) def test_minimax_h3_i2va_matches_official_reference( accuracy_artifact_root: Path, ) -> None: @@ -304,7 +302,7 @@ def test_minimax_h3_i2va_matches_official_reference( ) -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4) def test_minimax_h3_ref2va_matches_official_reference( accuracy_artifact_root: Path, ) -> None: diff --git a/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py b/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py index 7c48053e1..0acf21692 100644 --- a/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py +++ b/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py @@ -110,7 +110,7 @@ def _daily_omni_hub_inline_media(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_qwen3_omni_daily_omni_accuracy_bench(omni_server) -> None: _require_vllm_cli() @@ -122,7 +122,7 @@ def test_qwen3_omni_daily_omni_accuracy_bench(omni_server) -> None: assert _acc_bench.run_acc_benchmark(ns) == 0 -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_qwen3_omni_seed_tts_wer_bench(omni_server) -> None: _require_vllm_cli() diff --git a/tests/e2e/accuracy/sana_video/test_sana_video_golden.py b/tests/e2e/accuracy/sana_video/test_sana_video_golden.py index c0c468315..cb6ffe6c9 100644 --- a/tests/e2e/accuracy/sana_video/test_sana_video_golden.py +++ b/tests/e2e/accuracy/sana_video/test_sana_video_golden.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -44,7 +44,7 @@ SSIM_THRESHOLDS = { } pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.skipif( not GOLDEN_BASE_URL, @@ -169,6 +169,7 @@ def _validate_metadata(payload: Any, manifest: dict[str, Any], variant: str, tas def _download_case(variant: str, task: str, output_dir: Path, filenames: set[str]) -> dict[str, Any]: output_dir.mkdir(parents=True, exist_ok=True) + assert GOLDEN_BASE_URL is not None base_url = f"{GOLDEN_BASE_URL.rstrip('/')}/{variant}/{task}" response = requests.get(f"{base_url}/manifest.json", timeout=60) response.raise_for_status() diff --git a/tests/e2e/accuracy/sana_video/test_sana_video_i2v_alignment.py b/tests/e2e/accuracy/sana_video/test_sana_video_i2v_alignment.py index 8dd35eca2..ec7d67027 100644 --- a/tests/e2e/accuracy/sana_video/test_sana_video_i2v_alignment.py +++ b/tests/e2e/accuracy/sana_video/test_sana_video_i2v_alignment.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -11,7 +11,7 @@ import torch from tests.helpers.mark import hardware_marks pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.benchmark, ] diff --git a/tests/e2e/accuracy/sana_video/test_sana_video_pipeline_alignment.py b/tests/e2e/accuracy/sana_video/test_sana_video_pipeline_alignment.py index a88aec7e2..aabe166d2 100644 --- a/tests/e2e/accuracy/sana_video/test_sana_video_pipeline_alignment.py +++ b/tests/e2e/accuracy/sana_video/test_sana_video_pipeline_alignment.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -7,7 +7,7 @@ import gc import os from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, TypedDict import pytest import torch @@ -16,7 +16,7 @@ import torch.nn.functional as F from tests.helpers.mark import hardware_marks pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.benchmark, ] @@ -30,7 +30,23 @@ HEIGHT = 192 WIDTH = 320 NUM_FRAMES = 9 -VARIANTS = { + +class _VariantConfig(TypedDict): + model: str + revision: str + model_env: str + max_abs: float + relative_l2: float + cosine: float + cfg_max_abs: float + cfg_relative_l2: float + cfg_cosine: float + step_max_abs: float + step_relative_l2: float + step_cosine: float + + +VARIANTS: dict[str, _VariantConfig] = { "480p": { "model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers", "revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f", diff --git a/tests/e2e/accuracy/test_cosmos3_similarity.py b/tests/e2e/accuracy/test_cosmos3_similarity.py index efa095fb1..a7c9a9ece 100644 --- a/tests/e2e/accuracy/test_cosmos3_similarity.py +++ b/tests/e2e/accuracy/test_cosmos3_similarity.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -35,6 +35,7 @@ def _model_name() -> str: pytest.skip(f"Set {MODEL_ENV_VAR} to run Cosmos3 full-model smoke tests.") if not torch.cuda.is_available(): pytest.skip("Cosmos3 full-model smoke tests require CUDA.") + assert model return model @@ -56,7 +57,7 @@ def _image_data_url(image: Image.Image) -> str: @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_cosmos3_t2i_serving_smoke(accuracy_artifact_root: Path) -> None: output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID) with OmniServer(_model_name(), _server_args(), use_omni=True) as server: @@ -97,7 +98,7 @@ def test_cosmos3_t2i_serving_smoke(accuracy_artifact_root: Path) -> None: ], ) @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_cosmos3_video_serving_smoke( accuracy_artifact_root: Path, name: str, diff --git a/tests/e2e/accuracy/test_diffusers_backend_similarity.py b/tests/e2e/accuracy/test_diffusers_backend_similarity.py index b12e9c1e7..a582666d4 100644 --- a/tests/e2e/accuracy/test_diffusers_backend_similarity.py +++ b/tests/e2e/accuracy/test_diffusers_backend_similarity.py @@ -29,6 +29,7 @@ from tests.e2e.accuracy.helpers import ( assert_similarity, env_to_apply_ftfy_mock_in_subproc, model_output_dir, + resolve_device_threshold, ) from tests.e2e.accuracy.helpers import ( parse_psnr_score as _parse_psnr_score, @@ -49,16 +50,25 @@ from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter imp pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] -def _set_matched_attention_backend(pipe: DiffusionPipeline) -> None: - """Walk the same backend chain the omni server's diffusers adapter walks. +def _flash_attempt_backends() -> list[str]: + """Backends in ``CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS`` that can actually launch. - The server side (--diffusion-load-format diffusers) resolves its attention - backend through the adapter's preference chain, skipping backends that are - unavailable on the image (e.g. the FA3 hub kernel has no build variant for - the image's torch — build 2953). The reference run must resolve to the - same backend or the similarity comparison measures kernel differences. + Those names are Hopper FA2/FA3. ``set_attention_backend`` only checks import, + so a newer GPU still "selects" them and then dies with "no kernel image". + Do not probe by launching: a missing cubin can leave the CUDA context dead. + Capability >= 10 is outside that wheel, including future SKUs in that family. """ - for backend in CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS: + if not torch.cuda.is_available(): + return [] + major, _ = torch.cuda.get_device_capability() + if major >= 10: + return [] + return list(CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS) + + +def _set_matched_attention_backend(pipe: DiffusionPipeline) -> None: + """Pick a Diffusers attention backend the reference run can actually execute.""" + for backend in _flash_attempt_backends(): try: pipe.transformer.set_attention_backend(backend) return @@ -297,7 +307,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize("model_id", ["Qwen/Qwen-Image"]) def test_diffusers_backend_t2i_matches_diffusers(model_id: str, accuracy_artifact_root: Path) -> None: output_dir = model_output_dir(accuracy_artifact_root, model_id + "-diffusers-backend") @@ -336,7 +346,7 @@ def test_diffusers_backend_t2i_matches_diffusers(model_id: str, accuracy_artifac @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize( "model_id", [ @@ -363,7 +373,11 @@ def test_diffusers_backend_i2v_matches_diffusers( model=model_id, output_path=diffusers_path, conditioning_image=resized_image ) diffusers_latency = diffusers_latency * 1000 - latency_threshold_factor = 0.3 + # H100 keeps the historical 30% slack. B200 measured ~35.8% (6008 vs 4425 ms). + gpu_key, latency_threshold_factor = resolve_device_threshold( + {"H100": 0.3, "B200": 0.36}, + label="latency threshold factor", + ) latency_threshold = diffusers_latency * (1 + latency_threshold_factor) ssim_output = _run_ffmpeg_similarity("ssim", vllm_path, diffusers_path) @@ -372,7 +386,8 @@ def test_diffusers_backend_i2v_matches_diffusers( psnr_score = _parse_psnr_score(psnr_output) print(f"{model_id} latency metrics:") print( - f" Latency={vllm_latency:.2f}ms, threshold<={latency_threshold:.2f}ms, diffusers latency={diffusers_latency:.2f}ms, lower is better" + f" Latency={vllm_latency:.2f}ms, threshold<={latency_threshold:.2f}ms, " + f"diffusers latency={diffusers_latency:.2f}ms, slack={latency_threshold_factor:.0%} ({gpu_key}), lower is better" ) print(f"{model_id} similarity metrics:") print(f" SSIM: value={ssim_score:.6f}, threshold>={VIDEO_SSIM_THRESHOLD:.6f}, range=[-1, 1], higher is better") diff --git a/tests/e2e/accuracy/test_hunyuan_image3.py b/tests/e2e/accuracy/test_hunyuan_image3.py index 3e37f22b6..4258a470e 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3.py +++ b/tests/e2e/accuracy/test_hunyuan_image3.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations @@ -12,6 +12,7 @@ import tempfile import time from dataclasses import dataclass from pathlib import Path +from typing import Any, cast import pytest import requests @@ -288,7 +289,7 @@ def _quant_accuracy_cases() -> list[pytest.ParameterSet]: def _make_config(enable_kv_reuse: bool, path: Path) -> None: - config = copy.deepcopy(_DEPLOY_CONFIG) + config = cast(dict[str, Any], copy.deepcopy(_DEPLOY_CONFIG)) config["stages"][0]["omni_kv_config"] = {"need_send_cache": enable_kv_reuse} config["stages"][1]["omni_kv_config"] = {"need_recv_cache": enable_kv_reuse} path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False)) @@ -303,7 +304,7 @@ def _quant_tensor_parallel_size() -> int: def _make_quant_dit_config(path: Path) -> None: - config = copy.deepcopy(_QUANT_DIT_CONFIG) + config = cast(dict[str, Any], copy.deepcopy(_QUANT_DIT_CONFIG)) config["stages"][0]["devices"] = _quant_devices() config["stages"][0]["parallel_config"]["tensor_parallel_size"] = _quant_tensor_parallel_size() path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False)) @@ -448,7 +449,7 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None: """Online API test: same pipeline, same seed as offline → PSNR >= 10 dB.""" if importlib.util.find_spec("FlagEmbedding") is None: raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding") - from tabulate import tabulate + from tabulate import tabulate # type: ignore[import-untyped] output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME + "-online") @@ -556,7 +557,7 @@ def _run_dit_model( torch.accelerator.empty_cache() -@hardware_test(res={"cuda": "H100"}, num_cards=8) +@hardware_test(res={"cuda": "H200"}, num_cards=8) @pytest.mark.skipif( torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE, reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)", @@ -564,7 +565,7 @@ def _run_dit_model( def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None: if importlib.util.find_spec("FlagEmbedding") is None: raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding") - from tabulate import tabulate # lazy import + from tabulate import tabulate # type: ignore[import-untyped] """KV reuse ON vs OFF: same pipeline, same seed → PSNR >= 10 dB.""" output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME + "-offline-kv-reuse") diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 71dd6f308..bc656723b 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -299,7 +299,7 @@ def _assert_against_baseline(image: Image.Image, label: str) -> None: @pytest.mark.full_model -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=4) def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None: model = _model_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME) @@ -314,7 +314,7 @@ def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> N @pytest.mark.full_model -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=4) def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None: model = _model_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME) diff --git a/tests/e2e/accuracy/test_qwen_image.py b/tests/e2e/accuracy/test_qwen_image.py index 45fd20934..fb2f8cbc8 100644 --- a/tests/e2e/accuracy/test_qwen_image.py +++ b/tests/e2e/accuracy/test_qwen_image.py @@ -233,7 +233,7 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None: model = _model_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID) @@ -253,7 +253,7 @@ def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None: @pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_qwen_image_2512_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None: model = _model_2512_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_2512_ID) diff --git a/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py b/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py index 771ced5bc..419bf867d 100644 --- a/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py +++ b/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py @@ -195,7 +195,7 @@ def _generate_offline_video(*, image_source: str) -> tuple[Path, Path]: @pytest.mark.benchmark @pytest.mark.full_model -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_wan22_i2v_diffusers_offline_generates_video( wan22_i2v_image_source: str | None, ) -> None: @@ -217,7 +217,7 @@ def test_wan22_i2v_diffusers_offline_generates_video( @pytest.mark.benchmark @pytest.mark.full_model -@hardware_test(res={"cuda": "H100"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=2) @pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) def test_wan22_i2v_online_serving_generates_video( omni_server, @@ -244,7 +244,7 @@ def test_wan22_i2v_online_serving_generates_video( @pytest.mark.benchmark @pytest.mark.full_model -@hardware_test(res={"cuda": "H100"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=2) def test_wan22_i2v_serving_matches_diffusers_video_similarity( wan22_i2v_image_source: str | None, ) -> None: diff --git a/tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py b/tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py index a11c9b7e4..722d42a26 100644 --- a/tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py +++ b/tests/e2e/features/custom_pipeline/test_async_omni_collective_rpc.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ Regression tests for AsyncOmni collective_rpc in inline diffusion mode. diff --git a/tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py b/tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py index 12ad4f1b3..56a960121 100644 --- a/tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py +++ b/tests/e2e/features/custom_pipeline/test_async_omni_qwen_image_generate.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """E2E tests for AsyncOmni Qwen-Image generation with trajectory_* fields. diff --git a/tests/e2e/features/custom_pipeline/test_worker_extension.py b/tests/e2e/features/custom_pipeline/test_worker_extension.py index da7b0ac17..fff39cd1e 100644 --- a/tests/e2e/features/custom_pipeline/test_worker_extension.py +++ b/tests/e2e/features/custom_pipeline/test_worker_extension.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations diff --git a/tests/e2e/offline_inference/test_device_postprocess_equivalence.py b/tests/e2e/offline_inference/test_device_postprocess_equivalence.py index a7fbcafd9..019d90a65 100644 --- a/tests/e2e/offline_inference/test_device_postprocess_equivalence.py +++ b/tests/e2e/offline_inference/test_device_postprocess_equivalence.py @@ -83,7 +83,7 @@ def _as_uint8_frames(video: np.ndarray) -> np.ndarray: _MAX_DIFFERING_FRACTION = 0.35 -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_device_postprocess_produces_equivalent_frames() -> None: float_path = _generate(False) device_path = _generate(True) diff --git a/tests/e2e/offline_inference/test_dots_tts_expansion.py b/tests/e2e/offline_inference/test_dots_tts_expansion.py index 1c9362a23..4d13f03fd 100644 --- a/tests/e2e/offline_inference/test_dots_tts_expansion.py +++ b/tests/e2e/offline_inference/test_dots_tts_expansion.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """E2E tests for dots.tts native AR offline inference.""" from collections.abc import Mapping diff --git a/tests/e2e/offline_inference/test_gepard_tts.py b/tests/e2e/offline_inference/test_gepard_tts.py index 2dec32ffc..e1b68eb5c 100644 --- a/tests/e2e/offline_inference/test_gepard_tts.py +++ b/tests/e2e/offline_inference/test_gepard_tts.py @@ -246,7 +246,7 @@ def _transcribe_for(wav: torch.Tensor, tmp_dir: str, name: str, accepts) -> str: pytest.param( "The quick brown fox jumps over the lazy dog.", None, - marks=pytest.mark.full_model, + marks=pytest.mark.slow, id="second_text", ), ], @@ -274,7 +274,7 @@ def test_gepard_offline_zero_shot(omni_runner, run_level: str, text: str, keywor assert keyword in transcript, f"expected {keyword!r} in transcript, got {transcript!r}" -@pytest.mark.full_model +@pytest.mark.slow @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_gepard_offline_long_text_skips_repetition(omni_runner, run_level: str) -> None: """The other branch of the prompt layout, end to end. diff --git a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py index 7e7c63900..925833216 100644 --- a/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py +++ b/tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py @@ -57,7 +57,7 @@ def omni() -> Generator[Omni, None, None]: yield runner.omni -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4) def test_i2t_generates_text(omni: Omni) -> None: """Verify I2T output's first 20 token IDs match the HF greedy baseline.""" # Solid-color image keeps the input self-contained and reproducible. diff --git a/tests/e2e/offline_inference/test_lingbot_world_v2.py b/tests/e2e/offline_inference/test_lingbot_world_v2.py index 88e5979dc..bf1d2ca23 100644 --- a/tests/e2e/offline_inference/test_lingbot_world_v2.py +++ b/tests/e2e/offline_inference/test_lingbot_world_v2.py @@ -47,7 +47,7 @@ pytestmark = [ ] -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) def test_lingbot_world_v2_tp1_one_block(omni_runner_handler: OmniRunnerHandler) -> None: """Load the real checkpoint and generate one causal block on one GPU.""" diff --git a/tests/e2e/offline_inference/test_longcat_video_avatar.py b/tests/e2e/offline_inference/test_longcat_video_avatar.py index 404ad9485..59f9ae4e4 100644 --- a/tests/e2e/offline_inference/test_longcat_video_avatar.py +++ b/tests/e2e/offline_inference/test_longcat_video_avatar.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from collections.abc import Generator +from typing import Any import numpy as np import pytest @@ -105,15 +106,17 @@ def _generate_avatar_frames( num_frames: int = 5, sampling_extra_args: dict | None = None, ): + extra_info: dict[str, Any] = dict(additional_information or {}) + extra_info.update({"stage": stage, "resolution": "480p"}) + multi_modal_data: dict[str, Any] = {"audio": audio_path} + if image_path is not None: + multi_modal_data["image"] = image_path prompt = { "prompt": PROMPT, "negative_prompt": NEGATIVE_PROMPT, - "multi_modal_data": {"audio": audio_path}, - "additional_information": additional_information or {}, + "multi_modal_data": multi_modal_data, + "additional_information": extra_info, } - prompt["additional_information"].update({"stage": stage, "resolution": "480p"}) - if image_path is not None: - prompt["multi_modal_data"]["image"] = image_path outputs = omni_runner.generate( [prompt], diff --git a/tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py index e7b4d061e..3d370ec8b 100644 --- a/tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """E2E test for Qwen-Image AutoRound W4A16 quantized inference. Verifies that the W4A16 quantized Qwen-Image checkpoint loads end-to-end @@ -47,8 +47,9 @@ def _first_request_images(outputs) -> list: return req_out.images +@pytest.mark.full_model @pytest.mark.diffusion -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_qwen_image_autoround_w4a16_load(): """Load the W4A16 quantized Qwen-Image model and run a minimal generation. @@ -68,8 +69,9 @@ def test_qwen_image_autoround_w4a16_load(): assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)" +@pytest.mark.full_model @pytest.mark.diffusion -@hardware_test(res={"cuda": "L4"}) +@hardware_test(res={"cuda": ["L4", "B200"]}) def test_qwen_image_autoround_w4a16_generate(): """Full generation: 512×512, 20 steps, CFG=5.0. diff --git a/tests/e2e/offline_inference/test_voxcpm2_tts.py b/tests/e2e/offline_inference/test_voxcpm2_tts.py index af55c51aa..0b6ff9940 100644 --- a/tests/e2e/offline_inference/test_voxcpm2_tts.py +++ b/tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """E2E test for VoxCPM2 native AR offline inference.""" import os diff --git a/tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py index 4ac51a344..9594a0f99 100644 --- a/tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py @@ -156,7 +156,7 @@ def _generate_t2v_video(offline_client, prompt: str = "A cat sitting on a table" # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_runner", quant_i2v_params, indirect=True) def test_wan22_i2v_autoround_w4a16_generates_video(omni_runner, offline_client): """Load the W4A16 quantized Wan2.2 I2V model and verify it produces a valid video.""" @@ -180,7 +180,7 @@ def test_wan22_i2v_autoround_w4a16_generates_video(omni_runner, offline_client): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_runner", quant_t2v_params, indirect=True) def test_wan22_t2v_autoround_w4a16_generates_video(omni_runner, offline_client): """Load the W4A16 quantized Wan2.2 T2V model and verify it produces a valid video.""" @@ -202,7 +202,7 @@ def test_wan22_t2v_autoround_w4a16_generates_video(omni_runner, offline_client): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_runner", quant_i2v_params, indirect=True) def test_wan22_i2v_autoround_w4a16_quant_peak(omni_runner, offline_client): """Measure peak GPU memory of W4A16 quantized I2V model.""" @@ -218,7 +218,7 @@ def test_wan22_i2v_autoround_w4a16_quant_peak(omni_runner, offline_client): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_runner", baseline_i2v_params, indirect=True) def test_wan22_i2v_autoround_w4a16_baseline_peak(omni_runner, offline_client): """Measure peak GPU memory of BF16 baseline I2V model.""" @@ -234,7 +234,7 @@ def test_wan22_i2v_autoround_w4a16_baseline_peak(omni_runner, offline_client): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) def test_wan22_i2v_autoround_w4a16_memory_savings(): """Assert quantized I2V model uses meaningfully less memory than BF16 baseline.""" quant_peak = _memory_results["quant_i2v"] @@ -259,7 +259,7 @@ def test_wan22_i2v_autoround_w4a16_memory_savings(): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_runner", quant_t2v_params, indirect=True) def test_wan22_t2v_autoround_w4a16_quant_peak(omni_runner, offline_client): """Measure peak GPU memory of W4A16 quantized T2V model.""" @@ -275,7 +275,7 @@ def test_wan22_t2v_autoround_w4a16_quant_peak(omni_runner, offline_client): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_runner", baseline_t2v_params, indirect=True) def test_wan22_t2v_autoround_w4a16_baseline_peak(omni_runner, offline_client): """Measure peak GPU memory of BF16 baseline T2V model.""" @@ -291,7 +291,7 @@ def test_wan22_t2v_autoround_w4a16_baseline_peak(omni_runner, offline_client): # ------------------------------------------------------------------ -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) def test_wan22_t2v_autoround_w4a16_memory_savings(): """Assert quantized T2V model uses meaningfully less memory than BF16 baseline.""" quant_peak = _memory_results["quant_t2v"] diff --git a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py index 0bc3f449f..1d6df3317 100644 --- a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py +++ b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py @@ -33,19 +33,13 @@ NEGATIVE_PROMPT = "low quality, blurry, distorted" MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" -SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) -PARALLEL_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) - -# Only the most basic single-card deployment row is cheap enough for PR (L2) / merge (L3) -# CI; the CacheDiT / parallel feature combinations run nightly (L4). -_CORE_MARKS = [pytest.mark.core_model, pytest.mark.advanced_model] -_FULL_MARKS = [pytest.mark.full_model] - def _get_diffusion_feature_cases(model: str): """Return diffusion feature cases for HunyuanVideo-1.5. Designed for 2x H100 environment per issue #1832. + Only the CPU-offload row is cheap enough for PR (L2) / merge (L3); + CacheDiT / parallel combinations run nightly (L4). """ return [ # (1 GPU) CPU offload @@ -57,7 +51,11 @@ def _get_diffusion_feature_cases(model: str): ], ), id="single_card_cpu_offload", - marks=SINGLE_CARD_MARKS + _CORE_MARKS, + marks=[ + *hardware_marks(res={"cuda": "H100"}), + pytest.mark.core_model, + pytest.mark.advanced_model, + ], ), # (1 GPU) CacheDiT + Layerwise CPU offloading pytest.param( @@ -70,7 +68,10 @@ def _get_diffusion_feature_cases(model: str): ], ), id="single_card_cachedit_layerwise", - marks=SINGLE_CARD_MARKS + _FULL_MARKS, + marks=[ + *hardware_marks(res={"cuda": ["H100", "B200"]}), + pytest.mark.full_model, + ], ), # (2 GPUs) CacheDiT + TP=2 + VAE patch parallel=2 pytest.param( @@ -87,7 +88,10 @@ def _get_diffusion_feature_cases(model: str): ], ), id="parallel_cachedit_tp2_vae2", - marks=PARALLEL_MARKS + _FULL_MARKS, + marks=[ + *hardware_marks(res={"cuda": ["H100", "B200"]}, num_cards=2), + pytest.mark.full_model, + ], ), ] diff --git a/tests/e2e/online_serving/test_lingbot_video.py b/tests/e2e/online_serving/test_lingbot_video.py index 6ada319f5..e6f275bb9 100644 --- a/tests/e2e/online_serving/test_lingbot_video.py +++ b/tests/e2e/online_serving/test_lingbot_video.py @@ -20,7 +20,7 @@ PROMPT = "a robotic arm picks up a red block" NEGATIVE_PROMPT = "low quality, blurry, watermark, text" DEFAULT_SAMPLING_PARAMS = '{"0":{"num_frames":81,"num_inference_steps":40,"guidance_scale":6.0}}' -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}) def _get_diffusion_feature_cases(model: str): diff --git a/tests/e2e/online_serving/test_lingbot_video_expansion.py b/tests/e2e/online_serving/test_lingbot_video_expansion.py index fbf2bd5fa..4a411ddcf 100644 --- a/tests/e2e/online_serving/test_lingbot_video_expansion.py +++ b/tests/e2e/online_serving/test_lingbot_video_expansion.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ L4 expansion coverage for ``robbyant/lingbot-video-dense-1.3b``. @@ -26,7 +26,7 @@ MODEL = "robbyant/lingbot-video-dense-1.3b" PROMPT = "a robotic arm picks up a red block" NEGATIVE_PROMPT = "low quality, blurry, watermark, text" -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}) def _get_diffusion_feature_cases(model: str): diff --git a/tests/e2e/online_serving/test_lingbot_video_moe.py b/tests/e2e/online_serving/test_lingbot_video_moe.py index a75fa78e0..cc56161c9 100644 --- a/tests/e2e/online_serving/test_lingbot_video_moe.py +++ b/tests/e2e/online_serving/test_lingbot_video_moe.py @@ -20,7 +20,7 @@ PROMPT = "a robotic arm picks up a red block" NEGATIVE_PROMPT = "low quality, blurry, watermark, text" SMOKE_DEFAULT_SAMPLING_PARAMS = '{"0":{"num_frames":81,"num_inference_steps":2,"guidance_scale":3.0}}' -SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) +SINGLE_CARD_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}) def _get_server_cases(model: str): diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py index c3c6fd90a..628040001 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py @@ -51,7 +51,7 @@ def _cached_server_vad_artifact() -> None: ) -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_admission_and_expiry_reaper(omni_server, tmp_path: Path) -> None: args = multi_session_args( @@ -79,7 +79,7 @@ def test_duplex_admission_and_expiry_reaper(omni_server, tmp_path: Path) -> None # still talking, which only holds when the duplex pipeline sustains real-time # throughput. The current NPU stack runs several times slower than real time, # so it never reaches a mid-stream decision point. -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_soft_interrupt(omni_server, tmp_path: Path) -> None: input_wav = validated_soft_interrupt_wav() @@ -112,7 +112,7 @@ def test_duplex_soft_interrupt(omni_server, tmp_path: Path) -> None: assert result["followup_response_transcript_ok"] is True -@hardware_test(res={"cuda": "H100"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_server_vad_hard_interrupt(_cached_server_vad_artifact, omni_server) -> None: result = asyncio.run( diff --git a/tests/e2e/online_serving/test_pi0_expansion.py b/tests/e2e/online_serving/test_pi0_expansion.py index 3a9c3b177..6a16edc71 100644 --- a/tests/e2e/online_serving/test_pi0_expansion.py +++ b/tests/e2e/online_serving/test_pi0_expansion.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """E2E online serving test for π0 (Pi-Zero) OpenPI websocket serving. @@ -58,7 +58,7 @@ test_params = [ @pytest.mark.full_model @pytest.mark.diffusion -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": ["H100", "B200"]}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_pi0_openpi_online(omni_server): try: diff --git a/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py b/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py index 1a27a5449..a2d64de22 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py +++ b/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py @@ -91,7 +91,7 @@ def _assert_audio_outputs(responses: list[OmniResponse], *, expect_text: bool) - @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_only_batch_uses_multi_replica_talker(omni_server, online_client) -> None: request_config = { @@ -108,7 +108,7 @@ def test_text_only_batch_uses_multi_replica_talker(omni_server, online_client) - @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_stream_batch_uses_multi_replica_vocoder(omni_server, online_client) -> None: request_config = { @@ -125,7 +125,7 @@ def test_text_to_audio_stream_batch_uses_multi_replica_vocoder(omni_server, onli @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mixed_modal_stream_batch_generates_text_and_audio(omni_server, online_client) -> None: request_config = { diff --git a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py index e126877db..abf80fb85 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py @@ -69,7 +69,7 @@ tts_no_async_chunk_server_params = [ DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 180.0 -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize( "omni_server", tts_async_chunk_server_params + tts_no_async_chunk_server_params, @@ -99,7 +99,7 @@ def test_voice_clone_streaming_001(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config, request_num=get_max_batch_size("few")) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize( "omni_server", tts_async_chunk_server_params + tts_no_async_chunk_server_params, @@ -127,7 +127,7 @@ def test_response_format_001(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_async_chunk_server_params, indirect=True) def test_xvector_then_icl_same_ref_audio_keeps_engine_alive(omni_server, online_client) -> None: """Regression for #5049: x-vector-only then ICL on the same ``ref_audio``. diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py index 15198870e..e292ea4e8 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py @@ -60,7 +60,7 @@ tts_server_params = [ ] -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_001(omni_server, online_client) -> None: """ @@ -92,7 +92,7 @@ def test_voice_001(omni_server, online_client) -> None: raise -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_002(omni_server, online_client) -> None: """ @@ -115,7 +115,7 @@ def test_voice_002(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_003(omni_server, online_client) -> None: """ @@ -138,7 +138,7 @@ def test_voice_003(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_sample_rate_001(omni_server, online_client) -> None: """The speech API resamples Qwen3-TTS output from native 24 kHz to 8 kHz.""" @@ -155,7 +155,7 @@ def test_sample_rate_001(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": ["L4", "B200"]}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_language_001(omni_server, online_client) -> None: """ diff --git a/tests/e2e/online_serving/test_qwen_image_expansion.py b/tests/e2e/online_serving/test_qwen_image_expansion.py index 890b077d2..73e55789e 100644 --- a/tests/e2e/online_serving/test_qwen_image_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_expansion.py @@ -21,8 +21,8 @@ pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style." NEGATIVE_PROMPT = "blurry, low quality" -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) -PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}) +PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}, num_cards=2) MODEL_IMAGE = "Qwen/Qwen-Image" MODEL_2512 = "Qwen/Qwen-Image-2512" diff --git a/tests/e2e/online_serving/test_sana_wm.py b/tests/e2e/online_serving/test_sana_wm.py index 883e4c367..38277f08e 100644 --- a/tests/e2e/online_serving/test_sana_wm.py +++ b/tests/e2e/online_serving/test_sana_wm.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ Online serving smoke for SANA-WM image-to-video (first-frame I2V via ``/v1/videos``). diff --git a/tests/e2e/online_serving/test_wan22_expansion.py b/tests/e2e/online_serving/test_wan22_expansion.py index 322cde3cd..8734ddebb 100644 --- a/tests/e2e/online_serving/test_wan22_expansion.py +++ b/tests/e2e/online_serving/test_wan22_expansion.py @@ -30,8 +30,8 @@ PROMPT = "Two anthropomorphic cats in comfy boxing gear and bright gloves fight NEGATIVE_PROMPT = "low quality, blurry, distorted face, extra limbs, bad anatomy, watermark, logo, text, ugly, deformed, mutated, jpeg artifacts" # CUDA marks (original matrix, unchanged) -CUDA_SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) -CUDA_PARALLEL_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) +CUDA_SINGLE_CARD_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}) +CUDA_PARALLEL_MARKS = hardware_marks(res={"cuda": ["H100", "B200"]}, num_cards=2) # NPU marks NPU_TWO_CARD_MARKS = hardware_marks(res={"npu": "A2"}, num_cards=2) diff --git a/tests/e2e/online_serving/test_zimage_expansion.py b/tests/e2e/online_serving/test_zimage_expansion.py index c37a947f4..680cc3910 100644 --- a/tests/e2e/online_serving/test_zimage_expansion.py +++ b/tests/e2e/online_serving/test_zimage_expansion.py @@ -19,7 +19,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Tongyi-MAI/Z-Image-Turbo" PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard." diff --git a/tests/examples/offline_inference/test_image_to_video.py b/tests/examples/offline_inference/test_image_to_video.py index b53648cb3..182187298 100644 --- a/tests/examples/offline_inference/test_image_to_video.py +++ b/tests/examples/offline_inference/test_image_to_video.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Offline inference tests: image-to-video. See examples/offline_inference/image_to_video/README.md @@ -17,7 +20,7 @@ pytestmark = [ pytest.mark.usefixtures("clean_gpu_memory_between_tests"), pytest.mark.full_model, pytest.mark.example, - *hardware_marks(res={"cuda": "H100"}), + *hardware_marks(res={"cuda": ["H100", "B200"]}), ] I2V_SCRIPT = EXAMPLES / "offline_inference" / "image_to_video" / "image_to_video.py" diff --git a/tests/examples/offline_inference/test_internvla_a1.py b/tests/examples/offline_inference/test_internvla_a1.py index 791d380f0..f8ea50689 100644 --- a/tests/examples/offline_inference/test_internvla_a1.py +++ b/tests/examples/offline_inference/test_internvla_a1.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Offline E2E test for InternVLA-A1 open-loop inference.""" @@ -26,6 +26,7 @@ def _required_env(name: str) -> str: value = os.getenv(name) if not value: pytest.skip(f"{name} is not set") + assert value return value diff --git a/tests/examples/offline_inference/test_text_to_image.py b/tests/examples/offline_inference/test_text_to_image.py index 5ee2a2fae..6289dcd33 100644 --- a/tests/examples/offline_inference/test_text_to_image.py +++ b/tests/examples/offline_inference/test_text_to_image.py @@ -18,7 +18,7 @@ pytestmark = [ pytest.mark.usefixtures("clean_gpu_memory_between_tests"), pytest.mark.full_model, pytest.mark.example, - *hardware_marks(res={"cuda": "H100"}), + *hardware_marks(res={"cuda": ["H100", "B200"]}), ] T2I_SCRIPT = EXAMPLES / "offline_inference" / "text_to_image" / "text_to_image.py" diff --git a/tests/examples/offline_inference/test_vace_video_generation.py b/tests/examples/offline_inference/test_vace_video_generation.py index e4f6ca93d..fd23f0734 100644 --- a/tests/examples/offline_inference/test_vace_video_generation.py +++ b/tests/examples/offline_inference/test_vace_video_generation.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """VACE coverage through the shared text-to-video and image-to-video examples.""" from pathlib import Path @@ -14,7 +17,7 @@ pytestmark = [ pytest.mark.full_model, pytest.mark.diffusion, pytest.mark.example, - *hardware_marks(res={"cuda": "H100"}), + *hardware_marks(res={"cuda": ["H100", "B200"]}), ] TEXT_TO_VIDEO_README = EXAMPLES / "offline_inference" / "text_to_video" / "text_to_video.md" diff --git a/tests/examples/online_serving/test_qwen2_5_omni.py b/tests/examples/online_serving/test_qwen2_5_omni.py index 0d0ec6f46..d56b6f958 100644 --- a/tests/examples/online_serving/test_qwen2_5_omni.py +++ b/tests/examples/online_serving/test_qwen2_5_omni.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Online serving tests: Qwen2.5-Omni-7B. See examples/online_serving/qwen2_5_omni/README.md @@ -42,7 +45,7 @@ test_params = [ common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")] -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_001(omni_server) -> None: command = common_args + [ @@ -77,7 +80,7 @@ def test_send_multimodal_request_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_002(omni_server) -> None: command = common_args + [ @@ -112,7 +115,7 @@ def test_send_multimodal_request_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_003(omni_server) -> None: command = [ @@ -137,7 +140,7 @@ def test_send_multimodal_request_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_001(omni_server) -> None: command = common_args + [ @@ -165,7 +168,7 @@ def test_modality_control_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_002(omni_server) -> None: command = common_args + [ @@ -192,7 +195,7 @@ def test_modality_control_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_003(omni_server) -> None: command = common_args + [ @@ -228,7 +231,7 @@ def test_modality_control_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) +@hardware_test(res={"cuda": ["L4", "B200"], "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_stream_001(omni_server) -> None: command = common_args + [ diff --git a/tests/examples/online_serving/test_qwen3_omni.py b/tests/examples/online_serving/test_qwen3_omni.py index 711b7236a..23e2ba184 100644 --- a/tests/examples/online_serving/test_qwen3_omni.py +++ b/tests/examples/online_serving/test_qwen3_omni.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Online serving tests: Qwen3-Omni. See examples/online_serving/qwen3_omni/README.md @@ -41,7 +44,7 @@ test_params = [ common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")] -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_001(omni_server) -> None: command = common_args + [ @@ -69,7 +72,7 @@ def test_send_multimodal_request_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_002(omni_server) -> None: command = common_args + [ @@ -100,7 +103,7 @@ def test_send_multimodal_request_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_003(omni_server) -> None: command = ["bash", os.path.join(example_dir, "qwen3_omni/run_curl_multimodal_generation.sh"), "use_image"] @@ -115,7 +118,7 @@ def test_send_multimodal_request_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_001(omni_server) -> None: command = common_args + [ @@ -137,7 +140,7 @@ def test_modality_control_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_002(omni_server) -> None: command = common_args + [ @@ -159,7 +162,7 @@ def test_modality_control_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_003(omni_server) -> None: command = common_args + [ @@ -189,7 +192,7 @@ def test_modality_control_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_stream_001(omni_server) -> None: command = common_args + [ diff --git a/tests/examples/online_serving/test_text_to_image.py b/tests/examples/online_serving/test_text_to_image.py index 25970502b..f251cd16c 100644 --- a/tests/examples/online_serving/test_text_to_image.py +++ b/tests/examples/online_serving/test_text_to_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Online serving tests: text-to-image. See examples/online_serving/text_to_image/README.md @@ -18,7 +21,7 @@ from tests.helpers.assertions import assert_image_valid from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams -pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})] +pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": ["H100", "B200"]})] T2I_ONLINE_CLIENT = EXAMPLES / "online_serving" / "text_to_image" / "openai_chat_client.py" EXAMPLE_OUTPUT_SUBFOLDER = "example_online_t2i" diff --git a/tests/model_tests/diffusion/case_filtering.py b/tests/model_tests/diffusion/case_filtering.py index da1aaa552..9f53edce0 100644 --- a/tests/model_tests/diffusion/case_filtering.py +++ b/tests/model_tests/diffusion/case_filtering.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Analogous to: https://github.com/vllm-project/vllm/blob/v0.23.0/tests/models/multimodal/generation/vlm_utils/case_filtering.py @@ -45,7 +48,6 @@ def get_test_group_marks(model_name: str, test_group: list[DiffusionAccs] | None f"but the max CI device count is {MAX_CI_DEVICES}. " ) - marks.extend(hardware_marks(res={"cuda": "L4"}, num_cards=required_devices)) # hardware_marks only adds skipif for num_cards > 1, so we currently handle the single-device case # directly here. This should probably be handled in a more common way later on. assert current_omni_platform is not None and current_omni_platform.device_count is not None @@ -53,8 +55,10 @@ def get_test_group_marks(model_name: str, test_group: list[DiffusionAccs] | None if current_omni_platform.is_cuda() and device_count < required_devices: marks.append(pytest.mark.skip(reason=f"Need {required_devices} devices, got {device_count}")) if required_devices > 1: + marks.extend(hardware_marks(res={"cuda": ["L4", "B200"]}, num_cards=required_devices)) marks.append(pytest.mark.full_model) else: + marks.extend(hardware_marks(res={"cuda": "L4"}, num_cards=1)) marks.append(pytest.mark.core_model) return marks