[CI/Build] Share nightly CUDA YAML between H100/L4 and B200 (#7028)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] [Your Email]
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-14 14:27:10 +08:00
1 parent 58adeec05f
commit 64903b8e30
78 files changed
+437 -321

No files matched your search

+26
View File
@@ -135,6 +135,32 @@ def resolve_similarity_thresholds(
return table.get(profile, table[DEFAULT_DEVICE_PROFILE])
def resolve_device_threshold(
thresholds: float | int | Mapping[str, float],
*,
device_name: str | None = None,
label: str = "threshold",
) -> tuple[str, float]:
"""Resolve a scalar or per-GPU threshold for the current CUDA device.
Returns ``(profile_key, threshold)``. A bare number uses profile ``default``.
A mapping matches keys as substrings of the device name via
``resolve_device_profile`` (for example ``{"H100": 0.3, "B200": 0.36}``).
Prefer an explicit ``default`` entry when unmatched GPUs should fall back;
otherwise unmatched devices raise.
"""
if isinstance(thresholds, float | int):
return DEFAULT_DEVICE_PROFILE, float(thresholds)
profile = resolve_device_profile(device_name, profiles=thresholds)
if profile in thresholds:
return profile, float(thresholds[profile])
if DEFAULT_DEVICE_PROFILE in thresholds:
return DEFAULT_DEVICE_PROFILE, float(thresholds[DEFAULT_DEVICE_PROFILE])
known = ", ".join(str(key) for key in thresholds)
raise RuntimeError(f"no {label} entry for device profile {profile!r}; known: {known}")
_SSIM_RE = re.compile(r"All:(?P<score>[0-9.]+)")
_PSNR_RE = re.compile(r"average:(?P<score>[0-9.]+)")
@@ -183,7 +183,7 @@ def _generate_online_video(
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_hunyuanvideo15_i2v_diffusers_offline_generates_video(
hunyuanvideo15_i2v_image_source: str | None,
) -> None:
@@ -203,7 +203,7 @@ def test_hunyuanvideo15_i2v_diffusers_offline_generates_video(
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_hunyuanvideo15_i2v_online_serving_generates_video(
omni_server,
@@ -229,7 +229,7 @@ def test_hunyuanvideo15_i2v_online_serving_generates_video(
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_hunyuanvideo15_i2v_serving_matches_offline_video_similarity(
hunyuanvideo15_i2v_image_source: str | None,
) -> None:
@@ -144,7 +144,7 @@ def _generate_online_video(*, omni_server, online_client, timeout_seconds: int)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None:
if not torch.cuda.is_available():
pytest.skip("HunyuanVideo-1.5 T2V offline accuracy test requires CUDA.")
@@ -160,7 +160,7 @@ def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None:
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_hunyuanvideo15_t2v_online_serving_generates_video(
omni_server,
@@ -182,7 +182,7 @@ def test_hunyuanvideo15_t2v_online_serving_generates_video(
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_hunyuanvideo15_t2v_serving_matches_offline_video_similarity() -> None:
if not torch.cuda.is_available():
pytest.skip("HunyuanVideo-1.5 T2V video similarity test requires CUDA.")
@@ -141,7 +141,7 @@ def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None:
@requires_daily_omni_deps
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
_require_vllm_cli()
@@ -174,7 +174,7 @@ def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
_require_vllm_cli()
@@ -200,7 +200,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
@pytest.mark.parametrize("omni_server", duplex_accuracy_test_params, indirect=True)
def test_minicpmo_4_5_duplex_seed_tts_wer_bench(omni_server) -> None:
"""Gate Seed-TTS WER through the explicit Realtime TTS contract."""
@@ -138,8 +138,6 @@ def _server_args() -> list[str]:
"--vae-parallel-mode",
"tile",
"--vae-use-tiling",
"--diffusion-attention-backend",
"FLASH_ATTN",
"--stage-init-timeout",
"1800",
"--init-timeout",
@@ -228,7 +226,7 @@ def _assert_official_video(
)
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4)
def test_minimax_h3_i2va_matches_official_reference(
accuracy_artifact_root: Path,
) -> None:
@@ -304,7 +302,7 @@ def test_minimax_h3_i2va_matches_official_reference(
)
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=4)
def test_minimax_h3_ref2va_matches_official_reference(
accuracy_artifact_root: Path,
) -> None:
@@ -110,7 +110,7 @@ def _daily_omni_hub_inline_media(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_qwen3_omni_daily_omni_accuracy_bench(omni_server) -> None:
_require_vllm_cli()
@@ -122,7 +122,7 @@ def test_qwen3_omni_daily_omni_accuracy_bench(omni_server) -> None:
assert _acc_bench.run_acc_benchmark(ns) == 0
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_qwen3_omni_seed_tts_wer_bench(omni_server) -> None:
_require_vllm_cli()
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -44,7 +44,7 @@ SSIM_THRESHOLDS = {
}
pytestmark = [
pytest.mark.full_model,
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.skipif(
not GOLDEN_BASE_URL,
@@ -169,6 +169,7 @@ def _validate_metadata(payload: Any, manifest: dict[str, Any], variant: str, tas
def _download_case(variant: str, task: str, output_dir: Path, filenames: set[str]) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
assert GOLDEN_BASE_URL is not None
base_url = f"{GOLDEN_BASE_URL.rstrip('/')}/{variant}/{task}"
response = requests.get(f"{base_url}/manifest.json", timeout=60)
response.raise_for_status()
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -11,7 +11,7 @@ import torch
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.full_model,
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.benchmark,
]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -7,7 +7,7 @@ import gc
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, TypedDict
import pytest
import torch
@@ -16,7 +16,7 @@ import torch.nn.functional as F
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.full_model,
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.benchmark,
]
@@ -30,7 +30,23 @@ HEIGHT = 192
WIDTH = 320
NUM_FRAMES = 9
VARIANTS = {
class _VariantConfig(TypedDict):
model: str
revision: str
model_env: str
max_abs: float
relative_l2: float
cosine: float
cfg_max_abs: float
cfg_relative_l2: float
cfg_cosine: float
step_max_abs: float
step_relative_l2: float
step_cosine: float
VARIANTS: dict[str, _VariantConfig] = {
"480p": {
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
"revision": "fed3bce411c58a0f688a31afe8f52e61acc2b15f",
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -35,6 +35,7 @@ def _model_name() -> str:
pytest.skip(f"Set {MODEL_ENV_VAR} to run Cosmos3 full-model smoke tests.")
if not torch.cuda.is_available():
pytest.skip("Cosmos3 full-model smoke tests require CUDA.")
assert model
return model
@@ -56,7 +57,7 @@ def _image_data_url(image: Image.Image) -> str:
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_cosmos3_t2i_serving_smoke(accuracy_artifact_root: Path) -> None:
output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID)
with OmniServer(_model_name(), _server_args(), use_omni=True) as server:
@@ -97,7 +98,7 @@ def test_cosmos3_t2i_serving_smoke(accuracy_artifact_root: Path) -> None:
],
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_cosmos3_video_serving_smoke(
accuracy_artifact_root: Path,
name: str,
@@ -29,6 +29,7 @@ from tests.e2e.accuracy.helpers import (
assert_similarity,
env_to_apply_ftfy_mock_in_subproc,
model_output_dir,
resolve_device_threshold,
)
from tests.e2e.accuracy.helpers import (
parse_psnr_score as _parse_psnr_score,
@@ -49,16 +50,25 @@ from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter imp
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
def _set_matched_attention_backend(pipe: DiffusionPipeline) -> None:
"""Walk the same backend chain the omni server's diffusers adapter walks.
def _flash_attempt_backends() -> list[str]:
"""Backends in ``CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS`` that can actually launch.
The server side (--diffusion-load-format diffusers) resolves its attention
backend through the adapter's preference chain, skipping backends that are
unavailable on the image (e.g. the FA3 hub kernel has no build variant for
the image's torch — build 2953). The reference run must resolve to the
same backend or the similarity comparison measures kernel differences.
Those names are Hopper FA2/FA3. ``set_attention_backend`` only checks import,
so a newer GPU still "selects" them and then dies with "no kernel image".
Do not probe by launching: a missing cubin can leave the CUDA context dead.
Capability >= 10 is outside that wheel, including future SKUs in that family.
"""
for backend in CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS:
if not torch.cuda.is_available():
return []
major, _ = torch.cuda.get_device_capability()
if major >= 10:
return []
return list(CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS)
def _set_matched_attention_backend(pipe: DiffusionPipeline) -> None:
"""Pick a Diffusers attention backend the reference run can actually execute."""
for backend in _flash_attempt_backends():
try:
pipe.transformer.set_attention_backend(backend)
return
@@ -297,7 +307,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
@pytest.mark.parametrize("model_id", ["Qwen/Qwen-Image"])
def test_diffusers_backend_t2i_matches_diffusers(model_id: str, accuracy_artifact_root: Path) -> None:
output_dir = model_output_dir(accuracy_artifact_root, model_id + "-diffusers-backend")
@@ -336,7 +346,7 @@ def test_diffusers_backend_t2i_matches_diffusers(model_id: str, accuracy_artifac
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
@pytest.mark.parametrize(
"model_id",
[
@@ -363,7 +373,11 @@ def test_diffusers_backend_i2v_matches_diffusers(
model=model_id, output_path=diffusers_path, conditioning_image=resized_image
)
diffusers_latency = diffusers_latency * 1000
latency_threshold_factor = 0.3
# H100 keeps the historical 30% slack. B200 measured ~35.8% (6008 vs 4425 ms).
gpu_key, latency_threshold_factor = resolve_device_threshold(
{"H100": 0.3, "B200": 0.36},
label="latency threshold factor",
)
latency_threshold = diffusers_latency * (1 + latency_threshold_factor)
ssim_output = _run_ffmpeg_similarity("ssim", vllm_path, diffusers_path)
@@ -372,7 +386,8 @@ def test_diffusers_backend_i2v_matches_diffusers(
psnr_score = _parse_psnr_score(psnr_output)
print(f"{model_id} latency metrics:")
print(
f" Latency={vllm_latency:.2f}ms, threshold<={latency_threshold:.2f}ms, diffusers latency={diffusers_latency:.2f}ms, lower is better"
f" Latency={vllm_latency:.2f}ms, threshold<={latency_threshold:.2f}ms, "
f"diffusers latency={diffusers_latency:.2f}ms, slack={latency_threshold_factor:.0%} ({gpu_key}), lower is better"
)
print(f"{model_id} similarity metrics:")
print(f" SSIM: value={ssim_score:.6f}, threshold>={VIDEO_SSIM_THRESHOLD:.6f}, range=[-1, 1], higher is better")
+7 -6
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -12,6 +12,7 @@ import tempfile
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, cast
import pytest
import requests
@@ -288,7 +289,7 @@ def _quant_accuracy_cases() -> list[pytest.ParameterSet]:
def _make_config(enable_kv_reuse: bool, path: Path) -> None:
config = copy.deepcopy(_DEPLOY_CONFIG)
config = cast(dict[str, Any], copy.deepcopy(_DEPLOY_CONFIG))
config["stages"][0]["omni_kv_config"] = {"need_send_cache": enable_kv_reuse}
config["stages"][1]["omni_kv_config"] = {"need_recv_cache": enable_kv_reuse}
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
@@ -303,7 +304,7 @@ def _quant_tensor_parallel_size() -> int:
def _make_quant_dit_config(path: Path) -> None:
config = copy.deepcopy(_QUANT_DIT_CONFIG)
config = cast(dict[str, Any], copy.deepcopy(_QUANT_DIT_CONFIG))
config["stages"][0]["devices"] = _quant_devices()
config["stages"][0]["parallel_config"]["tensor_parallel_size"] = _quant_tensor_parallel_size()
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
@@ -448,7 +449,7 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None:
"""Online API test: same pipeline, same seed as offline → PSNR >= 10 dB."""
if importlib.util.find_spec("FlagEmbedding") is None:
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
from tabulate import tabulate
from tabulate import tabulate # type: ignore[import-untyped]
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME + "-online")
@@ -556,7 +557,7 @@ def _run_dit_model(
torch.accelerator.empty_cache()
@hardware_test(res={"cuda": "H100"}, num_cards=8)
@hardware_test(res={"cuda": "H200"}, num_cards=8)
@pytest.mark.skipif(
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
@@ -564,7 +565,7 @@ def _run_dit_model(
def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None:
if importlib.util.find_spec("FlagEmbedding") is None:
raise ImportError("Missing dependency: FlagEmbedding\nInstall with: pip install FlagEmbedding")
from tabulate import tabulate # lazy import
from tabulate import tabulate # type: ignore[import-untyped]
"""KV reuse ON vs OFF: same pipeline, same seed → PSNR >= 10 dB."""
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME + "-offline-kv-reuse")
@@ -299,7 +299,7 @@ def _assert_against_baseline(image: Image.Image, label: str) -> None:
@pytest.mark.full_model
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4)
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=4)
def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None:
model = _model_name()
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
@@ -314,7 +314,7 @@ def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> N
@pytest.mark.full_model
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4)
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=4)
def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
model = _model_name()
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
+2 -2
View File
@@ -233,7 +233,7 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:
model = _model_name()
output_dir = model_output_dir(accuracy_artifact_root, MODEL_ID)
@@ -253,7 +253,7 @@ def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_qwen_image_2512_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:
model = _model_2512_name()
output_dir = model_output_dir(accuracy_artifact_root, MODEL_2512_ID)
@@ -195,7 +195,7 @@ def _generate_offline_video(*, image_source: str) -> tuple[Path, Path]:
@pytest.mark.benchmark
@pytest.mark.full_model
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
def test_wan22_i2v_diffusers_offline_generates_video(
wan22_i2v_image_source: str | None,
) -> None:
@@ -217,7 +217,7 @@ def test_wan22_i2v_diffusers_offline_generates_video(
@pytest.mark.benchmark
@pytest.mark.full_model
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=2)
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_wan22_i2v_online_serving_generates_video(
omni_server,
@@ -244,7 +244,7 @@ def test_wan22_i2v_online_serving_generates_video(
@pytest.mark.benchmark
@pytest.mark.full_model
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=2)
def test_wan22_i2v_serving_matches_diffusers_video_similarity(
wan22_i2v_image_source: str | None,
) -> None: