[CI][Perf] migrate diffusion DFX benches to vllm bench serve --omni (#7737)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-09-21 19:16:39 -07:00
1 parent a4ad73e08b
commit 0570b0da7e
30 files changed
+2040 -729

No files matched your search

+82 -7
View File
@@ -6,11 +6,16 @@ Unit tests for metrics.py
"""
import math
from types import SimpleNamespace
import pytest
from vllm.benchmarks.serve import TaskType
from vllm_omni.benchmarks.metrics.metrics import calculate_metrics
from vllm_omni.benchmarks.metrics.metrics import (
aggregate_stage_durations,
calculate_metrics,
print_stage_durations_metrics,
)
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
@@ -379,12 +384,7 @@ class _EmptyAwareTokenizer:
"""
def __call__(self, text, add_special_tokens=False):
class _R:
pass
r = _R()
r.input_ids = [0] * len(text)
return r
return SimpleNamespace(input_ids=[0] * len(text))
def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput:
@@ -588,5 +588,80 @@ def test_image_with_generated_text_still_reports_text_result(capsys):
assert " Image Result " in out
def test_aggregate_stage_durations_mean_p50_p99() -> None:
ok_a = MixRequestFuncOutput()
ok_a.success = True
ok_a.stage_durations = {"diffuse": 1.0, "vae.decode": 0.2, "stage_0_gen_ms": 50.0}
ok_b = MixRequestFuncOutput()
ok_b.success = True
ok_b.stage_durations = {"diffuse": 3.0, "vae.decode": 0.4, "stage_1_gen_ms": 80.0}
failed = MixRequestFuncOutput()
failed.success = False
failed.stage_durations = {"diffuse": 99.0}
summaries = aggregate_stage_durations([ok_a, ok_b, failed])
assert summaries["stage_durations_mean"]["diffuse"] == pytest.approx(2.0)
assert summaries["stage_durations_p50"]["diffuse"] == pytest.approx(2.0)
assert summaries["stage_durations_mean"]["vae.decode"] == pytest.approx(0.3)
assert "stage_durations_p99" in summaries
assert "stage_0_gen_ms" not in summaries["stage_durations_mean"]
assert "stage_1_gen_ms" not in summaries["stage_durations_mean"]
def test_print_stage_durations_metrics(capsys) -> None:
output = MixRequestFuncOutput()
output.success = True
output.stage_durations = {
"Wan22I2VPipeline.diffuse": 1.25,
"Wan22I2VPipeline.text_encoder.forward": 0.4,
"queue_wait_ms": 0.5,
"stage_0_gen_ms": 1000.0,
}
summaries = aggregate_stage_durations([output])
metrics = SimpleNamespace(**summaries)
print_stage_durations_metrics([99.0], metrics)
out = capsys.readouterr().out
assert "Wan22I2VPipeline" not in out
assert "Mean Diffuse (s):" in out
assert "Median Diffuse (s):" in out
assert "P99 Diffuse (s):" in out
assert "Mean Text Encoder Forward (s):" in out
assert "Mean Queue Wait (ms):" in out
assert "Stage 0 Gen" not in out
assert "stage_0_gen" not in out
def test_profiler_stage_durations_print_without_print_stage(capsys) -> None:
output = MixRequestFuncOutput()
output.success = True
output.prompt_len = 8
output.latency = 1.0
output.stage_durations = {
"Wan22I2VPipeline.diffuse": 1.25,
"queue_wait_ms": 0.5,
"stage_0_gen_ms": 1000.0,
}
common = dict(
input_requests=[],
outputs=[output],
dur_s=1.0,
tokenizer=None,
selected_percentiles=[50.0, 99.0],
goodput_config_dict={},
task_type=TaskType.GENERATION,
selected_percentile_metrics=["e2el"],
max_concurrency=None,
request_rate=float("inf"),
benchmark_duration=1.0,
)
metrics, _ = calculate_metrics(**common, print_stage=False)
shown = capsys.readouterr().out
assert "Mean Diffuse (s):" in shown
assert "Mean Queue Wait (ms):" in shown
assert "Stage 0 Gen" not in shown
assert metrics.stage_durations_mean["Wan22I2VPipeline.diffuse"] == pytest.approx(1.25)
assert "stage_0_gen_ms" not in metrics.stage_durations_mean
if __name__ == "__main__":
pytest.main([__file__, "-v", "-s"])
+388 -8
View File
@@ -22,12 +22,17 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
)
from vllm_omni.benchmarks.patch.patch import (
MixRequestFuncOutput,
_add_combined_video_form_references,
_add_video_extra_body_to_form,
_add_video_reference_to_form,
_apply_image_metrics_from_payload,
_apply_stage0_token_timings,
_apply_video_metrics_from_payload,
_attach_seed_tts_to_request_func_input,
_build_benchmark_session,
_extract_stage_durations_from_payload,
_iter_image_reference_inputs,
_iter_video_reference_inputs,
_omni_request_timeout_s,
async_request_openai_chat_omni_completions,
async_request_openai_image_edits_omni,
@@ -1359,16 +1364,15 @@ async def test_image_edits_defaults_to_non_streaming_json(mocker: MockerFixture)
return None
captured_stream: list[str] = []
real_add_field = None
import aiohttp
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
if name == "stream":
captured_stream.append(str(value))
return real_add_field(self, name, value, **kwargs)
import aiohttp
real_add_field = aiohttp.FormData.add_field
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
mock_session = mocker.AsyncMock()
@@ -1512,8 +1516,7 @@ def test_video_local_image_reference_not_forwarded_as_raw_extra_field(tmp_path,
"reference",
[
{"image_url": "https://example.com/ref.png"},
{"file_id": "file-abc"},
[{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}],
[{"image_url": "https://example.com/a.png"}, {"image_url": "https://example.com/b.png"}],
],
)
def test_video_structured_image_reference_serialized_to_form(reference: object, mocker: MockerFixture) -> None:
@@ -1541,18 +1544,395 @@ def test_video_structured_image_reference_serialized_to_form(reference: object,
assert field_names.count("image_reference") == 1
assert "input_reference" not in field_names
payload = next(value for name, value in captured if name == "image_reference")
assert isinstance(payload, (str, bytes, bytearray))
assert json.loads(payload) == reference
def test_video_file_id_reference_is_rejected() -> None:
"""file_id is unsupported on the server and must not be sent as image_reference."""
import aiohttp
form = aiohttp.FormData()
with pytest.raises(ValueError, match="file_id is not supported yet"):
_add_video_reference_to_form(form, {"file_id": "file-abc"})
with pytest.raises(ValueError, match="file_id is not supported yet"):
_add_video_reference_to_form(form, [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}])
with pytest.raises(ValueError, match="file_id is not supported yet"):
_add_combined_video_form_references(form, None, {"video_reference": {"file_id": "file-vid"}})
def test_image_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None:
"""random-mm image_url parts keep an explicit image_reference type."""
import aiohttp
content = [
{
"type": "image_url",
"image_url": {"url": "https://example.com/ref.png"},
},
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{_MIN_PNG_B64}"},
},
]
refs = list(_iter_image_reference_inputs(content))
assert refs == [
{"image_url": "https://example.com/ref.png"},
{"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"},
]
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
form = aiohttp.FormData()
assert _add_video_reference_to_form(form, refs[0]) is True
assert _add_video_reference_to_form(form, refs[1]) is True
field_names = [name for name, _ in captured]
assert field_names.count("image_reference") == 2
assert "video_reference" not in field_names
payloads = []
for name, value in captured:
if name != "image_reference":
continue
assert isinstance(value, (str, bytes, bytearray))
payloads.append(json.loads(value))
assert payloads == refs
def test_video_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None:
"""random-mm data:video_url parts stay on video_reference."""
import aiohttp
content = [
{
"type": "video_url",
"video_url": {"url": "data:video/mp4;base64,AAAA"},
}
]
refs = list(_iter_video_reference_inputs(content))
assert refs == [{"video_url": "data:video/mp4;base64,AAAA"}]
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
form = aiohttp.FormData()
assert _add_video_reference_to_form(form, refs[0]) is True
field_names = [name for name, _ in captured]
assert field_names.count("video_reference") == 1
assert "input_references" not in field_names
assert "input_reference" not in field_names
assert "image_reference" not in field_names
payload = next(value for name, value in captured if name == "video_reference")
assert isinstance(payload, (str, bytes, bytearray))
assert json.loads(payload) == {"video_url": "data:video/mp4;base64,AAAA"}
def test_video_reference_https_url_from_random_mm_content(mocker: MockerFixture) -> None:
"""HTTP(S) video_url parts must stay on video_reference, not image_reference."""
import aiohttp
content = [
{
"type": "video_url",
"video_url": {"url": "https://example.com/ref.mp4"},
}
]
refs = list(_iter_video_reference_inputs(content))
assert refs == [{"video_url": "https://example.com/ref.mp4"}]
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
form = aiohttp.FormData()
assert _add_video_reference_to_form(form, refs[0]) is True
field_names = [name for name, _ in captured]
assert field_names.count("video_reference") == 1
assert "image_reference" not in field_names
assert "input_references" not in field_names
payload = next(value for name, value in captured if name == "video_reference")
assert isinstance(payload, (str, bytes, bytearray))
assert json.loads(payload) == {"video_url": "https://example.com/ref.mp4"}
@pytest.mark.parametrize(
("reference", "field_name", "expected"),
[
("data:image/png;base64,AAAA", "image_reference", {"image_url": "data:image/png;base64,AAAA"}),
("data:video/mp4;base64,AAAA", "video_reference", {"video_url": "data:video/mp4;base64,AAAA"}),
("https://example.com/ref.png", "image_reference", {"image_url": "https://example.com/ref.png"}),
("https://example.com/ref.mp4", "video_reference", {"video_url": "https://example.com/ref.mp4"}),
],
)
def test_bare_reference_string_uses_matching_json_field(
reference: str,
field_name: str,
expected: dict[str, str],
mocker: MockerFixture,
) -> None:
"""Image and video URLs use matching JSON fields; neither is a file upload."""
import aiohttp
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
assert _add_video_reference_to_form(aiohttp.FormData(), reference) is True
assert [name for name, _ in captured] == [field_name]
payload = captured[0][1]
assert isinstance(payload, (str, bytes, bytearray))
assert json.loads(payload) == expected
def test_bare_video_string_is_not_collected_as_image() -> None:
content = ["https://example.com/ref.mp4", "data:video/mp4;base64,AAAA"]
assert list(_iter_image_reference_inputs(content)) == []
assert list(_iter_video_reference_inputs(content)) == [
{"video_url": "https://example.com/ref.mp4"},
{"video_url": "data:video/mp4;base64,AAAA"},
]
def test_bare_http_without_media_extension_is_rejected() -> None:
import aiohttp
with pytest.raises(ValueError, match="image or video extension"):
_add_video_reference_to_form(aiohttp.FormData(), "https://example.com/ref")
def test_image_and_inline_video_use_combined_reference_fields(mocker: MockerFixture) -> None:
"""Image plus data:video must be image_reference + video_reference, not input_references."""
import aiohttp
content = [
{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}},
{"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}},
]
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
form = aiohttp.FormData()
_add_combined_video_form_references(form, content)
field_names = [name for name, _ in captured]
assert field_names.count("image_reference") == 1
assert field_names.count("video_reference") == 1
assert "input_references" not in field_names
assert "input_reference" not in field_names
image_payload = next(value for name, value in captured if name == "image_reference")
video_payload = next(value for name, value in captured if name == "video_reference")
assert isinstance(image_payload, (str, bytes, bytearray))
assert isinstance(video_payload, (str, bytes, bytearray))
assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"}
assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"}
def test_image_url_and_bare_data_video_string_use_json_fields(mocker: MockerFixture) -> None:
"""Image URL plus a bare data:video string must not upload input_references."""
import aiohttp
content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}]
extra_body = {"video_reference": "data:video/mp4;base64,AAAA"}
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
form = aiohttp.FormData()
_add_combined_video_form_references(form, content, extra_body)
field_names = [name for name, _ in captured]
assert field_names.count("image_reference") == 1
assert field_names.count("video_reference") == 1
assert "input_references" not in field_names
assert "input_reference" not in field_names
image_payload = next(value for name, value in captured if name == "image_reference")
video_payload = next(value for name, value in captured if name == "video_reference")
assert isinstance(image_payload, (str, bytes, bytearray))
assert isinstance(video_payload, (str, bytes, bytearray))
assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"}
assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"}
captured.clear()
assert _add_video_reference_to_form(form, "data:video/mp4;base64,AAAA") is True
assert [name for name, _ in captured] == ["video_reference"]
def _oversized_data_video_url() -> str:
"""data:video whose JSON text part is larger than 1MB."""
return "data:video/mp4;base64," + ("A" * (1024 * 1024))
def test_oversized_data_video_uploads_as_input_references(mocker: MockerFixture) -> None:
"""A lone data:video over the 1MB text limit is uploaded, not sent as JSON."""
import aiohttp
video_url = _oversized_data_video_url()
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
assert _add_video_reference_to_form(aiohttp.FormData(), {"video_url": video_url}) is True
uploaded = next(value for name, value in captured if name == "input_references")
assert uploaded == base64.b64decode("A" * (1024 * 1024))
field_names = [name for name, _ in captured]
assert "video_reference" not in field_names
assert "image_reference" not in field_names
def test_oversized_data_video_with_image_stays_json(mocker: MockerFixture) -> None:
"""input_references cannot be combined with image_reference, even for a large video."""
import aiohttp
content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}]
extra_body = {"video_reference": _oversized_data_video_url()}
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
_add_combined_video_form_references(aiohttp.FormData(), content, extra_body)
field_names = [name for name, _ in captured]
assert "image_reference" in field_names
assert "video_reference" in field_names
assert "input_references" not in field_names
def test_image_upload_bytes_and_structured_video_use_json_fields(mocker: MockerFixture) -> None:
"""Image upload bytes plus a video URL must not use singular input_reference."""
import aiohttp
image_bytes = base64.b64decode(_MIN_PNG_B64)
extra_body = {
"image_reference": {"bytes": image_bytes, "content_type": "image/png"},
"video_reference": {"video_url": "https://example.com/ref.mp4"},
}
captured: list[tuple[str, object]] = []
real_add_field = aiohttp.FormData.add_field
def tracking_add_field(self, name, value=None, **kwargs):
captured.append((str(name), value))
return real_add_field(self, name, value, **kwargs)
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
form = aiohttp.FormData()
_add_combined_video_form_references(form, None, extra_body)
field_names = [name for name, _ in captured]
assert field_names.count("image_reference") == 1
assert field_names.count("video_reference") == 1
assert "input_reference" not in field_names
assert "input_references" not in field_names
image_payload = next(value for name, value in captured if name == "image_reference")
video_payload = next(value for name, value in captured if name == "video_reference")
assert isinstance(image_payload, (str, bytes, bytearray))
assert isinstance(video_payload, (str, bytes, bytearray))
assert json.loads(image_payload) == {"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"}
assert json.loads(video_payload) == {"video_url": "https://example.com/ref.mp4"}
def test_video_unsupported_image_reference_raises() -> None:
import aiohttp
form = aiohttp.FormData()
with pytest.raises(ValueError, match="Unsupported image_reference"):
with pytest.raises(ValueError, match="Unsupported reference"):
_add_video_reference_to_form(form, {"not_a_supported_key": "x"})
with pytest.raises(ValueError, match="Unsupported image_reference"):
with pytest.raises(ValueError, match="Unsupported reference"):
_add_video_reference_to_form(form, "/tmp/does-not-exist-ref.png")
def test_extract_stage_durations_from_video_and_image_shapes() -> None:
video_payload = {
"stage_durations": {
"diffuse": 1.5,
"text_encoder.forward": 0.2,
"vae.decode": 0.1,
"stage_0_gen_ms": 4000.0,
}
}
assert _extract_stage_durations_from_payload(video_payload) == video_payload["stage_durations"]
image_stage_durations = {
"diffuse": 2.0,
"vae.decode": 0.3,
}
image_payload = {
"metrics": {"stage_durations": image_stage_durations},
"data": [{"b64_json": "x"}],
}
assert _extract_stage_durations_from_payload(image_payload) == image_stage_durations
def test_video_metrics_persist_full_stage_durations() -> None:
output = MixRequestFuncOutput()
output.latency = 4.2
stage_durations = {
"diffuse": 1.5,
"vae.decode": 0.25,
"stage_0_gen_ms": 4000.0,
}
payload = {
"duration_s": 2.0,
"num_frames": 48,
"fps": 24.0,
"stage_durations": stage_durations,
}
_apply_video_metrics_from_payload(output, payload, {})
assert output.stage_durations == stage_durations
assert output.video_generation_time_ms == pytest.approx(4000.0)
def test_image_metrics_persist_stage_durations_from_metrics() -> None:
output = MixRequestFuncOutput()
stage_durations = {
"diffuse": 1.1,
"text_encoder.forward": 0.4,
"vae.decode": 0.2,
}
payload = {
"created": 1,
"data": [{"b64_json": _MIN_PNG_B64}],
"metrics": {"stage_durations": stage_durations},
}
assert _apply_image_metrics_from_payload(output, payload) == 1
assert output.stage_durations == stage_durations
if __name__ == "__main__":
pytest.main([__file__, "-v", "-s"])
+43 -13
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import json
import os
import re
@@ -84,17 +87,20 @@ def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]:
raise ValueError(f"mark must be a list; got {type(mark_field).__name__}")
def _mark_names(mark_field: Any) -> set[str]:
if isinstance(mark_field, list):
return {str(item) for item in mark_field if isinstance(item, str)}
return set()
def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool:
"""True for perf JSON cases intended for ``run_diffusion_benchmark.py``."""
if cfg.get("server_type") is not None:
return True
return "diffusion" in _mark_names(cfg.get("mark"))
"""True for perf JSON cases intended for ``run_diffusion_benchmark.py``.
Schema split (not marks / endpoints):
- ``benchmark_params[].dataset`` → diffusion client (``run_diffusion_benchmark.py``)
- ``benchmark_params[].dataset_name`` → ``vllm bench serve --omni`` (``run_benchmark.py``)
"""
for params in cfg.get("benchmark_params") or []:
if not isinstance(params, dict):
continue
if "dataset" in params and "dataset_name" not in params:
return True
return False
def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]:
@@ -654,6 +660,7 @@ def run_benchmark(
random_output_len: Any | None = None,
resource_label: str | None = None,
num_warmups: int = 2,
benchmark_params_name: str | None = None,
) -> dict[str, Any]:
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
@@ -662,15 +669,35 @@ def run_benchmark(
maps keep every hardware bucket, but each metric is reduced to the value for
this concurrency / request-rate step. If the benchmark exits without writing a
result file, ``result_omni_template.json`` is used as a fallback.
``benchmark_params_name`` (the ``name`` field from ``benchmark_params``) is
persisted in the result JSON and filename so multiple generation configs
under one ``test_name`` stay distinguishable (e.g. Wan USP2 832x480 vs
1280x720).
"""
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
ri = _safe_filename_token(random_input_len)
ro = _safe_filename_token(random_output_len)
hw = resource_label_for_filename(resource_label)
name_token = _safe_filename_token(benchmark_params_name) if benchmark_params_name else ""
if name_token == "na":
name_token = ""
name_parts = [f"result_{test_name}"]
if hw:
result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
else:
result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
name_parts.append(hw)
if name_token:
name_parts.append(name_token)
name_parts.extend(
[
str(dataset_name),
str(flow),
str(num_prompt),
f"in{ri}",
f"out{ro}",
current_dt,
]
)
result_filename = "_".join(name_parts) + ".json"
if "--result-filename" in args:
print(f"The result file will be overwritten by {result_filename}")
command = (
@@ -740,6 +767,9 @@ def run_benchmark(
if random_output_len is not None:
result["random_output_len"] = random_output_len
result["Hardware"] = hardware_json_value(resource_label)
result["test_name"] = test_name
if benchmark_params_name:
result["name"] = benchmark_params_name
with open(result_path, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
+113 -8
View File
@@ -50,12 +50,19 @@ if CONFIG_FILE_PATH is None:
_all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR)
BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)]
print(
f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from "
f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s) from "
f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; "
f"use -m to filter, e.g. -m tts)"
)
else:
BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH)
_loaded = load_benchmark_configs(CONFIG_FILE_PATH)
BENCHMARK_CONFIGS = [cfg for cfg in _loaded if not is_diffusion_perf_config(cfg)]
skipped = len(_loaded) - len(BENCHMARK_CONFIGS)
if skipped:
print(
f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s); "
f"skipped {skipped} remaining diffusion case(s) (chat completions / custom jsonl)"
)
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
@@ -93,24 +100,107 @@ class _SingleActiveContext:
stack.close()
# OmniServer defaults for flags not already present in JSON ``serve_args`` /
# ``extra_cli_args``. Add new (flag, value) pairs here rather than special-casing.
_OMNI_DEFAULT_SERVER_ARGS: tuple[tuple[str, str], ...] = (
("--stage-init-timeout", "600"),
("--init-timeout", "900"),
)
def _cli_flag_names(cli_args: tuple[str, ...] | list[str]) -> set[str]:
"""Return long-option names present in a flat CLI argv list."""
names: set[str] = set()
for item in cli_args:
token = str(item)
if not token.startswith("--"):
continue
names.add(token.split("=", 1)[0])
return names
def _merge_omni_default_server_args(
extra_cli_args: tuple[str, ...] | list[str],
*,
use_omni: bool,
defaults: tuple[tuple[str, str], ...] = _OMNI_DEFAULT_SERVER_ARGS,
) -> list[str]:
"""Fill Omni defaults for flags not already set in JSON-derived CLI args.
JSON ``serve_args`` / ``extra_cli_args`` win; only missing flags are appended.
"""
if not use_omni:
return []
present = _cli_flag_names(extra_cli_args)
args: list[str] = []
for flag, value in defaults:
if flag not in present:
args += [flag, value]
return args
def _omni_server_env() -> dict[str, str]:
"""Writable video/image storage for ``/v1/videos`` and related generation APIs."""
result_dir = Path(os.environ.get("BENCHMARK_DIR", "tests/dfx/perf/results"))
storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(result_dir / "storage")))
storage_path.mkdir(parents=True, exist_ok=True)
return {"VLLM_OMNI_STORAGE_PATH": str(storage_path)}
def _resolve_offline_model(model: str) -> str:
"""Resolve HF ids / MiniMax env overrides the same way as the diffusion runner."""
import huggingface_hub
from vllm_omni.transformers_utils.repo_utils import hf_api
if not model or os.path.isdir(model):
return model
model_env_overrides = {
"MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL",
"MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL",
"MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL",
}
env_name = model_env_overrides.get(model)
if env_name:
env_model = os.environ.get(env_name)
if env_model:
return env_model
parts = model.split("/")
if len(parts) >= 3:
repo_id = "/".join(parts[:2])
subfolder = "/".join(parts[2:])
snapshot_root = hf_api().snapshot_download(
repo_id,
allow_patterns=[f"{subfolder}/**"],
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
)
return str(Path(snapshot_root) / subfolder)
if not huggingface_hub.constants.HF_HUB_OFFLINE:
return model
return hf_api().snapshot_download(model, local_files_only=True)
@contextmanager
def _start_omni_server(server_param):
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param
extra = tuple(extra_cli_args or ())
model = _resolve_offline_model(model)
print(f"Starting OmniServer with test: {test_name}, model: {model}")
server_args: list[str] = []
if use_omni:
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
server_args: list[str] = _merge_omni_default_server_args(extra, use_omni=use_omni)
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
if stage_config_path:
server_args = ["--deploy-config", stage_config_path] + server_args
if stage_overrides:
server_args = ["--stage-overrides", stage_overrides] + server_args
if extra_cli_args:
server_args = list(extra_cli_args) + server_args
with OmniServer(model, server_args, use_omni=use_omni) as server:
if extra:
server_args = list(extra) + server_args
with OmniServer(model, server_args, use_omni=use_omni, env_dict=_omni_server_env()) as server:
server.test_name = test_name
print("OmniServer started successfully")
yield server
@@ -259,11 +349,24 @@ def test_performance_benchmark(omni_server, benchmark_params):
"eval_phase",
"trust_remote_code",
"expected_duplex_audio_turns_per_session",
"name",
"enable_negative_prompt",
"random_request_config",
"num_input_images",
"warmup_requests",
"warmup_concurrency",
"warmup_num_inference_steps",
}
param_keys = {str(key).replace("-", "_") for key in params}
if "model" not in param_keys:
args.extend(["--model", str(model)])
for key, value in params.items():
if key in exclude_keys or value is None:
continue
if key in {"extra_body", "extra-body"} and value == {}:
continue
arg_name = f"--{key.replace('_', '-')}"
@@ -298,6 +401,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
num_warmups=_resolve_num_warmups(params, default=2),
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
)
assert_result(result, params, num_prompt)
@@ -316,5 +420,6 @@ def test_performance_benchmark(omni_server, benchmark_params):
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))),
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
)
assert_result(result, params, num_prompt)
@@ -204,11 +204,11 @@ def load_diffusion_benchmark_configs(
) -> list[dict[str, Any]]:
"""Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*."""
if config_path is not None:
configs = load_configs(config_path)
loaded = load_configs(config_path)
source = str(Path(config_path).resolve())
for cfg in configs:
for cfg in loaded:
cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source)
return configs
return loaded
if config_dir is None:
raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir")
configs: list[dict[str, Any]] = []
@@ -231,7 +231,15 @@ if CONFIG_FILE_PATH is None:
f"use -m to filter, e.g. -m diffusion)"
)
else:
BENCHMARK_CONFIGS = load_diffusion_benchmark_configs(CONFIG_FILE_PATH)
_loaded = load_diffusion_benchmark_configs(CONFIG_FILE_PATH)
BENCHMARK_CONFIGS = [cfg for cfg in _loaded if is_diffusion_perf_config(cfg)]
skipped = len(_loaded) - len(BENCHMARK_CONFIGS)
if skipped:
print(
f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s); "
f"skipped {skipped} omni-bench generation case(s) "
f"(/v1/images/edits, /v1/images/generations, /v1/videos → run_benchmark.py)"
)
_AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {}
+104 -85
View File
@@ -5,7 +5,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 2
}
@@ -14,8 +17,6 @@
"diffusion"
],
"description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/images/generations",
"server_params": {
"model": "nvidia/Cosmos3-Nano",
"serve_args": {
@@ -34,24 +35,28 @@
"benchmark_params": [
{
"name": "1024x1024_steps4",
"dataset": "random",
"task": "t2i",
"width": 1024,
"height": 1024,
"num-inference-steps": 4,
"num-prompts": 3,
"max-concurrency": 1,
"seed": 42,
"extra-body": {
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/images/generations",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 1024,
"height": 1024,
"num_inference_steps": 4,
"seed": 42,
"guidance_scale": 7.0,
"flow_shift": 3.0,
"guardrails": false
},
"baseline": {
"H100": {
"throughput_qps": 1.1958,
"latency_mean": 0.843,
"peak_memory_mb_mean": 80000.0
"request_throughput": 1.1958,
"mean_e2el_ms": 843.0,
"mean_peak_memory_mb": 80000.0
}
}
}
@@ -63,7 +68,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 2
}
@@ -72,8 +80,6 @@
"diffusion"
],
"description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "nvidia/Cosmos3-Nano",
"serve_args": {
@@ -92,23 +98,21 @@
"benchmark_params": [
{
"name": "1280x720_frames189_steps4",
"dataset": "random",
"task": "t2v",
"num-prompts": 3,
"max-concurrency": 1,
"seed": 42,
"random-request-config": [
{
"width": 1280,
"height": 720,
"prompt": "A robot arm is cleaning a plate in the kitchen.",
"num_inference_steps": 4,
"num_frames": 189,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 1280,
"height": 720,
"num_inference_steps": 4,
"num_frames": 189,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 10.0,
"max_sequence_length": 4096,
@@ -118,9 +122,9 @@
},
"baseline": {
"H100": {
"throughput_qps": 0.0307,
"latency_mean": 32.7665,
"peak_memory_mb_mean": 28762.4762
"request_throughput": 0.0307,
"mean_e2el_ms": 32766.5,
"mean_peak_memory_mb": 28762.4762
}
}
}
@@ -132,7 +136,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 2
}
@@ -141,8 +148,6 @@
"diffusion"
],
"description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "nvidia/Cosmos3-Nano",
"serve_args": {
@@ -161,24 +166,30 @@
"benchmark_params": [
{
"name": "1280x720_frames189_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 3,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"random-request-config": [
{
"width": 1280,
"height": 720,
"prompt": "The scene comes to life with smooth, natural motion.",
"num_inference_steps": 4,
"num_frames": 189,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(720, 1280, 1)": 1.0
},
"extra_body": {
"width": 1280,
"height": 720,
"num_inference_steps": 4,
"num_frames": 189,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 10.0,
"max_sequence_length": 4096,
@@ -186,9 +197,9 @@
},
"baseline": {
"H100": {
"throughput_qps": 0.0305,
"latency_mean": 33.1009,
"peak_memory_mb_mean": 29441.1429
"request_throughput": 0.0305,
"mean_e2el_ms": 33100.9,
"mean_peak_memory_mb": 29441.1429
}
}
}
@@ -200,7 +211,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 2
}
@@ -209,8 +223,6 @@
"diffusion"
],
"description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "nvidia/Cosmos3-Nano",
"serve_args": {
@@ -229,23 +241,30 @@
"benchmark_params": [
{
"name": "1280x720_frames189_steps4",
"dataset": "random",
"task": "v2v",
"num-prompts": 3,
"max-concurrency": 1,
"seed": 42,
"random-request-config": [
{
"width": 1280,
"height": 720,
"prompt": "Continue the same scene with smooth natural motion.",
"num_inference_steps": 4,
"num_frames": 189,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"video": 1
},
"random_mm_bucket_config": {
"(720, 1280, 189)": 1.0
},
"extra_body": {
"width": 1280,
"height": 720,
"num_inference_steps": 4,
"num_frames": 189,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 10.0,
"max_sequence_length": 4096,
@@ -258,9 +277,9 @@
},
"baseline": {
"H100": {
"throughput_qps": 0.0299,
"latency_mean": 33.925,
"peak_memory_mb_mean": 29443.1429
"request_throughput": 0.0299,
"mean_e2el_ms": 33925.0,
"mean_peak_memory_mb": 29443.1429
}
}
}
@@ -5,7 +5,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 1
}
@@ -14,7 +17,6 @@
"diffusion"
],
"description": "Single-device baseline",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
"serve_args": {
@@ -25,26 +27,33 @@
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(480, 832, 1)": 1.0
},
"extra_body": {
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 5.0
"flow_shift": 5.0,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
@@ -55,7 +64,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 2
}
@@ -64,7 +76,6 @@
"diffusion"
],
"description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
"serve_args": {
@@ -79,26 +90,33 @@
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(480, 832, 1)": 1.0
},
"extra_body": {
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 5.0
"flow_shift": 5.0,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
@@ -5,7 +5,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 1
}
@@ -14,7 +17,6 @@
"diffusion"
],
"description": "Single-device baseline",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
"serve_args": {
@@ -25,25 +27,24 @@
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "t2v",
"num-prompts": 10,
"max-concurrency": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 5.0
"flow_shift": 5.0,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
@@ -64,7 +65,6 @@
"diffusion"
],
"description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
"serve_args": {
@@ -79,25 +79,24 @@
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "t2v",
"num-prompts": 10,
"max-concurrency": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"seed": 42,
"guidance_scale": 6.0,
"flow_shift": 5.0
"flow_shift": 5.0,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
@@ -1,33 +1,50 @@
[
{
"test_name": "test_lingbot_video_single_device",
"description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.",
"server_type": "vllm-omni",
"server_params": {
"model": "robbyant/lingbot-video-dense-1.3b",
"serve_args": {
"model-class-name": "LingBotVideoPipeline",
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "320x192_frames9_steps2",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 320,
"height": 192,
"num-frames": 9,
"fps": 24,
"num-inference-steps": 2,
"num-prompts": 3,
"max-concurrency": 1,
"warmup-requests": 1,
"warmup-concurrency": 1,
"warmup-num-inference-steps": 2,
"enable-negative-prompt": true
}
]
}
{
"test_name": "test_lingbot_video_single_device",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": [
"H100",
"B200"
]
},
"num_cards": 1
}
},
"full_model",
"diffusion"
],
"description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.",
"server_params": {
"model": "robbyant/lingbot-video-dense-1.3b",
"serve_args": {
"model-class-name": "LingBotVideoPipeline",
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "320x192_frames9_steps2",
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"num_warmups": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 320,
"height": 192,
"num_frames": 9,
"fps": 24,
"num_inference_steps": 2,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
}
]
+160 -115
View File
@@ -14,7 +14,6 @@
"diffusion"
],
"description": "Single-device baseline with enforce-eager (no torch.compile)",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
@@ -25,31 +24,41 @@
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 256,
"height": 256,
"num_frames": 145,
"fps": 24,
"num_inference_steps": 6,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 768,
"height": 480,
"num_frames": 41,
"fps": 24,
"num_inference_steps": 20,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
},
@@ -68,7 +77,6 @@
"diffusion"
],
"description": "Single-device with torch.compile (default, no enforce-eager)",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
@@ -78,31 +86,41 @@
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 256,
"height": 256,
"num_frames": 145,
"fps": 24,
"num_inference_steps": 6,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 768,
"height": 480,
"num_frames": 41,
"fps": 24,
"num_inference_steps": 20,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
},
@@ -121,7 +139,6 @@
"diffusion"
],
"description": "CFG-parallel=2 with enforce-eager",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
@@ -133,31 +150,41 @@
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 256,
"height": 256,
"num_frames": 145,
"fps": 24,
"num_inference_steps": 6,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 768,
"height": 480,
"num_frames": 41,
"fps": 24,
"num_inference_steps": 20,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
},
@@ -176,7 +203,6 @@
"diffusion"
],
"description": "CFG-parallel=2 with torch.compile",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
@@ -187,31 +213,41 @@
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 256,
"height": 256,
"num_frames": 145,
"fps": 24,
"num_inference_steps": 6,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 768,
"height": 480,
"num_frames": 41,
"fps": 24,
"num_inference_steps": 20,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
},
@@ -230,7 +266,6 @@
"diffusion"
],
"description": "CacheDiT with enforce-eager",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
@@ -242,31 +277,41 @@
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 256,
"height": 256,
"num_frames": 145,
"fps": 24,
"num_inference_steps": 6,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 768,
"height": 480,
"num_frames": 41,
"fps": 24,
"num_inference_steps": 20,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
}
}
]
}
@@ -5,7 +5,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 4
}
@@ -14,8 +17,6 @@
"diffusion"
],
"description": "MiniMax-H3 T2VA text-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "MiniMaxAI/MiniMax-H3",
"serve_args": {
@@ -38,33 +39,30 @@
"benchmark_params": [
{
"name": "1344x768_frames209_steps8",
"dataset": "random",
"task": "t2v",
"num-prompts": 3,
"max-concurrency": 1,
"warmup-requests": 1,
"warmup-concurrency": 1,
"seed": 42,
"random-request-config": [
{
"width": 1344,
"height": 768,
"prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"num_warmups": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 1344,
"height": 768,
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"seed": 42,
"flow_shift": 12.0,
"extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
},
"baseline": {
"H100": {
"throughput_qps": 0.0211,
"latency_mean": 38.3252,
"peak_memory_mb_mean": 54219.6
"request_throughput": 0.0211,
"mean_e2el_ms": 38325.2,
"mean_peak_memory_mb": 54219.6
}
}
}
@@ -76,7 +74,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 4
}
@@ -85,8 +86,6 @@
"diffusion"
],
"description": "MiniMax-H3 FL2VA first-frame image-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "MiniMaxAI/MiniMax-H3",
"serve_args": {
@@ -109,34 +108,39 @@
"benchmark_params": [
{
"name": "1344x768_frames209_steps8",
"dataset": "random",
"task": "ti2v",
"num-prompts": 3,
"max-concurrency": 1,
"warmup-requests": 1,
"warmup-concurrency": 1,
"seed": 42,
"random-request-config": [
{
"width": 1344,
"height": 768,
"prompt": "Continue the scene: the camera slowly pushes in on the subject as the background city lights drift and blur, with the actor singing softly and emotionally.",
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"num_warmups": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(768, 1344, 1)": 1.0
},
"extra_body": {
"width": 1344,
"height": 768,
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"seed": 42,
"flow_shift": 12.0,
"extra_params": "{\"task\":\"fl2va\",\"duration\":8.7,\"audio_flow_shift\":3.0}"
},
"num-input-images": 1,
"baseline": {
"H100": {
"throughput_qps": 0.0212,
"latency_mean": 38.1385,
"peak_memory_mb_mean": 54622.8
"request_throughput": 0.0212,
"mean_e2el_ms": 38138.5,
"mean_peak_memory_mb": 54622.8
}
}
}
@@ -148,7 +152,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 4
}
@@ -157,8 +164,6 @@
"diffusion"
],
"description": "MiniMax-H3 Ref2VA video-reference-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "MiniMaxAI/MiniMax-H3",
"serve_args": {
@@ -181,33 +186,39 @@
"benchmark_params": [
{
"name": "1344x768_frames209_steps8",
"dataset": "random",
"task": "v2v",
"num-prompts": 3,
"max-concurrency": 1,
"warmup-requests": 1,
"warmup-concurrency": 1,
"seed": 42,
"random-request-config": [
{
"width": 1344,
"height": 768,
"prompt": "Restyle the reference footage into a cinematic night scene with neon reflections, keeping the same camera motion and subject.",
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"num_warmups": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"video": 1
},
"random_mm_bucket_config": {
"(480, 832, 209)": 1.0
},
"extra_body": {
"width": 1344,
"height": 768,
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"seed": 42,
"flow_shift": 12.0,
"extra_params": "{\"task\":\"ref2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
},
"baseline": {
"H100": {
"throughput_qps": 0.0077,
"latency_mean": 105.7508,
"peak_memory_mb_mean": 55180.8
"request_throughput": 0.0077,
"mean_e2el_ms": 105750.8,
"mean_peak_memory_mb": 55180.8
}
}
}
@@ -219,7 +230,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 4
}
@@ -228,8 +242,6 @@
"diffusion"
],
"description": "MiniMax-H3 T2VA with 4-card distributed layerwise offload (DLO), 8 denoise steps (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/videos",
"server_params": {
"model": "MiniMaxAI/MiniMax-H3",
"serve_args": {
@@ -256,33 +268,30 @@
"benchmark_params": [
{
"name": "1344x768_frames209_steps8",
"dataset": "random",
"task": "t2v",
"num-prompts": 3,
"max-concurrency": 1,
"warmup-requests": 1,
"warmup-concurrency": 1,
"seed": 42,
"random-request-config": [
{
"width": 1344,
"height": 768,
"prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"dataset_name": "random",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 3,
"max_concurrency": 1,
"num_warmups": 1,
"random_input_len": 8,
"random_output_len": 1,
"percentile_metrics": "e2el",
"extra_body": {
"width": 1344,
"height": 768,
"num_inference_steps": 8,
"num_frames": 209,
"fps": 24,
"seed": 42,
"flow_shift": 12.0,
"extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
},
"baseline": {
"H100": {
"throughput_qps": 0.0212,
"latency_mean": 38.2089,
"peak_memory_mb_mean": 36218.4
"request_throughput": 0.0212,
"mean_e2el_ms": 38208.9,
"mean_peak_memory_mb": 36218.4
}
}
}
+117 -7
View File
@@ -205,15 +205,53 @@ def test_is_diffusion_perf_config():
from tests.dfx.conftest import is_diffusion_perf_config
assert not is_diffusion_perf_config(
{"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]}
{
"test_name": "omni_a",
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"],
"benchmark_params": [{"dataset_name": "random", "endpoint": "/v1/chat/completions"}],
}
)
assert is_diffusion_perf_config(
{
"test_name": "diff_a",
"server_type": "vllm-omni",
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
"benchmark_params": [{"task": "t2i", "dataset": "random"}],
}
)
videos_cfg = {
"test_name": "diff_videos",
"server_type": "vllm-omni",
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
"benchmark_params": [{"task": "t2v", "dataset_name": "random", "endpoint": "/v1/videos"}],
}
assert not is_diffusion_perf_config(videos_cfg)
custom_edits_cfg = {
"test_name": "diff_custom_edits",
"server_type": "vllm-omni",
"benchmark_endpoint": "/v1/images/edits",
"benchmark_params": [{"dataset": "custom", "task": "ti2i"}],
}
assert is_diffusion_perf_config(custom_edits_cfg)
def test_merge_omni_default_server_args_respects_json():
from tests.dfx.perf.scripts.run_benchmark import _merge_omni_default_server_args
extra = ("--stage-init-timeout", "1800", "--init-timeout", "1800", "--usp", "4")
assert _merge_omni_default_server_args(extra, use_omni=True) == []
assert _merge_omni_default_server_args((), use_omni=True) == [
"--stage-init-timeout",
"600",
"--init-timeout",
"900",
]
assert _merge_omni_default_server_args((), use_omni=False) == []
# Only fill the missing default; keep JSON's other timeouts.
assert _merge_omni_default_server_args(("--stage-init-timeout=1800",), use_omni=True) == [
"--init-timeout",
"900",
]
def test_benchmark_param_id_suffix_from_task_eval_phase():
@@ -303,12 +341,84 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat
finally:
active_context.close()
assert events == [
("start", omni_p0_server),
("stop", omni_p0_server),
("start", tts_server),
("stop", tts_server),
]
def test_run_benchmark_persists_distinct_benchmark_params_name(tmp_path, monkeypatch):
"""Two benchmark_params under one test_name must keep distinct saved identity."""
import io
from pathlib import Path
from tests.dfx import conftest as dfx_conftest
result_dir = tmp_path / "results"
result_dir.mkdir()
monkeypatch.setenv("BENCHMARK_DIR", str(result_dir))
class _FakePopen:
def __init__(self, command, **kwargs):
self.stdout = io.StringIO("")
self.stderr = io.StringIO("")
result_dir_idx = command.index("--result-dir")
filename_idx = command.index("--result-filename")
out = Path(command[result_dir_idx + 1]) / command[filename_idx + 1]
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps({"completed": 3, "request_throughput": 0.01}), encoding="utf-8")
def wait(self):
return 0
monkeypatch.setattr(dfx_conftest.subprocess, "Popen", _FakePopen)
shared_test = "test_wan22_i2v_usp2"
name_a = "832x480_frames81_steps4"
name_b = "1280x720_frames121_steps4"
result_a = dfx_conftest.run_benchmark(
args=["--host", "127.0.0.1", "--port", "8000"],
test_name=shared_test,
flow=1,
dataset_name="random-mm",
num_prompt=10,
random_input_len=8,
random_output_len=1,
resource_label="H800",
benchmark_params_name=name_a,
)
result_b = dfx_conftest.run_benchmark(
args=["--host", "127.0.0.1", "--port", "8000"],
test_name=shared_test,
flow=1,
dataset_name="random-mm",
num_prompt=10,
random_input_len=8,
random_output_len=1,
resource_label="H800",
benchmark_params_name=name_b,
)
assert result_a["test_name"] == shared_test
assert result_b["test_name"] == shared_test
assert result_a["name"] == name_a
assert result_b["name"] == name_b
assert "benchmark_params" not in result_a
assert "benchmark_params" not in result_b
files = sorted(p.name for p in result_dir.glob("result_*.json"))
assert len(files) == 2
assert any(name_a in name for name in files)
assert any(name_b in name for name in files)
assert files[0] != files[1]
def omni_group_key(record: dict) -> tuple:
return (
record.get("model_id") or "",
record.get("test_name") or "",
record.get("name") or "",
record.get("dataset_name") or "",
record.get("max_concurrency") if record.get("max_concurrency") is not None else 0,
record.get("num_prompts") if record.get("num_prompts") is not None else 0,
)
assert omni_group_key(result_a) != omni_group_key(result_b)
def test_is_hardware_nested_baseline():
@@ -5,7 +5,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"]
"cuda": [
"H100",
"B200"
]
},
"num_cards": 1
}
@@ -14,7 +17,6 @@
"diffusion"
],
"description": "Single-device baseline",
"server_type": "vllm-omni",
"server_params": {
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
"serve_args": {
@@ -24,28 +26,37 @@
"benchmark_params": [
{
"name": "832x480_frames81_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 81,
"fps": 16,
"weight": 1
}
],
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(480, 832, 1)": 1.0
},
"extra_body": {
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 81,
"fps": 16,
"seed": 42,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
},
"baseline": {
"H100": {
"throughput_qps": 0.0361,
"latency_mean": 27.6206,
"peak_memory_mb_mean": 80548.0
"request_throughput": 0.0361,
"mean_e2el_ms": 27620.6,
"mean_peak_memory_mb": 80548.0
}
}
}
@@ -57,7 +68,10 @@
{
"hardware_marks": {
"res": {
"cuda": ["H100", "B200"],
"cuda": [
"H100",
"B200"
],
"npu": "A3"
},
"num_cards": 2
@@ -67,7 +81,6 @@
"diffusion"
],
"description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
"server_type": "vllm-omni",
"server_params": {
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
"serve_args": {
@@ -81,55 +94,73 @@
"benchmark_params": [
{
"name": "832x480_frames81_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 81,
"fps": 16,
"weight": 1
}
],
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(480, 832, 1)": 1.0
},
"extra_body": {
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 81,
"fps": 16,
"seed": 42,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
},
"baseline": {
"H100": {
"throughput_qps": 0.05263,
"latency_mean": 18.9415,
"peak_memory_mb_mean": 50053.7
"request_throughput": 0.05263,
"mean_e2el_ms": 18941.5,
"mean_peak_memory_mb": 50053.7
}
}
},
{
"name": "1280x720_frames121_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 1280,
"height": 720,
"num_inference_steps": 4,
"num_frames": 121,
"fps": 16,
"weight": 1
}
],
"dataset_name": "random-mm",
"tokenizer": "gpt2",
"endpoint": "/v1/videos",
"num_prompts": 10,
"max_concurrency": 1,
"random_input_len": 8,
"random_output_len": 1,
"random_range_ratio": 0.0,
"percentile_metrics": "e2el",
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1
},
"random_mm_bucket_config": {
"(720, 1280, 1)": 1.0
},
"extra_body": {
"width": 1280,
"height": 720,
"num_inference_steps": 4,
"num_frames": 121,
"fps": 16,
"seed": 42,
"negative_prompt": "Negative prompt for benchmarking diffusion models"
},
"baseline": {
"H100": {
"throughput_qps": 0.009595,
"latency_mean": 104.1058,
"peak_memory_mb_mean": 59718.0
"request_throughput": 0.009595,
"mean_e2el_ms": 104105.8,
"mean_peak_memory_mb": 59718.0
}
}
}