[CI][Perf] migrate diffusion DFX benches to vllm bench serve --omni (#7737)
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
1 parent
a4ad73e08b
commit
0570b0da7e
30 files changed
+2040
-729
No files matched your search
@@ -6,11 +6,16 @@ Unit tests for metrics.py
|
||||
"""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm.benchmarks.serve import TaskType
|
||||
|
||||
from vllm_omni.benchmarks.metrics.metrics import calculate_metrics
|
||||
from vllm_omni.benchmarks.metrics.metrics import (
|
||||
aggregate_stage_durations,
|
||||
calculate_metrics,
|
||||
print_stage_durations_metrics,
|
||||
)
|
||||
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||
@@ -379,12 +384,7 @@ class _EmptyAwareTokenizer:
|
||||
"""
|
||||
|
||||
def __call__(self, text, add_special_tokens=False):
|
||||
class _R:
|
||||
pass
|
||||
|
||||
r = _R()
|
||||
r.input_ids = [0] * len(text)
|
||||
return r
|
||||
return SimpleNamespace(input_ids=[0] * len(text))
|
||||
|
||||
|
||||
def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput:
|
||||
@@ -588,5 +588,80 @@ def test_image_with_generated_text_still_reports_text_result(capsys):
|
||||
assert " Image Result " in out
|
||||
|
||||
|
||||
def test_aggregate_stage_durations_mean_p50_p99() -> None:
|
||||
ok_a = MixRequestFuncOutput()
|
||||
ok_a.success = True
|
||||
ok_a.stage_durations = {"diffuse": 1.0, "vae.decode": 0.2, "stage_0_gen_ms": 50.0}
|
||||
ok_b = MixRequestFuncOutput()
|
||||
ok_b.success = True
|
||||
ok_b.stage_durations = {"diffuse": 3.0, "vae.decode": 0.4, "stage_1_gen_ms": 80.0}
|
||||
failed = MixRequestFuncOutput()
|
||||
failed.success = False
|
||||
failed.stage_durations = {"diffuse": 99.0}
|
||||
|
||||
summaries = aggregate_stage_durations([ok_a, ok_b, failed])
|
||||
assert summaries["stage_durations_mean"]["diffuse"] == pytest.approx(2.0)
|
||||
assert summaries["stage_durations_p50"]["diffuse"] == pytest.approx(2.0)
|
||||
assert summaries["stage_durations_mean"]["vae.decode"] == pytest.approx(0.3)
|
||||
assert "stage_durations_p99" in summaries
|
||||
assert "stage_0_gen_ms" not in summaries["stage_durations_mean"]
|
||||
assert "stage_1_gen_ms" not in summaries["stage_durations_mean"]
|
||||
|
||||
|
||||
def test_print_stage_durations_metrics(capsys) -> None:
|
||||
output = MixRequestFuncOutput()
|
||||
output.success = True
|
||||
output.stage_durations = {
|
||||
"Wan22I2VPipeline.diffuse": 1.25,
|
||||
"Wan22I2VPipeline.text_encoder.forward": 0.4,
|
||||
"queue_wait_ms": 0.5,
|
||||
"stage_0_gen_ms": 1000.0,
|
||||
}
|
||||
summaries = aggregate_stage_durations([output])
|
||||
metrics = SimpleNamespace(**summaries)
|
||||
print_stage_durations_metrics([99.0], metrics)
|
||||
out = capsys.readouterr().out
|
||||
assert "Wan22I2VPipeline" not in out
|
||||
assert "Mean Diffuse (s):" in out
|
||||
assert "Median Diffuse (s):" in out
|
||||
assert "P99 Diffuse (s):" in out
|
||||
assert "Mean Text Encoder Forward (s):" in out
|
||||
assert "Mean Queue Wait (ms):" in out
|
||||
assert "Stage 0 Gen" not in out
|
||||
assert "stage_0_gen" not in out
|
||||
|
||||
|
||||
def test_profiler_stage_durations_print_without_print_stage(capsys) -> None:
|
||||
output = MixRequestFuncOutput()
|
||||
output.success = True
|
||||
output.prompt_len = 8
|
||||
output.latency = 1.0
|
||||
output.stage_durations = {
|
||||
"Wan22I2VPipeline.diffuse": 1.25,
|
||||
"queue_wait_ms": 0.5,
|
||||
"stage_0_gen_ms": 1000.0,
|
||||
}
|
||||
common = dict(
|
||||
input_requests=[],
|
||||
outputs=[output],
|
||||
dur_s=1.0,
|
||||
tokenizer=None,
|
||||
selected_percentiles=[50.0, 99.0],
|
||||
goodput_config_dict={},
|
||||
task_type=TaskType.GENERATION,
|
||||
selected_percentile_metrics=["e2el"],
|
||||
max_concurrency=None,
|
||||
request_rate=float("inf"),
|
||||
benchmark_duration=1.0,
|
||||
)
|
||||
metrics, _ = calculate_metrics(**common, print_stage=False)
|
||||
shown = capsys.readouterr().out
|
||||
assert "Mean Diffuse (s):" in shown
|
||||
assert "Mean Queue Wait (ms):" in shown
|
||||
assert "Stage 0 Gen" not in shown
|
||||
assert metrics.stage_durations_mean["Wan22I2VPipeline.diffuse"] == pytest.approx(1.25)
|
||||
assert "stage_0_gen_ms" not in metrics.stage_durations_mean
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
@@ -22,12 +22,17 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
||||
)
|
||||
from vllm_omni.benchmarks.patch.patch import (
|
||||
MixRequestFuncOutput,
|
||||
_add_combined_video_form_references,
|
||||
_add_video_extra_body_to_form,
|
||||
_add_video_reference_to_form,
|
||||
_apply_image_metrics_from_payload,
|
||||
_apply_stage0_token_timings,
|
||||
_apply_video_metrics_from_payload,
|
||||
_attach_seed_tts_to_request_func_input,
|
||||
_build_benchmark_session,
|
||||
_extract_stage_durations_from_payload,
|
||||
_iter_image_reference_inputs,
|
||||
_iter_video_reference_inputs,
|
||||
_omni_request_timeout_s,
|
||||
async_request_openai_chat_omni_completions,
|
||||
async_request_openai_image_edits_omni,
|
||||
@@ -1359,16 +1364,15 @@ async def test_image_edits_defaults_to_non_streaming_json(mocker: MockerFixture)
|
||||
return None
|
||||
|
||||
captured_stream: list[str] = []
|
||||
real_add_field = None
|
||||
import aiohttp
|
||||
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
if name == "stream":
|
||||
captured_stream.append(str(value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
import aiohttp
|
||||
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
|
||||
mock_session = mocker.AsyncMock()
|
||||
@@ -1512,8 +1516,7 @@ def test_video_local_image_reference_not_forwarded_as_raw_extra_field(tmp_path,
|
||||
"reference",
|
||||
[
|
||||
{"image_url": "https://example.com/ref.png"},
|
||||
{"file_id": "file-abc"},
|
||||
[{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}],
|
||||
[{"image_url": "https://example.com/a.png"}, {"image_url": "https://example.com/b.png"}],
|
||||
],
|
||||
)
|
||||
def test_video_structured_image_reference_serialized_to_form(reference: object, mocker: MockerFixture) -> None:
|
||||
@@ -1541,18 +1544,395 @@ def test_video_structured_image_reference_serialized_to_form(reference: object,
|
||||
assert field_names.count("image_reference") == 1
|
||||
assert "input_reference" not in field_names
|
||||
payload = next(value for name, value in captured if name == "image_reference")
|
||||
assert isinstance(payload, (str, bytes, bytearray))
|
||||
assert json.loads(payload) == reference
|
||||
|
||||
|
||||
def test_video_file_id_reference_is_rejected() -> None:
|
||||
"""file_id is unsupported on the server and must not be sent as image_reference."""
|
||||
import aiohttp
|
||||
|
||||
form = aiohttp.FormData()
|
||||
with pytest.raises(ValueError, match="file_id is not supported yet"):
|
||||
_add_video_reference_to_form(form, {"file_id": "file-abc"})
|
||||
with pytest.raises(ValueError, match="file_id is not supported yet"):
|
||||
_add_video_reference_to_form(form, [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}])
|
||||
with pytest.raises(ValueError, match="file_id is not supported yet"):
|
||||
_add_combined_video_form_references(form, None, {"video_reference": {"file_id": "file-vid"}})
|
||||
|
||||
|
||||
def test_image_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None:
|
||||
"""random-mm image_url parts keep an explicit image_reference type."""
|
||||
import aiohttp
|
||||
|
||||
content = [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/ref.png"},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{_MIN_PNG_B64}"},
|
||||
},
|
||||
]
|
||||
refs = list(_iter_image_reference_inputs(content))
|
||||
assert refs == [
|
||||
{"image_url": "https://example.com/ref.png"},
|
||||
{"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"},
|
||||
]
|
||||
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
form = aiohttp.FormData()
|
||||
assert _add_video_reference_to_form(form, refs[0]) is True
|
||||
assert _add_video_reference_to_form(form, refs[1]) is True
|
||||
|
||||
field_names = [name for name, _ in captured]
|
||||
assert field_names.count("image_reference") == 2
|
||||
assert "video_reference" not in field_names
|
||||
payloads = []
|
||||
for name, value in captured:
|
||||
if name != "image_reference":
|
||||
continue
|
||||
assert isinstance(value, (str, bytes, bytearray))
|
||||
payloads.append(json.loads(value))
|
||||
assert payloads == refs
|
||||
|
||||
|
||||
def test_video_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None:
|
||||
"""random-mm data:video_url parts stay on video_reference."""
|
||||
import aiohttp
|
||||
|
||||
content = [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": "data:video/mp4;base64,AAAA"},
|
||||
}
|
||||
]
|
||||
refs = list(_iter_video_reference_inputs(content))
|
||||
assert refs == [{"video_url": "data:video/mp4;base64,AAAA"}]
|
||||
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
form = aiohttp.FormData()
|
||||
assert _add_video_reference_to_form(form, refs[0]) is True
|
||||
|
||||
field_names = [name for name, _ in captured]
|
||||
assert field_names.count("video_reference") == 1
|
||||
assert "input_references" not in field_names
|
||||
assert "input_reference" not in field_names
|
||||
assert "image_reference" not in field_names
|
||||
payload = next(value for name, value in captured if name == "video_reference")
|
||||
assert isinstance(payload, (str, bytes, bytearray))
|
||||
assert json.loads(payload) == {"video_url": "data:video/mp4;base64,AAAA"}
|
||||
|
||||
|
||||
def test_video_reference_https_url_from_random_mm_content(mocker: MockerFixture) -> None:
|
||||
"""HTTP(S) video_url parts must stay on video_reference, not image_reference."""
|
||||
import aiohttp
|
||||
|
||||
content = [
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": "https://example.com/ref.mp4"},
|
||||
}
|
||||
]
|
||||
refs = list(_iter_video_reference_inputs(content))
|
||||
assert refs == [{"video_url": "https://example.com/ref.mp4"}]
|
||||
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
form = aiohttp.FormData()
|
||||
assert _add_video_reference_to_form(form, refs[0]) is True
|
||||
|
||||
field_names = [name for name, _ in captured]
|
||||
assert field_names.count("video_reference") == 1
|
||||
assert "image_reference" not in field_names
|
||||
assert "input_references" not in field_names
|
||||
payload = next(value for name, value in captured if name == "video_reference")
|
||||
assert isinstance(payload, (str, bytes, bytearray))
|
||||
assert json.loads(payload) == {"video_url": "https://example.com/ref.mp4"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reference", "field_name", "expected"),
|
||||
[
|
||||
("data:image/png;base64,AAAA", "image_reference", {"image_url": "data:image/png;base64,AAAA"}),
|
||||
("data:video/mp4;base64,AAAA", "video_reference", {"video_url": "data:video/mp4;base64,AAAA"}),
|
||||
("https://example.com/ref.png", "image_reference", {"image_url": "https://example.com/ref.png"}),
|
||||
("https://example.com/ref.mp4", "video_reference", {"video_url": "https://example.com/ref.mp4"}),
|
||||
],
|
||||
)
|
||||
def test_bare_reference_string_uses_matching_json_field(
|
||||
reference: str,
|
||||
field_name: str,
|
||||
expected: dict[str, str],
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""Image and video URLs use matching JSON fields; neither is a file upload."""
|
||||
import aiohttp
|
||||
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
assert _add_video_reference_to_form(aiohttp.FormData(), reference) is True
|
||||
assert [name for name, _ in captured] == [field_name]
|
||||
payload = captured[0][1]
|
||||
assert isinstance(payload, (str, bytes, bytearray))
|
||||
assert json.loads(payload) == expected
|
||||
|
||||
|
||||
def test_bare_video_string_is_not_collected_as_image() -> None:
|
||||
content = ["https://example.com/ref.mp4", "data:video/mp4;base64,AAAA"]
|
||||
assert list(_iter_image_reference_inputs(content)) == []
|
||||
assert list(_iter_video_reference_inputs(content)) == [
|
||||
{"video_url": "https://example.com/ref.mp4"},
|
||||
{"video_url": "data:video/mp4;base64,AAAA"},
|
||||
]
|
||||
|
||||
|
||||
def test_bare_http_without_media_extension_is_rejected() -> None:
|
||||
import aiohttp
|
||||
|
||||
with pytest.raises(ValueError, match="image or video extension"):
|
||||
_add_video_reference_to_form(aiohttp.FormData(), "https://example.com/ref")
|
||||
|
||||
|
||||
def test_image_and_inline_video_use_combined_reference_fields(mocker: MockerFixture) -> None:
|
||||
"""Image plus data:video must be image_reference + video_reference, not input_references."""
|
||||
import aiohttp
|
||||
|
||||
content = [
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}},
|
||||
{"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}},
|
||||
]
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
form = aiohttp.FormData()
|
||||
_add_combined_video_form_references(form, content)
|
||||
|
||||
field_names = [name for name, _ in captured]
|
||||
assert field_names.count("image_reference") == 1
|
||||
assert field_names.count("video_reference") == 1
|
||||
assert "input_references" not in field_names
|
||||
assert "input_reference" not in field_names
|
||||
image_payload = next(value for name, value in captured if name == "image_reference")
|
||||
video_payload = next(value for name, value in captured if name == "video_reference")
|
||||
assert isinstance(image_payload, (str, bytes, bytearray))
|
||||
assert isinstance(video_payload, (str, bytes, bytearray))
|
||||
assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"}
|
||||
assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"}
|
||||
|
||||
|
||||
def test_image_url_and_bare_data_video_string_use_json_fields(mocker: MockerFixture) -> None:
|
||||
"""Image URL plus a bare data:video string must not upload input_references."""
|
||||
import aiohttp
|
||||
|
||||
content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}]
|
||||
extra_body = {"video_reference": "data:video/mp4;base64,AAAA"}
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
form = aiohttp.FormData()
|
||||
_add_combined_video_form_references(form, content, extra_body)
|
||||
|
||||
field_names = [name for name, _ in captured]
|
||||
assert field_names.count("image_reference") == 1
|
||||
assert field_names.count("video_reference") == 1
|
||||
assert "input_references" not in field_names
|
||||
assert "input_reference" not in field_names
|
||||
image_payload = next(value for name, value in captured if name == "image_reference")
|
||||
video_payload = next(value for name, value in captured if name == "video_reference")
|
||||
assert isinstance(image_payload, (str, bytes, bytearray))
|
||||
assert isinstance(video_payload, (str, bytes, bytearray))
|
||||
assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"}
|
||||
assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"}
|
||||
|
||||
captured.clear()
|
||||
assert _add_video_reference_to_form(form, "data:video/mp4;base64,AAAA") is True
|
||||
assert [name for name, _ in captured] == ["video_reference"]
|
||||
|
||||
|
||||
def _oversized_data_video_url() -> str:
|
||||
"""data:video whose JSON text part is larger than 1MB."""
|
||||
return "data:video/mp4;base64," + ("A" * (1024 * 1024))
|
||||
|
||||
|
||||
def test_oversized_data_video_uploads_as_input_references(mocker: MockerFixture) -> None:
|
||||
"""A lone data:video over the 1MB text limit is uploaded, not sent as JSON."""
|
||||
import aiohttp
|
||||
|
||||
video_url = _oversized_data_video_url()
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
assert _add_video_reference_to_form(aiohttp.FormData(), {"video_url": video_url}) is True
|
||||
uploaded = next(value for name, value in captured if name == "input_references")
|
||||
assert uploaded == base64.b64decode("A" * (1024 * 1024))
|
||||
field_names = [name for name, _ in captured]
|
||||
assert "video_reference" not in field_names
|
||||
assert "image_reference" not in field_names
|
||||
|
||||
|
||||
def test_oversized_data_video_with_image_stays_json(mocker: MockerFixture) -> None:
|
||||
"""input_references cannot be combined with image_reference, even for a large video."""
|
||||
import aiohttp
|
||||
|
||||
content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}]
|
||||
extra_body = {"video_reference": _oversized_data_video_url()}
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
_add_combined_video_form_references(aiohttp.FormData(), content, extra_body)
|
||||
field_names = [name for name, _ in captured]
|
||||
assert "image_reference" in field_names
|
||||
assert "video_reference" in field_names
|
||||
assert "input_references" not in field_names
|
||||
|
||||
|
||||
def test_image_upload_bytes_and_structured_video_use_json_fields(mocker: MockerFixture) -> None:
|
||||
"""Image upload bytes plus a video URL must not use singular input_reference."""
|
||||
import aiohttp
|
||||
|
||||
image_bytes = base64.b64decode(_MIN_PNG_B64)
|
||||
extra_body = {
|
||||
"image_reference": {"bytes": image_bytes, "content_type": "image/png"},
|
||||
"video_reference": {"video_url": "https://example.com/ref.mp4"},
|
||||
}
|
||||
captured: list[tuple[str, object]] = []
|
||||
real_add_field = aiohttp.FormData.add_field
|
||||
|
||||
def tracking_add_field(self, name, value=None, **kwargs):
|
||||
captured.append((str(name), value))
|
||||
return real_add_field(self, name, value, **kwargs)
|
||||
|
||||
mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
|
||||
form = aiohttp.FormData()
|
||||
_add_combined_video_form_references(form, None, extra_body)
|
||||
|
||||
field_names = [name for name, _ in captured]
|
||||
assert field_names.count("image_reference") == 1
|
||||
assert field_names.count("video_reference") == 1
|
||||
assert "input_reference" not in field_names
|
||||
assert "input_references" not in field_names
|
||||
image_payload = next(value for name, value in captured if name == "image_reference")
|
||||
video_payload = next(value for name, value in captured if name == "video_reference")
|
||||
assert isinstance(image_payload, (str, bytes, bytearray))
|
||||
assert isinstance(video_payload, (str, bytes, bytearray))
|
||||
assert json.loads(image_payload) == {"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"}
|
||||
assert json.loads(video_payload) == {"video_url": "https://example.com/ref.mp4"}
|
||||
|
||||
|
||||
def test_video_unsupported_image_reference_raises() -> None:
|
||||
import aiohttp
|
||||
|
||||
form = aiohttp.FormData()
|
||||
with pytest.raises(ValueError, match="Unsupported image_reference"):
|
||||
with pytest.raises(ValueError, match="Unsupported reference"):
|
||||
_add_video_reference_to_form(form, {"not_a_supported_key": "x"})
|
||||
with pytest.raises(ValueError, match="Unsupported image_reference"):
|
||||
with pytest.raises(ValueError, match="Unsupported reference"):
|
||||
_add_video_reference_to_form(form, "/tmp/does-not-exist-ref.png")
|
||||
|
||||
|
||||
def test_extract_stage_durations_from_video_and_image_shapes() -> None:
|
||||
video_payload = {
|
||||
"stage_durations": {
|
||||
"diffuse": 1.5,
|
||||
"text_encoder.forward": 0.2,
|
||||
"vae.decode": 0.1,
|
||||
"stage_0_gen_ms": 4000.0,
|
||||
}
|
||||
}
|
||||
assert _extract_stage_durations_from_payload(video_payload) == video_payload["stage_durations"]
|
||||
|
||||
image_stage_durations = {
|
||||
"diffuse": 2.0,
|
||||
"vae.decode": 0.3,
|
||||
}
|
||||
image_payload = {
|
||||
"metrics": {"stage_durations": image_stage_durations},
|
||||
"data": [{"b64_json": "x"}],
|
||||
}
|
||||
assert _extract_stage_durations_from_payload(image_payload) == image_stage_durations
|
||||
|
||||
|
||||
def test_video_metrics_persist_full_stage_durations() -> None:
|
||||
output = MixRequestFuncOutput()
|
||||
output.latency = 4.2
|
||||
stage_durations = {
|
||||
"diffuse": 1.5,
|
||||
"vae.decode": 0.25,
|
||||
"stage_0_gen_ms": 4000.0,
|
||||
}
|
||||
payload = {
|
||||
"duration_s": 2.0,
|
||||
"num_frames": 48,
|
||||
"fps": 24.0,
|
||||
"stage_durations": stage_durations,
|
||||
}
|
||||
_apply_video_metrics_from_payload(output, payload, {})
|
||||
assert output.stage_durations == stage_durations
|
||||
assert output.video_generation_time_ms == pytest.approx(4000.0)
|
||||
|
||||
|
||||
def test_image_metrics_persist_stage_durations_from_metrics() -> None:
|
||||
output = MixRequestFuncOutput()
|
||||
stage_durations = {
|
||||
"diffuse": 1.1,
|
||||
"text_encoder.forward": 0.4,
|
||||
"vae.decode": 0.2,
|
||||
}
|
||||
payload = {
|
||||
"created": 1,
|
||||
"data": [{"b64_json": _MIN_PNG_B64}],
|
||||
"metrics": {"stage_durations": stage_durations},
|
||||
}
|
||||
assert _apply_image_metrics_from_payload(output, payload) == 1
|
||||
assert output.stage_durations == stage_durations
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
+43
-13
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -84,17 +87,20 @@ def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]:
|
||||
raise ValueError(f"mark must be a list; got {type(mark_field).__name__}")
|
||||
|
||||
|
||||
def _mark_names(mark_field: Any) -> set[str]:
|
||||
if isinstance(mark_field, list):
|
||||
return {str(item) for item in mark_field if isinstance(item, str)}
|
||||
return set()
|
||||
|
||||
|
||||
def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool:
|
||||
"""True for perf JSON cases intended for ``run_diffusion_benchmark.py``."""
|
||||
if cfg.get("server_type") is not None:
|
||||
return True
|
||||
return "diffusion" in _mark_names(cfg.get("mark"))
|
||||
"""True for perf JSON cases intended for ``run_diffusion_benchmark.py``.
|
||||
|
||||
Schema split (not marks / endpoints):
|
||||
|
||||
- ``benchmark_params[].dataset`` → diffusion client (``run_diffusion_benchmark.py``)
|
||||
- ``benchmark_params[].dataset_name`` → ``vllm bench serve --omni`` (``run_benchmark.py``)
|
||||
"""
|
||||
for params in cfg.get("benchmark_params") or []:
|
||||
if not isinstance(params, dict):
|
||||
continue
|
||||
if "dataset" in params and "dataset_name" not in params:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]:
|
||||
@@ -654,6 +660,7 @@ def run_benchmark(
|
||||
random_output_len: Any | None = None,
|
||||
resource_label: str | None = None,
|
||||
num_warmups: int = 2,
|
||||
benchmark_params_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
|
||||
|
||||
@@ -662,15 +669,35 @@ def run_benchmark(
|
||||
maps keep every hardware bucket, but each metric is reduced to the value for
|
||||
this concurrency / request-rate step. If the benchmark exits without writing a
|
||||
result file, ``result_omni_template.json`` is used as a fallback.
|
||||
|
||||
``benchmark_params_name`` (the ``name`` field from ``benchmark_params``) is
|
||||
persisted in the result JSON and filename so multiple generation configs
|
||||
under one ``test_name`` stay distinguishable (e.g. Wan USP2 832x480 vs
|
||||
1280x720).
|
||||
"""
|
||||
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
ri = _safe_filename_token(random_input_len)
|
||||
ro = _safe_filename_token(random_output_len)
|
||||
hw = resource_label_for_filename(resource_label)
|
||||
name_token = _safe_filename_token(benchmark_params_name) if benchmark_params_name else ""
|
||||
if name_token == "na":
|
||||
name_token = ""
|
||||
name_parts = [f"result_{test_name}"]
|
||||
if hw:
|
||||
result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
|
||||
else:
|
||||
result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
|
||||
name_parts.append(hw)
|
||||
if name_token:
|
||||
name_parts.append(name_token)
|
||||
name_parts.extend(
|
||||
[
|
||||
str(dataset_name),
|
||||
str(flow),
|
||||
str(num_prompt),
|
||||
f"in{ri}",
|
||||
f"out{ro}",
|
||||
current_dt,
|
||||
]
|
||||
)
|
||||
result_filename = "_".join(name_parts) + ".json"
|
||||
if "--result-filename" in args:
|
||||
print(f"The result file will be overwritten by {result_filename}")
|
||||
command = (
|
||||
@@ -740,6 +767,9 @@ def run_benchmark(
|
||||
if random_output_len is not None:
|
||||
result["random_output_len"] = random_output_len
|
||||
result["Hardware"] = hardware_json_value(resource_label)
|
||||
result["test_name"] = test_name
|
||||
if benchmark_params_name:
|
||||
result["name"] = benchmark_params_name
|
||||
with open(result_path, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
return result
|
||||
|
||||
@@ -50,12 +50,19 @@ if CONFIG_FILE_PATH is None:
|
||||
_all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR)
|
||||
BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)]
|
||||
print(
|
||||
f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from "
|
||||
f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s) from "
|
||||
f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; "
|
||||
f"use -m to filter, e.g. -m tts)"
|
||||
)
|
||||
else:
|
||||
BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH)
|
||||
_loaded = load_benchmark_configs(CONFIG_FILE_PATH)
|
||||
BENCHMARK_CONFIGS = [cfg for cfg in _loaded if not is_diffusion_perf_config(cfg)]
|
||||
skipped = len(_loaded) - len(BENCHMARK_CONFIGS)
|
||||
if skipped:
|
||||
print(
|
||||
f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s); "
|
||||
f"skipped {skipped} remaining diffusion case(s) (chat completions / custom jsonl)"
|
||||
)
|
||||
|
||||
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
|
||||
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
|
||||
@@ -93,24 +100,107 @@ class _SingleActiveContext:
|
||||
stack.close()
|
||||
|
||||
|
||||
# OmniServer defaults for flags not already present in JSON ``serve_args`` /
|
||||
# ``extra_cli_args``. Add new (flag, value) pairs here rather than special-casing.
|
||||
_OMNI_DEFAULT_SERVER_ARGS: tuple[tuple[str, str], ...] = (
|
||||
("--stage-init-timeout", "600"),
|
||||
("--init-timeout", "900"),
|
||||
)
|
||||
|
||||
|
||||
def _cli_flag_names(cli_args: tuple[str, ...] | list[str]) -> set[str]:
|
||||
"""Return long-option names present in a flat CLI argv list."""
|
||||
names: set[str] = set()
|
||||
for item in cli_args:
|
||||
token = str(item)
|
||||
if not token.startswith("--"):
|
||||
continue
|
||||
names.add(token.split("=", 1)[0])
|
||||
return names
|
||||
|
||||
|
||||
def _merge_omni_default_server_args(
|
||||
extra_cli_args: tuple[str, ...] | list[str],
|
||||
*,
|
||||
use_omni: bool,
|
||||
defaults: tuple[tuple[str, str], ...] = _OMNI_DEFAULT_SERVER_ARGS,
|
||||
) -> list[str]:
|
||||
"""Fill Omni defaults for flags not already set in JSON-derived CLI args.
|
||||
|
||||
JSON ``serve_args`` / ``extra_cli_args`` win; only missing flags are appended.
|
||||
"""
|
||||
if not use_omni:
|
||||
return []
|
||||
present = _cli_flag_names(extra_cli_args)
|
||||
args: list[str] = []
|
||||
for flag, value in defaults:
|
||||
if flag not in present:
|
||||
args += [flag, value]
|
||||
return args
|
||||
|
||||
|
||||
def _omni_server_env() -> dict[str, str]:
|
||||
"""Writable video/image storage for ``/v1/videos`` and related generation APIs."""
|
||||
result_dir = Path(os.environ.get("BENCHMARK_DIR", "tests/dfx/perf/results"))
|
||||
storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(result_dir / "storage")))
|
||||
storage_path.mkdir(parents=True, exist_ok=True)
|
||||
return {"VLLM_OMNI_STORAGE_PATH": str(storage_path)}
|
||||
|
||||
|
||||
def _resolve_offline_model(model: str) -> str:
|
||||
"""Resolve HF ids / MiniMax env overrides the same way as the diffusion runner."""
|
||||
import huggingface_hub
|
||||
|
||||
from vllm_omni.transformers_utils.repo_utils import hf_api
|
||||
|
||||
if not model or os.path.isdir(model):
|
||||
return model
|
||||
|
||||
model_env_overrides = {
|
||||
"MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL",
|
||||
"MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL",
|
||||
"MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL",
|
||||
}
|
||||
env_name = model_env_overrides.get(model)
|
||||
if env_name:
|
||||
env_model = os.environ.get(env_name)
|
||||
if env_model:
|
||||
return env_model
|
||||
|
||||
parts = model.split("/")
|
||||
if len(parts) >= 3:
|
||||
repo_id = "/".join(parts[:2])
|
||||
subfolder = "/".join(parts[2:])
|
||||
snapshot_root = hf_api().snapshot_download(
|
||||
repo_id,
|
||||
allow_patterns=[f"{subfolder}/**"],
|
||||
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
|
||||
)
|
||||
return str(Path(snapshot_root) / subfolder)
|
||||
|
||||
if not huggingface_hub.constants.HF_HUB_OFFLINE:
|
||||
return model
|
||||
return hf_api().snapshot_download(model, local_files_only=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _start_omni_server(server_param):
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param
|
||||
extra = tuple(extra_cli_args or ())
|
||||
model = _resolve_offline_model(model)
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
server_args: list[str] = []
|
||||
if use_omni:
|
||||
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
server_args: list[str] = _merge_omni_default_server_args(extra, use_omni=use_omni)
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args, use_omni=use_omni) as server:
|
||||
if extra:
|
||||
server_args = list(extra) + server_args
|
||||
with OmniServer(model, server_args, use_omni=use_omni, env_dict=_omni_server_env()) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
@@ -259,11 +349,24 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
"name",
|
||||
"enable_negative_prompt",
|
||||
"random_request_config",
|
||||
"num_input_images",
|
||||
"warmup_requests",
|
||||
"warmup_concurrency",
|
||||
"warmup_num_inference_steps",
|
||||
}
|
||||
|
||||
param_keys = {str(key).replace("-", "_") for key in params}
|
||||
if "model" not in param_keys:
|
||||
args.extend(["--model", str(model)])
|
||||
|
||||
for key, value in params.items():
|
||||
if key in exclude_keys or value is None:
|
||||
continue
|
||||
if key in {"extra_body", "extra-body"} and value == {}:
|
||||
continue
|
||||
|
||||
arg_name = f"--{key.replace('_', '-')}"
|
||||
|
||||
@@ -298,6 +401,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
num_warmups=_resolve_num_warmups(params, default=2),
|
||||
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
|
||||
@@ -316,5 +420,6 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))),
|
||||
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
@@ -204,11 +204,11 @@ def load_diffusion_benchmark_configs(
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*."""
|
||||
if config_path is not None:
|
||||
configs = load_configs(config_path)
|
||||
loaded = load_configs(config_path)
|
||||
source = str(Path(config_path).resolve())
|
||||
for cfg in configs:
|
||||
for cfg in loaded:
|
||||
cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source)
|
||||
return configs
|
||||
return loaded
|
||||
if config_dir is None:
|
||||
raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir")
|
||||
configs: list[dict[str, Any]] = []
|
||||
@@ -231,7 +231,15 @@ if CONFIG_FILE_PATH is None:
|
||||
f"use -m to filter, e.g. -m diffusion)"
|
||||
)
|
||||
else:
|
||||
BENCHMARK_CONFIGS = load_diffusion_benchmark_configs(CONFIG_FILE_PATH)
|
||||
_loaded = load_diffusion_benchmark_configs(CONFIG_FILE_PATH)
|
||||
BENCHMARK_CONFIGS = [cfg for cfg in _loaded if is_diffusion_perf_config(cfg)]
|
||||
skipped = len(_loaded) - len(BENCHMARK_CONFIGS)
|
||||
if skipped:
|
||||
print(
|
||||
f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s); "
|
||||
f"skipped {skipped} omni-bench generation case(s) "
|
||||
f"(/v1/images/edits, /v1/images/generations, /v1/videos → run_benchmark.py)"
|
||||
)
|
||||
|
||||
_AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {}
|
||||
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -14,8 +17,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/generations",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
@@ -34,24 +35,28 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1024x1024_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num-inference-steps": 4,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"extra-body": {
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/images/generations",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"num_inference_steps": 4,
|
||||
"seed": 42,
|
||||
"guidance_scale": 7.0,
|
||||
"flow_shift": 3.0,
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 1.1958,
|
||||
"latency_mean": 0.843,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
"request_throughput": 1.1958,
|
||||
"mean_e2el_ms": 843.0,
|
||||
"mean_peak_memory_mb": 80000.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,7 +68,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -72,8 +80,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
@@ -92,23 +98,21 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "A robot arm is cleaning a plate in the kitchen.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
@@ -118,9 +122,9 @@
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0307,
|
||||
"latency_mean": 32.7665,
|
||||
"peak_memory_mb_mean": 28762.4762
|
||||
"request_throughput": 0.0307,
|
||||
"mean_e2el_ms": 32766.5,
|
||||
"mean_peak_memory_mb": 28762.4762
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,7 +136,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -141,8 +148,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
@@ -161,24 +166,30 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "The scene comes to life with smooth, natural motion.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(720, 1280, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
@@ -186,9 +197,9 @@
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0305,
|
||||
"latency_mean": 33.1009,
|
||||
"peak_memory_mb_mean": 29441.1429
|
||||
"request_throughput": 0.0305,
|
||||
"mean_e2el_ms": 33100.9,
|
||||
"mean_peak_memory_mb": 29441.1429
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -200,7 +211,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -209,8 +223,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "nvidia/Cosmos3-Nano",
|
||||
"serve_args": {
|
||||
@@ -229,23 +241,30 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1280x720_frames189_steps4",
|
||||
"dataset": "random",
|
||||
"task": "v2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"prompt": "Continue the same scene with smooth natural motion.",
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(720, 1280, 189)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 189,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"max_sequence_length": 4096,
|
||||
@@ -258,9 +277,9 @@
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0299,
|
||||
"latency_mean": 33.925,
|
||||
"peak_memory_mb_mean": 29443.1429
|
||||
"request_throughput": 0.0299,
|
||||
"mean_e2el_ms": 33925.0,
|
||||
"mean_peak_memory_mb": 29443.1429
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
@@ -14,7 +17,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
|
||||
"serve_args": {
|
||||
@@ -25,26 +27,33 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames33_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(480, 832, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 5.0
|
||||
"flow_shift": 5.0,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -55,7 +64,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -64,7 +76,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
|
||||
"serve_args": {
|
||||
@@ -79,26 +90,33 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames33_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(480, 832, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 5.0
|
||||
"flow_shift": 5.0,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
@@ -14,7 +17,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||||
"serve_args": {
|
||||
@@ -25,25 +27,24 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames33_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 5.0
|
||||
"flow_shift": 5.0,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -64,7 +65,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||||
"serve_args": {
|
||||
@@ -79,25 +79,24 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames33_steps4",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 33,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 5.0
|
||||
"flow_shift": 5.0,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,33 +1,50 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_lingbot_video_single_device",
|
||||
"description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "robbyant/lingbot-video-dense-1.3b",
|
||||
"serve_args": {
|
||||
"model-class-name": "LingBotVideoPipeline",
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "320x192_frames9_steps2",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 320,
|
||||
"height": 192,
|
||||
"num-frames": 9,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 2,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"warmup-num-inference-steps": 2,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"test_name": "test_lingbot_video_single_device",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.",
|
||||
"server_params": {
|
||||
"model": "robbyant/lingbot-video-dense-1.3b",
|
||||
"serve_args": {
|
||||
"model-class-name": "LingBotVideoPipeline",
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "320x192_frames9_steps2",
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"num_warmups": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 320,
|
||||
"height": 192,
|
||||
"num_frames": 9,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -14,7 +14,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline with enforce-eager (no torch.compile)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
@@ -25,31 +24,41 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num_frames": 145,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 6,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num_frames": 41,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 20,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -68,7 +77,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device with torch.compile (default, no enforce-eager)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
@@ -78,31 +86,41 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num_frames": 145,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 6,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num_frames": 41,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 20,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -121,7 +139,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CFG-parallel=2 with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
@@ -133,31 +150,41 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num_frames": 145,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 6,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num_frames": 41,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 20,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -176,7 +203,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CFG-parallel=2 with torch.compile",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
@@ -187,31 +213,41 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num_frames": 145,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 6,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num_frames": 41,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 20,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -230,7 +266,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CacheDiT with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
@@ -242,31 +277,41 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num_frames": 145,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 6,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num_frames": 41,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 20,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
@@ -14,8 +17,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "MiniMax-H3 T2VA text-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "MiniMaxAI/MiniMax-H3",
|
||||
"serve_args": {
|
||||
@@ -38,33 +39,30 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1344x768_frames209_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"num_warmups": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"flow_shift": 12.0,
|
||||
"extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0211,
|
||||
"latency_mean": 38.3252,
|
||||
"peak_memory_mb_mean": 54219.6
|
||||
"request_throughput": 0.0211,
|
||||
"mean_e2el_ms": 38325.2,
|
||||
"mean_peak_memory_mb": 54219.6
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,7 +74,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
@@ -85,8 +86,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "MiniMax-H3 FL2VA first-frame image-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "MiniMaxAI/MiniMax-H3",
|
||||
"serve_args": {
|
||||
@@ -109,34 +108,39 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1344x768_frames209_steps8",
|
||||
"dataset": "random",
|
||||
"task": "ti2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"prompt": "Continue the scene: the camera slowly pushes in on the subject as the background city lights drift and blur, with the actor singing softly and emotionally.",
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"num_warmups": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(768, 1344, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"flow_shift": 12.0,
|
||||
"extra_params": "{\"task\":\"fl2va\",\"duration\":8.7,\"audio_flow_shift\":3.0}"
|
||||
},
|
||||
"num-input-images": 1,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0212,
|
||||
"latency_mean": 38.1385,
|
||||
"peak_memory_mb_mean": 54622.8
|
||||
"request_throughput": 0.0212,
|
||||
"mean_e2el_ms": 38138.5,
|
||||
"mean_peak_memory_mb": 54622.8
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -148,7 +152,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
@@ -157,8 +164,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "MiniMax-H3 Ref2VA video-reference-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "MiniMaxAI/MiniMax-H3",
|
||||
"serve_args": {
|
||||
@@ -181,33 +186,39 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1344x768_frames209_steps8",
|
||||
"dataset": "random",
|
||||
"task": "v2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"prompt": "Restyle the reference footage into a cinematic night scene with neon reflections, keeping the same camera motion and subject.",
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"num_warmups": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"video": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(480, 832, 209)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"flow_shift": 12.0,
|
||||
"extra_params": "{\"task\":\"ref2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0077,
|
||||
"latency_mean": 105.7508,
|
||||
"peak_memory_mb_mean": 55180.8
|
||||
"request_throughput": 0.0077,
|
||||
"mean_e2el_ms": 105750.8,
|
||||
"mean_peak_memory_mb": 55180.8
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,7 +230,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
@@ -228,8 +242,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "MiniMax-H3 T2VA with 4-card distributed layerwise offload (DLO), 8 denoise steps (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/videos",
|
||||
"server_params": {
|
||||
"model": "MiniMaxAI/MiniMax-H3",
|
||||
"serve_args": {
|
||||
@@ -256,33 +268,30 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "1344x768_frames209_steps8",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"seed": 42,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"extra-body": {
|
||||
"dataset_name": "random",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 3,
|
||||
"max_concurrency": 1,
|
||||
"num_warmups": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"percentile_metrics": "e2el",
|
||||
"extra_body": {
|
||||
"width": 1344,
|
||||
"height": 768,
|
||||
"num_inference_steps": 8,
|
||||
"num_frames": 209,
|
||||
"fps": 24,
|
||||
"seed": 42,
|
||||
"flow_shift": 12.0,
|
||||
"extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0212,
|
||||
"latency_mean": 38.2089,
|
||||
"peak_memory_mb_mean": 36218.4
|
||||
"request_throughput": 0.0212,
|
||||
"mean_e2el_ms": 38208.9,
|
||||
"mean_peak_memory_mb": 36218.4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,15 +205,53 @@ def test_is_diffusion_perf_config():
|
||||
from tests.dfx.conftest import is_diffusion_perf_config
|
||||
|
||||
assert not is_diffusion_perf_config(
|
||||
{"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]}
|
||||
{
|
||||
"test_name": "omni_a",
|
||||
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"],
|
||||
"benchmark_params": [{"dataset_name": "random", "endpoint": "/v1/chat/completions"}],
|
||||
}
|
||||
)
|
||||
assert is_diffusion_perf_config(
|
||||
{
|
||||
"test_name": "diff_a",
|
||||
"server_type": "vllm-omni",
|
||||
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
|
||||
"benchmark_params": [{"task": "t2i", "dataset": "random"}],
|
||||
}
|
||||
)
|
||||
videos_cfg = {
|
||||
"test_name": "diff_videos",
|
||||
"server_type": "vllm-omni",
|
||||
"mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
|
||||
"benchmark_params": [{"task": "t2v", "dataset_name": "random", "endpoint": "/v1/videos"}],
|
||||
}
|
||||
assert not is_diffusion_perf_config(videos_cfg)
|
||||
custom_edits_cfg = {
|
||||
"test_name": "diff_custom_edits",
|
||||
"server_type": "vllm-omni",
|
||||
"benchmark_endpoint": "/v1/images/edits",
|
||||
"benchmark_params": [{"dataset": "custom", "task": "ti2i"}],
|
||||
}
|
||||
assert is_diffusion_perf_config(custom_edits_cfg)
|
||||
|
||||
|
||||
def test_merge_omni_default_server_args_respects_json():
|
||||
from tests.dfx.perf.scripts.run_benchmark import _merge_omni_default_server_args
|
||||
|
||||
extra = ("--stage-init-timeout", "1800", "--init-timeout", "1800", "--usp", "4")
|
||||
assert _merge_omni_default_server_args(extra, use_omni=True) == []
|
||||
assert _merge_omni_default_server_args((), use_omni=True) == [
|
||||
"--stage-init-timeout",
|
||||
"600",
|
||||
"--init-timeout",
|
||||
"900",
|
||||
]
|
||||
assert _merge_omni_default_server_args((), use_omni=False) == []
|
||||
# Only fill the missing default; keep JSON's other timeouts.
|
||||
assert _merge_omni_default_server_args(("--stage-init-timeout=1800",), use_omni=True) == [
|
||||
"--init-timeout",
|
||||
"900",
|
||||
]
|
||||
|
||||
|
||||
def test_benchmark_param_id_suffix_from_task_eval_phase():
|
||||
@@ -303,12 +341,84 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat
|
||||
finally:
|
||||
active_context.close()
|
||||
|
||||
assert events == [
|
||||
("start", omni_p0_server),
|
||||
("stop", omni_p0_server),
|
||||
("start", tts_server),
|
||||
("stop", tts_server),
|
||||
]
|
||||
|
||||
def test_run_benchmark_persists_distinct_benchmark_params_name(tmp_path, monkeypatch):
|
||||
"""Two benchmark_params under one test_name must keep distinct saved identity."""
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
from tests.dfx import conftest as dfx_conftest
|
||||
|
||||
result_dir = tmp_path / "results"
|
||||
result_dir.mkdir()
|
||||
monkeypatch.setenv("BENCHMARK_DIR", str(result_dir))
|
||||
|
||||
class _FakePopen:
|
||||
def __init__(self, command, **kwargs):
|
||||
self.stdout = io.StringIO("")
|
||||
self.stderr = io.StringIO("")
|
||||
result_dir_idx = command.index("--result-dir")
|
||||
filename_idx = command.index("--result-filename")
|
||||
out = Path(command[result_dir_idx + 1]) / command[filename_idx + 1]
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps({"completed": 3, "request_throughput": 0.01}), encoding="utf-8")
|
||||
|
||||
def wait(self):
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(dfx_conftest.subprocess, "Popen", _FakePopen)
|
||||
|
||||
shared_test = "test_wan22_i2v_usp2"
|
||||
name_a = "832x480_frames81_steps4"
|
||||
name_b = "1280x720_frames121_steps4"
|
||||
|
||||
result_a = dfx_conftest.run_benchmark(
|
||||
args=["--host", "127.0.0.1", "--port", "8000"],
|
||||
test_name=shared_test,
|
||||
flow=1,
|
||||
dataset_name="random-mm",
|
||||
num_prompt=10,
|
||||
random_input_len=8,
|
||||
random_output_len=1,
|
||||
resource_label="H800",
|
||||
benchmark_params_name=name_a,
|
||||
)
|
||||
result_b = dfx_conftest.run_benchmark(
|
||||
args=["--host", "127.0.0.1", "--port", "8000"],
|
||||
test_name=shared_test,
|
||||
flow=1,
|
||||
dataset_name="random-mm",
|
||||
num_prompt=10,
|
||||
random_input_len=8,
|
||||
random_output_len=1,
|
||||
resource_label="H800",
|
||||
benchmark_params_name=name_b,
|
||||
)
|
||||
|
||||
assert result_a["test_name"] == shared_test
|
||||
assert result_b["test_name"] == shared_test
|
||||
assert result_a["name"] == name_a
|
||||
assert result_b["name"] == name_b
|
||||
assert "benchmark_params" not in result_a
|
||||
assert "benchmark_params" not in result_b
|
||||
|
||||
files = sorted(p.name for p in result_dir.glob("result_*.json"))
|
||||
assert len(files) == 2
|
||||
assert any(name_a in name for name in files)
|
||||
assert any(name_b in name for name in files)
|
||||
assert files[0] != files[1]
|
||||
|
||||
def omni_group_key(record: dict) -> tuple:
|
||||
return (
|
||||
record.get("model_id") or "",
|
||||
record.get("test_name") or "",
|
||||
record.get("name") or "",
|
||||
record.get("dataset_name") or "",
|
||||
record.get("max_concurrency") if record.get("max_concurrency") is not None else 0,
|
||||
record.get("num_prompts") if record.get("num_prompts") is not None else 0,
|
||||
)
|
||||
|
||||
assert omni_group_key(result_a) != omni_group_key(result_b)
|
||||
|
||||
|
||||
def test_is_hardware_nested_baseline():
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"]
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
]
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
@@ -14,7 +17,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"serve_args": {
|
||||
@@ -24,28 +26,37 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames81_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(480, 832, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"seed": 42,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0361,
|
||||
"latency_mean": 27.6206,
|
||||
"peak_memory_mb_mean": 80548.0
|
||||
"request_throughput": 0.0361,
|
||||
"mean_e2el_ms": 27620.6,
|
||||
"mean_peak_memory_mb": 80548.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -57,7 +68,10 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": ["H100", "B200"],
|
||||
"cuda": [
|
||||
"H100",
|
||||
"B200"
|
||||
],
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 2
|
||||
@@ -67,7 +81,6 @@
|
||||
"diffusion"
|
||||
],
|
||||
"description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"serve_args": {
|
||||
@@ -81,55 +94,73 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "832x480_frames81_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(480, 832, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 832,
|
||||
"height": 480,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 81,
|
||||
"fps": 16,
|
||||
"seed": 42,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.05263,
|
||||
"latency_mean": 18.9415,
|
||||
"peak_memory_mb_mean": 50053.7
|
||||
"request_throughput": 0.05263,
|
||||
"mean_e2el_ms": 18941.5,
|
||||
"mean_peak_memory_mb": 50053.7
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1280x720_frames121_steps4",
|
||||
"dataset": "random",
|
||||
"task": "i2v",
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 1,
|
||||
"seed": 42,
|
||||
"enable-negative-prompt": true,
|
||||
"random-request-config": [
|
||||
{
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 121,
|
||||
"fps": 16,
|
||||
"weight": 1
|
||||
}
|
||||
],
|
||||
"dataset_name": "random-mm",
|
||||
"tokenizer": "gpt2",
|
||||
"endpoint": "/v1/videos",
|
||||
"num_prompts": 10,
|
||||
"max_concurrency": 1,
|
||||
"random_input_len": 8,
|
||||
"random_output_len": 1,
|
||||
"random_range_ratio": 0.0,
|
||||
"percentile_metrics": "e2el",
|
||||
"random_mm_base_items_per_request": 1,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(720, 1280, 1)": 1.0
|
||||
},
|
||||
"extra_body": {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"num_inference_steps": 4,
|
||||
"num_frames": 121,
|
||||
"fps": 16,
|
||||
"seed": 42,
|
||||
"negative_prompt": "Negative prompt for benchmarking diffusion models"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.009595,
|
||||
"latency_mean": 104.1058,
|
||||
"peak_memory_mb_mean": 59718.0
|
||||
"request_throughput": 0.009595,
|
||||
"mean_e2el_ms": 104105.8,
|
||||
"mean_peak_memory_mb": 59718.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user