[CI][Perf] migrate diffusion DFX benches to vllm bench serve --omni (#7737)

Signed-off-by: wangyu <410167048@qq.com>

tests/benchmarks/patch/test_patch.py

@@ -22,12 +22,17 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
 )
 from vllm_omni.benchmarks.patch.patch import (
     MixRequestFuncOutput,
+    _add_combined_video_form_references,
     _add_video_extra_body_to_form,
     _add_video_reference_to_form,
+    _apply_image_metrics_from_payload,
     _apply_stage0_token_timings,
     _apply_video_metrics_from_payload,
     _attach_seed_tts_to_request_func_input,
     _build_benchmark_session,
+    _extract_stage_durations_from_payload,
+    _iter_image_reference_inputs,
+    _iter_video_reference_inputs,
     _omni_request_timeout_s,
     async_request_openai_chat_omni_completions,
     async_request_openai_image_edits_omni,
@@ -1359,16 +1364,15 @@ async def test_image_edits_defaults_to_non_streaming_json(mocker: MockerFixture)
             return None
 
     captured_stream: list[str] = []
-    real_add_field = None
+    import aiohttp
+
+    real_add_field = aiohttp.FormData.add_field
 
     def tracking_add_field(self, name, value=None, **kwargs):
         if name == "stream":
             captured_stream.append(str(value))
         return real_add_field(self, name, value, **kwargs)
 
-    import aiohttp
-
-    real_add_field = aiohttp.FormData.add_field
     mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
 
     mock_session = mocker.AsyncMock()
@@ -1512,8 +1516,7 @@ def test_video_local_image_reference_not_forwarded_as_raw_extra_field(tmp_path,
     "reference",
     [
         {"image_url": "https://example.com/ref.png"},
-        {"file_id": "file-abc"},
-        [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}],
+        [{"image_url": "https://example.com/a.png"}, {"image_url": "https://example.com/b.png"}],
     ],
 )
 def test_video_structured_image_reference_serialized_to_form(reference: object, mocker: MockerFixture) -> None:
@@ -1541,18 +1544,395 @@ def test_video_structured_image_reference_serialized_to_form(reference: object,
     assert field_names.count("image_reference") == 1
     assert "input_reference" not in field_names
     payload = next(value for name, value in captured if name == "image_reference")
+    assert isinstance(payload, (str, bytes, bytearray))
     assert json.loads(payload) == reference
 
 
+def test_video_file_id_reference_is_rejected() -> None:
+    """file_id is unsupported on the server and must not be sent as image_reference."""
+    import aiohttp
+
+    form = aiohttp.FormData()
+    with pytest.raises(ValueError, match="file_id is not supported yet"):
+        _add_video_reference_to_form(form, {"file_id": "file-abc"})
+    with pytest.raises(ValueError, match="file_id is not supported yet"):
+        _add_video_reference_to_form(form, [{"image_url": "https://example.com/a.png"}, {"file_id": "file-xyz"}])
+    with pytest.raises(ValueError, match="file_id is not supported yet"):
+        _add_combined_video_form_references(form, None, {"video_reference": {"file_id": "file-vid"}})
+
+
+def test_image_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None:
+    """random-mm image_url parts keep an explicit image_reference type."""
+    import aiohttp
+
+    content = [
+        {
+            "type": "image_url",
+            "image_url": {"url": "https://example.com/ref.png"},
+        },
+        {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{_MIN_PNG_B64}"},
+        },
+    ]
+    refs = list(_iter_image_reference_inputs(content))
+    assert refs == [
+        {"image_url": "https://example.com/ref.png"},
+        {"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"},
+    ]
+
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    form = aiohttp.FormData()
+    assert _add_video_reference_to_form(form, refs[0]) is True
+    assert _add_video_reference_to_form(form, refs[1]) is True
+
+    field_names = [name for name, _ in captured]
+    assert field_names.count("image_reference") == 2
+    assert "video_reference" not in field_names
+    payloads = []
+    for name, value in captured:
+        if name != "image_reference":
+            continue
+        assert isinstance(value, (str, bytes, bytearray))
+        payloads.append(json.loads(value))
+    assert payloads == refs
+
+
+def test_video_reference_urls_from_random_mm_content(mocker: MockerFixture) -> None:
+    """random-mm data:video_url parts stay on video_reference."""
+    import aiohttp
+
+    content = [
+        {
+            "type": "video_url",
+            "video_url": {"url": "data:video/mp4;base64,AAAA"},
+        }
+    ]
+    refs = list(_iter_video_reference_inputs(content))
+    assert refs == [{"video_url": "data:video/mp4;base64,AAAA"}]
+
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    form = aiohttp.FormData()
+    assert _add_video_reference_to_form(form, refs[0]) is True
+
+    field_names = [name for name, _ in captured]
+    assert field_names.count("video_reference") == 1
+    assert "input_references" not in field_names
+    assert "input_reference" not in field_names
+    assert "image_reference" not in field_names
+    payload = next(value for name, value in captured if name == "video_reference")
+    assert isinstance(payload, (str, bytes, bytearray))
+    assert json.loads(payload) == {"video_url": "data:video/mp4;base64,AAAA"}
+
+
+def test_video_reference_https_url_from_random_mm_content(mocker: MockerFixture) -> None:
+    """HTTP(S) video_url parts must stay on video_reference, not image_reference."""
+    import aiohttp
+
+    content = [
+        {
+            "type": "video_url",
+            "video_url": {"url": "https://example.com/ref.mp4"},
+        }
+    ]
+    refs = list(_iter_video_reference_inputs(content))
+    assert refs == [{"video_url": "https://example.com/ref.mp4"}]
+
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    form = aiohttp.FormData()
+    assert _add_video_reference_to_form(form, refs[0]) is True
+
+    field_names = [name for name, _ in captured]
+    assert field_names.count("video_reference") == 1
+    assert "image_reference" not in field_names
+    assert "input_references" not in field_names
+    payload = next(value for name, value in captured if name == "video_reference")
+    assert isinstance(payload, (str, bytes, bytearray))
+    assert json.loads(payload) == {"video_url": "https://example.com/ref.mp4"}
+
+
+@pytest.mark.parametrize(
+    ("reference", "field_name", "expected"),
+    [
+        ("data:image/png;base64,AAAA", "image_reference", {"image_url": "data:image/png;base64,AAAA"}),
+        ("data:video/mp4;base64,AAAA", "video_reference", {"video_url": "data:video/mp4;base64,AAAA"}),
+        ("https://example.com/ref.png", "image_reference", {"image_url": "https://example.com/ref.png"}),
+        ("https://example.com/ref.mp4", "video_reference", {"video_url": "https://example.com/ref.mp4"}),
+    ],
+)
+def test_bare_reference_string_uses_matching_json_field(
+    reference: str,
+    field_name: str,
+    expected: dict[str, str],
+    mocker: MockerFixture,
+) -> None:
+    """Image and video URLs use matching JSON fields; neither is a file upload."""
+    import aiohttp
+
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    assert _add_video_reference_to_form(aiohttp.FormData(), reference) is True
+    assert [name for name, _ in captured] == [field_name]
+    payload = captured[0][1]
+    assert isinstance(payload, (str, bytes, bytearray))
+    assert json.loads(payload) == expected
+
+
+def test_bare_video_string_is_not_collected_as_image() -> None:
+    content = ["https://example.com/ref.mp4", "data:video/mp4;base64,AAAA"]
+    assert list(_iter_image_reference_inputs(content)) == []
+    assert list(_iter_video_reference_inputs(content)) == [
+        {"video_url": "https://example.com/ref.mp4"},
+        {"video_url": "data:video/mp4;base64,AAAA"},
+    ]
+
+
+def test_bare_http_without_media_extension_is_rejected() -> None:
+    import aiohttp
+
+    with pytest.raises(ValueError, match="image or video extension"):
+        _add_video_reference_to_form(aiohttp.FormData(), "https://example.com/ref")
+
+
+def test_image_and_inline_video_use_combined_reference_fields(mocker: MockerFixture) -> None:
+    """Image plus data:video must be image_reference + video_reference, not input_references."""
+    import aiohttp
+
+    content = [
+        {"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}},
+        {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}},
+    ]
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    form = aiohttp.FormData()
+    _add_combined_video_form_references(form, content)
+
+    field_names = [name for name, _ in captured]
+    assert field_names.count("image_reference") == 1
+    assert field_names.count("video_reference") == 1
+    assert "input_references" not in field_names
+    assert "input_reference" not in field_names
+    image_payload = next(value for name, value in captured if name == "image_reference")
+    video_payload = next(value for name, value in captured if name == "video_reference")
+    assert isinstance(image_payload, (str, bytes, bytearray))
+    assert isinstance(video_payload, (str, bytes, bytearray))
+    assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"}
+    assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"}
+
+
+def test_image_url_and_bare_data_video_string_use_json_fields(mocker: MockerFixture) -> None:
+    """Image URL plus a bare data:video string must not upload input_references."""
+    import aiohttp
+
+    content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}]
+    extra_body = {"video_reference": "data:video/mp4;base64,AAAA"}
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    form = aiohttp.FormData()
+    _add_combined_video_form_references(form, content, extra_body)
+
+    field_names = [name for name, _ in captured]
+    assert field_names.count("image_reference") == 1
+    assert field_names.count("video_reference") == 1
+    assert "input_references" not in field_names
+    assert "input_reference" not in field_names
+    image_payload = next(value for name, value in captured if name == "image_reference")
+    video_payload = next(value for name, value in captured if name == "video_reference")
+    assert isinstance(image_payload, (str, bytes, bytearray))
+    assert isinstance(video_payload, (str, bytes, bytearray))
+    assert json.loads(image_payload) == {"image_url": "https://example.com/ref.png"}
+    assert json.loads(video_payload) == {"video_url": "data:video/mp4;base64,AAAA"}
+
+    captured.clear()
+    assert _add_video_reference_to_form(form, "data:video/mp4;base64,AAAA") is True
+    assert [name for name, _ in captured] == ["video_reference"]
+
+
+def _oversized_data_video_url() -> str:
+    """data:video whose JSON text part is larger than 1MB."""
+    return "data:video/mp4;base64," + ("A" * (1024 * 1024))
+
+
+def test_oversized_data_video_uploads_as_input_references(mocker: MockerFixture) -> None:
+    """A lone data:video over the 1MB text limit is uploaded, not sent as JSON."""
+    import aiohttp
+
+    video_url = _oversized_data_video_url()
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    assert _add_video_reference_to_form(aiohttp.FormData(), {"video_url": video_url}) is True
+    uploaded = next(value for name, value in captured if name == "input_references")
+    assert uploaded == base64.b64decode("A" * (1024 * 1024))
+    field_names = [name for name, _ in captured]
+    assert "video_reference" not in field_names
+    assert "image_reference" not in field_names
+
+
+def test_oversized_data_video_with_image_stays_json(mocker: MockerFixture) -> None:
+    """input_references cannot be combined with image_reference, even for a large video."""
+    import aiohttp
+
+    content = [{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}]
+    extra_body = {"video_reference": _oversized_data_video_url()}
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    _add_combined_video_form_references(aiohttp.FormData(), content, extra_body)
+    field_names = [name for name, _ in captured]
+    assert "image_reference" in field_names
+    assert "video_reference" in field_names
+    assert "input_references" not in field_names
+
+
+def test_image_upload_bytes_and_structured_video_use_json_fields(mocker: MockerFixture) -> None:
+    """Image upload bytes plus a video URL must not use singular input_reference."""
+    import aiohttp
+
+    image_bytes = base64.b64decode(_MIN_PNG_B64)
+    extra_body = {
+        "image_reference": {"bytes": image_bytes, "content_type": "image/png"},
+        "video_reference": {"video_url": "https://example.com/ref.mp4"},
+    }
+    captured: list[tuple[str, object]] = []
+    real_add_field = aiohttp.FormData.add_field
+
+    def tracking_add_field(self, name, value=None, **kwargs):
+        captured.append((str(name), value))
+        return real_add_field(self, name, value, **kwargs)
+
+    mocker.patch.object(aiohttp.FormData, "add_field", tracking_add_field)
+    form = aiohttp.FormData()
+    _add_combined_video_form_references(form, None, extra_body)
+
+    field_names = [name for name, _ in captured]
+    assert field_names.count("image_reference") == 1
+    assert field_names.count("video_reference") == 1
+    assert "input_reference" not in field_names
+    assert "input_references" not in field_names
+    image_payload = next(value for name, value in captured if name == "image_reference")
+    video_payload = next(value for name, value in captured if name == "video_reference")
+    assert isinstance(image_payload, (str, bytes, bytearray))
+    assert isinstance(video_payload, (str, bytes, bytearray))
+    assert json.loads(image_payload) == {"image_url": f"data:image/png;base64,{_MIN_PNG_B64}"}
+    assert json.loads(video_payload) == {"video_url": "https://example.com/ref.mp4"}
+
+
 def test_video_unsupported_image_reference_raises() -> None:
     import aiohttp
 
     form = aiohttp.FormData()
-    with pytest.raises(ValueError, match="Unsupported image_reference"):
+    with pytest.raises(ValueError, match="Unsupported reference"):
         _add_video_reference_to_form(form, {"not_a_supported_key": "x"})
-    with pytest.raises(ValueError, match="Unsupported image_reference"):
+    with pytest.raises(ValueError, match="Unsupported reference"):
         _add_video_reference_to_form(form, "/tmp/does-not-exist-ref.png")
 
 
+def test_extract_stage_durations_from_video_and_image_shapes() -> None:
+    video_payload = {
+        "stage_durations": {
+            "diffuse": 1.5,
+            "text_encoder.forward": 0.2,
+            "vae.decode": 0.1,
+            "stage_0_gen_ms": 4000.0,
+        }
+    }
+    assert _extract_stage_durations_from_payload(video_payload) == video_payload["stage_durations"]
+
+    image_stage_durations = {
+        "diffuse": 2.0,
+        "vae.decode": 0.3,
+    }
+    image_payload = {
+        "metrics": {"stage_durations": image_stage_durations},
+        "data": [{"b64_json": "x"}],
+    }
+    assert _extract_stage_durations_from_payload(image_payload) == image_stage_durations
+
+
+def test_video_metrics_persist_full_stage_durations() -> None:
+    output = MixRequestFuncOutput()
+    output.latency = 4.2
+    stage_durations = {
+        "diffuse": 1.5,
+        "vae.decode": 0.25,
+        "stage_0_gen_ms": 4000.0,
+    }
+    payload = {
+        "duration_s": 2.0,
+        "num_frames": 48,
+        "fps": 24.0,
+        "stage_durations": stage_durations,
+    }
+    _apply_video_metrics_from_payload(output, payload, {})
+    assert output.stage_durations == stage_durations
+    assert output.video_generation_time_ms == pytest.approx(4000.0)
+
+
+def test_image_metrics_persist_stage_durations_from_metrics() -> None:
+    output = MixRequestFuncOutput()
+    stage_durations = {
+        "diffuse": 1.1,
+        "text_encoder.forward": 0.4,
+        "vae.decode": 0.2,
+    }
+    payload = {
+        "created": 1,
+        "data": [{"b64_json": _MIN_PNG_B64}],
+        "metrics": {"stage_durations": stage_durations},
+    }
+    assert _apply_image_metrics_from_payload(output, payload) == 1
+    assert output.stage_durations == stage_durations
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v", "-s"])