[Bugfix][Model] Fix Qwen3-Omni video metadata handling for use_audio_in_video (#4959)

Signed-off-by: ZhengWG <zwg0606@gmail.com>
This commit is contained in:
Zheng Wengang authored and GitHub committed 2026-07-10 14:24:18 +08:00
1 parent 020792e6be
commit 9fe26cfacb
4 files changed
+206 -7

No files matched your search

@@ -410,6 +410,44 @@ def test_audio_in_video_002(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_audio_in_video_default_loader_sampling_regression(omni_server, openai_client) -> None:
"""
Regression: ``use_audio_in_video`` with default video loader sampling (no
``media_io_kwargs.video.fps``) after a prior request that pinned fps.
Before the fix the second request could partial-hit mm-cache (audio cached,
video re-processed without audio) and fail with ``StopIteration`` in
``replace_multimodal_special_tokens``.
"""
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 128, embed_audio=True)['base64']}"
messages = dummy_messages_from_mix_data(
system_prompt=get_system_prompt(),
video_data_url=video_data_url,
content_text="Reply with one word: OK.",
)
base_config = {
"model": omni_server.model,
"messages": messages,
"stream": False,
"use_audio_in_video": True,
"modalities": ["text"],
}
# Prime mm-cache with explicit loader fps (historical failure mode).
openai_client.send_omni_request(
{
**base_config,
"extra_body": {"media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}}},
}
)
# Default loader sampling: must succeed without explicit media_io fps.
openai_client.send_omni_request(dict(base_config))
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_one_word_prompt_001(omni_server, openai_client) -> None: