diff --git a/tests/examples/conftest.py b/tests/examples/conftest.py index 8a7705d92..a66db9040 100644 --- a/tests/examples/conftest.py +++ b/tests/examples/conftest.py @@ -336,3 +336,17 @@ def extract_content_after_keyword(keywords: str, text: str) -> str: if not matches: raise AssertionError(f"Keywords {keywords} not found in provided text output") return matches[0] + + +def strip_trailing_audio_saved_line(text: str) -> str: + """Drop trailing ``Audio saved to ...`` lines from captured client stdout. + + ``openai_chat_completion_client_for_multimodal_generation.py`` may print + ``Chat completion output from text:`` for one choice and ``Audio saved to`` + for another; :func:`extract_content_after_keyword` uses ``re.DOTALL`` and + would otherwise keep the audio progress line inside the *text* segment. + """ + lines = text.splitlines() + while lines and lines[-1].strip().startswith("Audio saved to"): + lines.pop() + return "\n".join(lines).strip() diff --git a/tests/examples/online_serving/test_qwen2_5_omni.py b/tests/examples/online_serving/test_qwen2_5_omni.py index 370f2251f..31b9188a4 100644 --- a/tests/examples/online_serving/test_qwen2_5_omni.py +++ b/tests/examples/online_serving/test_qwen2_5_omni.py @@ -14,7 +14,11 @@ from pathlib import Path import pytest from tests.conftest import OmniServerParams, convert_audio_file_to_text, cosine_similarity_text -from tests.examples.conftest import extract_content_after_keyword, run_cmd +from tests.examples.conftest import ( + extract_content_after_keyword, + run_cmd, + strip_trailing_audio_saved_line, +) from tests.utils import hardware_test pytestmark = [pytest.mark.advanced_model, pytest.mark.example] @@ -29,7 +33,7 @@ if current_omni_platform.is_xpu(): str(Path(__file__).parent.parent.parent / "e2e" / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml") ] -example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "qwen2_5_omni") +example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving") # Create parameter combinations for model and stage config test_params = [ OmniServerParams(model=model, port=8091, stage_config_path=stage_config) @@ -37,25 +41,30 @@ test_params = [ for stage_config in stage_configs ] +common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")] + +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_001(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", - "mixed_modalities", + "use_mixed_modalities", ] result = run_cmd(command) - text_content = extract_content_after_keyword("Chat completion output from text:", result) + text_content_tmp = extract_content_after_keyword("Chat completion output from text:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") print(f"audio content is: {audio_content}") @@ -73,25 +82,28 @@ def test_send_multimodal_request_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_002(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", - "mixed_modalities", + "use_mixed_modalities", "--prompt", "Analyze all the media content and provide a comprehensive summary.", ] result = run_cmd(command) - text_content = extract_content_after_keyword("Chat completion output from text:", result) + text_content_tmp = extract_content_after_keyword("Chat completion output from text:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") assert all(keyword in text_content for keyword in ["baby", "book"]), ( "The output does not contain any of the keywords in video description." @@ -108,12 +120,17 @@ def test_send_multimodal_request_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_003(omni_server) -> None: - command = ["bash", os.path.join(example_dir, "run_curl_multimodal_generation.sh"), "mixed_modalities"] + command = [ + "bash", + os.path.join(example_dir, "qwen2_5_omni/run_curl_multimodal_generation.sh"), + "use_mixed_modalities", + ] result = run_cmd(command) @@ -131,16 +148,17 @@ def test_send_multimodal_request_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_001(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", - "mixed_modalities", + "use_mixed_modalities", "--modalities", "text", ] @@ -161,23 +179,25 @@ def test_modality_control_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_002(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", - "mixed_modalities", + "use_mixed_modalities", "--modalities", "audio", ] - run_cmd(command) + result = run_cmd(command) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"audio content is: {audio_content}") assert all(keyword in audio_content for keyword in ["baby", "book"]), ( "The output does not contain any of the keywords in video description." @@ -189,26 +209,29 @@ def test_modality_control_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_003(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", - "mixed_modalities", + "use_mixed_modalities", "--modalities", "audio,text", ] result = run_cmd(command) - text_content = extract_content_after_keyword("Chat completion output from text:", result) + text_content_tmp = extract_content_after_keyword("Chat completion output from text:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") assert all(keyword in text_content for keyword in ["baby", "book"]), ( "The output does not contain any of the keywords in video description." @@ -225,25 +248,28 @@ def test_modality_control_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. +@pytest.mark.skip(reason="issue #2168") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_stream_001(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", - "mixed_modalities", + "use_mixed_modalities", "--stream", ] result = run_cmd(command) - text_content = extract_content_after_keyword("content:", result) + text_content_tmp = extract_content_after_keyword("content:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") assert all(keyword in text_content for keyword in ["baby", "book"]), ( "The output does not contain any of the keywords in video description." diff --git a/tests/examples/online_serving/test_qwen3_omni.py b/tests/examples/online_serving/test_qwen3_omni.py index 7743ae552..65f99d7bf 100644 --- a/tests/examples/online_serving/test_qwen3_omni.py +++ b/tests/examples/online_serving/test_qwen3_omni.py @@ -14,7 +14,11 @@ from pathlib import Path import pytest from tests.conftest import OmniServerParams, convert_audio_file_to_text, cosine_similarity_text -from tests.examples.conftest import extract_content_after_keyword, run_cmd +from tests.examples.conftest import ( + extract_content_after_keyword, + run_cmd, + strip_trailing_audio_saved_line, +) from tests.utils import hardware_test pytestmark = [pytest.mark.advanced_model, pytest.mark.example] @@ -28,13 +32,14 @@ if current_omni_platform.is_xpu(): stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")] -example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "qwen3_omni") +example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving") # Create parameter combinations for model and stage config test_params = [ OmniServerParams(model=model, port=8091, stage_config_path=stage_config) for model in models for stage_config in stage_configs ] +common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")] @pytest.mark.advanced_model @@ -42,17 +47,19 @@ test_params = [ @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_001(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ "--query-type", "use_image", + "--model", + omni_server.model, ] result = run_cmd(command) - text_content = extract_content_after_keyword("Chat completion output from text:", result) + text_content_tmp = extract_content_after_keyword("Chat completion output from text:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) + wav_path = extract_content_after_keyword("Audio saved to", result) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") print(f"audio content is: {audio_content}") @@ -70,20 +77,22 @@ def test_send_multimodal_request_001(omni_server) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_002(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ "--query-type", "use_video", + "--model", + omni_server.model, "--prompt", "What are the main activities shown in this video?", ] result = run_cmd(command) - text_content = extract_content_after_keyword("Chat completion output from text:", result) + text_content_tmp = extract_content_after_keyword("Chat completion output from text:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") print(f"audio content is: {audio_content}") assert all(keyword in text_content for keyword in ["baby", "book"]), ( @@ -101,7 +110,7 @@ def test_send_multimodal_request_002(omni_server) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_003(omni_server) -> None: - command = ["bash", os.path.join(example_dir, "run_curl_multimodal_generation.sh"), "use_image"] + command = ["bash", os.path.join(example_dir, "qwen3_omni/run_curl_multimodal_generation.sh"), "use_image"] result = run_cmd(command) @@ -118,11 +127,11 @@ def test_send_multimodal_request_003(omni_server) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_001(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ "--query-type", "use_image", + "--model", + omni_server.model, "--modalities", "text", ] @@ -142,18 +151,19 @@ def test_modality_control_001(omni_server) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_002(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", "use_image", "--modalities", "audio", ] - run_cmd(command) + result = run_cmd(command) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"audio content is: {audio_content}") assert "cherry blossom" in audio_content, "The output does not contain any of the keywords." @@ -165,9 +175,9 @@ def test_modality_control_002(omni_server) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_003(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", "use_image", "--modalities", @@ -176,10 +186,12 @@ def test_modality_control_003(omni_server) -> None: result = run_cmd(command) - text_content = extract_content_after_keyword("Chat completion output from text:", result) + text_content_tmp = extract_content_after_keyword("Chat completion output from text:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") assert "cherry blossom" in audio_content, "The output does not contain any of the keywords." print(f"audio content is: {audio_content}") @@ -195,9 +207,9 @@ def test_modality_control_003(omni_server) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_stream_001(omni_server) -> None: - command = [ - "python", - os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py"), + command = common_args + [ + "--model", + omni_server.model, "--query-type", "use_image", "--stream", @@ -205,10 +217,12 @@ def test_stream_001(omni_server) -> None: result = run_cmd(command) - text_content = extract_content_after_keyword("content:", result) + text_content_tmp = extract_content_after_keyword("content:", result) + text_content = strip_trailing_audio_saved_line(text_content_tmp) # Verify text output same as audio output - audio_content = convert_audio_file_to_text(output_path="./audio_0.wav") + wav_path = extract_content_after_keyword("Audio saved to", result) + audio_content = convert_audio_file_to_text(output_path=f"./{wav_path.strip()}") print(f"text content is: {text_content}") assert "cherry blossom" in audio_content, "The output does not contain any of the keywords." print(f"audio content is: {audio_content}")