[Bugfix] Use Seed-TTS reference audio with MiniCPM-o 4.5 (#6628)
Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
1 parent
4cca28e9bd
commit
f321cdbe16
10 files changed
+529
-56
No files matched your search
@@ -1,12 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output.
|
||||
|
||||
Exercises async chunk streaming (``--async-chunk``) across separate Thinker,
|
||||
Talker, and Code2Wav stages.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.logger import DEFAULT_LOGGING_CONFIG
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||
@@ -23,6 +31,23 @@ _CI_DEPLOY = modify_stage_config(
|
||||
},
|
||||
)
|
||||
|
||||
_PROMPT_LOG_TEMP_DIR = tempfile.TemporaryDirectory(prefix="minicpmo45-prompt-selection-")
|
||||
_PROMPT_SELECTION_LOG = Path(_PROMPT_LOG_TEMP_DIR.name) / "prompt-selection.log"
|
||||
_PROMPT_LOGGING_CONFIG_PATH = Path(_PROMPT_LOG_TEMP_DIR.name) / "logging.json"
|
||||
_PROMPT_LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG)
|
||||
_PROMPT_LOGGING_CONFIG["handlers"]["minicpmo_prompt_selection"] = {
|
||||
"class": "logging.FileHandler",
|
||||
"filename": str(_PROMPT_SELECTION_LOG),
|
||||
"formatter": "vllm",
|
||||
"level": "DEBUG",
|
||||
}
|
||||
_PROMPT_LOGGING_CONFIG["loggers"]["vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav"] = {
|
||||
"handlers": ["minicpmo_prompt_selection"],
|
||||
"level": "DEBUG",
|
||||
"propagate": True,
|
||||
}
|
||||
_PROMPT_LOGGING_CONFIG_PATH.write_text(json.dumps(_PROMPT_LOGGING_CONFIG), encoding="utf-8")
|
||||
|
||||
test_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
@@ -30,6 +55,10 @@ test_params = [
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
use_stage_cli=False,
|
||||
server_args=["--trust-remote-code", "--async-chunk"],
|
||||
env_dict={
|
||||
"VLLM_CONFIGURE_LOGGING": "1",
|
||||
"VLLM_LOGGING_CONFIG_PATH": str(_PROMPT_LOGGING_CONFIG_PATH),
|
||||
},
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
@@ -74,6 +103,18 @@ _TTS_EXTRA_BODY = {
|
||||
}
|
||||
|
||||
|
||||
def _prompt_selection_log_offset() -> int:
|
||||
return _PROMPT_SELECTION_LOG.stat().st_size if _PROMPT_SELECTION_LOG.exists() else 0
|
||||
|
||||
|
||||
def _read_prompt_selection_log(offset: int) -> str:
|
||||
if not _PROMPT_SELECTION_LOG.exists():
|
||||
return ""
|
||||
with open(_PROMPT_SELECTION_LOG, "rb") as log_file:
|
||||
log_file.seek(offset)
|
||||
return log_file.read().decode("utf-8")
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@@ -126,6 +167,51 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_audio_with_reference_audio(omni_server, openai_client) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()),
|
||||
"stream": True,
|
||||
"modalities": ["text", "audio"],
|
||||
"extra_body": {
|
||||
**_TTS_EXTRA_BODY,
|
||||
"ref_audio": f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}",
|
||||
},
|
||||
}
|
||||
|
||||
log_offset = _prompt_selection_log_offset()
|
||||
responses = openai_client.send_omni_request(request_config)
|
||||
prompt_log = _read_prompt_selection_log(log_offset)
|
||||
|
||||
assert responses[0].success
|
||||
assert responses[0].audio_bytes
|
||||
assert "prompt_cache_id=runtime-ref-" in prompt_log
|
||||
assert "minicpmo45_ref_" in prompt_log
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_audio_with_default_reference(omni_server, openai_client) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()),
|
||||
"stream": True,
|
||||
"modalities": ["text", "audio"],
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
|
||||
responses = openai_client.send_omni_request(request_config)
|
||||
|
||||
assert responses[0].success
|
||||
assert responses[0].audio_bytes
|
||||
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
|
||||
Reference in new issue
Block a user