[BugFix][MiniCPM-o] Restore audio output without async chunk (#5455)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
e9bc7fc06a
commit
82172b112d
9 files changed
+233
-17
No files matched your search
@@ -6,7 +6,7 @@ Covers:
|
||||
- pipeline declared in the central registry
|
||||
- lazy loader returns the expected ``PipelineConfig``
|
||||
- split 3-stage topology with no fused compatibility registration
|
||||
- stage 1 routes through ``llm2tts`` and the async Code2Wav producer
|
||||
- stage 1 routes through ``llm2tts`` and mode-specific Code2Wav producers
|
||||
- ``hf_architectures`` covers both the shared ``MiniCPMO`` alias and the
|
||||
explicit 4.5 arch
|
||||
- ``hf_config_predicate`` selects MiniCPM-o 4.5 only and rejects 2.6
|
||||
@@ -105,7 +105,9 @@ class TestPipelineTopology:
|
||||
assert talker.engine_output_type == "latent"
|
||||
# scope KV cache / mrope sizing to talker sub-config
|
||||
assert talker.hf_config_name == "tts_config"
|
||||
assert talker.custom_process_next_stage_input_func is None
|
||||
assert talker.custom_process_next_stage_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_full_payload"
|
||||
)
|
||||
assert talker.async_chunk_process_next_stage_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_async_chunk"
|
||||
)
|
||||
@@ -130,7 +132,9 @@ class TestPipelineTopology:
|
||||
assert code2wav.final_output_type == "audio"
|
||||
assert code2wav.engine_output_type == "audio"
|
||||
assert code2wav.model_arch == "MiniCPMO45Code2Wav"
|
||||
assert code2wav.sync_process_input_func is None
|
||||
assert code2wav.sync_process_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_token_only"
|
||||
)
|
||||
|
||||
|
||||
class TestDeployTopology:
|
||||
@@ -156,6 +160,8 @@ class TestDeployTopology:
|
||||
assert connector["extra"]["codec_left_context_frames"] == 3
|
||||
assert connector["extra"]["connector_get_max_wait_first_chunk"] == 3000
|
||||
assert connector["extra"]["connector_get_max_wait"] == 300
|
||||
expected_processor = "tts2code2wav_async_chunk" if deploy.async_chunk else "tts2code2wav_full_payload"
|
||||
assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor)
|
||||
if filename == "minicpmo_4_5.yaml":
|
||||
assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4]
|
||||
assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [
|
||||
@@ -171,14 +177,21 @@ class TestDeployTopology:
|
||||
0.35,
|
||||
]
|
||||
|
||||
def test_pipeline_exposes_no_full_payload_or_token_placeholder_hooks(self) -> None:
|
||||
def test_pipeline_exposes_full_and_async_payload_hooks(self) -> None:
|
||||
pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
|
||||
talker = pipeline.get_stage(1)
|
||||
code2wav = pipeline.get_stage(2)
|
||||
assert talker is not None
|
||||
assert code2wav is not None
|
||||
assert talker.custom_process_next_stage_input_func is None
|
||||
assert code2wav.sync_process_input_func is None
|
||||
assert talker.custom_process_next_stage_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_full_payload"
|
||||
)
|
||||
assert talker.async_chunk_process_next_stage_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_async_chunk"
|
||||
)
|
||||
assert code2wav.sync_process_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_token_only"
|
||||
)
|
||||
|
||||
|
||||
def test_code2wav_model_is_lazily_registered() -> None:
|
||||
|
||||
@@ -10,6 +10,8 @@ from vllm.v1.request import RequestStatus
|
||||
|
||||
from vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni import (
|
||||
tts2code2wav_async_chunk,
|
||||
tts2code2wav_full_payload,
|
||||
tts2code2wav_token_only,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -132,6 +134,67 @@ def test_first_chunk_forwards_reference_voice_and_duplex_identity() -> None:
|
||||
assert payload.meta.turn_end is True
|
||||
|
||||
|
||||
def test_full_payload_forwards_all_codes_and_request_metadata() -> None:
|
||||
manager = _manager()
|
||||
request = _request("req")
|
||||
request.additional_information = {
|
||||
"codes": {"ref": [0.1, -0.1]},
|
||||
"meta": {
|
||||
"ref_audio_sr": 16000,
|
||||
"native_duplex_segment_text": "hello",
|
||||
"segment_end": True,
|
||||
"turn_end": True,
|
||||
},
|
||||
"duplex": {"epoch": 3, "model_turn_id": 7},
|
||||
}
|
||||
|
||||
payload = tts2code2wav_full_payload(
|
||||
transfer_manager=manager,
|
||||
pooling_output={
|
||||
"codes.audio": torch.arange(7, dtype=torch.long).reshape(-1, 1),
|
||||
"meta.finished": torch.tensor(True),
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert _codes(payload) == [4218, 4218, 4218, *range(7)]
|
||||
assert payload.codes.ref.tolist() == pytest.approx([0.1, -0.1])
|
||||
assert payload.meta.request_id == "req"
|
||||
assert payload.meta.chunk_seq == 0
|
||||
assert payload.meta.code_flat_numel == 10
|
||||
assert payload.meta.codec_chunk_frames == 7
|
||||
assert payload.meta.codec_left_context_frames == 3
|
||||
assert payload.meta.left_context_size == 3
|
||||
assert payload.meta.last_chunk is True
|
||||
assert payload.meta.finished.item() is True
|
||||
assert payload.meta.ref_audio_sr == 16000
|
||||
assert payload.meta.native_duplex_segment_text == "hello"
|
||||
assert payload.meta.duplex_epoch == 3
|
||||
assert payload.meta.duplex_turn_id == 7
|
||||
assert payload.meta.segment_end is True
|
||||
assert payload.meta.turn_end is True
|
||||
|
||||
|
||||
def test_sync_token_only_reserves_codec_and_silence_slots() -> None:
|
||||
output = SimpleNamespace(
|
||||
finished=True,
|
||||
outputs=[
|
||||
SimpleNamespace(
|
||||
multimodal_output={
|
||||
"codes.audio": torch.arange(7, dtype=torch.long).reshape(-1, 1),
|
||||
"meta.finished": torch.tensor(True),
|
||||
}
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
prompts = tts2code2wav_token_only([output])
|
||||
|
||||
assert len(prompts) == 1
|
||||
assert prompts[0]["prompt_token_ids"] == [0] * 10
|
||||
assert prompts[0]["additional_information"] is None
|
||||
|
||||
|
||||
def test_empty_final_releases_wait_gate_once() -> None:
|
||||
manager = _manager()
|
||||
request = _request("req")
|
||||
|
||||
Reference in new issue
Block a user