[Bugfix] Fix MiniCPM-o 4.5 native duplex speech alignment (#7974)
Signed-off-by: natureofnature <wzliu@connect.hku.hk> Co-authored-by: liuqihao <liuqihao970610@gmail.com>
This commit is contained in:
1 parent
d486bf097a
commit
7a8d956ece
13 files changed
+687
-118
No files matched your search
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
@@ -16,6 +18,49 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_multimodal_runner_preserves_forwarded_token_identities(monkeypatch):
|
||||
stage_model = torch.nn.Module()
|
||||
stage_model.make_empty_intermediate_tensors = lambda: None
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni.init_vllm_registered_model",
|
||||
lambda **kwargs: stage_model,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat.patch_minicpmo_remote_config",
|
||||
lambda config: None,
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(), multimodal_config=None, model_stage="llm")
|
||||
)
|
||||
wrapper = MiniCPMO45OmniForConditionalGeneration(vllm_config=config)
|
||||
hidden = torch.randn(2, 4)
|
||||
wrapper.thinker.forward = MagicMock(return_value=hidden)
|
||||
token_buffer = torch.tensor([99692, 3837, 0, 0])
|
||||
embed_buffer = torch.randn(4, 4)
|
||||
runner = SimpleNamespace(
|
||||
model=wrapper,
|
||||
input_ids=SimpleNamespace(gpu=token_buffer),
|
||||
inputs_embeds=SimpleNamespace(gpu=embed_buffer),
|
||||
)
|
||||
|
||||
# This same upstream path prepares normal multimodal forwards and graph
|
||||
# capture inputs. Embeddings must not replace the real token identities.
|
||||
input_ids, inputs_embeds = GPUModelRunner._prepare_mm_inputs(runner, 2)
|
||||
out = wrapper.forward(input_ids=input_ids, positions=torch.tensor([10, 11]), inputs_embeds=inputs_embeds)
|
||||
|
||||
torch.testing.assert_close(out.multimodal_outputs["latent_input_ids"], token_buffer[:2, None])
|
||||
torch.testing.assert_close(out.multimodal_outputs["latent_positions"], torch.tensor([[10], [11]]))
|
||||
torch.testing.assert_close(wrapper.thinker.forward.call_args.kwargs["inputs_embeds"], embed_buffer[:2])
|
||||
|
||||
# The codec Talker has no text vocabulary for randomized raw-token
|
||||
# profiling. It must retain the existing embeddings-only input path.
|
||||
config.model_config.model_stage = "tts"
|
||||
runner.model = MiniCPMO45OmniForConditionalGeneration(vllm_config=config)
|
||||
talker_ids, talker_embeds = GPUModelRunner._prepare_mm_inputs(runner, 2)
|
||||
assert talker_ids is None
|
||||
torch.testing.assert_close(talker_embeds, embed_buffer[:2])
|
||||
|
||||
|
||||
def test_thinker_forward_returns_bare_hidden_states():
|
||||
"""Verify that MiniCPMO45OmniLLMForConditionalGeneration.forward returns
|
||||
a bare hidden_states Tensor (not a tuple), which stock vLLM GPUModelRunner
|
||||
|
||||
@@ -25,6 +25,11 @@ def _output(
|
||||
):
|
||||
mm_output = dict(multimodal_output or {})
|
||||
mm_output["latent"] = latent
|
||||
if "duplex_prompt_token_ids" in mm_output and "latent_input_ids" not in mm_output:
|
||||
forwarded_ids = [*prompt_ids, *output_ids]
|
||||
assert len(forwarded_ids) == latent.shape[0]
|
||||
mm_output["latent_input_ids"] = torch.tensor(forwarded_ids).reshape(-1, 1)
|
||||
mm_output["latent_positions"] = torch.arange(len(forwarded_ids)).reshape(-1, 1)
|
||||
completion = SimpleNamespace(
|
||||
token_ids=output_ids if token_list is None else token_list,
|
||||
text="hello",
|
||||
@@ -142,6 +147,246 @@ def test_native_duplex_speak_segment_reaches_split_talker() -> None:
|
||||
assert info["duplex"]["turn_id"] == 7
|
||||
|
||||
|
||||
def test_native_duplex_uses_forwarded_row_ledger_after_later_audio_prefill() -> None:
|
||||
prompt_ids = [101, 102]
|
||||
output_ids = [9304, 21, 22, 9308]
|
||||
row_ids = [101, 102, 9304, 21, 22, 999, 999]
|
||||
latent = torch.arange(len(row_ids) * 4, dtype=torch.float32).reshape(-1, 4)
|
||||
metadata = {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
"listen_token_id": 9303,
|
||||
"speak_token_id": 9304,
|
||||
"chunk_eos_token_id": 9308,
|
||||
"chunk_tts_eos_token_id": 9309,
|
||||
"turn_eos_token_id": 9310,
|
||||
}
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"latent_input_ids": torch.tensor(row_ids).reshape(-1, 1),
|
||||
"latent_positions": torch.arange(len(row_ids)).reshape(-1, 1),
|
||||
"meta": metadata,
|
||||
},
|
||||
)
|
||||
context = SimpleNamespace(bridge_states={"duplex": {"epoch": 3, "model_turn_id": 7}})
|
||||
|
||||
converted = llm2tts([source], prompt=[{}], _streaming_context=context)[0]
|
||||
|
||||
info = converted["model_intermediate_buffer"]
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[3:5])
|
||||
|
||||
|
||||
def test_native_duplex_selects_latest_contiguous_forwarded_span() -> None:
|
||||
prompt_ids = [101, 102]
|
||||
output_ids = [9304, 21, 22, 9308]
|
||||
row_ids = [101, 21, 22, 102, 9304, 21, 22, 999]
|
||||
positions = [0, 1, 3, 4, 5, 6, 7, 8]
|
||||
latent = torch.arange(len(row_ids) * 4, dtype=torch.float32).reshape(-1, 4)
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"latent_input_ids": torch.tensor(row_ids).reshape(-1, 1),
|
||||
"latent_positions": torch.tensor(positions).reshape(-1, 1),
|
||||
"meta": {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
"listen_token_id": 9303,
|
||||
"speak_token_id": 9304,
|
||||
"chunk_eos_token_id": 9308,
|
||||
"chunk_tts_eos_token_id": 9309,
|
||||
"turn_eos_token_id": 9310,
|
||||
},
|
||||
},
|
||||
)
|
||||
context = SimpleNamespace(bridge_states={"duplex": {"epoch": 3, "model_turn_id": 7}})
|
||||
|
||||
converted = llm2tts([source], prompt=[{}], _streaming_context=context)[0]
|
||||
|
||||
info = converted["model_intermediate_buffer"]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[5:7])
|
||||
|
||||
|
||||
def test_native_duplex_rejects_missing_forwarded_terminal_token() -> None:
|
||||
prompt_ids = [101, 102]
|
||||
output_ids = [9304, 21, 9310, 9308]
|
||||
row_ids = [101, 102, 9304, 21, 999, 999]
|
||||
latent = torch.arange(len(row_ids) * 4, dtype=torch.float32).reshape(-1, 4)
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"latent_input_ids": torch.tensor(row_ids).reshape(-1, 1),
|
||||
"latent_positions": torch.arange(len(row_ids)).reshape(-1, 1),
|
||||
"meta": {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
"listen_token_id": 9303,
|
||||
"speak_token_id": 9304,
|
||||
"chunk_eos_token_id": 9308,
|
||||
"chunk_tts_eos_token_id": 9309,
|
||||
"turn_eos_token_id": 9310,
|
||||
},
|
||||
},
|
||||
)
|
||||
context = SimpleNamespace(bridge_states={"duplex": {"epoch": 3, "model_turn_id": 7}})
|
||||
|
||||
with pytest.raises(ValueError, match="missing own-token hidden states"):
|
||||
llm2tts([source], prompt=[{}], _streaming_context=context)
|
||||
|
||||
|
||||
_NATIVE_META = {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
"listen_token_id": 9303,
|
||||
"speak_token_id": 9304,
|
||||
"chunk_eos_token_id": 9308,
|
||||
"chunk_tts_eos_token_id": 9309,
|
||||
"turn_eos_token_id": 9310,
|
||||
}
|
||||
|
||||
|
||||
def _native_source(prompt_ids, output_ids, row_ids, positions=None):
|
||||
latent = torch.arange(len(row_ids) * 4, dtype=torch.float32).reshape(-1, 4)
|
||||
positions = list(range(len(row_ids))) if positions is None else positions
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"latent_input_ids": torch.tensor(row_ids).reshape(-1, 1),
|
||||
"latent_positions": torch.tensor(positions).reshape(-1, 1),
|
||||
"meta": dict(_NATIVE_META),
|
||||
},
|
||||
)
|
||||
return source, latent
|
||||
|
||||
|
||||
def _native_handoff(source):
|
||||
context = SimpleNamespace(bridge_states={"duplex": {"epoch": 3, "model_turn_id": 7}})
|
||||
return llm2tts([source], prompt=[{}], _streaming_context=context)[0]["model_intermediate_buffer"]
|
||||
|
||||
|
||||
def test_native_duplex_ledger_tolerates_async_lookahead_row_after_terminator() -> None:
|
||||
"""An async scheduler forwards the sampled chunk terminator in one extra
|
||||
frame before the segment stop lands; its row trails the unit's rows and
|
||||
must not shift the Talker slice."""
|
||||
source, latent = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9304, 21, 22, 9308],
|
||||
row_ids=[101, 102, 9304, 21, 22, 9308],
|
||||
)
|
||||
|
||||
info = _native_handoff(source)
|
||||
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[3:5])
|
||||
|
||||
|
||||
def test_native_duplex_ledger_without_forwarded_terminator() -> None:
|
||||
"""Synchronous scheduling never forwards the chunk terminator."""
|
||||
source, latent = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9304, 21, 22, 9308],
|
||||
row_ids=[101, 102, 9304, 21, 22],
|
||||
)
|
||||
|
||||
info = _native_handoff(source)
|
||||
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[3:5])
|
||||
|
||||
|
||||
def test_native_duplex_turn_eos_row_is_handed_to_talker() -> None:
|
||||
"""<|turn_eos|> is forwarded (it is not a stop token) and the model keeps
|
||||
sampling until a chunk terminator; the Talker gets the turn_eos row and
|
||||
nothing after it."""
|
||||
source, latent = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9304, 21, 9310, 77, 9308],
|
||||
row_ids=[101, 102, 9304, 21, 9310, 77],
|
||||
)
|
||||
|
||||
info = _native_handoff(source)
|
||||
|
||||
assert info["ids"]["tts"] == [21, 9310]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[3:5])
|
||||
assert info["meta"]["turn_end"] is True
|
||||
|
||||
|
||||
def test_native_duplex_ledger_anchors_on_unit_after_reinjected_listens() -> None:
|
||||
"""Listens from earlier units sit at the head of the segment delta but
|
||||
were re-injected as prompt tokens, so they are not contiguous with this
|
||||
unit's decode rows in the ledger. Anchor on the unit, not the segment."""
|
||||
unit_end, unit, audio = 2, 1, 11
|
||||
source, latent = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9303, 9303, 9304, 21, 22, 9308],
|
||||
row_ids=[9303, unit_end, unit, audio, 9303, unit_end, unit, audio, 9304, 21, 22],
|
||||
)
|
||||
|
||||
info = _native_handoff(source)
|
||||
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[9:11])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("previous_terminator_forwarded", [False, True])
|
||||
@pytest.mark.parametrize("current_terminator_forwarded", [False, True])
|
||||
def test_native_duplex_ledger_prefers_latest_repeat_of_the_unit(
|
||||
previous_terminator_forwarded: bool, current_terminator_forwarded: bool
|
||||
) -> None:
|
||||
previous_ids = [9304, 21, 22] + ([9308] if previous_terminator_forwarded else [])
|
||||
current_ids = [9304, 21, 22] + ([9308] if current_terminator_forwarded else [])
|
||||
current_start = len(previous_ids) + 1
|
||||
source, latent = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9304, 21, 22, 9308],
|
||||
row_ids=[*previous_ids, 555, *current_ids],
|
||||
positions=[*range(current_start), *range(10, 10 + len(current_ids))],
|
||||
)
|
||||
|
||||
info = _native_handoff(source)
|
||||
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[current_start + 1 : current_start + 3])
|
||||
|
||||
|
||||
def test_native_duplex_ledger_rejects_non_contiguous_positions() -> None:
|
||||
source, _ = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9304, 21, 22, 9308],
|
||||
row_ids=[101, 102, 9304, 21, 22],
|
||||
positions=[0, 1, 2, 3, 9],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="missing own-token hidden states"):
|
||||
_native_handoff(source)
|
||||
|
||||
|
||||
def test_native_duplex_mid_turn_tts_bos_slices_after_boundary() -> None:
|
||||
source, latent = _native_source(
|
||||
prompt_ids=[101, 102],
|
||||
output_ids=[9301, 21, 22, 9308],
|
||||
row_ids=[101, 102, 9301, 21, 22],
|
||||
)
|
||||
|
||||
info = _native_handoff(source)
|
||||
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
assert torch.equal(torch.tensor(info["hidden_states"]["tts"]), latent[3:5])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("folded_decisions", [0, 1, 2])
|
||||
@pytest.mark.parametrize("bos_in_prompt", [False, True])
|
||||
def test_native_duplex_tts_bos_aligns_after_window_rebuild(folded_decisions, bos_in_prompt) -> None:
|
||||
@@ -152,12 +397,17 @@ def test_native_duplex_tts_bos_aligns_after_window_rebuild(folded_decisions, bos
|
||||
output_ids = [21, 22, 9308]
|
||||
rows = len(prompt_ids) + len(output_ids) - folded_decisions
|
||||
latent = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4)
|
||||
forwarded_ids = (
|
||||
prompt_ids[folded_decisions:] + output_ids if bos_in_prompt else prompt_ids + output_ids[folded_decisions:]
|
||||
)
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"latent_input_ids": torch.tensor(forwarded_ids).reshape(-1, 1),
|
||||
"latent_positions": torch.arange(len(forwarded_ids)).reshape(-1, 1),
|
||||
"meta": {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
|
||||
Reference in new issue
Block a user