From 6df62d51573ca5be7826321ba2de1cd071d997f8 Mon Sep 17 00:00:00 2001 From: chickeyton Date: Fri, 18 Sep 2026 16:58:33 +0800 Subject: [PATCH] [CI/Build][MiniCPM-o] Fix the Nightly tests (#7758) Signed-off-by: chickeyton Co-authored-by: Claude Fable 5.1 --- tests/dfx/perf/tests/test_minicpmo_4_5.json | 86 +++++++++++++++++-- .../perf/tests/test_minicpmo_4_5_ready.json | 40 ++++++++- .../test_nemotron_voicechat_duplex.py | 11 ++- .../minicpmo_4_5/test_talker_batching.py | 46 +++++++++- .../distributed/omni_connectors/adapter.py | 9 +- .../models/minicpmo_4_5/__init__.py | 6 ++ .../minicpmo_4_5/minicpmo_4_5_omni_tts.py | 42 +++++++-- .../minicpmo_4_5_omni.py | 11 ++- 8 files changed, 233 insertions(+), 18 deletions(-) diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index b384d61a0..7d7ca08ad 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -26,8 +26,8 @@ { "dataset_name": "seed-tts", "dataset_path": "zhaochenyang20/seed-tts-eval", - "backend": "openai-realtime-tts", - "endpoint": "/v1/realtime", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", "num_prompts": [ 32, 64, @@ -42,7 +42,73 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", - "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", + "baseline": { + "H100": { + "request_throughput": [ + 0.9731, + 1.1986, + 1.686 + ], + "mean_ttft_ms": [ + 152.5665, + 270.2559, + 301.455 + ], + "mean_e2el_ms": [ + 1027.2953, + 3306.3735, + 4686.5366 + ], + "mean_audio_ttfp_ms": [ + 458.1553, + 2030.2335, + 2068.6233 + ], + "mean_audio_rtf": [ + 0.2357, + 0.791, + 1.0324 + ] + }, + "A3": { + "request_throughput": [ + 0.5383, + 0.6042, + 0.7547 + ], + "mean_ttft_ms": [ + 333.2633, + 533.8656, + 547.3388 + ], + "mean_e2el_ms": [ + 1857.2154, + 6600.5101, + 10450.6811 + ], + "mean_audio_ttfp_ms": [ + 986.4666, + 3411.0754, + 3352.7515 + ], + "mean_audio_rtf": [ + 0.4423, + 1.5734, + 2.3024 + ] + } + } } ] }, @@ -72,8 +138,8 @@ { "dataset_name": "seed-tts", "dataset_path": "zhaochenyang20/seed-tts-eval", - "backend": "openai-realtime-tts", - "endpoint": "/v1/realtime", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", "num_prompts": [ 32, 64, @@ -88,6 +154,16 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" } ] diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json index 4fd78d681..26dc54aab 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json @@ -26,8 +26,8 @@ { "dataset_name": "seed-tts", "dataset_path": "zhaochenyang20/seed-tts-eval", - "backend": "openai-realtime-tts", - "endpoint": "/v1/realtime", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", "num_prompts": [ 10, 40 @@ -40,7 +40,41 @@ "disable_shuffle": true, "trust_remote_code": true, "seed_tts_locale": "en", - "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", + "baseline": { + "H100": { + "request_throughput": [ + 0.9731, + 1.1986 + ], + "mean_ttft_ms": [ + 152.5665, + 270.2559 + ], + "mean_e2el_ms": [ + 1027.2953, + 3306.3735 + ], + "mean_audio_ttfp_ms": [ + 458.1553, + 2030.2335 + ], + "mean_audio_rtf": [ + 0.2357, + 0.791 + ] + } + } } ] } diff --git a/tests/e2e/online_serving/test_nemotron_voicechat_duplex.py b/tests/e2e/online_serving/test_nemotron_voicechat_duplex.py index 6f97eaee2..dfb3044e8 100644 --- a/tests/e2e/online_serving/test_nemotron_voicechat_duplex.py +++ b/tests/e2e/online_serving/test_nemotron_voicechat_duplex.py @@ -29,7 +29,16 @@ _SERVER_ENV: dict[str, str] = {"VLLM_OMNI_INPUT_WAIT_TIMEOUT_S": str(SERVER_INPU if TOKENIZER: _SERVER_ENV["NEMOTRON_VOICECHAT_LLM_PATH"] = TOKENIZER -pytestmark = [pytest.mark.full_model, pytest.mark.omni] +pytestmark = [ + pytest.mark.full_model, + pytest.mark.omni, + # Since #7413 a duplex-capable model is served by the duplex engine only when + # its pipeline declares a duplex_plugin. The Nemotron VoiceChat pipeline does + # not, so the server starts the turn engine and the /v1/realtime websocket + # falls through to vllm's speech-to-text realtime handler, which rejects the + # native duplex client's session.update ("Missing required field: model"). + pytest.mark.skip(reason="Nemotron VoiceChat is not served by the duplex engine yet (no duplex_plugin)"), +] @hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) diff --git a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py index 0c5b97ae2..4335bb88e 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py +++ b/tests/model_executor/models/minicpmo_4_5/test_talker_batching.py @@ -20,12 +20,14 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import ( _CODEC_PENALTY_WINDOW, _DUPLEX_CODEC_TOKENS_PER_CHUNK, + _DUPLEX_TURN_END_CODEC_TOKENS, _OFFLINE_CODEC_MAX_NEW_TOKENS, _REPETITION_PENALTY_CHUNK_SIZE, MiniCPMO45OmniTTSForConditionalGeneration, _apply_batched_repetition_penalty, _native_duplex_chunk_budget, _restore_weight_norm_weight, + _turn_end_boundary_eos_masked, blank_scheduler_prompt_for_penalties, ) from vllm_omni.utils.mm_outputs import to_payload_element @@ -453,7 +455,9 @@ def test_sample_leaves_unforced_rows_to_the_sampler(mocker) -> None: def test_native_duplex_chunk_budget_masks_eos_until_generate_chunk_is_full() -> None: assert _native_duplex_chunk_budget({}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, _DUPLEX_CODEC_TOKENS_PER_CHUNK) assert _native_duplex_chunk_budget({"turn_start": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0) - assert _native_duplex_chunk_budget({"turn_end": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0) + # The turn-end chunk drains the owed text: no floor, multi-unit ceiling. + assert _native_duplex_chunk_budget({"turn_end": True}) == (_DUPLEX_TURN_END_CODEC_TOKENS, 0) + assert _DUPLEX_TURN_END_CODEC_TOKENS > _DUPLEX_CODEC_TOKENS_PER_CHUNK talker = _make_talker() talker._request_audio_states["req-chunk"] = { @@ -1097,3 +1101,43 @@ def test_request_cleanup_evicts_decode_state() -> None: assert "req-done" not in talker._request_audio_states assert "req-done" not in talker._request_condition_states + + +def test_turn_end_drain_masks_only_the_cadence_eos() -> None: + # Early EOS (text spoken before the first boundary) is allowed. + assert not _turn_end_boundary_eos_masked(0) + assert not _turn_end_boundary_eos_masked(18) + assert not _turn_end_boundary_eos_masked(24) + # The cadence EOS right after each 25-frame boundary is masked for a short window. + assert _turn_end_boundary_eos_masked(25) + assert _turn_end_boundary_eos_masked(29) + assert not _turn_end_boundary_eos_masked(30) + assert not _turn_end_boundary_eos_masked(49) + assert _turn_end_boundary_eos_masked(50) + assert not _turn_end_boundary_eos_masked(60) + + talker = _make_talker() + talker._request_audio_states["req-drain"] = { + "finished": False, + "step": 25, + "max_tokens": _DUPLEX_TURN_END_CODEC_TOKENS, + "min_tokens": 0, + "turn_end_drain": True, + } + output = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[{"request_id": "req-drain", "codes": {"audio": torch.empty(0)}}], + request_token_spans=[(0, 1)], + ) + logits = talker.compute_logits(output.text_hidden_states) + assert logits[0, 7].item() == float("-inf") + assert output.multimodal_outputs["meta"]["finished"][0].item() is False + + talker._request_audio_states["req-drain"]["step"] = 30 + output = talker.make_omni_output( + torch.ones(1, 2), + model_intermediate_buffer=[{"request_id": "req-drain", "codes": {"audio": torch.empty(0)}}], + request_token_spans=[(0, 1)], + ) + logits = talker.compute_logits(output.text_hidden_states) + assert torch.isfinite(logits[0, 7]) diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index 79a1d7887..dc3be166f 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -10,6 +10,11 @@ from vllm_omni.metrics import OrchestratorAggregator from .utils.logging import get_connector_logger +# Upper bound on a previous streaming chunk's generation reserve: a duplex +# turn-end chunk may retain more codec ids than the steady-state per-unit +# reserve (see MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS). +_MAX_PREVIOUS_CHUNK_GENERATION_RESERVE = 26 * 4 + logger = get_connector_logger(__name__) @@ -345,7 +350,9 @@ def construct_next_stage_streaming_input_prompt( f"confirmed={confirmed_num_computed_tokens}, prompt={request.num_prompt_tokens}" ) previous_codec_ids = list(request._all_token_ids[request.num_prompt_tokens : confirmed_num_computed_tokens]) - max_confirmed_codec_tokens = generation_reserve - 1 + # The previous chunk may have been a turn-end unit that ran to the + # Talker's EOS with a larger reserve than this chunk's. + max_confirmed_codec_tokens = max(generation_reserve, _MAX_PREVIOUS_CHUNK_GENERATION_RESERVE) - 1 if len(previous_codec_ids) > max_confirmed_codec_tokens: raise ValueError( "streaming prompt recompute retained too many codec tokens: " diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py b/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py index 6e8677cf2..77ee3b9a0 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/__init__.py @@ -9,3 +9,9 @@ # One MiniCPMTTS.generate_chunk emits 25 codec frames plus its terminator. MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK = 26 +# Codec budget for the Talker chunk that carries the Thinker's turn end. The +# Talker paces ~1 s of speech per unit and emits its chunk EOS at the 25-frame +# cadence, while the text it still owes at turn end can exceed one unit. The +# turn-end chunk may run for several units so the tail of the turn is spoken +# (see ``_native_duplex_chunk_budget`` and the boundary EOS mask in the Talker). +MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK * 4 diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py index bdc418f05..67843833d 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py @@ -28,7 +28,10 @@ from vllm.model_executor.models.utils import maybe_prefix from vllm.v1.sample.sampler import Sampler from vllm_omni.engine.duplex.intermediate import get_tts_handoff -from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK +from vllm_omni.model_executor.models.minicpmo_4_5 import ( + MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK, + MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS, +) from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.platforms import current_omni_platform @@ -48,13 +51,38 @@ _OFFLINE_CODEC_MAX_NEW_TOKENS = 2048 # Without this, the single-vocab Sampler keeps the stage-1 request alive # until codec EOS / 4096 and Thinker never starts the next model turn. _DUPLEX_CODEC_TOKENS_PER_CHUNK = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK +_DUPLEX_TURN_END_CODEC_TOKENS = MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS +#: Frames the Talker forwards per generate_chunk before its cadence EOS. +_DUPLEX_CODEC_FRAMES_PER_CHUNK = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK - 1 +#: On a turn-end chunk the Talker's EOS is masked for this many steps after +#: each 25-frame boundary: the model emits a cadence EOS there whether or not +#: its text is spoken, while a genuine end of text shows up as an EOS anywhere +#: else in the window. +_DUPLEX_TURN_END_BOUNDARY_MASK_STEPS = 5 def _native_duplex_chunk_budget(meta: Mapping[str, Any] | None) -> tuple[int, int]: """Return ``(max_tokens, min_tokens)`` for one native-duplex Talker request.""" - boundary = isinstance(meta, Mapping) and (bool(meta.get("turn_start")) or bool(meta.get("turn_end"))) + turn_start = isinstance(meta, Mapping) and bool(meta.get("turn_start")) + turn_end = isinstance(meta, Mapping) and bool(meta.get("turn_end")) + if turn_end: + # The turn-end chunk drains the text the Talker still owes: no floor + # (an early EOS means the text is spoken) and a multi-unit ceiling. + return _DUPLEX_TURN_END_CODEC_TOKENS, 0 ceiling = _DUPLEX_CODEC_TOKENS_PER_CHUNK - return ceiling, 0 if boundary else ceiling + return ceiling, 0 if turn_start else ceiling + + +def _turn_end_boundary_eos_masked(step: int) -> bool: + """Whether a turn-end chunk masks codec EOS at ``step`` forwarded frames. + + The Talker emits a cadence EOS after every 25 frames regardless of the text + left, so a turn-end chunk ignores EOS in a short window after each boundary + and lets the model continue; EOS elsewhere ends the chunk as usual. + """ + if step < _DUPLEX_CODEC_FRAMES_PER_CHUNK: + return False + return step % _DUPLEX_CODEC_FRAMES_PER_CHUNK < _DUPLEX_TURN_END_BOUNDARY_MASK_STEPS def blank_scheduler_prompt_for_penalties( @@ -367,7 +395,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): code_ids = torch.as_tensor(previous_codes, device=self.emb_code[0].weight.device, dtype=torch.long) else: raise ValueError("streaming prompt recompute is missing confirmed codec ids") - if code_ids.numel() > _DUPLEX_CODEC_TOKENS_PER_CHUNK - 1: + if code_ids.numel() > _DUPLEX_TURN_END_CODEC_TOKENS - 1: raise ValueError(f"streaming prompt recompute has too many codec ids: {code_ids.numel()}") if code_ids.numel() and bool(((code_ids < 0) | (code_ids >= self._codec_eos_id)).any()): raise ValueError("streaming prompt recompute codec ids include an invalid or terminal token") @@ -500,6 +528,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): "step": 0, "max_tokens": max_tokens, "min_tokens": min_tokens, + "turn_end_drain": bool(native_duplex and isinstance(meta, Mapping) and bool(meta.get("turn_end"))), } if retained_codes: state["recent_codes"] = retained_codes @@ -653,7 +682,10 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP): # plane does not wait for a follow-up empty decode. state["finished"] = True force_eos_rows[index] = chunk_done - mask_eos_rows[index] = not force_eos_rows[index] and min_tokens is not None and step < int(min_tokens) + mask_eos_rows[index] = not force_eos_rows[index] and ( + (min_tokens is not None and step < int(min_tokens)) + or (bool(state.get("turn_end_drain")) and _turn_end_boundary_eos_masked(step)) + ) terminal_flags[index] = torch.tensor(chunk_done, dtype=torch.bool) # Empty-speech rows, finished duplex chunks, and offline requests that diff --git a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py index 1c374be85..4d818b79f 100644 --- a/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py @@ -16,7 +16,10 @@ from vllm_omni.engine.duplex.intermediate import ( set_tts_handoff, ) from vllm_omni.inputs.data import OmniTokensPrompt -from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK +from vllm_omni.model_executor.models.minicpmo_4_5 import ( + MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK, + MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS, +) from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY logger = logging.getLogger(__name__) @@ -1010,7 +1013,11 @@ def llm2tts( handoff_meta = model_intermediate_buffer.setdefault("meta", {}) handoff_meta["next_stage_prompt_len"] = condition_length if is_native_duplex_handoff: - handoff_meta["next_stage_generation_tokens"] = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK + handoff_meta["next_stage_generation_tokens"] = ( + MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS + if native_turn_end_handoff + else MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK + ) bridge_states = getattr(_streaming_context, "bridge_states", None) handoff_state = bridge_states.get("minicpmo45_tts_handoff") if isinstance(bridge_states, dict) else None if not isinstance(handoff_state, dict) or handoff_state.get("request_id") != str(llm_output.request_id):