anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI/Build][MiniCPM-o] Fix the Nightly tests (#7758)

Browse Source 
Signed-off-by: chickeyton <ngton2014@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Fable 5.1 authored and GitHub committed 2026-09-18 08:58:33 +00:00
1 parent aa2fadac42
commit 6df62d5157
8 files changed
+233 -18
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/dfx/perf/tests/test_minicpmo_4_5.json
+81 -5
View File
@@ -26,8 +26,8 @@
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
@@ -42,7 +42,73 @@
"disable_shuffle": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"H100": {
"request_throughput": [
0.9731,
1.1986,
1.686
],
"mean_ttft_ms": [
152.5665,
270.2559,
301.455
],
"mean_e2el_ms": [
1027.2953,
3306.3735,
4686.5366
],
"mean_audio_ttfp_ms": [
458.1553,
2030.2335,
2068.6233
],
"mean_audio_rtf": [
0.2357,
0.791,
1.0324
]
},
"A3": {
"request_throughput": [
0.5383,
0.6042,
0.7547
],
"mean_ttft_ms": [
333.2633,
533.8656,
547.3388
],
"mean_e2el_ms": [
1857.2154,
6600.5101,
10450.6811
],
"mean_audio_ttfp_ms": [
986.4666,
3411.0754,
3352.7515
],
"mean_audio_rtf": [
0.4423,
1.5734,
2.3024
]
}
}
}
]
},
@@ -72,8 +138,8 @@
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
32,
64,
@@ -88,6 +154,16 @@
"disable_shuffle": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
}
]
tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+37 -3
View File
@@ -26,8 +26,8 @@
{
"dataset_name": "seed-tts",
"dataset_path": "zhaochenyang20/seed-tts-eval",
"backend": "openai-realtime-tts",
"endpoint": "/v1/realtime",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
10,
40
@@ -40,7 +40,41 @@
"disable_shuffle": true,
"trust_remote_code": true,
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
"extra_body": {
"modalities": [
"text",
"audio"
],
"chat_template_kwargs": {
"enable_thinking": false,
"use_tts_template": true
}
},
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
"baseline": {
"H100": {
"request_throughput": [
0.9731,
1.1986
],
"mean_ttft_ms": [
152.5665,
270.2559
],
"mean_e2el_ms": [
1027.2953,
3306.3735
],
"mean_audio_ttfp_ms": [
458.1553,
2030.2335
],
"mean_audio_rtf": [
0.2357,
0.791
]
}
}
}
]
}
tests/e2e/online_serving/test_nemotron_voicechat_duplex.py
+10 -1
View File
@@ -29,7 +29,16 @@ _SERVER_ENV: dict[str, str] = {"VLLM_OMNI_INPUT_WAIT_TIMEOUT_S": str(SERVER_INPU
if TOKENIZER:
_SERVER_ENV["NEMOTRON_VOICECHAT_LLM_PATH"] = TOKENIZER
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
pytestmark = [
pytest.mark.full_model,
pytest.mark.omni,
# Since #7413 a duplex-capable model is served by the duplex engine only when
# its pipeline declares a duplex_plugin. The Nemotron VoiceChat pipeline does
# not, so the server starts the turn engine and the /v1/realtime websocket
# falls through to vllm's speech-to-text realtime handler, which rejects the
# native duplex client's session.update ("Missing required field: model").
pytest.mark.skip(reason="Nemotron VoiceChat is not served by the duplex engine yet (no duplex_plugin)"),
]
@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1)
tests/model_executor/models/minicpmo_4_5/test_talker_batching.py
+45 -1
View File
@@ -20,12 +20,14 @@ from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
_CODEC_PENALTY_WINDOW,
_DUPLEX_CODEC_TOKENS_PER_CHUNK,
_DUPLEX_TURN_END_CODEC_TOKENS,
_OFFLINE_CODEC_MAX_NEW_TOKENS,
_REPETITION_PENALTY_CHUNK_SIZE,
MiniCPMO45OmniTTSForConditionalGeneration,
_apply_batched_repetition_penalty,
_native_duplex_chunk_budget,
_restore_weight_norm_weight,
_turn_end_boundary_eos_masked,
blank_scheduler_prompt_for_penalties,
)
from vllm_omni.utils.mm_outputs import to_payload_element
@@ -453,7 +455,9 @@ def test_sample_leaves_unforced_rows_to_the_sampler(mocker) -> None:
def test_native_duplex_chunk_budget_masks_eos_until_generate_chunk_is_full() -> None:
assert _native_duplex_chunk_budget({}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, _DUPLEX_CODEC_TOKENS_PER_CHUNK)
assert _native_duplex_chunk_budget({"turn_start": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0)
assert _native_duplex_chunk_budget({"turn_end": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0)
# The turn-end chunk drains the owed text: no floor, multi-unit ceiling.
assert _native_duplex_chunk_budget({"turn_end": True}) == (_DUPLEX_TURN_END_CODEC_TOKENS, 0)
assert _DUPLEX_TURN_END_CODEC_TOKENS > _DUPLEX_CODEC_TOKENS_PER_CHUNK
talker = _make_talker()
talker._request_audio_states["req-chunk"] = {
@@ -1097,3 +1101,43 @@ def test_request_cleanup_evicts_decode_state() -> None:
assert "req-done" not in talker._request_audio_states
assert "req-done" not in talker._request_condition_states
def test_turn_end_drain_masks_only_the_cadence_eos() -> None:
# Early EOS (text spoken before the first boundary) is allowed.
assert not _turn_end_boundary_eos_masked(0)
assert not _turn_end_boundary_eos_masked(18)
assert not _turn_end_boundary_eos_masked(24)
# The cadence EOS right after each 25-frame boundary is masked for a short window.
assert _turn_end_boundary_eos_masked(25)
assert _turn_end_boundary_eos_masked(29)
assert not _turn_end_boundary_eos_masked(30)
assert not _turn_end_boundary_eos_masked(49)
assert _turn_end_boundary_eos_masked(50)
assert not _turn_end_boundary_eos_masked(60)
talker = _make_talker()
talker._request_audio_states["req-drain"] = {
"finished": False,
"step": 25,
"max_tokens": _DUPLEX_TURN_END_CODEC_TOKENS,
"min_tokens": 0,
"turn_end_drain": True,
}
output = talker.make_omni_output(
torch.ones(1, 2),
model_intermediate_buffer=[{"request_id": "req-drain", "codes": {"audio": torch.empty(0)}}],
request_token_spans=[(0, 1)],
)
logits = talker.compute_logits(output.text_hidden_states)
assert logits[0, 7].item() == float("-inf")
assert output.multimodal_outputs["meta"]["finished"][0].item() is False
talker._request_audio_states["req-drain"]["step"] = 30
output = talker.make_omni_output(
torch.ones(1, 2),
model_intermediate_buffer=[{"request_id": "req-drain", "codes": {"audio": torch.empty(0)}}],
request_token_spans=[(0, 1)],
)
logits = talker.compute_logits(output.text_hidden_states)
assert torch.isfinite(logits[0, 7])
vllm_omni/distributed/omni_connectors/adapter.py
+8 -1
View File
@@ -10,6 +10,11 @@ from vllm_omni.metrics import OrchestratorAggregator
from .utils.logging import get_connector_logger
# Upper bound on a previous streaming chunk's generation reserve: a duplex
# turn-end chunk may retain more codec ids than the steady-state per-unit
# reserve (see MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS).
_MAX_PREVIOUS_CHUNK_GENERATION_RESERVE = 26 * 4
logger = get_connector_logger(__name__)
@@ -345,7 +350,9 @@ def construct_next_stage_streaming_input_prompt(
f"confirmed={confirmed_num_computed_tokens}, prompt={request.num_prompt_tokens}"
)
previous_codec_ids = list(request._all_token_ids[request.num_prompt_tokens : confirmed_num_computed_tokens])
max_confirmed_codec_tokens = generation_reserve - 1
# The previous chunk may have been a turn-end unit that ran to the
# Talker's EOS with a larger reserve than this chunk's.
max_confirmed_codec_tokens = max(generation_reserve, _MAX_PREVIOUS_CHUNK_GENERATION_RESERVE) - 1
if len(previous_codec_ids) > max_confirmed_codec_tokens:
raise ValueError(
"streaming prompt recompute retained too many codec tokens: "
vllm_omni/model_executor/models/minicpmo_4_5/__init__.py
+6
View File
@@ -9,3 +9,9 @@
# One MiniCPMTTS.generate_chunk emits 25 codec frames plus its terminator.
MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK = 26
# Codec budget for the Talker chunk that carries the Thinker's turn end. The
# Talker paces ~1 s of speech per unit and emits its chunk EOS at the 25-frame
# cadence, while the text it still owes at turn end can exceed one unit. The
# turn-end chunk may run for several units so the tail of the turn is spoken
# (see ``_native_duplex_chunk_budget`` and the boundary EOS mask in the Talker).
MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK * 4
vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py
+37 -5
View File
@@ -28,7 +28,10 @@ from vllm.model_executor.models.utils import maybe_prefix
from vllm.v1.sample.sampler import Sampler
from vllm_omni.engine.duplex.intermediate import get_tts_handoff
from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
from vllm_omni.model_executor.models.minicpmo_4_5 import (
MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK,
MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.platforms import current_omni_platform
@@ -48,13 +51,38 @@ _OFFLINE_CODEC_MAX_NEW_TOKENS = 2048
# Without this, the single-vocab Sampler keeps the stage-1 request alive
# until codec EOS / 4096 and Thinker never starts the next model turn.
_DUPLEX_CODEC_TOKENS_PER_CHUNK = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
_DUPLEX_TURN_END_CODEC_TOKENS = MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS
#: Frames the Talker forwards per generate_chunk before its cadence EOS.
_DUPLEX_CODEC_FRAMES_PER_CHUNK = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK - 1
#: On a turn-end chunk the Talker's EOS is masked for this many steps after
#: each 25-frame boundary: the model emits a cadence EOS there whether or not
#: its text is spoken, while a genuine end of text shows up as an EOS anywhere
#: else in the window.
_DUPLEX_TURN_END_BOUNDARY_MASK_STEPS = 5
def _native_duplex_chunk_budget(meta: Mapping[str, Any] | None) -> tuple[int, int]:
"""Return ``(max_tokens, min_tokens)`` for one native-duplex Talker request."""
boundary = isinstance(meta, Mapping) and (bool(meta.get("turn_start")) or bool(meta.get("turn_end")))
turn_start = isinstance(meta, Mapping) and bool(meta.get("turn_start"))
turn_end = isinstance(meta, Mapping) and bool(meta.get("turn_end"))
if turn_end:
# The turn-end chunk drains the text the Talker still owes: no floor
# (an early EOS means the text is spoken) and a multi-unit ceiling.
return _DUPLEX_TURN_END_CODEC_TOKENS, 0
ceiling = _DUPLEX_CODEC_TOKENS_PER_CHUNK
return ceiling, 0 if boundary else ceiling
return ceiling, 0 if turn_start else ceiling
def _turn_end_boundary_eos_masked(step: int) -> bool:
"""Whether a turn-end chunk masks codec EOS at ``step`` forwarded frames.
The Talker emits a cadence EOS after every 25 frames regardless of the text
left, so a turn-end chunk ignores EOS in a short window after each boundary
and lets the model continue; EOS elsewhere ends the chunk as usual.
"""
if step < _DUPLEX_CODEC_FRAMES_PER_CHUNK:
return False
return step % _DUPLEX_CODEC_FRAMES_PER_CHUNK < _DUPLEX_TURN_END_BOUNDARY_MASK_STEPS
def blank_scheduler_prompt_for_penalties(
@@ -367,7 +395,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
code_ids = torch.as_tensor(previous_codes, device=self.emb_code[0].weight.device, dtype=torch.long)
else:
raise ValueError("streaming prompt recompute is missing confirmed codec ids")
if code_ids.numel() > _DUPLEX_CODEC_TOKENS_PER_CHUNK - 1:
if code_ids.numel() > _DUPLEX_TURN_END_CODEC_TOKENS - 1:
raise ValueError(f"streaming prompt recompute has too many codec ids: {code_ids.numel()}")
if code_ids.numel() and bool(((code_ids < 0) | (code_ids >= self._codec_eos_id)).any()):
raise ValueError("streaming prompt recompute codec ids include an invalid or terminal token")
@@ -500,6 +528,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
"step": 0,
"max_tokens": max_tokens,
"min_tokens": min_tokens,
"turn_end_drain": bool(native_duplex and isinstance(meta, Mapping) and bool(meta.get("turn_end"))),
}
if retained_codes:
state["recent_codes"] = retained_codes
@@ -653,7 +682,10 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
# plane does not wait for a follow-up empty decode.
state["finished"] = True
force_eos_rows[index] = chunk_done
mask_eos_rows[index] = not force_eos_rows[index] and min_tokens is not None and step < int(min_tokens)
mask_eos_rows[index] = not force_eos_rows[index] and (
(min_tokens is not None and step < int(min_tokens))
or (bool(state.get("turn_end_drain")) and _turn_end_boundary_eos_masked(step))
)
terminal_flags[index] = torch.tensor(chunk_done, dtype=torch.bool)
# Empty-speech rows, finished duplex chunks, and offline requests that
vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+9 -2
View File
@@ -16,7 +16,10 @@ from vllm_omni.engine.duplex.intermediate import (
set_tts_handoff,
)
from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.model_executor.models.minicpmo_4_5 import MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
from vllm_omni.model_executor.models.minicpmo_4_5 import (
MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK,
MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS,
)
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO45_REFERENCE_AUDIO_KEY
logger = logging.getLogger(__name__)
@@ -1010,7 +1013,11 @@ def llm2tts(
handoff_meta = model_intermediate_buffer.setdefault("meta", {})
handoff_meta["next_stage_prompt_len"] = condition_length
if is_native_duplex_handoff:
handoff_meta["next_stage_generation_tokens"] = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
handoff_meta["next_stage_generation_tokens"] = (
MINICPMO45_DUPLEX_TURN_END_CODEC_TOKENS
if native_turn_end_handoff
else MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
)
bridge_states = getattr(_streaming_context, "bridge_states", None)
handoff_state = bridge_states.get("minicpmo45_tts_handoff") if isinstance(bridge_states, dict) else None
if not isinstance(handoff_state, dict) or handoff_state.get("request_id") != str(llm_output.request_id):