[Bugfix][MiniCPM-o] Stop Talker mid-utterance truncation and load cod… (#6346)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
47b1893676
commit
9c61d5b67b
27 files changed
+1118
-678
No files matched your search
@@ -1127,9 +1127,11 @@ class TestDeployConfigLoading:
|
||||
assert all("Async" not in (stage.scheduler_cls or "") for stage in stages)
|
||||
assert [stage.devices for stage in deploy.stages] == ["0", "0", "0"]
|
||||
assert deploy.stages[1].enforce_eager is False
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 2048
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["max_tokens"] == 4096
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["min_tokens"] == 0
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [1]
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["temperature"] == 0.8
|
||||
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [6561]
|
||||
assert "codec_sampling_params" not in stages[1].yaml_engine_args
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "stage0_devices", "stage1_devices", "stage2_devices", "stage1_replicas"),
|
||||
@@ -2111,6 +2113,23 @@ class TestMingFlashOmniPipeline:
|
||||
class TestBaseConfigInheritance:
|
||||
"""Test deploy YAML base_config inheritance."""
|
||||
|
||||
def test_minicpmo_overlays_inherit_talker_sampling_params(self):
|
||||
"""Overlays must keep the base Talker codec Sampler knobs."""
|
||||
for filename in (
|
||||
"minicpmo_4_5_duplex.yaml",
|
||||
"minicpmo_4_5_3gpu_stage1_replicas.yaml",
|
||||
"minicpmo_4_5_4gpu_stage1_replicas.yaml",
|
||||
"minicpmo_4_5_8x4090_stage1_replicas.yaml",
|
||||
):
|
||||
path = Path(get_deploy_config_path(filename))
|
||||
if not path.exists():
|
||||
pytest.skip(f"{filename} not found")
|
||||
deploy = load_deploy_config(path)
|
||||
sampling = deploy.stages[1].default_sampling_params
|
||||
assert sampling is not None, f"{filename} stage 1 lost default_sampling_params"
|
||||
assert sampling["temperature"] == 0.8, filename
|
||||
assert sampling["top_k"] == 100, filename
|
||||
|
||||
def test_ci_inherits_from_main(self):
|
||||
ci_path = Path(get_deploy_config_path("ci/qwen3_omni_moe.yaml"))
|
||||
if not ci_path.exists():
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
@@ -79,9 +80,92 @@
|
||||
0.8568,
|
||||
1.5425
|
||||
]
|
||||
},
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.5383,
|
||||
0.6042,
|
||||
0.7547
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
333.2633,
|
||||
533.8656,
|
||||
547.3388
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1857.2154,
|
||||
6600.5101,
|
||||
10450.6811
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
986.4666,
|
||||
3411.0754,
|
||||
3352.7515
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.4423,
|
||||
1.5734,
|
||||
2.3024
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_a2",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"npu": "A2"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -35,6 +35,7 @@
|
||||
1
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"seed_tts_turns_per_session": 4,
|
||||
@@ -60,6 +61,23 @@
|
||||
"mean_audio_rtf": [
|
||||
0.46163
|
||||
]
|
||||
},
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.1594
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
274.164
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6250.7696
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
994.422
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.46163
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
4
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
|
||||
@@ -65,6 +65,15 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
# Close <think> and emit <|tts_bos|> so Talker speaks the answer, not reasoning.
|
||||
_TTS_EXTRA_BODY = {
|
||||
"chat_template_kwargs": {
|
||||
"use_tts_template": True,
|
||||
"enable_thinking": False,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@@ -111,6 +120,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {"audio": ["Beijing"]},
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config)
|
||||
@@ -141,12 +151,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"stream": True,
|
||||
"key_words": {"text": ["Beijing"]},
|
||||
"modalities": ["text", "audio"],
|
||||
"extra_body": {
|
||||
"chat_template_kwargs": {
|
||||
"use_tts_template": True,
|
||||
"enable_thinking": False,
|
||||
}
|
||||
},
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -175,6 +180,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -203,6 +209,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -236,6 +243,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -1,7 +1,7 @@
|
||||
"""E2E online expansion tests for MiniCPM-o 4.5.
|
||||
|
||||
These cover modality combinations, separate Code2Wav behavior, sequential
|
||||
request isolation, longer audio output, and Chinese speech.
|
||||
request isolation, longer audio output, long-form generation, and Chinese speech.
|
||||
|
||||
Like Qwen3-Omni expansion, ``default`` exercises ``--no-async-chunk`` and
|
||||
``async_chunk`` exercises ``--async-chunk``.
|
||||
@@ -75,6 +75,15 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
# Close <think> and emit <|tts_bos|> so Talker speaks the answer, not reasoning.
|
||||
_TTS_EXTRA_BODY = {
|
||||
"chat_template_kwargs": {
|
||||
"use_tts_template": True,
|
||||
"enable_thinking": False,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||
@@ -127,6 +136,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"messages": messages_1,
|
||||
"stream": True,
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
},
|
||||
request_num=1,
|
||||
)
|
||||
@@ -138,6 +148,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
"messages": messages_2,
|
||||
"stream": True,
|
||||
"key_words": {"text": ["Beijing"]},
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
},
|
||||
request_num=1,
|
||||
)
|
||||
@@ -164,20 +175,39 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||
"messages": messages,
|
||||
"modalities": ["text", "audio"],
|
||||
"stream": True,
|
||||
# Delimit the assistant answer with the TTS template so the Talker
|
||||
# does not spend its codec-token budget speaking hidden reasoning.
|
||||
"extra_body": {
|
||||
"chat_template_kwargs": {
|
||||
"use_tts_template": True,
|
||||
"enable_thinking": False,
|
||||
}
|
||||
},
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
"key_words": {"audio": ["Beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_audio_long_form_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Input Modal: text only (long-form generation prompt).
|
||||
Output Modal: text, audio (default ``modalities``);
|
||||
Input Setting: stream=True
|
||||
Datasets: single request
|
||||
"""
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
content_text="帮我讲一个300字的故事.",
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
responses = openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
text = responses[0].text_content if responses else ""
|
||||
word_count = len(text.split())
|
||||
assert word_count >= 200, f"Expected at least 200 words in long output, got {word_count}"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_chinese_text_to_audio(omni_server, openai_client) -> None:
|
||||
@@ -193,5 +223,6 @@ def test_chinese_text_to_audio(omni_server, openai_client) -> None:
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {"text": ["北京"]},
|
||||
"extra_body": _TTS_EXTRA_BODY,
|
||||
}
|
||||
openai_client.send_omni_request(request_config)
|
||||
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Token-budget clamping when the orchestrator builds a stage request."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.engine.orchestrator import build_engine_core_request_from_tokens
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _StageModelConfig:
|
||||
"""Only the field ``build_engine_core_request_from_tokens`` reads."""
|
||||
|
||||
max_model_len = 4096
|
||||
|
||||
|
||||
class TestStageRequestTokenBudget:
|
||||
def test_min_tokens_is_clamped_to_the_remaining_context(self):
|
||||
# A min_tokens wider than the leftover window makes ``check_stop``
|
||||
# return early forever: the request can neither stop nor be granted
|
||||
# more tokens, so it stalls instead of being length-capped.
|
||||
params = SamplingParams(max_tokens=4096, min_tokens=50)
|
||||
|
||||
request = build_engine_core_request_from_tokens(
|
||||
request_id="req-clamped",
|
||||
prompt={"prompt_token_ids": [0] * 4050},
|
||||
params=params,
|
||||
model_config=_StageModelConfig(),
|
||||
)
|
||||
|
||||
assert request.sampling_params.min_tokens == 46
|
||||
assert params.min_tokens == 50
|
||||
|
||||
def test_min_tokens_survives_when_the_prompt_leaves_room(self):
|
||||
params = SamplingParams(max_tokens=4096, min_tokens=50)
|
||||
|
||||
request = build_engine_core_request_from_tokens(
|
||||
request_id="req-untouched",
|
||||
prompt={"prompt_token_ids": [0] * 68},
|
||||
params=params,
|
||||
model_config=_StageModelConfig(),
|
||||
)
|
||||
|
||||
assert request.sampling_params.min_tokens == 50
|
||||
assert request.sampling_params.max_tokens == 4096
|
||||
|
||||
def test_min_tokens_floors_at_zero_when_the_prompt_fills_the_context(self):
|
||||
params = SamplingParams(max_tokens=16, min_tokens=8)
|
||||
|
||||
request = build_engine_core_request_from_tokens(
|
||||
request_id="req-full",
|
||||
prompt={"prompt_token_ids": [0] * 4096},
|
||||
params=params,
|
||||
model_config=_StageModelConfig(),
|
||||
)
|
||||
|
||||
assert request.sampling_params.min_tokens == 0
|
||||
|
||||
def test_absent_max_tokens_still_defaults_to_the_remaining_context(self):
|
||||
params = SamplingParams(min_tokens=0)
|
||||
params.max_tokens = None
|
||||
|
||||
request = build_engine_core_request_from_tokens(
|
||||
request_id="req-default",
|
||||
prompt={"prompt_token_ids": [0] * 96},
|
||||
params=params,
|
||||
model_config=_StageModelConfig(),
|
||||
)
|
||||
|
||||
assert request.sampling_params.max_tokens == 4000
|
||||
@@ -7,6 +7,8 @@ import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import (
|
||||
BatchedToken2Wav,
|
||||
plan_token2wav_encode_slices,
|
||||
relpos_encode_token_budget,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_code2wav import (
|
||||
MiniCPMO45Code2Wav,
|
||||
@@ -217,6 +219,43 @@ def test_adapter_runs_true_batch_cfg_and_splits_request_caches():
|
||||
assert cache1[0, 0, 0, 0, 0].item() == 20
|
||||
|
||||
|
||||
def test_relpos_encode_token_budget_leaves_room_for_upsample_and_cache():
|
||||
# The NPU crash was 6968 vs 985: 3333 tokens * 2 + cache overflowed max_pos=5000.
|
||||
assert relpos_encode_token_budget(max_pos=5000, stride=2, cache_offset=150, lookahead=3) <= 1024
|
||||
assert relpos_encode_token_budget(max_pos=5000, stride=2, cache_offset=150, lookahead=3, cap=3000) == 2346
|
||||
assert plan_token2wav_encode_slices(3333, max_frames=1024, min_nonfinal=4, last_chunk=True) == [
|
||||
(0, 1024),
|
||||
(1024, 2048),
|
||||
(2048, 3072),
|
||||
(3072, 3333),
|
||||
]
|
||||
assert plan_token2wav_encode_slices(50, max_frames=40, min_nonfinal=20, last_chunk=False) == [
|
||||
(0, 30),
|
||||
(30, 50),
|
||||
]
|
||||
|
||||
|
||||
def test_decode_batch_splits_overlong_prefill_inside_relpos_budget():
|
||||
token2wav = _FakeToken2Wav()
|
||||
adapter = BatchedToken2Wav(token2wav)
|
||||
prompt = adapter.prepare_prompt("shared", "/fake/prompt.wav")
|
||||
states = adapter.setup_batch(prompt, 1)
|
||||
setup_encodes = len(token2wav.flow.encoder.calls)
|
||||
adapter._max_encode_token_frames = lambda _states: 4
|
||||
|
||||
audios, _ = adapter.decode_batch(
|
||||
torch.arange(10, dtype=torch.long).reshape(1, -1),
|
||||
prompt,
|
||||
states,
|
||||
last_chunk=True,
|
||||
)
|
||||
|
||||
assert token2wav.flow.encoder.calls[setup_encodes:] == [1, 1, 1]
|
||||
assert token2wav.flow.encoder.last_chunk_calls[setup_encodes:] == [False, False, True]
|
||||
assert len(audios) == 1
|
||||
assert audios[0].numel() > 0
|
||||
|
||||
|
||||
def test_fade_in_out_limits_overlap_to_available_previous_audio():
|
||||
speech = torch.arange(6, dtype=torch.float32).reshape(1, -1)
|
||||
previous = torch.full((1, 3), 2.0)
|
||||
|
||||
@@ -105,6 +105,10 @@ class TestPipelineTopology:
|
||||
assert talker.engine_output_type == "latent"
|
||||
# scope KV cache / mrope sizing to talker sub-config
|
||||
assert talker.hf_config_name == "tts_config"
|
||||
assert talker.sampling_constraints == {
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [6561],
|
||||
}
|
||||
assert talker.custom_process_next_stage_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.minicpmo_4_5_omni.tts2code2wav_full_payload"
|
||||
)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -11,18 +13,30 @@ import torch.nn as nn
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
|
||||
ConditionalChatTTSConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts import (
|
||||
_DUPLEX_CODEC_TOKENS_PER_CHUNK,
|
||||
_REPETITION_PENALTY_CHUNK_SIZE,
|
||||
MiniCPMO45OmniTTSForConditionalGeneration,
|
||||
_apply_batched_repetition_penalty,
|
||||
_max_audio_tokens,
|
||||
_native_duplex_chunk_budget,
|
||||
_restore_weight_norm_weight,
|
||||
blank_scheduler_prompt_for_penalties,
|
||||
)
|
||||
from vllm_omni.utils.mm_outputs import to_payload_element
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PenaltyMetadata:
|
||||
"""Stand-in for the penalty-relevant slice of vLLM's SamplingMetadata."""
|
||||
|
||||
prompt_token_ids: torch.Tensor | None
|
||||
|
||||
|
||||
class _FakeNativeTalker(nn.Module):
|
||||
has_preprocess = True
|
||||
|
||||
@@ -35,6 +49,12 @@ class _FakeNativeTalker(nn.Module):
|
||||
return torch.ones(2, 4)
|
||||
|
||||
|
||||
def test_tts_config_exposes_codec_vocab_to_vllm() -> None:
|
||||
cfg = ConditionalChatTTSConfig(num_audio_tokens=6562)
|
||||
assert cfg.vocab_size == 6562
|
||||
assert cfg.eos_token_id == 6561
|
||||
|
||||
|
||||
def test_wrapper_always_delegates_talker_to_native_ar_path() -> None:
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
nn.Module.__init__(model)
|
||||
@@ -55,12 +75,14 @@ def _make_talker() -> MiniCPMO45OmniTTSForConditionalGeneration:
|
||||
talker = MiniCPMO45OmniTTSForConditionalGeneration.__new__(MiniCPMO45OmniTTSForConditionalGeneration)
|
||||
nn.Module.__init__(talker)
|
||||
talker._num_audio_tokens = 8
|
||||
talker._batch_stop_logits = None
|
||||
talker._request_generators = {}
|
||||
talker._codec_eos_id = 7
|
||||
talker._force_eos_rows = None
|
||||
talker._mask_eos_rows = None
|
||||
talker._request_audio_states = {}
|
||||
talker._deferred_cleanup_ids = set()
|
||||
talker._codec_min_tokens = 50
|
||||
talker._codec_seed = 42
|
||||
talker.head_code = nn.ModuleList([nn.Linear(2, 8, bias=False)])
|
||||
with torch.no_grad():
|
||||
talker.head_code[0].weight.copy_(torch.eye(8, 2))
|
||||
return talker
|
||||
|
||||
|
||||
@@ -90,46 +112,6 @@ def _reference_repetition_penalty(
|
||||
return torch.where(logits < 0, logits * alpha, logits / alpha)
|
||||
|
||||
|
||||
def _make_sampling_talker() -> MiniCPMO45OmniTTSForConditionalGeneration:
|
||||
talker = _make_talker()
|
||||
talker._codec_temperature = 0.8
|
||||
talker._codec_top_k = 5
|
||||
talker._codec_top_p = 0.85
|
||||
talker._codec_repetition_penalty = 1.05
|
||||
talker._request_audio_states = {
|
||||
"req-a": {"min_tokens": 2},
|
||||
"req-b": {"min_tokens": 0},
|
||||
}
|
||||
talker.head_code = nn.ModuleList([nn.Linear(2, 8, bias=False)])
|
||||
with torch.no_grad():
|
||||
talker.head_code[0].weight.copy_(
|
||||
torch.tensor(
|
||||
[
|
||||
[0.1, -0.3],
|
||||
[0.2, 0.4],
|
||||
[-0.5, 0.2],
|
||||
[0.7, -0.1],
|
||||
[-0.2, 0.8],
|
||||
[0.6, 0.3],
|
||||
[-0.4, -0.7],
|
||||
[0.3, 0.5],
|
||||
]
|
||||
)
|
||||
)
|
||||
return talker
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("condition_tokens", "expected"),
|
||||
[(3, 64), (100, 1000), (1000, 2048)],
|
||||
)
|
||||
def test_audio_token_limit_scales_with_condition_length(
|
||||
condition_tokens: int,
|
||||
expected: int,
|
||||
) -> None:
|
||||
assert _max_audio_tokens(condition_tokens) == expected
|
||||
|
||||
|
||||
def test_weight_norm_restore_matches_checkpoint_parametrization_in_bfloat16() -> None:
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
weight_v = torch.randn(8, 16, generator=generator, dtype=torch.bfloat16)
|
||||
@@ -220,133 +202,212 @@ def test_batched_repetition_penalty_matches_rows_across_chunks(mocker) -> None:
|
||||
]
|
||||
|
||||
|
||||
def test_batched_codec_sampling_is_request_order_independent() -> None:
|
||||
talker = _make_sampling_talker()
|
||||
histories = [torch.tensor([1, 1, 2]), torch.tensor([3, 4, 4])]
|
||||
hidden = torch.tensor([[0.25, -0.75], [-0.5, 0.5]])
|
||||
def test_scheduler_prompt_is_fully_blanked_for_penalties() -> None:
|
||||
# The handoff path fills the prompt with 0s and the non-handoff path passes
|
||||
# thinker token ids; neither is codec history, so every position is padded.
|
||||
prompt = torch.tensor([[0, 0, 0], [12, 0, 340]])
|
||||
|
||||
forward_order = talker._sample_audio_codes(
|
||||
hidden,
|
||||
histories,
|
||||
["req-a", "req-b"],
|
||||
[0, 0],
|
||||
)
|
||||
talker._request_generators = {}
|
||||
reverse_order = talker._sample_audio_codes(
|
||||
hidden.flip(0),
|
||||
list(reversed(histories)),
|
||||
["req-b", "req-a"],
|
||||
[0, 0],
|
||||
blanked = blank_scheduler_prompt_for_penalties(prompt, vocab_size=8)
|
||||
|
||||
assert blanked.tolist() == [[8, 8, 8], [8, 8, 8]]
|
||||
assert prompt.tolist() == [[0, 0, 0], [12, 0, 340]]
|
||||
|
||||
|
||||
def test_blanked_prompt_leaves_every_codec_logit_unpenalized() -> None:
|
||||
from vllm.model_executor.layers.utils import apply_penalties
|
||||
|
||||
vocab_size = 8
|
||||
logits = torch.arange(1.0, vocab_size + 1.0).unsqueeze(0)
|
||||
prompt = blank_scheduler_prompt_for_penalties(torch.tensor([[0, 0, 3]]), vocab_size)
|
||||
|
||||
penalized = apply_penalties(
|
||||
logits.clone(),
|
||||
prompt,
|
||||
torch.full((1, 1), vocab_size, dtype=torch.long),
|
||||
torch.zeros(1),
|
||||
torch.zeros(1),
|
||||
torch.full((1,), 1.02),
|
||||
)
|
||||
|
||||
assert torch.equal(forward_order[0], reverse_order[1])
|
||||
assert torch.equal(forward_order[1], reverse_order[0])
|
||||
torch.testing.assert_close(penalized, logits)
|
||||
|
||||
|
||||
def test_batched_codec_sampling_preserves_request_rng_after_compaction() -> None:
|
||||
talker = _make_sampling_talker()
|
||||
history_a = torch.tensor([1, 1, 2])
|
||||
history_b = torch.tensor([3, 4, 4])
|
||||
hidden_a = torch.tensor([[0.25, -0.75]])
|
||||
hidden_b = torch.tensor([[-0.5, 0.5]])
|
||||
|
||||
first_batch = talker._sample_audio_codes(
|
||||
torch.cat([hidden_a, hidden_b]),
|
||||
[history_a, history_b],
|
||||
["req-a", "req-b"],
|
||||
[0, 0],
|
||||
)
|
||||
compacted_b = talker._sample_audio_codes(
|
||||
hidden_b,
|
||||
[history_b],
|
||||
["req-b"],
|
||||
[1],
|
||||
)
|
||||
|
||||
talker._request_generators = {}
|
||||
standalone_b_first = talker._sample_audio_codes(
|
||||
hidden_b,
|
||||
[history_b],
|
||||
["req-b"],
|
||||
[0],
|
||||
)[0]
|
||||
standalone_b_second = talker._sample_audio_codes(
|
||||
hidden_b,
|
||||
[history_b],
|
||||
["req-b"],
|
||||
[1],
|
||||
)[0]
|
||||
|
||||
assert torch.equal(first_batch[1], standalone_b_first)
|
||||
assert torch.equal(compacted_b[0], standalone_b_second)
|
||||
|
||||
|
||||
def test_talker_emits_request_aligned_codec_deltas_after_compaction(mocker) -> None:
|
||||
def test_talker_sample_blanks_penalty_prompt_without_touching_runner_state(mocker) -> None:
|
||||
talker = _make_talker()
|
||||
seen: list[tuple[str, list[float], list[int]]] = []
|
||||
metadata = _PenaltyMetadata(prompt_token_ids=torch.tensor([[0, 2, 0]]))
|
||||
captured: dict[str, torch.Tensor] = {}
|
||||
|
||||
def sample(hidden, histories, request_ids, steps):
|
||||
assert steps == [0, 0]
|
||||
seen.extend(
|
||||
(request_id, hidden[row].reshape(-1).tolist(), histories[row].tolist())
|
||||
for row, request_id in enumerate(request_ids)
|
||||
)
|
||||
return torch.tensor([2, 3])
|
||||
class _Recorder:
|
||||
def __call__(self, logits, sampling_metadata):
|
||||
captured["prompt"] = sampling_metadata.prompt_token_ids
|
||||
return None
|
||||
|
||||
batched_sample = mocker.patch.object(talker, "_sample_audio_codes", side_effect=sample)
|
||||
mocker.patch(
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_tts.Sampler",
|
||||
return_value=_Recorder(),
|
||||
)
|
||||
talker.sample(torch.zeros(1, 8), metadata)
|
||||
|
||||
assert captured["prompt"].tolist() == [[8, 8, 8]]
|
||||
assert metadata.prompt_token_ids.tolist() == [[0, 2, 0]]
|
||||
|
||||
|
||||
def test_compute_logits_returns_codec_vocab() -> None:
|
||||
talker = _make_talker()
|
||||
hidden = torch.tensor([[1.0, 0.0], [0.0, 1.0]])
|
||||
|
||||
logits = talker.compute_logits(hidden)
|
||||
|
||||
assert logits.shape == (2, 8)
|
||||
assert torch.allclose(logits[0], talker.head_code[0](hidden[0]))
|
||||
|
||||
|
||||
def test_compute_logits_forces_eos_for_empty_speech() -> None:
|
||||
talker = _make_talker()
|
||||
talker._request_audio_states["req-empty"] = {"finished": True}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[{"request_id": "req-empty", "codes": {"audio": torch.empty(0)}}],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
logits = talker.compute_logits(output.text_hidden_states)
|
||||
|
||||
assert output.multimodal_outputs["codes"]["audio"][0].numel() == 0
|
||||
assert output.multimodal_outputs["meta"]["finished"][0].item() is True
|
||||
assert int(logits.argmax(dim=-1)) == 7
|
||||
|
||||
|
||||
def test_native_duplex_chunk_budget_masks_eos_until_generate_chunk_is_full() -> None:
|
||||
assert _native_duplex_chunk_budget({}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, _DUPLEX_CODEC_TOKENS_PER_CHUNK)
|
||||
assert _native_duplex_chunk_budget({"turn_start": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0)
|
||||
assert _native_duplex_chunk_budget({"turn_end": True}) == (_DUPLEX_CODEC_TOKENS_PER_CHUNK, 0)
|
||||
|
||||
talker = _make_talker()
|
||||
talker._request_audio_states["req-chunk"] = {
|
||||
"finished": False,
|
||||
"step": 0,
|
||||
"max_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK,
|
||||
"min_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK,
|
||||
}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[{"request_id": "req-chunk", "codes": {"audio": torch.empty(0)}}],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
logits = talker.compute_logits(output.text_hidden_states)
|
||||
|
||||
assert logits[0, 7].item() == float("-inf")
|
||||
assert torch.isfinite(logits[0, 0])
|
||||
assert output.multimodal_outputs["meta"]["finished"][0].item() is False
|
||||
|
||||
|
||||
def test_native_duplex_chunk_budget_forces_eos_after_twenty_five_frames() -> None:
|
||||
talker = _make_talker()
|
||||
talker._request_audio_states["req-full"] = {
|
||||
"finished": False,
|
||||
"step": 24,
|
||||
"max_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK,
|
||||
"min_tokens": _DUPLEX_CODEC_TOKENS_PER_CHUNK,
|
||||
}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[{"request_id": "req-full", "codes": {"audio": torch.tensor([[3]])}}],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
logits = talker.compute_logits(output.text_hidden_states)
|
||||
|
||||
assert talker._request_audio_states["req-full"]["step"] == 25
|
||||
assert talker._request_audio_states["req-full"]["finished"] is True
|
||||
assert int(logits.argmax(dim=-1)) == 7
|
||||
assert output.multimodal_outputs["meta"]["finished"][0].item() is True
|
||||
|
||||
|
||||
def test_simplex_talker_does_not_force_eos_after_a_duplex_sized_chunk() -> None:
|
||||
talker = _make_talker()
|
||||
talker._request_audio_states["req-simplex"] = {"finished": False, "step": 0}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.tensor([[1.0, 0.0]]),
|
||||
model_intermediate_buffer=[{"request_id": "req-simplex", "codes": {"audio": torch.tensor([[3]])}}],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
logits = talker.compute_logits(output.text_hidden_states)
|
||||
|
||||
assert talker._request_audio_states["req-simplex"]["step"] == 0
|
||||
assert torch.allclose(logits[0], talker.head_code[0](output.text_hidden_states[0]).float())
|
||||
|
||||
|
||||
def test_native_duplex_prefill_records_generate_chunk_budget() -> None:
|
||||
talker = _make_talker()
|
||||
talker.emb_text = nn.Embedding(1, 2)
|
||||
talker.emb_code = nn.ModuleList([nn.Embedding(8, 2)])
|
||||
talker._text_eos_id = 0
|
||||
talker._tts_bos_id = 0
|
||||
|
||||
_, _, mid_turn = talker.preprocess(
|
||||
torch.zeros(2, dtype=torch.long),
|
||||
None,
|
||||
_omni_is_prefill=True,
|
||||
request_id="req-mid",
|
||||
native_duplex=True,
|
||||
tts_token_ids=torch.empty(0, dtype=torch.long),
|
||||
tts_hidden_states=torch.empty(0, 2),
|
||||
)
|
||||
_, _, boundary = talker.preprocess(
|
||||
torch.zeros(2, dtype=torch.long),
|
||||
None,
|
||||
_omni_is_prefill=True,
|
||||
request_id="req-end",
|
||||
native_duplex=True,
|
||||
meta={"turn_end": True},
|
||||
tts_token_ids=torch.empty(0, dtype=torch.long),
|
||||
tts_hidden_states=torch.empty(0, 2),
|
||||
)
|
||||
|
||||
assert mid_turn["audio_state"]["max_tokens"] == _DUPLEX_CODEC_TOKENS_PER_CHUNK
|
||||
assert mid_turn["audio_state"]["min_tokens"] == _DUPLEX_CODEC_TOKENS_PER_CHUNK
|
||||
assert boundary["audio_state"]["min_tokens"] == 0
|
||||
|
||||
|
||||
def test_make_omni_output_packs_the_previous_codec_id() -> None:
|
||||
talker = _make_talker()
|
||||
infos = [
|
||||
{"request_id": "req-a", "audio_codes": {"accumulated": torch.tensor([1])}},
|
||||
{"request_id": "req-b", "audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)}},
|
||||
{"request_id": "req-a", "codes": {"audio": torch.tensor([[2]])}},
|
||||
{"request_id": "req-b", "codes": {"audio": torch.tensor([[3]])}},
|
||||
]
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.tensor([[1.0, 0.0], [2.0, 0.0], [3.0, 0.0]]),
|
||||
torch.ones(2, 2),
|
||||
model_intermediate_buffer=infos,
|
||||
request_token_spans=[(0, 2), (2, 3)],
|
||||
request_token_spans=[(0, 1), (1, 2)],
|
||||
)
|
||||
|
||||
assert seen == [
|
||||
("req-a", [2.0, 0.0], [1]),
|
||||
("req-b", [3.0, 0.0], []),
|
||||
]
|
||||
batched_sample.assert_called_once()
|
||||
assert infos[0]["audio_codes"]["accumulated"].tolist() == [1, 2]
|
||||
assert infos[1]["audio_codes"]["accumulated"].tolist() == [3]
|
||||
assert set(output.multimodal_outputs) == {"codes", "meta"}
|
||||
assert "model_outputs" not in output.multimodal_outputs
|
||||
assert "sr" not in output.multimodal_outputs
|
||||
assert _routed(output, 0)["codes"]["audio"].tolist() == [[2]]
|
||||
assert _routed(output, 1)["codes"]["audio"].tolist() == [[3]]
|
||||
assert _routed(output, 0)["meta"]["finished"].item() is False
|
||||
assert set(output.multimodal_outputs["meta"]) == {"finished"}
|
||||
assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [0, 0]
|
||||
|
||||
|
||||
def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None:
|
||||
def test_talker_projects_request_aligned_duplex_metadata() -> None:
|
||||
talker = _make_talker()
|
||||
mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([2, 2]))
|
||||
infos = [
|
||||
{
|
||||
"request_id": "req-a",
|
||||
"native_duplex": True,
|
||||
"duplex": {"epoch": 3, "turn_id": 7},
|
||||
"ids": {"tts": [41]},
|
||||
"meta": {
|
||||
"native_duplex_segment_text": "first",
|
||||
"turn_eos_token_id": 99,
|
||||
},
|
||||
"audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)},
|
||||
"meta": {"native_duplex_segment_text": "first", "turn_eos_token_id": 99},
|
||||
"codes": {"audio": torch.empty(0)},
|
||||
},
|
||||
{
|
||||
"request_id": "req-b",
|
||||
"native_duplex": True,
|
||||
"duplex": {"epoch": 4, "turn_id": 8},
|
||||
"ids": {"tts": [42, 99]},
|
||||
"meta": {
|
||||
"native_duplex_segment_text": "second",
|
||||
"turn_eos_token_id": 99,
|
||||
},
|
||||
"audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)},
|
||||
"meta": {"native_duplex_segment_text": "second", "turn_eos_token_id": 99},
|
||||
"codes": {"audio": torch.empty(0)},
|
||||
},
|
||||
]
|
||||
|
||||
@@ -360,7 +421,6 @@ def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None:
|
||||
assert [value.item() for value in meta["native_duplex"]] == [True, True]
|
||||
assert [value.item() for value in meta["duplex_epoch"]] == [3, 4]
|
||||
assert [value.item() for value in meta["duplex_turn_id"]] == [7, 8]
|
||||
assert "native_duplex_segment_text" not in meta
|
||||
assert [bytes(value.tolist()).decode("utf-8") for value in meta["llm_output_text_utf8"]] == [
|
||||
"first",
|
||||
"second",
|
||||
@@ -368,14 +428,9 @@ def test_talker_projects_request_aligned_duplex_metadata(mocker) -> None:
|
||||
assert [value.item() for value in meta["turn_end"]] == [False, True]
|
||||
|
||||
|
||||
def test_talker_rejects_native_duplex_without_fence_identity(mocker) -> None:
|
||||
def test_talker_rejects_native_duplex_without_fence_identity() -> None:
|
||||
talker = _make_talker()
|
||||
mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([2]))
|
||||
info = {
|
||||
"request_id": "req-missing-fence",
|
||||
"native_duplex": True,
|
||||
"audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)},
|
||||
}
|
||||
info = {"request_id": "req-missing-fence", "native_duplex": True, "codes": {"audio": torch.empty(0)}}
|
||||
|
||||
with pytest.raises(RuntimeError, match="requires non-negative integer epoch and turn_id"):
|
||||
talker.make_omni_output(
|
||||
@@ -385,113 +440,35 @@ def test_talker_rejects_native_duplex_without_fence_identity(mocker) -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_incomplete_prefill_emits_no_code_and_does_not_advance_state(mocker) -> None:
|
||||
def test_decode_preprocess_embeds_the_sampled_codec_id() -> None:
|
||||
talker = _make_talker()
|
||||
sample = mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([2]))
|
||||
infos = [
|
||||
{
|
||||
"request_id": "req-prefill",
|
||||
"audio_state": {"step": 0},
|
||||
"audio_codes": {"accumulated": torch.empty(0, dtype=torch.long)},
|
||||
},
|
||||
{
|
||||
"request_id": "req-decode",
|
||||
"audio_state": {"step": 4},
|
||||
"audio_codes": {"accumulated": torch.tensor([1])},
|
||||
},
|
||||
]
|
||||
talker.emb_code = nn.ModuleList([nn.Embedding(8, 4)])
|
||||
with torch.no_grad():
|
||||
talker.emb_code[0].weight.copy_(torch.arange(32, dtype=torch.float32).reshape(8, 4))
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.tensor([[1.0, 0.0], [2.0, 0.0], [3.0, 0.0]]),
|
||||
model_intermediate_buffer=infos,
|
||||
request_token_spans=[(0, 2), (2, 3)],
|
||||
request_sample_eligible=[False, True],
|
||||
_, embeds, updates = talker.preprocess(
|
||||
torch.tensor([3]),
|
||||
None,
|
||||
request_id="req-decode",
|
||||
audio_state={"finished": False},
|
||||
)
|
||||
|
||||
sample.assert_called_once()
|
||||
assert sample.call_args.args[2] == ["req-decode"]
|
||||
assert infos[0]["audio_state"]["step"] == 0
|
||||
assert infos[0]["audio_codes"]["accumulated"].numel() == 0
|
||||
assert infos[1]["audio_state"]["step"] == 5
|
||||
assert _routed(output, 0)["codes"]["audio"].shape == (0, 1)
|
||||
assert _routed(output, 1)["codes"]["audio"].tolist() == [[2]]
|
||||
assert torch.equal(embeds, talker.emb_code[0](torch.tensor([3])))
|
||||
assert updates["codes"]["audio"].tolist() == [[3]]
|
||||
|
||||
|
||||
def test_eos_is_terminal_once_and_never_enters_codec_history(mocker) -> None:
|
||||
def test_decode_preprocess_drops_codec_eos() -> None:
|
||||
talker = _make_talker()
|
||||
sample = mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([7]))
|
||||
info = {
|
||||
"request_id": "req-stop",
|
||||
"audio_state": {"step": 3},
|
||||
"audio_codes": {"accumulated": torch.tensor([4, 5])},
|
||||
}
|
||||
talker.emb_code = nn.ModuleList([nn.Embedding(8, 4)])
|
||||
|
||||
first = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[info],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
first_logits = talker.compute_logits(first.text_hidden_states)
|
||||
second = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[info],
|
||||
request_token_spans=[(0, 1)],
|
||||
_, _, updates = talker.preprocess(
|
||||
torch.tensor([7]),
|
||||
None,
|
||||
request_id="req-eos",
|
||||
audio_state={"finished": False},
|
||||
)
|
||||
|
||||
sample.assert_called_once()
|
||||
assert info["audio_codes"]["accumulated"].tolist() == [4, 5]
|
||||
assert first.multimodal_outputs["codes"]["audio"][0].shape == (0, 1)
|
||||
assert first.multimodal_outputs["meta"]["finished"][0].item() is True
|
||||
assert second.multimodal_outputs["meta"]["finished"][0].item() is False
|
||||
assert first_logits.argmax(dim=-1).tolist() == [1]
|
||||
assert talker.compute_logits(second.text_hidden_states).argmax(dim=-1).tolist() == [1]
|
||||
|
||||
|
||||
def test_max_token_terminal_drops_unconsumed_codec_delta(mocker) -> None:
|
||||
talker = _make_talker()
|
||||
mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([3]))
|
||||
info = {
|
||||
"request_id": "req-limit",
|
||||
"audio_state": {"step": 1, "max_tokens": 2},
|
||||
"audio_codes": {"accumulated": torch.tensor([4, 5])},
|
||||
}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[info],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
|
||||
# MiniCPMTTS.generate_chunk samples once at the max-token boundary to
|
||||
# advance RNG state, but the sampled code is not fed into KV or returned.
|
||||
assert info["audio_codes"]["accumulated"].tolist() == [4, 5]
|
||||
assert output.multimodal_outputs["codes"]["audio"][0].shape == (0, 1)
|
||||
assert output.multimodal_outputs["meta"]["finished"][0].item() is True
|
||||
assert talker.compute_logits(output.text_hidden_states).argmax(dim=-1).tolist() == [1]
|
||||
|
||||
|
||||
def test_request_local_state_survives_missing_runner_buffer_update(mocker) -> None:
|
||||
talker = _make_talker()
|
||||
mocker.patch.object(talker, "_sample_audio_codes", return_value=torch.tensor([3]))
|
||||
first_info = {
|
||||
"request_id": "req-local-state",
|
||||
"audio_state": {"step": 1, "max_tokens": 3},
|
||||
"audio_codes": {"accumulated": torch.tensor([4])},
|
||||
}
|
||||
|
||||
talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[first_info],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
second = talker.make_omni_output(
|
||||
torch.ones(1, 2),
|
||||
model_intermediate_buffer=[{"request_id": "req-local-state"}],
|
||||
request_token_spans=[(0, 1)],
|
||||
)
|
||||
|
||||
assert second.multimodal_outputs["meta"]["finished"][0].item() is True
|
||||
assert talker._request_audio_states["req-local-state"]["step"] == 3
|
||||
assert updates["codes"]["audio"].numel() == 0
|
||||
|
||||
|
||||
def test_missing_conditioning_fails_clearly() -> None:
|
||||
@@ -524,18 +501,19 @@ def test_empty_speech_segment_finishes_without_sampling_codes() -> None:
|
||||
|
||||
assert torch.equal(embeds, talker.emb_text(torch.tensor([5, 6])))
|
||||
assert updates["audio_state"]["finished"] is True
|
||||
assert updates["codes"]["audio"].numel() == 0
|
||||
|
||||
# Stage 1's sampling min_tokens keeps scheduling decode steps until the stop
|
||||
# token becomes eligible, and those steps have no previous code to embed.
|
||||
_, decode_embeds, _ = talker.preprocess(
|
||||
# min_tokens can keep scheduling leftover decode rows after an empty
|
||||
# speech segment. Those rows have no previous codec id to embed.
|
||||
_, decode_embeds, decode_updates = talker.preprocess(
|
||||
torch.zeros(1, dtype=torch.long),
|
||||
None,
|
||||
request_id="req-empty",
|
||||
audio_state=updates["audio_state"],
|
||||
audio_codes=updates["audio_codes"],
|
||||
)
|
||||
|
||||
assert decode_embeds.shape == (1, 4)
|
||||
assert decode_updates["codes"]["audio"].numel() == 0
|
||||
|
||||
|
||||
def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> None:
|
||||
@@ -544,7 +522,7 @@ def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> Non
|
||||
condition = torch.arange(18, dtype=torch.float32).reshape(9, 2)
|
||||
mocker.patch.object(talker, "_build_condition_embeddings", return_value=condition)
|
||||
|
||||
_, embeds, _ = talker.preprocess(
|
||||
_, embeds, updates = talker.preprocess(
|
||||
torch.zeros(9, dtype=torch.long),
|
||||
None,
|
||||
_omni_is_prefill=True,
|
||||
@@ -556,46 +534,9 @@ def test_chunked_prefill_tail_aligns_condition_with_prompt_length(mocker) -> Non
|
||||
)
|
||||
|
||||
assert torch.equal(embeds, condition)
|
||||
assert updates["codes"]["audio"].numel() == 0
|
||||
state = talker._request_audio_states["req-chunked-prefill"]
|
||||
assert state["min_tokens"] == 50
|
||||
assert state["max_tokens"] == 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("meta", "expected_min_tokens"),
|
||||
[
|
||||
({"turn_start": True}, 0),
|
||||
({}, 26),
|
||||
({"turn_end": True}, 0),
|
||||
],
|
||||
)
|
||||
def test_native_duplex_prefill_uses_official_chunk_limits(
|
||||
mocker,
|
||||
meta,
|
||||
expected_min_tokens,
|
||||
) -> None:
|
||||
talker = _make_talker()
|
||||
talker.emb_text = nn.Embedding(1, 2)
|
||||
mocker.patch.object(
|
||||
talker,
|
||||
"_build_condition_embeddings",
|
||||
return_value=torch.ones(3, 2),
|
||||
)
|
||||
|
||||
talker.preprocess(
|
||||
torch.zeros(3, dtype=torch.long),
|
||||
None,
|
||||
_omni_is_prefill=True,
|
||||
request_id="req-duplex-chunk",
|
||||
native_duplex=True,
|
||||
meta=meta,
|
||||
tts_token_ids=torch.tensor([1]),
|
||||
tts_hidden_states=torch.ones(1, 2),
|
||||
)
|
||||
|
||||
state = talker._request_audio_states["req-duplex-chunk"]
|
||||
assert state["min_tokens"] == expected_min_tokens
|
||||
assert state["max_tokens"] == 26
|
||||
assert state["finished"] is False
|
||||
|
||||
|
||||
def test_native_duplex_condition_matches_official_text_plus_audio_bos() -> None:
|
||||
@@ -626,13 +567,11 @@ def test_native_duplex_condition_matches_official_text_plus_audio_bos() -> None:
|
||||
assert condition.shape[0] == token_ids.shape[0] + 1
|
||||
|
||||
|
||||
def test_request_cleanup_evicts_ar_rng_and_decode_state() -> None:
|
||||
def test_request_cleanup_evicts_decode_state() -> None:
|
||||
talker = _make_talker()
|
||||
talker._request_generators["req-done"] = torch.Generator()
|
||||
talker._request_audio_states["req-done"] = {"step": 1}
|
||||
talker._request_audio_states["req-done"] = {"finished": False}
|
||||
|
||||
talker.on_requests_finished(["req-done"])
|
||||
talker._flush_deferred_cleanup()
|
||||
|
||||
assert "req-done" not in talker._request_generators
|
||||
assert "req-done" not in talker._request_audio_states
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -2466,3 +2467,128 @@ def test_minicpmo_stage0_session_context_includes_resolved_ref_audio():
|
||||
|
||||
assert state.context_token_ids == [1, 2, 3, 151683, 151683, 4, 5]
|
||||
assert len(state.context_embeds) == 6
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-platform wiring: every AR model runner must reach the duplex hook.
|
||||
#
|
||||
# GPUARModelRunner and NPUARModelRunner are siblings -- neither inherits the
|
||||
# other's _sample -- so the NPU runner once copied GPU _sample without the
|
||||
# prepare_duplex_sampling call. Parse source instead of importing NPUARModelRunner:
|
||||
# that class needs vllm_ascend, which CPU test images do not have.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_REQUIRED_DUPLEX_CALLS = {
|
||||
"__init__": "_init_duplex_sampling_state",
|
||||
"load_model": "_resolve_duplex_sampling_hook",
|
||||
"_update_states": "_update_duplex_sampling_states",
|
||||
"_sample": "_apply_duplex_sampling",
|
||||
}
|
||||
|
||||
_MODEL_SAMPLER_CALL = "model_sample"
|
||||
|
||||
|
||||
def _repo_root():
|
||||
from pathlib import Path
|
||||
|
||||
return Path(__file__).parents[2]
|
||||
|
||||
|
||||
def _ar_runner_sources():
|
||||
root = _repo_root()
|
||||
paths = [root / "vllm_omni/worker/gpu_ar_model_runner.py"]
|
||||
paths.extend(sorted(root.glob("vllm_omni/platforms/*/worker/*_ar_model_runner.py")))
|
||||
return paths
|
||||
|
||||
|
||||
def _ar_runner_classdefs():
|
||||
root = _repo_root()
|
||||
found = {}
|
||||
for path in _ar_runner_sources():
|
||||
module = ast.parse(path.read_text())
|
||||
for node in module.body:
|
||||
if isinstance(node, ast.ClassDef) and node.name.endswith("ARModelRunner"):
|
||||
found[node.name] = (node, str(path.relative_to(root)))
|
||||
assert found, "no AR model runners discovered; the glob in _ar_runner_sources() is stale"
|
||||
return found
|
||||
|
||||
|
||||
def _method_calls(method):
|
||||
calls = {}
|
||||
for node in ast.walk(method):
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
||||
calls.setdefault(node.func.attr, node.lineno)
|
||||
return calls
|
||||
|
||||
|
||||
def _wires_duplex_sampling_directly(classdef) -> bool:
|
||||
bases = {base.id for base in classdef.bases if isinstance(base, ast.Name)}
|
||||
if "DuplexSamplingRunnerMixin" not in bases:
|
||||
return False
|
||||
methods = {node.name: node for node in classdef.body if isinstance(node, ast.FunctionDef)}
|
||||
return all(
|
||||
method_name in methods and required_call in _method_calls(methods[method_name])
|
||||
for method_name, required_call in _REQUIRED_DUPLEX_CALLS.items()
|
||||
)
|
||||
|
||||
|
||||
def test_every_ar_runner_reaches_the_duplex_sampling_hook():
|
||||
classdefs = _ar_runner_classdefs()
|
||||
wired = {name for name, (classdef, _path) in classdefs.items() if _wires_duplex_sampling_directly(classdef)}
|
||||
|
||||
for class_name, (classdef, path) in sorted(classdefs.items()):
|
||||
if class_name in wired:
|
||||
continue
|
||||
base_names = {base.id for base in classdef.bases if isinstance(base, ast.Name)}
|
||||
assert base_names & wired, (
|
||||
f"{path}::{class_name} neither wires the duplex sampling hook nor derives from a "
|
||||
f"runner that does (wired runners: {sorted(wired)}). Inherit DuplexSamplingRunnerMixin "
|
||||
f"and call {sorted(_REQUIRED_DUPLEX_CALLS.values())} from "
|
||||
f"{sorted(_REQUIRED_DUPLEX_CALLS)} respectively."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("class_name", ["GPUARModelRunner", "NPUARModelRunner"])
|
||||
def test_ar_runner_calls_every_duplex_sampling_hook_site(class_name: str):
|
||||
classdefs = _ar_runner_classdefs()
|
||||
assert class_name in classdefs, f"{class_name} not found by _ar_runner_sources()"
|
||||
classdef, path = classdefs[class_name]
|
||||
|
||||
bases = {base.id for base in classdef.bases if isinstance(base, ast.Name)}
|
||||
assert "DuplexSamplingRunnerMixin" in bases, (
|
||||
f"{path}::{class_name} must inherit DuplexSamplingRunnerMixin so the duplex "
|
||||
f"sampling hook is wired the same way on every platform"
|
||||
)
|
||||
|
||||
methods = {node.name: node for node in classdef.body if isinstance(node, ast.FunctionDef)}
|
||||
for method_name, required_call in _REQUIRED_DUPLEX_CALLS.items():
|
||||
method = methods.get(method_name)
|
||||
assert method is not None, f"{path}::{class_name}.{method_name} is missing; it must call {required_call}"
|
||||
calls = _method_calls(method)
|
||||
assert required_call in calls, f"{path}::{class_name}.{method_name} must call self.{required_call}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("class_name", ["GPUARModelRunner", "NPUARModelRunner"])
|
||||
def test_ar_runner_applies_duplex_sampling_before_the_model_sampler(class_name: str):
|
||||
classdef, path = _ar_runner_classdefs()[class_name]
|
||||
sample = next(node for node in classdef.body if isinstance(node, ast.FunctionDef) and node.name == "_sample")
|
||||
|
||||
apply_lineno = _method_calls(sample).get("_apply_duplex_sampling")
|
||||
sampler_lineno = next(
|
||||
(
|
||||
node.lineno
|
||||
for node in ast.walk(sample)
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == _MODEL_SAMPLER_CALL
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert apply_lineno is not None, f"{path}::{class_name}._sample must call self._apply_duplex_sampling"
|
||||
assert sampler_lineno is not None, (
|
||||
f"{path}::{class_name}._sample no longer calls {_MODEL_SAMPLER_CALL}(); update _MODEL_SAMPLER_CALL "
|
||||
f"so the ordering check keeps testing something"
|
||||
)
|
||||
assert apply_lineno < sampler_lineno, (
|
||||
f"{path}::{class_name}._sample calls self._apply_duplex_sampling at line {apply_lineno}, after "
|
||||
f"{_MODEL_SAMPLER_CALL}() at line {sampler_lineno}. The hook masks the logits and publishes the "
|
||||
f"row -> session map the sampler reads, so running it afterwards is a silent no-op."
|
||||
)
|
||||
Reference in new issue
Block a user