Merge main and preserve multi-API serving mode selection

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-17 11:49:47 +08:00
commit 79c629885a
155 files changed
+13313 -3163

No files matched your search

@@ -1,28 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Guards for the duplex-only API server (unified full-duplex framework).
A model whose pipeline declares ``duplex_plugin`` is served through
``DuplexOmni`` and nothing else: sessions run over ``/v1/realtime?duplex=1``
(alias ``/v1/duplex``) and every turn-based route reports "not available".
These tests lock that shape, which ``test_api_server_guards.py`` covers for the
diffusion and multi-stage servers:
* the pipeline probe that decides duplex vs turn-based at startup,
* the ``app.state`` snapshot of a duplex server (what is wired, what stays
``None``),
* both duplex websocket routes reaching the session handler, and
``/v1/realtime`` without the flag still reporting the turn-based Realtime API
as unavailable,
* the startup-warmup gate that holds real clients while the warmup session runs.
CPU-only: no engine is started; ``DuplexOmni`` is subclassed with just the
accessors the app-state wiring reads.
"""
"""CPU tests for serving-mode selection, duplex API wiring and warmup."""
from __future__ import annotations
import asyncio
from argparse import Namespace
from dataclasses import replace
from types import SimpleNamespace
import pytest
@@ -31,12 +15,14 @@ from fastapi.testclient import TestClient
from starlette.datastructures import State
from starlette.websockets import WebSocketDisconnect
from vllm_omni.config import stage_config
from vllm_omni.config.config_factory import StageConfigFactory
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig, PipelineConfig, StagePipelineConfig
from vllm_omni.engine.duplex.config import DuplexCapabilities
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
from vllm_omni.entrypoints.openai import api_server
from vllm_omni.utils.tracking_parser import TrackingNamespace
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -202,9 +188,10 @@ def _minimal_args(**overrides) -> SimpleNamespace:
(None, False),
],
)
def test_duplex_model_probe_follows_the_pipeline_plugin(monkeypatch, pipeline_config, expected: bool) -> None:
"""``duplex_plugin`` alone decides whether ``serve`` builds ``DuplexOmni``."""
def test_duplex_serving_requires_a_pipeline_plugin(monkeypatch, tmp_path, pipeline_config, expected: bool) -> None:
seen: dict[str, object] = {}
deploy_path = tmp_path / "deploy.yaml"
deploy_path.write_text("session_mode: duplex\n")
def fake_get_pipeline_config(*, model, trust_remote_code, deploy_config_path=None, **_kwargs):
seen.update(model=model, trust_remote_code=trust_remote_code, deploy_config_path=deploy_config_path)
@@ -212,12 +199,12 @@ def test_duplex_model_probe_follows_the_pipeline_plugin(monkeypatch, pipeline_co
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", fake_get_pipeline_config)
kwargs = {"trust_remote_code": True, "deploy_config": "deploy.yaml"}
assert api_server._is_duplex_model("demo-duplex-model", kwargs) is expected
kwargs = {"trust_remote_code": True, "deploy_config": str(deploy_path)}
assert api_server._should_serve_duplex("demo-duplex-model", kwargs) is expected
assert seen == {
"model": "demo-duplex-model",
"trust_remote_code": True,
"deploy_config_path": "deploy.yaml",
"deploy_config_path": str(deploy_path),
}
@@ -230,7 +217,98 @@ def test_duplex_model_probe_propagates_resolution_errors(monkeypatch) -> None:
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", fake_get_pipeline_config)
with pytest.raises(ValueError, match="trust_remote_code"):
api_server._is_duplex_model("demo-duplex-model", {})
api_server._should_serve_duplex("demo-duplex-model", {})
@pytest.fixture
def duplex_pipeline():
return PipelineConfig(
model_type="demo",
stages=(StagePipelineConfig(stage_id=0, model_stage="demo", final_output=True),),
duplex_plugin="pkg.mod.Plugin",
)
@pytest.mark.parametrize("session_mode", ["turn", "duplex"])
@pytest.mark.parametrize("config_source", ["explicit", "default", "basename", "inherited"])
@pytest.mark.asyncio
async def test_serving_mode_selects_engine_and_preserves_stage_config(
monkeypatch, tmp_path, duplex_pipeline, session_mode, config_source
):
base_path = tmp_path / "base.yaml"
base_path.write_text("session_mode: duplex\nstages:\n - stage_id: 0\n max_num_seqs: 4\n")
deploy_path = tmp_path / "deploy.yaml"
deploy_path.write_text(f"base_config: base.yaml\nsession_mode: {session_mode}\n")
if config_source == "inherited":
deploy_path = tmp_path / "overlay.yaml"
deploy_path.write_text("base_config: deploy.yaml\n")
pipeline = replace(duplex_pipeline, default_deploy_config_name=deploy_path.name)
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: pipeline)
monkeypatch.setattr(stage_config, "_DEPLOY_DIR", tmp_path)
kwargs = {}
if config_source != "default":
kwargs["deploy_config"] = deploy_path.name if config_source == "basename" else str(deploy_path)
created = []
class TurnEngine:
def __init__(self, model, **engine_kwargs):
self.model = model
self.kwargs = engine_kwargs
self.closed = False
created.append(self)
def shutdown(self):
self.closed = True
class DuplexEngine(TurnEngine):
pass
monkeypatch.setattr(api_server, "AsyncOmni", TurnEngine)
monkeypatch.setattr(api_server, "DuplexOmni", DuplexEngine)
args = TrackingNamespace(
Namespace(model="demo-model", disable_log_stats=True, **kwargs), frozenset({"model", *kwargs})
)
async with api_server.build_async_omni_from_stage_config(args) as engine:
assert type(engine) is (DuplexEngine if session_mode == "duplex" else TurnEngine)
assert engine.model == "demo-model"
assert engine.kwargs == {**kwargs, "log_stats": False}
assert not engine.closed
assert len(created) == 1
assert engine.closed
deploy = stage_config.load_deploy_config(deploy_path)
assert deploy.session_mode == session_mode
assert deploy.stages[0].max_num_seqs == 4
@pytest.mark.parametrize("contents", ["{}\n", "session_mode: null\n", "session_mode: typo\n"])
def test_duplex_serving_rejects_missing_or_invalid_mode(monkeypatch, tmp_path, duplex_pipeline, contents):
deploy_path = tmp_path / "deploy.yaml"
deploy_path.write_text(contents)
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: duplex_pipeline)
with pytest.raises(ValueError, match="session_mode: turn or duplex"):
api_server._should_serve_duplex("demo-model", {"deploy_config": str(deploy_path)})
def test_duplex_serving_requires_a_deploy_config(monkeypatch, duplex_pipeline):
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: duplex_pipeline)
with pytest.raises(ValueError, match="requires a deploy config"):
api_server._should_serve_duplex("demo-model", {})
def test_duplex_serving_propagates_deploy_load_errors(monkeypatch, tmp_path, duplex_pipeline):
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: duplex_pipeline)
with pytest.raises(FileNotFoundError):
api_server._should_serve_duplex("demo-model", {"deploy_config": str(tmp_path / "missing.yaml")})
@pytest.mark.parametrize("deploy_name,expected", [("minicpmo_4_5.yaml", True), ("minicpmo_4_5_turn.yaml", False)])
def test_minicpmo_serving_profiles(monkeypatch, deploy_name, expected):
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: MINICPMO_4_5_PIPELINE)
assert api_server._should_serve_duplex("openbmb/MiniCPM-o-4_5", {"deploy_config": deploy_name}) is expected
assert api_server._should_serve_duplex("openbmb/MiniCPM-o-4_5", {}) is True
# --------------------------------------------------------------------------- #
@@ -0,0 +1,302 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import torch
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.tts_adapters import moss_tts as adapter_module
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import MossTTSAdapter
from vllm_omni.model_executor.models.moss_tts import reference_encoder
from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
("hf_config", "expected_codec_path"),
[
(SimpleNamespace(), "OpenMOSS-Team/MOSS-Audio-Tokenizer"),
(
SimpleNamespace(codec_model_name_or_path="/models/custom-moss-codec"),
"/models/custom-moss-codec",
),
],
ids=["default-codec", "configured-codec"],
)
def test_realtime_components_use_the_realtime_model_and_codec(
monkeypatch: pytest.MonkeyPatch,
hf_config: SimpleNamespace,
expected_codec_path: str,
) -> None:
engine_client = SimpleNamespace(
model_config=SimpleNamespace(
model="OpenMOSS-Team/MOSS-TTS-Realtime",
hf_config=hf_config,
)
)
server = MossTTSAdapter(SpeechServingContext(server=object(), engine_client=engine_client))
tokenizer = object()
codec = type("Codec", (), {"to": lambda self, device: self, "eval": lambda self: self})()
processor_calls = []
class Processor:
def __init__(self, *, tokenizer):
processor_calls.append(tokenizer)
class_calls = []
tokenizer_calls = []
codec_calls = []
def fake_get_class(class_reference, model_id):
class_calls.append((class_reference, model_id))
return Processor
def fake_load_tokenizer(model_id, *, trust_remote_code):
tokenizer_calls.append((model_id, trust_remote_code))
return tokenizer
def fake_load_codec(model_id, *, trust_remote_code):
codec_calls.append((model_id, trust_remote_code))
return codec
monkeypatch.setattr(adapter_module, "get_class_from_dynamic_module", fake_get_class)
monkeypatch.setattr(adapter_module.AutoTokenizer, "from_pretrained", fake_load_tokenizer)
monkeypatch.setattr(adapter_module.AutoModel, "from_pretrained", fake_load_codec)
components = server._get_moss_realtime_components()
assert components[0] is tokenizer
assert components[2] is codec
assert server._get_moss_realtime_components() is components
assert class_calls == [
(
"processing_mossttsrealtime.MossTTSRealtimeProcessor",
"OpenMOSS-Team/MOSS-TTS-Realtime",
)
]
assert tokenizer_calls == [("OpenMOSS-Team/MOSS-TTS-Realtime", True)]
assert codec_calls == [(expected_codec_path, True)]
assert processor_calls == [tokenizer]
def test_realtime_serving_builds_the_talker_prompt(monkeypatch: pytest.MonkeyPatch) -> None:
server = MossTTSAdapter(SpeechServingContext(server=SimpleNamespace(uploaded_speakers={})))
server._moss_variant = "realtime"
tokenizer = object()
processor = object()
codec = object()
request_thread = threading.get_ident()
def load_components():
assert threading.get_ident() != request_thread
return tokenizer, processor, codec
server._get_moss_realtime_components = load_components
reference_codes = torch.arange(64, dtype=torch.int64).reshape(4, 16)
encode = AsyncMock(return_value=([reference_codes], {0: "reference-cache-key"}))
monkeypatch.setattr(server, "_encode_moss_references", encode)
build_call = None
def fake_build(actual_tokenizer, actual_processor, text, actual_codes):
nonlocal build_call
build_call = (actual_tokenizer, actual_processor, text, actual_codes)
return {
"prompt_token_ids": [10, 11],
"codes": {"ref": actual_codes},
"ids": {"all": [12]},
}
monkeypatch.setattr(adapter_module, "build_realtime_prompt", fake_build)
params = asyncio.run(
server._build_moss_tts_params(
OpenAICreateSpeechRequest(
input="speak this text",
ref_audio="data:audio/wav;base64,AAAA",
max_new_tokens=50,
)
)
)
encode.assert_awaited_once()
assert encode.call_args.kwargs == {"has_inline_ref_audio": False, "two_speaker": False}
assert encode.call_args.args[0].ref_audio == "data:audio/wav;base64,AAAA"
assert build_call == (tokenizer, processor, "speak this text", reference_codes)
assert params == {
"prompt_token_ids": [10, 11],
"codes": {"ref": reference_codes},
"ids": {"all": [12]},
"max_new_frames": [50],
"ref_audio_cache_key": "reference-cache-key",
}
assert "prompt_audio_array" not in params
def test_cancelled_component_waiter_does_not_duplicate_codec_load(monkeypatch):
adapter = MossTTSAdapter(
SpeechServingContext(
server=object(),
engine_client=SimpleNamespace(
model_config=SimpleNamespace(model="OpenMOSS-Team/MOSS-TTS-Realtime", hf_config=SimpleNamespace())
),
)
)
started = threading.Event()
release = threading.Event()
second_started = threading.Event()
loads = []
codec = type("Codec", (), {"to": lambda self, device: self, "eval": lambda self: self})()
def load_codec(*args, **kwargs):
loads.append(args)
started.set()
assert release.wait(5), "codec loader was not released"
return codec
monkeypatch.setattr(adapter_module, "get_class_from_dynamic_module", lambda *args: lambda **kwargs: object())
monkeypatch.setattr(adapter_module.AutoTokenizer, "from_pretrained", lambda *args, **kwargs: object())
monkeypatch.setattr(adapter_module.AutoModel, "from_pretrained", load_codec)
def second_load():
second_started.set()
return adapter._get_moss_realtime_components()
async def run():
first = asyncio.create_task(asyncio.to_thread(adapter._get_moss_realtime_components))
try:
assert await asyncio.to_thread(started.wait, 5)
first.cancel()
with pytest.raises(asyncio.CancelledError):
await first
second = asyncio.create_task(asyncio.to_thread(second_load))
try:
assert await asyncio.to_thread(second_started.wait, 5)
finally:
release.set()
components = await asyncio.wait_for(second, 5)
assert components[2] is codec
assert adapter._get_moss_realtime_components() is components
assert len(loads) == 1
finally:
release.set()
asyncio.run(run())
@pytest.mark.parametrize("voice_name", [None, "speaker"])
def test_realtime_reference_cache_reuses_codes(voice_name):
cache = SpeakerEmbeddingCache()
resolve = AsyncMock(return_value=([0.0], 24000, "audio-key"))
request_thread = threading.get_ident()
encode_calls = []
expected = torch.arange(64).reshape(4, 16)
def batch_encode(wavs, *, num_quantizers):
assert threading.get_ident() != request_thread
assert num_quantizers == 16
encode_calls.append(wavs)
return SimpleNamespace(audio_codes=expected.T.unsqueeze(1), audio_codes_lengths=torch.tensor([4]))
encoder = reference_encoder.build_reference_encoder(
SimpleNamespace(batch_encode=batch_encode), variant="realtime", speaker_cache=cache
)
async def run():
kwargs = dict(
resolve_ref_audio=resolve,
get_artifact_key=lambda key: "content-key",
voice_name=voice_name,
voice_created_at=1,
)
try:
first, first_key = await encoder.encode("reference", **kwargs)
second, second_key = await encoder.encode("reference", **kwargs)
assert torch.equal(first, expected)
assert torch.equal(second, expected)
assert first.dtype == torch.int64
assert first_key == "audio-key"
assert second_key == (None if voice_name else "audio-key")
second.zero_()
third, _ = await encoder.encode("another-reference", **kwargs)
assert torch.equal(third, expected)
assert len(encode_calls) == 1
assert resolve.await_count == (1 if voice_name else 3)
key = cache.make_cache_key(
voice_name or "ref:content-key", "moss_tts_realtime_nq16", 1 if voice_name else 0
)
assert cache.get(key)["codes"].dtype == torch.int32
if voice_name:
kwargs["voice_created_at"] = 2
await encoder.encode("reference", **kwargs)
assert len(encode_calls) == 2
assert resolve.await_count == 2
finally:
await encoder.aclose()
asyncio.run(run())
def test_realtime_reference_batch_trims_each_item():
cache = SpeakerEmbeddingCache()
codes = torch.arange(16 * 2 * 5).reshape(16, 2, 5)
calls = []
def batch_encode(wavs, *, num_quantizers):
calls.append(wavs)
assert num_quantizers == 16
assert [wav.shape for wav in wavs] == [torch.Size([8]), torch.Size([12])]
return SimpleNamespace(audio_codes=codes, audio_codes_lengths=torch.tensor([3, 5]))
encoder = reference_encoder.build_reference_encoder(
SimpleNamespace(batch_encode=batch_encode), variant="realtime", speaker_cache=cache
)
async def resolve(ref):
return [0.0] * (8 if ref == "short" else 12), 24000, ref
async def run():
try:
short, long = await asyncio.gather(
encoder.encode("short", resolve_ref_audio=resolve, get_artifact_key=lambda key: key),
encoder.encode("long", resolve_ref_audio=resolve, get_artifact_key=lambda key: key),
)
torch.testing.assert_close(short[0], codes[:, 0, :3].T)
torch.testing.assert_close(long[0], codes[:, 1, :5].T)
assert len(calls) == 1
finally:
await encoder.aclose()
asyncio.run(run())
def test_failed_realtime_reference_encoding_does_not_populate_cache():
cache = SpeakerEmbeddingCache()
resolve = AsyncMock(return_value=([0.0], 24000, "audio-key"))
def fail(*args, **kwargs):
raise ValueError("invalid reference")
encoder = reference_encoder.build_reference_encoder(
SimpleNamespace(batch_encode=fail), variant="realtime", speaker_cache=cache
)
async def run():
try:
with pytest.raises(ValueError, match="invalid reference"):
await encoder.encode("reference", resolve_ref_audio=resolve, get_artifact_key=lambda key: None)
assert cache.memory_bytes() == 0
finally:
await encoder.aclose()
asyncio.run(run())
@@ -5213,6 +5213,52 @@ class TestTTSAsyncOffloading:
for call in log_info.call_args_list
)
@pytest.mark.parametrize("word_timestamps", [False, True])
def test_streaming_timestamps_retain_complete_audio(self, qwen3_tts_server, mocker, word_timestamps):
from vllm import SamplingParams
from vllm.sampling_params import RequestOutputKind
defaults = [SamplingParams(), SamplingParams()]
qwen3_tts_server.engine_client.default_sampling_params_list = defaults
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock(
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
)
qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512)
request = OpenAICreateSpeechRequest(
input="hello", stream=True, response_format="pcm", word_timestamps=word_timestamps
)
asyncio.run(qwen3_tts_server._prepare_speech_generation(request))
params = qwen3_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"]
expected = RequestOutputKind.CUMULATIVE if word_timestamps else RequestOutputKind.DELTA
assert all(sp.output_kind == expected for sp in params)
assert all(sp.output_kind == RequestOutputKind.CUMULATIVE for sp in defaults)
assert all(sp is not original for sp, original in zip(params, defaults))
@pytest.mark.asyncio
async def test_cumulative_timestamp_audio_emits_each_sample_once(self, qwen3_tts_server):
audio = torch.linspace(-0.5, 0.5, 2400)
async def cumulative_generator():
for end in (800, 1600, 2400, 2400):
yield OmniRequestOutput(
request_id="req-cumulative",
final_output_type="audio",
_multimodal_output={"audio": audio[:end], "sr": 24000},
)
chunks = [
chunk
async for chunk in qwen3_tts_server._generate_pcm_chunks(
cumulative_generator(), "req-cumulative", cumulative_audio=True
)
]
assert len(chunks) == 3
assert len(b"".join(chunks)) == 2400 * 2
decoded = np.frombuffer(b"".join(chunks), dtype=np.int16)
assert np.all(np.diff(decoded.astype(np.int32)) >= 0)
def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
"""stream_format=sse should request delta-style multimodal outputs."""
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
@@ -56,6 +56,7 @@ def _build_test_app(
include_sample_rate=False,
tts_params=None,
collect=None,
cumulative_audio=False,
):
for chunk in (b"\x01\x02", b"\x03\x04\x05"):
yield (chunk, 24000) if include_sample_rate else chunk
@@ -386,9 +387,11 @@ class TestStreamingSpeechWebSocket:
include_sample_rate=False,
tts_params=None,
collect=None,
cumulative_audio=False,
):
assert request_start_s is not None
assert request_arrival_ts is not None
assert cumulative_audio is True
for chunk in (first_chunk, second_chunk):
yield (chunk, 1000) if include_sample_rate else chunk
if collect is not None:
@@ -480,6 +483,7 @@ class TestStreamingSpeechWebSocket:
include_sample_rate=False,
tts_params=None,
collect=None,
cumulative_audio=False,
):
chunk = b"\x01" * 1000
yield (chunk, 1000) if include_sample_rate else chunk