diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index eae3dee27..4d0715a7e 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -432,13 +432,19 @@ For a single H200, the optional deployment places both the talker and codec on logical GPU 0: ```bash -CUDA_VISIBLE_DEVICES=0 vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 \ +CUDA_VISIBLE_DEVICES=0 VLLM_OMNI_EVENT_DRIVEN_ORCH=1 \ + vllm serve OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 \ --omni --trust-remote-code \ - --deploy-config vllm_omni/deploy/moss_tts_local_h200.yaml + --deploy-config vllm_omni/deploy/moss_tts_local_h200.yaml \ + --disable-log-stats ``` Run from the repository root and select an available physical GPU with -`CUDA_VISIBLE_DEVICES`. This preset configures 256 request slots per stage, +`CUDA_VISIBLE_DEVICES`. The command enables event-driven orchestration and +disables detailed per-request statistics logging to reduce CPU overhead at +high concurrency. Remove `--disable-log-stats` when those statistics are needed; +keep these settings identical when comparing performance. +This preset configures 256 request slots per stage, a 32 GiB talker KV cache, talker CUDA Graph buckets through 512 scheduled tokens, and codec batch buckets through 256. It requires more than 80 GiB of GPU memory; request capacity also depends on input and generated lengths. The default diff --git a/tests/entrypoints/test_omni_entrypoints.py b/tests/entrypoints/test_omni_entrypoints.py index 4f9fc592b..a93ae0dbf 100644 --- a/tests/entrypoints/test_omni_entrypoints.py +++ b/tests/entrypoints/test_omni_entrypoints.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from __future__ import annotations import queue @@ -97,9 +100,11 @@ class FakeAsyncOmniEngine: ) -> None: self.model = model self.config_path = None - self.stage_configs: list[Any] = [] self.stage_metadata = stage_metadata or [THREE_STAGE_META[-1]] self.num_stages = len(self.stage_metadata) + self.stage_configs = [ + StageConfig(stage_id=i, model_stage="dummy-model").to_omegaconf() for i in range(self.num_stages) + ] self.default_sampling_params_list = default_sampling_params_list or [ SamplingParams(max_tokens=8) for _ in range(self.num_stages) ] diff --git a/tests/model_executor/models/moss_tts/test_local_output_policy.py b/tests/model_executor/models/moss_tts/test_local_output_policy.py new file mode 100644 index 000000000..4c367c049 --- /dev/null +++ b/tests/model_executor/models/moss_tts/test_local_output_policy.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import pytest +from vllm import SamplingParams +from vllm.sampling_params import RequestOutputKind + +from vllm_omni.entrypoints.omni_base import OmniBase +from vllm_omni.entrypoints.utils import coerce_param_message_types +from vllm_omni.model_executor.models.moss_tts.pipeline import MOSS_TTS_LOCAL_PIPELINE + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.parametrize("use_defaults", [False, True]) +def test_local_talker_control_outputs_preserve_codec_streaming(mocker, use_defaults): + base = object.__new__(OmniBase) + base.engine = mocker.Mock(num_stages=2) + base.sampling_constraints_list = [dict(stage.sampling_constraints) for stage in MOSS_TTS_LOCAL_PIPELINE.stages] + base.default_sampling_params_list = [ + base._apply_sampling_constraints(SamplingParams(max_tokens=8, seed=11), constraints) + for constraints in base.sampling_constraints_list + ] + caller = coerce_param_message_types( + [SamplingParams(max_tokens=8, seed=11), SamplingParams(max_tokens=8, seed=11)], True + ) + result = base.resolve_sampling_params_list(None if use_defaults else caller, allow_delta_coercion=True) + + assert result[0].output_kind == RequestOutputKind.FINAL_ONLY + assert result[0].detokenize is False + assert result[0].max_tokens == 8 + assert result[0].seed == 11 + assert result[1].output_kind == RequestOutputKind.DELTA + assert caller[0].output_kind == RequestOutputKind.DELTA + assert base.default_sampling_params_list[0].output_kind == RequestOutputKind.FINAL_ONLY + assert MOSS_TTS_LOCAL_PIPELINE.stages[0].final_output is False + assert MOSS_TTS_LOCAL_PIPELINE.stages[1].final_output_type == "audio" diff --git a/vllm_omni/entrypoints/omni_base.py b/vllm_omni/entrypoints/omni_base.py index 0aa6832d0..5b13fc246 100644 --- a/vllm_omni/entrypoints/omni_base.py +++ b/vllm_omni/entrypoints/omni_base.py @@ -354,7 +354,9 @@ class OmniBase(PDDisaggregationMixin): if len(normalized) != self.num_stages: raise ValueError(f"Expected {self.num_stages} sampling params, got {len(normalized)}") - if sampling_params_list is not None: + # Streaming coercion may also change a constrained output kind in + # the defaults (for example, an internal TTS stage's FINAL_ONLY). + if sampling_params_list is not None or allow_delta_coercion: normalized = [ self._apply_sampling_constraints(params, constraints) for params, constraints in zip(normalized, self.sampling_constraints_list, strict=True) diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index 4b91f752a..1f4f78c4e 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -868,9 +868,9 @@ def maybe_coerce_to_message_type(params: SamplingParams, is_streaming: bool): if params.output_kind == target_type: return params elif is_streaming and params.output_kind == RequestOutputKind.FINAL_ONLY: - logger.warning("Request appears to be streaming, but got request type final only!") + logger.debug("Coercing FINAL_ONLY output to DELTA for streaming") elif not is_streaming and params.output_kind == RequestOutputKind.DELTA: - logger.warning("Request appears to not be streaming, but got request type delta!") + logger.debug("Coercing DELTA output to FINAL_ONLY for non-streaming") if not params.skip_clone: params = params.clone() diff --git a/vllm_omni/model_executor/models/moss_tts/pipeline.py b/vllm_omni/model_executor/models/moss_tts/pipeline.py index f499654a4..0d6151313 100644 --- a/vllm_omni/model_executor/models/moss_tts/pipeline.py +++ b/vllm_omni/model_executor/models/moss_tts/pipeline.py @@ -1,8 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + # Copyright 2026 OpenMOSS and the vLLM-Omni team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). """Pipeline topology for all MOSS-TTS variants (2-stage: talker → codec).""" +from vllm.sampling_params import RequestOutputKind + from vllm_omni.config.stage_config import ( PipelineConfig, StageExecutionType, @@ -100,6 +105,10 @@ MOSS_TTS_LOCAL_PIPELINE = PipelineConfig( sampling_constraints={ "detokenize": False, "stop_token_ids": [151645], + # The worker connector streams codes directly to the codec. + # This internal stage only needs to publish its terminal + # result; codec audio output remains incremental. + "output_kind": RequestOutputKind.FINAL_ONLY, }, ), StagePipelineConfig(