From 65b3d41b3ad9525bdfa2cbe4bd0bd099282d55b2 Mon Sep 17 00:00:00 2001 From: boatman <1930807094@qq.com> Date: Mon, 17 Aug 2026 12:14:26 +0800 Subject: [PATCH] [Migrate] Move sampling parameter overrides from legacy dispatches to TTS adapters (#5272) Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- .../openai_api/test_audex_serving_guards.py | 33 +- .../openai_api/test_serving_speech.py | 114 ++++++- .../openai_api/test_tts_detection.py | 17 +- tests/tools/test_check_tts_adapter.py | 10 - tests/worker/test_gpu_ar_model_runner.py | 10 + tools/pre_commit/check_tts_adapter.py | 44 +-- .../entrypoints/openai/serving_speech.py | 281 +----------------- .../openai/tts_adapters/__init__.py | 16 +- .../entrypoints/openai/tts_adapters/audex.py | 70 ++++- .../openai/tts_adapters/audex_tta.py | 84 +++++- .../entrypoints/openai/tts_adapters/base.py | 48 ++- .../openai/tts_adapters/cosyvoice3.py | 68 ++++- .../openai/tts_adapters/fish_speech.py | 18 +- .../openai/tts_adapters/glm_tts.py | 67 ++++- .../openai/tts_adapters/higgs_audio_v2.py | 13 +- .../openai/tts_adapters/higgs_audio_v3.py | 13 +- .../openai/tts_adapters/indextts2.py | 11 +- .../openai/tts_adapters/ming_tts.py | 31 +- .../openai/tts_adapters/qwen3_tts.py | 13 +- .../openai/tts_adapters/voxcpm2.py | 13 +- .../openai/tts_adapters/voxtral.py | 13 +- 22 files changed, 565 insertions(+), 424 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 491123225..535bc1452 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,7 +66,7 @@ repos: entry: python tools/pre_commit/check_tts_adapter.py language: python pass_filenames: false - files: ^(vllm_omni/entrypoints/openai/(serving_speech\.py|tts_adapters/__init__\.py)|tools/pre_commit/check_tts_adapter\.py)$ + files: ^(vllm_omni/entrypoints/openai/serving_speech\.py|tools/pre_commit/check_tts_adapter\.py)$ # The budget-is-now-slack reminder is printed on a PASSING run, and # pre-commit hides output from passing hooks unless the hook is verbose. verbose: true diff --git a/tests/entrypoints/openai_api/test_audex_serving_guards.py b/tests/entrypoints/openai_api/test_audex_serving_guards.py index 68661865f..27ed11356 100644 --- a/tests/entrypoints/openai_api/test_audex_serving_guards.py +++ b/tests/entrypoints/openai_api/test_audex_serving_guards.py @@ -26,8 +26,13 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _serving(model_type: str = "audex") -> OmniOpenAIServingSpeech: + from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter + from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext + serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech) serving._tts_model_type = model_type + adapter_cls = resolve_adapter(model_type) + serving._adapter = adapter_cls(SpeechServingContext(server=serving)) if adapter_cls is not None else None serving._mark_ref_audio_artifact_ready_for_request = lambda request_id: None serving._discard_ref_audio_artifact_warmup = lambda request_id: None return serving @@ -154,7 +159,7 @@ class TestAudexCfgValidation: assert _adapter().validate(_speech_request(cfg_scale=None)) is None serving = TestAudexCfgInjection()._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": None}) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": "p"}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": "p"}, params) assert out.extra_args == {} @pytest.mark.parametrize("key", ["cfg_role", "cfg_pair_id", "cfg_null_prompt"]) @@ -177,21 +182,21 @@ class TestAudexCfgInjection: def test_no_extra_args_is_untouched(self): serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args=None) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) assert out.extra_args is None assert params.extra_args is None def test_scale_one_leaves_params_byte_identical(self): serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": 1.0}) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params) assert out.extra_args == {} def test_guided_request_gains_pair_contract(self): serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": 1.5}) cond = self._cond_prompt("Some transcription text here.") - out = serving._inject_audex_cfg_pair_args("req-42", {"prompt": cond}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-42", {"prompt": cond}, params) extra = out.extra_args assert extra["cfg_scale"] == 1.5 @@ -207,14 +212,14 @@ class TestAudexCfgInjection: serving = self._serving_with_fake_tokenizer() params = SimpleNamespace(extra_args={"cfg_scale": 1.5}) with pytest.raises(ValueError, match="adapter-built text prompt"): - serving._inject_audex_cfg_pair_args("req-1", {"prompt_token_ids": [1]}, params) + serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt_token_ids": [1]}, params) def test_injection_never_mutates_input_params(self): """The input may be the engine's SHARED default params (review P2): pair state must land only on the returned clone.""" serving = self._serving_with_fake_tokenizer() shared = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1) - out = serving._inject_audex_cfg_pair_args("req-7", {"prompt": self._cond_prompt()}, shared) + out = serving._adapter._inject_audex_cfg_pair_args("req-7", {"prompt": self._cond_prompt()}, shared) assert out is not shared assert shared.extra_args == {"cfg_scale": 1.5} assert shared.temperature == 0.1 @@ -252,11 +257,11 @@ def test_guided_injection_sets_guided_temperature(): serving = TestAudexCfgInjection()._serving_with_fake_tokenizer() cond = TestAudexCfgInjection()._cond_prompt() params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1) - out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": cond}, params) + out = serving._adapter._inject_audex_cfg_pair_args("req-1", {"prompt": cond}, params) assert out.temperature == 0.05 unguided = SimpleNamespace(extra_args={"cfg_scale": 1.0}, temperature=0.1) - out = serving._inject_audex_cfg_pair_args("req-2", {"prompt": cond}, unguided) + out = serving._adapter._inject_audex_cfg_pair_args("req-2", {"prompt": cond}, unguided) assert out.temperature == 0.1 @@ -340,7 +345,7 @@ class TestAudexTTAInjection: def test_default_injection_applies_rvq_and_cfg3(self): serving = self._serving() params = SimpleNamespace(extra_args=None) - out = serving._inject_audex_tta_args("req-9", self._prompt(), params) + out = serving._adapter._inject_audex_tta_args("req-9", self._prompt(), params) extra = out.extra_args rvq = extra["tta_rvq"] @@ -355,15 +360,15 @@ class TestAudexTTAInjection: def test_explicit_scale_one_disables_cfg_but_keeps_rvq(self): serving = self._serving() params = SimpleNamespace(extra_args={"cfg_scale": 1.0}) - out = serving._inject_audex_tta_args("req-1", self._prompt(), params) + out = serving._adapter._inject_audex_tta_args("req-1", self._prompt(), params) assert "tta_rvq" in out.extra_args assert "cfg_scale" not in out.extra_args assert "cfg_role" not in out.extra_args def test_rvq_contract_cached_per_process(self): serving = self._serving() - out_a = serving._inject_audex_tta_args("a", self._prompt(), SimpleNamespace(extra_args=None)) - out_b = serving._inject_audex_tta_args("b", self._prompt(), SimpleNamespace(extra_args=None)) + out_a = serving._adapter._inject_audex_tta_args("a", self._prompt(), SimpleNamespace(extra_args=None)) + out_b = serving._adapter._inject_audex_tta_args("b", self._prompt(), SimpleNamespace(extra_args=None)) assert out_a.extra_args["tta_rvq"] is out_b.extra_args["tta_rvq"] def test_injection_never_mutates_input_params(self): @@ -372,8 +377,8 @@ class TestAudexTTAInjection: per-request pair state must land only on the returned clone.""" serving = self._serving() shared = SimpleNamespace(extra_args=None) - out_1 = serving._inject_audex_tta_args("req-1", self._prompt(), shared) - out_2 = serving._inject_audex_tta_args("req-2", self._prompt(), shared) + out_1 = serving._adapter._inject_audex_tta_args("req-1", self._prompt(), shared) + out_2 = serving._adapter._inject_audex_tta_args("req-2", self._prompt(), shared) assert shared.extra_args is None assert out_1 is not shared and out_2 is not shared assert out_1.extra_args["cfg_pair_id"] == "req-1" diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 94d84d041..934c0ac64 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -52,6 +52,7 @@ from vllm_omni.model_executor.models.ming_tts.constants import ( KEY_SPEAKER_WAVEFORM, KEY_SPEAKER_WAVEFORM_LENGTHS, SPEAKER_EMBEDDING_DIM, + TEXT_EOS_TOKEN_ID, ) from vllm_omni.outputs import OmniRequestOutput @@ -3618,7 +3619,21 @@ class TestCosyVoice3Serving: assert error is not None assert "max_new_tokens" in error - def test_prepare_speech_generation_cosyvoice3(self, cosyvoice3_server, mocker: MockerFixture): + @pytest.mark.parametrize( + ("max_new_tokens", "expected_min_tokens", "expected_max_tokens"), + [ + (None, 10, 2000), + (5, 5, 5), + ], + ) + def test_prepare_speech_generation_cosyvoice3( + self, + cosyvoice3_server, + mocker: MockerFixture, + max_new_tokens: int | None, + expected_min_tokens: int, + expected_max_tokens: int, + ): cosyvoice3_server._build_cosyvoice3_prompt = mocker.AsyncMock( return_value={ "prompt": "Hello", @@ -3626,15 +3641,23 @@ class TestCosyVoice3Serving: "mm_processor_kwargs": {"prompt_text": "ref text", "sample_rate": 24000}, } ) - cosyvoice3_server._apply_cosyvoice3_dynamic_tokens = mocker.MagicMock(side_effect=lambda spl, req: spl) + cosyvoice3_server.model_config.hf_config = SimpleNamespace( + min_token_text_ratio=1, max_token_text_ratio=200, allowed_special=True + ) + cosyvoice3_server._cosyvoice3_tokenizer = mocker.MagicMock() + mocker.patch("vllm_omni.model_executor.models.cosyvoice3.utils.extract_text_token", return_value=[None, 10]) request = OpenAICreateSpeechRequest( input="Hello", ref_audio="data:audio/wav;base64,abc", ref_text="Reference text", + max_new_tokens=max_new_tokens, ) request_id, generator, tts_params = asyncio.run(cosyvoice3_server._prepare_speech_generation(request)) + sampling_params = cosyvoice3_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] + assert sampling_params.min_tokens == expected_min_tokens + assert sampling_params.max_tokens == expected_max_tokens assert request_id.startswith("speech-") assert generator == "generator" assert tts_params == {} @@ -3718,6 +3741,90 @@ class TestGLMTTSServing: assert text_token_len == 3 load_tokenizer.assert_called_once() + def test_prepare_speech_generation_glm_tts(self, glm_tts_server, mocker: MockerFixture): + glm_tts_server._build_glm_tts_prompt = mocker.AsyncMock( + return_value={ + "prompt": "Hello", + "multi_modal_data": {"audio": (np.zeros(24000), 24000)}, + "mm_processor_kwargs": {"prompt_text": "ref text", "sample_rate": 24000}, + } + ) + request = OpenAICreateSpeechRequest( + input="Hello", + ref_audio="data:audio/wav;base64,abc", + ref_text="Reference text", + ) + glm_tts_server.model_config.hf_config = SimpleNamespace(min_token_text_ratio=1, max_token_text_ratio=200) + glm_tts_server._estimate_glm_tts_text_token_len = mocker.MagicMock(return_value=10) + request_id, generator, tts_params = asyncio.run(glm_tts_server._prepare_speech_generation(request)) + sampling_params = glm_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] + + assert sampling_params.min_tokens == 10 + assert sampling_params.max_tokens == 2000 + assert request_id.startswith("speech-") + assert generator == "generator" + assert tts_params == {} + glm_tts_server._build_glm_tts_prompt.assert_awaited_once() + + +@pytest.fixture +def ming_tts_server(mocker: MockerFixture): + mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set()) + mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock( + model="inclusionAI/Ming-omni-tts-0.5B", + hf_config=SimpleNamespace(min_token_text_ratio=2, max_token_text_ratio=20), + ) + mock_engine_client.default_sampling_params_list = [ + SimpleNamespace(max_tokens=2048, min_tokens=None, extra_args=None) + ] + mock_engine_client.tts_batch_max_items = 32 + mock_engine_client.generate = mocker.MagicMock(return_value="generator") + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace( + model_stage="llm", model_arch="MingTTSForConditionalGeneration", worker_type="ar" + ), + tts_args={}, + ) + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + + +class TestMingTTSServing: + def test_prepare_speech_generation_ming(self, ming_tts_server, mocker: MockerFixture): + ming_tts_server._adapter.build = mocker.AsyncMock( + return_value=PreparedRequest( + prompt={"prompt": "hello"}, + tts_params={}, + model_type="ming_tts", + ) + ) + + request = OpenAICreateSpeechRequest( + input="Hello", + ref_audio="data:audio/wav;base64,abc", + ref_text="Reference text", + max_new_tokens=7, + ) + asyncio.run(ming_tts_server._prepare_speech_generation(request)) + + assert ming_tts_server._tts_model_type == "ming_tts" + ming_tts_server._adapter.build.assert_awaited_once() + sampling_params = ming_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"][0] + assert sampling_params.stop_token_ids == [TEXT_EOS_TOKEN_ID] + assert sampling_params.max_tokens == 8 + @pytest.fixture def ming_flash_omni_tts_server(mocker: MockerFixture): @@ -3901,6 +4008,9 @@ class TestTTSAsyncOffloading: model_type=adapter_model_type, ) + def apply_sampling_overrides(self, sampling_params_list, request, prompt, request_id=None): + return sampling_params_list + voxtral_server._tts_model_type = legacy_tts_model_type mocker.patch.object(voxtral_server, "_get_tts_adapter", return_value=FakeAdapter()) log_info = mocker.patch("vllm_omni.entrypoints.openai.serving_speech.logger.info") diff --git a/tests/entrypoints/openai_api/test_tts_detection.py b/tests/entrypoints/openai_api/test_tts_detection.py index c63e5c8e4..2ac27bb18 100644 --- a/tests/entrypoints/openai_api/test_tts_detection.py +++ b/tests/entrypoints/openai_api/test_tts_detection.py @@ -14,7 +14,6 @@ import itertools import pytest from vllm_omni.entrypoints.openai.tts_adapters import ( - LEGACY_TTS_DETECTORS, TTS_ADAPTER_REGISTRY, all_tts_stage_keys, detect_tts_model_type, @@ -282,27 +281,17 @@ def test_entry_stage_archs_is_ming_only(): assert tts_entry_stage_archs() == frozenset({"MingTTSForConditionalGeneration"}) -def test_every_detected_type_is_adapter_backed_or_declared_legacy(): - """Detection may only name a model that has an adapter or an explicit - :data:`LEGACY_TTS_DETECTORS` entry — never an undeclared string.""" - legacy_names = {d.name for d in LEGACY_TTS_DETECTORS} +def test_every_detected_type_is_adapter_backed(): + """Detection may only name a model that has a registered adapter.""" for stage, arch in itertools.product(_STAGES, _ARCHS): # full cross product on purpose detected = detect_tts_model_type(stage, arch) if detected is None: continue - assert detected in TTS_ADAPTER_REGISTRY or detected in legacy_names, ( + assert detected in TTS_ADAPTER_REGISTRY, ( f"detection produced undeclared model type {detected!r} for stage={stage!r} arch={arch!r}" ) -def test_legacy_detectors_have_no_adapter(): - """A legacy entry that gained an adapter must be deleted from the list.""" - for detector in LEGACY_TTS_DETECTORS: - assert resolve_adapter(detector.name) is None, ( - f"{detector.name!r} now has an adapter; remove it from LEGACY_TTS_DETECTORS" - ) - - def test_no_ambiguous_detectors_at_equal_priority(): """Two detectors that can match the same deployment must be ordered. diff --git a/tests/tools/test_check_tts_adapter.py b/tests/tools/test_check_tts_adapter.py index 486a629d5..47c6ac89f 100644 --- a/tests/tools/test_check_tts_adapter.py +++ b/tests/tools/test_check_tts_adapter.py @@ -17,9 +17,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools" / "pre_commit")) from check_tts_adapter import ( # noqa: E402 - MAX_LEGACY_DETECTORS, MAX_MODEL_TYPE_BRANCHES, - _legacy_detectors, _model_type_branches, main, ) @@ -60,12 +58,6 @@ def test_dict_dispatch_is_a_known_gap(): assert _count('TABLE = {"foo": 1}\nvalue = TABLE[self._tts_model_type]') == 0 -def test_legacy_detector_counting(): - src = 'X = (LegacyDetector(name="a", stage_keys=frozenset({"s"})), LegacyDetector(name="b"))' - found = _legacy_detectors(ast.parse(src)) - assert [name for _, name in found] == ["a", "b"] - - def test_gate_passes_on_current_tree(): """The recorded budgets must match reality, or the ratchet is decorative.""" assert main([]) == 0 @@ -74,6 +66,4 @@ def test_gate_passes_on_current_tree(): def test_budgets_are_the_real_counts(): root = Path(__file__).resolve().parents[2] serving = root / "vllm_omni" / "entrypoints" / "openai" / "serving_speech.py" - adapters = root / "vllm_omni" / "entrypoints" / "openai" / "tts_adapters" / "__init__.py" assert len(_model_type_branches(ast.parse(serving.read_text()))) == MAX_MODEL_TYPE_BRANCHES - assert len(_legacy_detectors(ast.parse(adapters.read_text()))) == MAX_LEGACY_DETECTORS diff --git a/tests/worker/test_gpu_ar_model_runner.py b/tests/worker/test_gpu_ar_model_runner.py index 116b5ee4d..967407169 100644 --- a/tests/worker/test_gpu_ar_model_runner.py +++ b/tests/worker/test_gpu_ar_model_runner.py @@ -2,6 +2,7 @@ import asyncio import copy from collections.abc import Sequence from types import SimpleNamespace +from typing import Any import numpy as np import pytest @@ -200,6 +201,15 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc async def build(self, request, sampling_params_list, has_inline_ref_audio): return PreparedRequest(prompt={"prompt": request.input}, tts_params={}, model_type="higgs_audio_v3") + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return sampling_params_list + engine = SimpleNamespace( errored=False, default_sampling_params_list=[SamplingParams(temperature=1.0, top_p=0.95, top_k=50)], diff --git a/tools/pre_commit/check_tts_adapter.py b/tools/pre_commit/check_tts_adapter.py index 626323b4f..bdd62f823 100644 --- a/tools/pre_commit/check_tts_adapter.py +++ b/tools/pre_commit/check_tts_adapter.py @@ -6,10 +6,8 @@ Per-model behaviour is being moved out of ``serving_speech.py`` and into nothing stopped a new model from adding one more ``self._tts_model_type == ...`` branch faster than the migration removed them. -This gate makes the two debt counters monotonically non-increasing: - -1. ``self._tts_model_type`` comparisons in ``serving_speech.py`` -2. entries in ``tts_adapters.LEGACY_TTS_DETECTORS`` (models with no adapter) +This gate makes the number of ``self._tts_model_type`` comparisons in +``serving_speech.py`` monotonically non-increasing. Lowering a budget after removing branches is expected and encouraged — the check insists on it, so the recorded numbers cannot silently drift upward. Raising one @@ -28,11 +26,9 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] SERVING_SPEECH = Path("vllm_omni/entrypoints/openai/serving_speech.py") -ADAPTERS_INIT = Path("vllm_omni/entrypoints/openai/tts_adapters/__init__.py") # Budgets. These may only go DOWN. See the module docstring. -MAX_MODEL_TYPE_BRANCHES = 27 -MAX_LEGACY_DETECTORS = 0 +MAX_MODEL_TYPE_BRANCHES = 20 _GUIDANCE = """ Move the per-model behaviour into its adapter under @@ -109,20 +105,6 @@ def _model_type_branches(tree: ast.AST) -> list[tuple[int, str]]: return found -def _legacy_detectors(tree: ast.AST) -> list[tuple[int, str]]: - """Every ``LegacyDetector(...)`` construction.""" - found: list[tuple[int, str]] = [] - for node in ast.walk(tree): - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "LegacyDetector": - name = "?" - for kw in node.keywords: - if kw.arg == "name" and isinstance(kw.value, ast.Constant): - name = str(kw.value.value) - found.append((node.lineno, name)) - found.sort() - return found - - def _check( label: str, path: Path, @@ -151,21 +133,18 @@ def _check( def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - # pre-commit passes the staged filenames; we always audit the same two files, + # pre-commit passes the staged filenames; we always audit the same file, # so they are accepted and ignored. parser.add_argument("filenames", nargs="*") parser.parse_args(argv) serving_path = REPO_ROOT / SERVING_SPEECH - adapters_path = REPO_ROOT / ADAPTERS_INIT - for path in (serving_path, adapters_path): - if not path.is_file(): - print(f"check_tts_adapter: expected file not found: {path}", file=sys.stderr) - return 1 + if not serving_path.is_file(): + print(f"check_tts_adapter: expected file not found: {serving_path}", file=sys.stderr) + return 1 notices: list[str] = [] branches = _model_type_branches(ast.parse(serving_path.read_text())) - detectors = _legacy_detectors(ast.parse(adapters_path.read_text())) errors = _check( "`_tts_model_type` comparisons", @@ -176,15 +155,6 @@ def main(argv: list[str] | None = None) -> int: "MAX_MODEL_TYPE_BRANCHES", notices, ) - errors += _check( - "unmigrated model types in LEGACY_TTS_DETECTORS", - ADAPTERS_INIT, - len(detectors), - MAX_LEGACY_DETECTORS, - [f"line {lineno}: {name}" for lineno, name in detectors], - "MAX_LEGACY_DETECTORS", - notices, - ) for notice in notices: print(f"check_tts_adapter: {notice}", file=sys.stderr) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 3612a2ff1..1b44d397d 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -92,19 +92,7 @@ logger = init_logger(__name__) # reference (issue #4644). Matches the offline example/test and upstream demo. _COSYVOICE3_PROMPT_DELIMITER = "<|endofprompt|>" _COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DELIMITER}" -_SAMPLING_MAX_TOKENS_TTS_MODEL_TYPES = { - "fish_tts", - "qwen3_tts", - "voxtral_tts", - "cosyvoice3", - "voxcpm2", - "higgs_audio_v2", - "higgs_audio_v3", - "indextts2", - "indextts2_5", - "audex", - "audex_tta", -} + # Audex contract: zero-codec / invalid generations arrive as empty terminal # payloads and must fail the request, never serialize as a successful empty # WAV. Covers both the TTS ("audex") and TTA ("audex_tta") pipelines. @@ -444,7 +432,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Cached per process: the CosyVoice3 Qwen tokenizer + resolved model # path used for dynamic-token sizing. Without this, every request # re-ran snapshot_download + reloaded the tokenizer (~100 ms on the - # TTFP critical path) in _apply_cosyvoice3_dynamic_tokens. + # TTFP critical path) in the CosyVoice3 adapter's sampling override. self._cosyvoice3_tokenizer = None # Determine TTS model type or None @@ -502,9 +490,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Resolve the per-model serving adapter (RFC #4327), keyed on the # detected model-type. Every dedicated TTS model has an adapter; the - # adapter owns request validation + prompt/param building. Sampling - # overrides and the model-type label remain in the orchestrator tail - # (keyed on ``_tts_model_type``) during this incremental migration. + # adapter owns request validation, prompt/param building, and sampling + # overrides; the model-type label remains available for compatibility. self._adapter = None if self._tts_stage is not None: adapter_cls = resolve_adapter(self._tts_model_type) @@ -2825,154 +2812,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._audex_tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, folder)) return self._audex_tokenizer - def _inject_audex_tta_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: - """Complete the Audex TTA contract on a CLONE of the stage-0 params. - - Always attaches the RVQ phase-mask contract (required for token - validity) and the CFG pair contract at the official default scale - 3.0 unless the caller passed ``cfg_scale`` (1.0 disables guidance). - - Returns the cloned params; the input is never mutated. On the - no-extra_params online path the input is an element of the engine's - SHARED ``default_sampling_params_list`` — writing per-request pair - state (``cfg_pair_id``/``cfg_null_prompt``) into it would race - concurrent requests and leak stale CFG metadata into later ones. - """ - import copy - - from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt - from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids - - cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt - if not isinstance(cond_prompt, str) or not cond_prompt: - raise ValueError("Audex TTA requires the adapter-built caption prompt") - - stage0_params = copy.deepcopy(stage0_params) - tokenizer = self._get_audex_tokenizer() - if self._audex_tta_rvq is None: - phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer) - self._audex_tta_rvq = { - "phase_token_ids": phase_token_ids, - "start_tid": start_tid, - "end_tid": end_tid, - # Official generation cap (decode truncates at 500 frames). - "codec_cap": 4000, - # The TTA prompt ends with . - "start_in_prompt": True, - } - - extra_args = getattr(stage0_params, "extra_args", None) - if extra_args is None: - extra_args = {} - stage0_params.extra_args = extra_args - extra_args["tta_rvq"] = self._audex_tta_rvq - - cfg_scale = extra_args.get("cfg_scale") - cfg_scale = 3.0 if cfg_scale is None else float(cfg_scale) - if cfg_scale <= 1.0: - extra_args.pop("cfg_scale", None) - return stage0_params - extra_args.update( - { - "cfg_scale": cfg_scale, - "cfg_role": "cond", - "cfg_pair_id": request_id, - "cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer), - } - ) - return stage0_params - - def _inject_audex_cfg_pair_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: - """Complete the Audex CFG contract on a CLONE of the stage-0 params. - - The adapter validated ``cfg_scale``; here (where the final request id - exists) the cond role, pair id, and the length-matched null prompt are - attached. ``cfg_scale`` absent/1.0 returns a clone value-identical to - the non-CFG flow (the key is dropped rather than forwarded). - - Returns the cloned params; the input is never mutated (it may be an - element of the engine's shared ``default_sampling_params_list`` — - see ``_inject_audex_tta_args``). - """ - import copy - - stage0_params = copy.deepcopy(stage0_params) - extra_args = getattr(stage0_params, "extra_args", None) - if not extra_args or extra_args.get("cfg_scale") is None: - if extra_args: - extra_args.pop("cfg_scale", None) - return stage0_params - cfg_scale = float(extra_args["cfg_scale"]) - if cfg_scale <= 1.0: - extra_args.pop("cfg_scale", None) - return stage0_params - - from vllm_omni.model_executor.models.audex.prompt import build_null_prompt - - cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt - if not isinstance(cond_prompt, str) or not cond_prompt: - raise ValueError("Audex CFG requires the adapter-built text prompt") - null_prompt = build_null_prompt(cond_prompt, self._get_audex_tokenizer()) - extra_args.update( - { - "cfg_scale": cfg_scale, - "cfg_role": "cond", - "cfg_pair_id": request_id, - "cfg_null_prompt": null_prompt, - } - ) - # Guided decoding sharpens the distribution; the unguided default - # temperature (0.1) adds excess sampling noise under CFG. Measured - # on the en-24 gate corpus at cfg 1.5: temp 0.1 -> CER 7.31%, - # temp 0.05 -> CER 6.87% (unguided baseline 7.24%). - stage0_params.temperature = 0.05 - return stage0_params - - def _apply_cosyvoice3_dynamic_tokens( - self, - sampling_params_list: list, - request: OpenAICreateSpeechRequest, - ) -> list: - """Set min/max tokens from tokenized text length (ratios target tokens, not chars).""" - import copy - - from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer - from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token - - sampling_params_list = copy.deepcopy(sampling_params_list) - hf_cfg = self.model_config.hf_config - # Build the Qwen tokenizer once per process (resolving the model dir via - # snapshot_download at most once) and reuse it across requests. - tokenizer = self._cosyvoice3_tokenizer - if tokenizer is None: - model_path = self.engine_client.model_config.model - if not os.path.isdir(model_path): - from huggingface_hub import snapshot_download - - model_path = snapshot_download(model_path) - tokenizer = get_qwen_tokenizer( - token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path), - skip_special_tokens=hf_cfg.skip_special_tokens, - version=hf_cfg.version, - ) - self._cosyvoice3_tokenizer = tokenizer - _, text_token_len = extract_text_token( - request.input, - tokenizer, - hf_cfg.allowed_special, - ) - min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) - max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) - sampling_params_list[0].min_tokens = max(1, int(text_token_len * min_ratio)) - sampling_params_list[0].max_tokens = min(2048, int(text_token_len * max_ratio)) - logger.info( - "CosyVoice3 dynamic tokens: text_tokens=%d, min_tokens=%d, max_tokens=%d", - text_token_len, - sampling_params_list[0].min_tokens, - sampling_params_list[0].max_tokens, - ) - return sampling_params_list - # ---- GLM-TTS helpers ---- async def _build_glm_tts_prompt( @@ -3172,9 +3011,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Build prompt + tts_params via the per-model adapter (RFC #4327). Every # dedicated TTS model resolves to an adapter that owns its validation, - # uploaded-speaker handling, and prompt/param building. Sampling - # overrides and the model-type label remain in the orchestrator tail - # below (keyed on ``_tts_model_type``) during this incremental migration. + # uploaded-speaker handling, prompt/param building, and sampling + # overrides. The model-type label remains available for compatibility. # Non-TTS deployments (no adapter) fall through to the rejection below. # Capture inline-ref-audio status BEFORE validate(): several adapters # apply uploaded speakers inside validate(), which sets request.ref_audio @@ -3225,63 +3063,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): model_type, ) - # CosyVoice3: set dynamic min/max tokens based on text length. - # The official model requires min_token_text_ratio to prevent early - # EOS and max_token_text_ratio to cap generation length. - if self._tts_model_type == "cosyvoice3" and sampling_params_list: - sampling_params_list = self._apply_cosyvoice3_dynamic_tokens(sampling_params_list, request) - - # GLM-TTS: set dynamic min/max tokens based on text length. - if self._tts_model_type == "glm_tts" and sampling_params_list: - import copy - - sampling_params_list = copy.deepcopy(sampling_params_list) - glm_metadata = prompt.get("additional_information") if isinstance(prompt, dict) else None - text_len_value = None - if isinstance(glm_metadata, dict): - text_len_value = glm_metadata.get("glm_tts_text_token_len") - if isinstance(text_len_value, list) and text_len_value: - text_len_value = text_len_value[0] - text_token_len = ( - int(text_len_value) - if text_len_value is not None - else self._estimate_glm_tts_text_token_len(request.input) - ) - hf_cfg = self.model_config.hf_config - min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) - max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) - stage_min_tokens = getattr(sampling_params_list[0], "min_tokens", None) - stage_max_tokens = getattr(sampling_params_list[0], "max_tokens", None) - cap_candidates = [int(cap) for cap in (stage_max_tokens, request.max_new_tokens) if cap is not None] - hard_cap = min(cap_candidates) if cap_candidates else None - - min_tokens = max(1, int(text_token_len * min_ratio)) - if stage_min_tokens is not None: - min_tokens = max(min_tokens, int(stage_min_tokens)) - if hard_cap is not None: - min_tokens = min(min_tokens, hard_cap) - - max_tokens = max(min_tokens, int(text_token_len * max_ratio)) - if hard_cap is not None: - max_tokens = min(max_tokens, hard_cap) - sampling_params_list[0].min_tokens = min_tokens - sampling_params_list[0].max_tokens = max_tokens - seed = getattr(request, "seed", None) - if seed is not None: - sampling_params_list[0].seed = seed - logger.info( - "GLM-TTS dynamic tokens: text_tokens=%d, min_ratio=%s, max_ratio=%s, " - "stage_min=%s, stage_max=%s, request_max=%s, min_tokens=%d, max_tokens=%d", - text_token_len, - min_ratio, - max_ratio, - stage_min_tokens, - stage_max_tokens, - request.max_new_tokens, - min_tokens, - max_tokens, - ) - # Apply model-specific extra parameters if request.extra_params is not None and sampling_params_list: if not isinstance(request.extra_params, dict): @@ -3300,52 +3081,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): sampling_params_list[0].extra_args.update(request.extra_params) logger.info("Applied extra_params: %s", request.extra_params) - # Audex CFG: turn an adapter-validated cfg_scale into the engine-side - # pair contract. This must run here (not in the adapter) because the - # pair id is the final request_id, which the adapter never sees. - # The injectors return a CLONE: without extra_params above, - # sampling_params_list still holds the engine's shared default params, - # which must never carry per-request CFG state. - if self._tts_model_type == "audex" and sampling_params_list: - sampling_params_list[0] = self._inject_audex_cfg_pair_args(request_id, prompt, sampling_params_list[0]) - elif self._tts_model_type == "audex_tta" and sampling_params_list: - sampling_params_list[0] = self._inject_audex_tta_args(request_id, prompt, sampling_params_list[0]) - - # Some TTS model defaults come from deploy YAML. Their AR - # generation length is controlled by SamplingParams.max_tokens, so only - # override it when the caller explicitly requests max_new_tokens. - if ( - self._tts_model_type in _SAMPLING_MAX_TOKENS_TTS_MODEL_TYPES - and request.max_new_tokens is not None - and sampling_params_list - ): - import copy - - sampling_params_list = copy.deepcopy(sampling_params_list) - sampling_params_list[0].max_tokens = request.max_new_tokens - if self._tts_model_type == "cosyvoice3": - sampling_params_list[0].min_tokens = min( - getattr(sampling_params_list[0], "min_tokens", 0), - request.max_new_tokens, - ) - elif self._tts_model_type == "ming_tts" and sampling_params_list: - import copy - - from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( - MOE_TEXT_EOS_TOKEN_ID, - TEXT_EOS_TOKEN_ID, - ) - - hf_config = self.engine_client.model_config.hf_config - is_moe = getattr(hf_config, "model_type", "") == "bailingmm" - stop_token_id = MOE_TEXT_EOS_TOKEN_ID if is_moe else TEXT_EOS_TOKEN_ID - - sampling_params_list = copy.deepcopy(sampling_params_list) - sampling_params_list[0].stop_token_ids = [int(stop_token_id)] - if request.max_new_tokens is not None: - # Ming emits TEXT_EOS after the latent decode budget is exhausted, so - # Stage-0 needs one extra token beyond ming_max_decode_steps. - sampling_params_list[0].max_tokens = int(request.max_new_tokens) + 1 + # Apply adapter-owned sampling overrides, including request-level token + # limits and model-specific dynamic token or stop-token configuration. + if sampling_params_list and (adapter := self._get_tts_adapter()) is not None: + sampling_params_list = adapter.apply_sampling_overrides(sampling_params_list, request, prompt, request_id) if request.seed is not None and sampling_params_list: import copy diff --git a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py index cd44f0ff5..74597a2d1 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/__init__.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/__init__.py @@ -9,8 +9,6 @@ key alone cannot distinguish. That metadata is the single source of truth. :func:`detect_tts_model_type` and :func:`all_tts_stage_keys` derive the whole stage -> model-type mapping from it, so a new TTS model is one adapter file and needs no edit to ``serving_speech.py``. -Models still awaiting an adapter are listed explicitly in -:data:`LEGACY_TTS_DETECTORS`. """ from vllm.logger import init_logger @@ -18,7 +16,6 @@ from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters.base import ( ARTTSAdapter, DiffusionTTSAdapter, - LegacyDetector, OutputPolicy, PreparedRequest, SpeechServingContext, @@ -29,10 +26,6 @@ logger = init_logger(__name__) TTS_ADAPTER_REGISTRY: dict[str, type[TTSModelAdapter]] = {} -#: Model types the serving layer detects that have no adapter yet. See -#: :class:`LegacyDetector`. This list must only ever shrink. -LEGACY_TTS_DETECTORS: tuple[LegacyDetector, ...] = () - def register_tts_adapter(cls: type[TTSModelAdapter]) -> type[TTSModelAdapter]: """Class decorator: index ``cls`` under its ``name`` (model-type).""" @@ -57,16 +50,13 @@ def resolve_adapter(model_type: str | None) -> type[TTSModelAdapter] | None: return TTS_ADAPTER_REGISTRY.get(model_type) -def iter_tts_detectors() -> list[type[TTSModelAdapter] | LegacyDetector]: +def iter_tts_detectors() -> list[type[TTSModelAdapter]]: """Every detector, in resolution order. Sorted by ``detect_priority`` then ``name``, so the order is total and does not depend on adapter import order. """ - detectors: list[type[TTSModelAdapter] | LegacyDetector] = [ - *TTS_ADAPTER_REGISTRY.values(), - *LEGACY_TTS_DETECTORS, - ] + detectors = list(TTS_ADAPTER_REGISTRY.values()) detectors.sort(key=lambda d: (d.detect_priority, d.name)) return detectors @@ -132,12 +122,10 @@ from vllm_omni.entrypoints.openai.tts_adapters import ( # noqa: E402,F401 __all__ = [ "ARTTSAdapter", "DiffusionTTSAdapter", - "LegacyDetector", "OutputPolicy", "PreparedRequest", "SpeechServingContext", "TTSModelAdapter", - "LEGACY_TTS_DETECTORS", "TTS_ADAPTER_REGISTRY", "all_tts_model_types", "all_tts_stage_keys", diff --git a/vllm_omni/entrypoints/openai/tts_adapters/audex.py b/vllm_omni/entrypoints/openai/tts_adapters/audex.py index 4533a420b..ab7c7c25e 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/audex.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/audex.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Audex (Nemotron-Labs-Audex-2B) TTS serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt if TYPE_CHECKING: @@ -17,8 +17,8 @@ if TYPE_CHECKING: AUDEX_CFG_SCALE_MIN = 1.0 AUDEX_CFG_SCALE_MAX = 10.0 -# Internal CFG pair-plumbing keys; injected by the serving layer once the -# request id exists, never accepted from callers. +# Internal CFG pair-plumbing keys; injected by this adapter once it receives +# the final request id, never accepted from callers. _AUDEX_INTERNAL_CFG_KEYS = ("cfg_role", "cfg_pair_id", "cfg_null_prompt") @@ -74,3 +74,65 @@ class AudexAdapter(ARTTSAdapter): ) -> PreparedRequest: prompt = {"prompt": build_cond_prompt(request.input)} return PreparedRequest(prompt=prompt, tts_params={}, model_type="audex") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + if request_id is None: + raise ValueError("Audex CFG requires a request id") + stage0_params = sampling_params_list[0] + sampling_params_list[0] = self._inject_audex_cfg_pair_args( + request_id=request_id, prompt=prompt, stage0_params=stage0_params + ) + return apply_max_new_tokens(sampling_params_list, request) + + def _inject_audex_cfg_pair_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: + """Complete the Audex CFG contract on a CLONE of the stage-0 params. + + The adapter validated ``cfg_scale``; here (where the final request id + exists) the cond role, pair id, and the length-matched null prompt are + attached. ``cfg_scale`` absent/1.0 returns a clone value-identical to + the non-CFG flow (the key is dropped rather than forwarded). + + Returns the cloned params; the input is never mutated (it may be an + element of the engine's shared ``default_sampling_params_list`` — + see ``_inject_audex_tta_args``). + """ + server = self.ctx.server + import copy + + stage0_params = copy.deepcopy(stage0_params) + extra_args = getattr(stage0_params, "extra_args", None) + if not extra_args or extra_args.get("cfg_scale") is None: + if extra_args: + extra_args.pop("cfg_scale", None) + return stage0_params + cfg_scale = float(extra_args["cfg_scale"]) + if cfg_scale <= 1.0: + extra_args.pop("cfg_scale", None) + return stage0_params + + from vllm_omni.model_executor.models.audex.prompt import build_null_prompt + + cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt + if not isinstance(cond_prompt, str) or not cond_prompt: + raise ValueError("Audex CFG requires the adapter-built text prompt") + null_prompt = build_null_prompt(cond_prompt, server._get_audex_tokenizer()) + extra_args.update( + { + "cfg_scale": cfg_scale, + "cfg_role": "cond", + "cfg_pair_id": request_id, + "cfg_null_prompt": null_prompt, + } + ) + # Guided decoding sharpens the distribution; the unguided default + # temperature (0.1) adds excess sampling noise under CFG. Measured + # on the en-24 gate corpus at cfg 1.5: temp 0.1 -> CER 7.31%, + # temp 0.05 -> CER 6.87% (unguided baseline 7.24%). + stage0_params.temperature = 0.05 + return stage0_params diff --git a/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py b/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py index f754f0158..33dd2294a 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/audex_tta.py @@ -2,16 +2,16 @@ """Audex (Nemotron-Labs-Audex-2B) text-to-audio serving adapter. Caption → general audio through the ``audex_tta`` pipeline. -The adapter builds the TTA prompt; the serving layer completes the request -after the final request id exists: the RVQ phase-mask contract +The adapter builds the TTA prompt and, after receiving the final request id, +completes the sampling parameters with the RVQ phase-mask contract (``extra_args["tta_rvq"]``) and the CFG pair contract (official default scale 3.0 — guidance is effectively mandatory for TTA quality). """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt if TYPE_CHECKING: @@ -21,8 +21,7 @@ AUDEX_TTA_DEFAULT_CFG_SCALE = 3.0 AUDEX_TTA_CFG_SCALE_MIN = 1.0 AUDEX_TTA_CFG_SCALE_MAX = 10.0 -# Internal plumbing keys; injected by the serving layer, never accepted -# from callers. +# Internal plumbing keys; injected by this adapter, never accepted from callers. _INTERNAL_KEYS = ("cfg_role", "cfg_pair_id", "cfg_null_prompt", "tta_rvq") @@ -67,3 +66,76 @@ class AudexTTAAdapter(ARTTSAdapter): ) -> PreparedRequest: prompt = {"prompt": build_tta_cond_prompt(request.input)} return PreparedRequest(prompt=prompt, tts_params={}, model_type="audex_tta") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + if request_id is None: + raise ValueError("Audex TTA requires a request id") + stage0_params = sampling_params_list[0] + sampling_params_list[0] = self._inject_audex_tta_args( + request_id=request_id, prompt=prompt, stage0_params=stage0_params + ) + return apply_max_new_tokens(sampling_params_list, request) + + def _inject_audex_tta_args(self, request_id: str, prompt: Any, stage0_params: Any) -> Any: + """Complete the Audex TTA contract on a CLONE of the stage-0 params. + + Always attaches the RVQ phase-mask contract (required for token + validity) and the CFG pair contract at the official default scale + 3.0 unless the caller passed ``cfg_scale`` (1.0 disables guidance). + + Returns the cloned params; the input is never mutated. On the + no-extra_params online path the input is an element of the engine's + SHARED ``default_sampling_params_list`` — writing per-request pair + state (``cfg_pair_id``/``cfg_null_prompt``) into it would race + concurrent requests and leak stale CFG metadata into later ones. + """ + server = self.ctx.server + import copy + + from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt + from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids + + cond_prompt = prompt.get("prompt") if isinstance(prompt, dict) else prompt + if not isinstance(cond_prompt, str) or not cond_prompt: + raise ValueError("Audex TTA requires the adapter-built caption prompt") + + stage0_params = copy.deepcopy(stage0_params) + tokenizer = server._get_audex_tokenizer() + if server._audex_tta_rvq is None: + phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer) + server._audex_tta_rvq = { + "phase_token_ids": phase_token_ids, + "start_tid": start_tid, + "end_tid": end_tid, + # Official generation cap (decode truncates at 500 frames). + "codec_cap": 4000, + # The TTA prompt ends with . + "start_in_prompt": True, + } + + extra_args = getattr(stage0_params, "extra_args", None) + if extra_args is None: + extra_args = {} + stage0_params.extra_args = extra_args + extra_args["tta_rvq"] = server._audex_tta_rvq + + cfg_scale = extra_args.get("cfg_scale") + cfg_scale = 3.0 if cfg_scale is None else float(cfg_scale) + if cfg_scale <= 1.0: + extra_args.pop("cfg_scale", None) + return stage0_params + extra_args.update( + { + "cfg_scale": cfg_scale, + "cfg_role": "cond", + "cfg_pair_id": request_id, + "cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer), + } + ) + return stage0_params diff --git a/vllm_omni/entrypoints/openai/tts_adapters/base.py b/vllm_omni/entrypoints/openai/tts_adapters/base.py index b8679041c..643d71ad6 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/base.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/base.py @@ -7,9 +7,7 @@ model's request normalization, validation, prompt/param building, sampling overrides, and output policy, so adding a model means writing one adapter file instead of editing the shared serving module in ~10 scattered places. -See the RFC for the full design (issue #4327). This is the foundation landed in -the first migration PR; Qwen3-TTS is the first model routed through it while the -remaining models stay on the legacy path until individually migrated. +See the RFC for the full design (issue #4327). """ from abc import ABC, abstractmethod @@ -39,6 +37,21 @@ def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: di return _conditioning_cache_salt_fn(request, tts_params) +def apply_max_new_tokens( + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", +) -> list: + """Apply a request-level ``max_new_tokens`` limit.""" + if request.max_new_tokens is None: + return sampling_params_list + + import copy + + sampling_params_list = copy.deepcopy(sampling_params_list) + sampling_params_list[0].max_tokens = request.max_new_tokens + return sampling_params_list + + @dataclass class OutputPolicy: """How the orchestrator aggregates engine output for a model. @@ -184,6 +197,8 @@ class TTSModelAdapter(ABC): self, sampling_params_list: list, request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, ) -> list: """Apply model-specific sampling mutations. @@ -228,37 +243,10 @@ class DiffusionTTSAdapter(TTSModelAdapter): return frozenset(params) if params is not None else frozenset() -@dataclass(frozen=True) -class LegacyDetector: - """A model type the serving layer detects but has no adapter for yet. - - Detection has to keep naming these models so ``serving_speech.py`` can route - them down its legacy path, but they own none of the adapter contract. Each - entry is a migration debt: delete it when the model gets an adapter. The - pre-commit gate (``tools/pre_commit/check_tts_adapter.py``) fails if the list - grows. - - Exposes the subset of the :class:`TTSModelAdapter` class surface that - detection reads, so both kinds live in one sorted sequence. - """ - - name: str - stage_keys: frozenset[str] = frozenset() - model_archs: frozenset[str] = frozenset() - arch_identifies_entry_stage: bool = False - detect_priority: int = 100 - - def matches(self, model_stage: str | None, model_arch: str | None) -> bool: - if model_arch is not None and model_arch in self.model_archs: - return True - return model_stage is not None and model_stage in self.stage_keys - - # Re-exported here to avoid import cycles at call sites. __all__ = [ "ARTTSAdapter", "DiffusionTTSAdapter", - "LegacyDetector", "OutputPolicy", "PreparedRequest", "SpeechServingContext", diff --git a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py index 85648442f..ec1c145c7 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/cosyvoice3.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """CosyVoice3 TTS serving adapter.""" -from typing import TYPE_CHECKING +import os +from typing import TYPE_CHECKING, Any + +from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest @@ -9,6 +12,8 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +logger = init_logger(__name__) + @register_tts_adapter class CosyVoice3Adapter(ARTTSAdapter): @@ -47,6 +52,63 @@ class CosyVoice3Adapter(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: prompt = await self.ctx.server._build_cosyvoice3_prompt(request, has_inline_ref_audio=has_inline_ref_audio) - # NOTE: CosyVoice3 dynamic-token sampling stays in the orchestrator tail - # (keyed on _tts_model_type) during this incremental migration. return PreparedRequest(prompt=prompt, tts_params={}, model_type="cosyvoice3") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + """Set dynamic min/max tokens based on tokenized text length. + + The official model uses ``min_token_text_ratio`` to prevent early EOS + and ``max_token_text_ratio`` to cap the generated token count. + """ + import copy + + from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer + from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token + + server = self.ctx.server + sampling_params_list = copy.deepcopy(sampling_params_list) + hf_cfg = server.model_config.hf_config + # Build the Qwen tokenizer once per process (resolving the model dir via + # snapshot_download at most once) and reuse it across requests. + tokenizer = server._cosyvoice3_tokenizer + if tokenizer is None: + model_path = server.engine_client.model_config.model + if not os.path.isdir(model_path): + from huggingface_hub import snapshot_download + + model_path = snapshot_download(model_path) + tokenizer = get_qwen_tokenizer( + token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path), + skip_special_tokens=hf_cfg.skip_special_tokens, + version=hf_cfg.version, + ) + server._cosyvoice3_tokenizer = tokenizer + _, text_token_len = extract_text_token( + request.input, + tokenizer, + hf_cfg.allowed_special, + ) + min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) + max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) + sampling_params_list[0].min_tokens = max(1, int(text_token_len * min_ratio)) + sampling_params_list[0].max_tokens = min(2048, int(text_token_len * max_ratio)) + + if request.max_new_tokens is not None: + sampling_params_list[0].max_tokens = request.max_new_tokens + sampling_params_list[0].min_tokens = min( + getattr(sampling_params_list[0], "min_tokens", 0), + request.max_new_tokens, + ) + logger.info( + "CosyVoice3 dynamic tokens: text_tokens=%d, min_tokens=%d, max_tokens=%d", + text_token_len, + sampling_params_list[0].min_tokens, + sampling_params_list[0].max_tokens, + ) + return sampling_params_list diff --git a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py index e8080a427..48d5886c4 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py @@ -1,10 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 """Fish Speech serving adapter (retires the legacy ``_is_fish_speech`` flag).""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, conditioning_cache_salt +from vllm_omni.entrypoints.openai.tts_adapters.base import ( + ARTTSAdapter, + PreparedRequest, + apply_max_new_tokens, + conditioning_cache_salt, +) if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -53,3 +58,12 @@ class FishSpeechAdapter(ARTTSAdapter): tts_params = {} prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="fish_speech") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py index ddfd194e2..9ef342666 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 """GLM-TTS serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + +from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest @@ -9,6 +11,8 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +logger = init_logger(__name__) + @register_tts_adapter class GlmTTSAdapter(ARTTSAdapter): @@ -43,6 +47,63 @@ class GlmTTSAdapter(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: prompt = await self.ctx.server._build_glm_tts_prompt(request, has_inline_ref_audio=has_inline_ref_audio) - # GLM-TTS dynamic-token sampling stays in the orchestrator tail - # (keyed on _tts_model_type) during this incremental migration. return PreparedRequest(prompt=prompt, tts_params={}, model_type="glm_tts") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + # GLM-TTS: set dynamic min/max tokens based on text length. + import copy + + server = self.ctx.server + sampling_params_list = copy.deepcopy(sampling_params_list) + glm_metadata = prompt.get("additional_information") if isinstance(prompt, dict) else None + text_len_value = None + if isinstance(glm_metadata, dict): + text_len_value = glm_metadata.get("glm_tts_text_token_len") + if isinstance(text_len_value, list) and text_len_value: + text_len_value = text_len_value[0] + text_token_len = ( + int(text_len_value) + if text_len_value is not None + else server._estimate_glm_tts_text_token_len(request.input) + ) + hf_cfg = server.model_config.hf_config + min_ratio = getattr(hf_cfg, "min_token_text_ratio", 2) + max_ratio = getattr(hf_cfg, "max_token_text_ratio", 20) + stage_min_tokens = getattr(sampling_params_list[0], "min_tokens", None) + stage_max_tokens = getattr(sampling_params_list[0], "max_tokens", None) + cap_candidates = [int(cap) for cap in (stage_max_tokens, request.max_new_tokens) if cap is not None] + hard_cap = min(cap_candidates) if cap_candidates else None + + min_tokens = max(1, int(text_token_len * min_ratio)) + if stage_min_tokens is not None: + min_tokens = max(min_tokens, int(stage_min_tokens)) + if hard_cap is not None: + min_tokens = min(min_tokens, hard_cap) + + max_tokens = max(min_tokens, int(text_token_len * max_ratio)) + if hard_cap is not None: + max_tokens = min(max_tokens, hard_cap) + sampling_params_list[0].min_tokens = min_tokens + sampling_params_list[0].max_tokens = max_tokens + seed = getattr(request, "seed", None) + if seed is not None: + sampling_params_list[0].seed = seed + logger.info( + "GLM-TTS dynamic tokens: text_tokens=%d, min_ratio=%s, max_ratio=%s, " + "stage_min=%s, stage_max=%s, request_max=%s, min_tokens=%d, max_tokens=%d", + text_token_len, + min_ratio, + max_ratio, + stage_min_tokens, + stage_max_tokens, + request.max_new_tokens, + min_tokens, + max_tokens, + ) + return sampling_params_list diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py index 99949d8b3..74ec591cc 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Higgs-Audio v2 serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -99,3 +99,12 @@ class HiggsAudioV2Adapter(ARTTSAdapter): additional["voice_name"] = voice_lower additional["voice_created_at"] = server._voice_created_at(voice_lower) return PreparedRequest(prompt=prompt, tts_params={}, model_type="higgs_audio_v2") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py index 9cae9aa2d..411d97f31 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v3.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Higgs-Audio v3 serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -36,3 +36,12 @@ class HiggsAudioV3Adapter(ARTTSAdapter): ) -> PreparedRequest: prompt = await self.ctx.server._build_higgs_audio_v3_params(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="higgs_audio_v3") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py b/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py index d1cb2a469..de70cd5b8 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py @@ -14,7 +14,7 @@ from vllm.inputs import tokens_input from vllm.utils import random_uuid from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens from vllm_omni.model_executor.models.indextts2.configuration_indextts2 import ( INDEXTTS25_MAX_DURATION_FACTOR, INDEXTTS25_MIN_DURATION_FACTOR, @@ -241,6 +241,15 @@ class IndexTTS2Adapter(ARTTSAdapter): prompt["cache_salt"] = indextts2_conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type=self.name) + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: OpenAICreateSpeechRequest, + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) + async def _build_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: server = self.ctx.server params: dict[str, Any] = {"text": [request.input]} diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index dd8b8b621..cf76cdafe 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 """Ming-TTS (dense) serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm.logger import init_logger @@ -73,9 +73,36 @@ class MingTTSAdapter(ARTTSAdapter): voice_created_at=uploaded_audio_created_at, ) tts_params = prompt.get("additional_information", {}) - # Ming stop-token / max_tokens sampling stays in the orchestrator tail. return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts") + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + import copy + + server = self.ctx.server + + from vllm_omni.model_executor.models.ming_tts.config_ming_tts import ( + MOE_TEXT_EOS_TOKEN_ID, + TEXT_EOS_TOKEN_ID, + ) + + hf_config = server.engine_client.model_config.hf_config + is_moe = getattr(hf_config, "model_type", "") == "bailingmm" + stop_token_id = MOE_TEXT_EOS_TOKEN_ID if is_moe else TEXT_EOS_TOKEN_ID + + sampling_params_list = copy.deepcopy(sampling_params_list) + sampling_params_list[0].stop_token_ids = [int(stop_token_id)] + if request.max_new_tokens is not None: + # Ming emits TEXT_EOS after the latent decode budget is exhausted, so + # Stage-0 needs one extra token beyond ming_max_decode_steps. + sampling_params_list[0].max_tokens = int(request.max_new_tokens) + 1 + return sampling_params_list + def _validate_ming_tts_single_speaker_request(self, request: "OpenAICreateSpeechRequest") -> str | None: server = self.ctx.server if request.ref_audio is not None: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py index c0be8c685..c3dcf6a54 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py @@ -2,12 +2,12 @@ """Qwen3-TTS serving adapter.""" from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm.logger import init_logger from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -163,3 +163,12 @@ class Qwen3TTSAdapter(ARTTSAdapter): model_type=tts_params.get("task_type", ["unknown"])[0], warmup_artifact_key=warmup_key, ) + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py index b0f188fbb..92b84774b 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """VoxCPM2 serving adapter (AR base-LM + diffusion side-computation).""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -70,3 +70,12 @@ class VoxCPM2Adapter(ARTTSAdapter): additional["voice_name"] = voice_lower additional["voice_created_at"] = server._voice_created_at(voice_lower) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="voxcpm2") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request) diff --git a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py index 0b2d400b1..09de585d8 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/voxtral.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Voxtral TTS serving adapter.""" -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter -from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest +from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens if TYPE_CHECKING: from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest @@ -48,3 +48,12 @@ class VoxtralTTSAdapter(ARTTSAdapter): ) -> PreparedRequest: prompt = await self.ctx.server._build_voxtral_prompt_async(request) return PreparedRequest(prompt=prompt, tts_params={}, model_type="voxtral_tts") + + def apply_sampling_overrides( + self, + sampling_params_list: list, + request: "OpenAICreateSpeechRequest", + prompt: dict[str, Any] | None = None, + request_id: str | None = None, + ) -> list: + return apply_max_new_tokens(sampling_params_list, request)