diff --git a/examples/offline_inference/moss_tts_nano/README.md b/examples/offline_inference/moss_tts_nano/README.md index f7ace3b86..8534d0549 100644 --- a/examples/offline_inference/moss_tts_nano/README.md +++ b/examples/offline_inference/moss_tts_nano/README.md @@ -31,7 +31,7 @@ Options: --text-temperature F Text layer temperature (default: 1.0) --batch Run a built-in batch of diverse samples (ZH/EN/FR) --output-dir DIR Directory for WAV outputs (default: /tmp/moss_tts_nano_output) - --stage-config PATH Path to stage config YAML + --deploy-config PATH Override deploy YAML (defaults to vllm_omni/deploy/moss_tts_nano.yaml) --stage-init-timeout INT Timeout in seconds for stage init (default: 120) ``` @@ -77,15 +77,17 @@ python end2end.py --text "Deterministic test." --seed 42 | `Hina` | JA | | `Mei` | JA | -## Stage Config +## Deploy Config -The stage config is at `vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml`. Key settings: +Runtime knobs live in `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded; +override with `--deploy-config PATH`). Key stage-level settings: ```yaml -engine_args: - gpu_memory_utilization: 0.3 # ~2 GB VRAM; increase for faster init - max_num_seqs: 4 # concurrent requests - max_model_len: 4096 +stages: + - stage_id: 0 + gpu_memory_utilization: 0.3 # ~2 GB VRAM; increase for faster init + max_num_seqs: 4 # concurrent requests + max_model_len: 4096 ``` ## Output Format diff --git a/examples/offline_inference/moss_tts_nano/end2end.py b/examples/offline_inference/moss_tts_nano/end2end.py index adb8cadbb..177e03791 100644 --- a/examples/offline_inference/moss_tts_nano/end2end.py +++ b/examples/offline_inference/moss_tts_nano/end2end.py @@ -39,9 +39,6 @@ from vllm_omni import Omni # noqa: E402 MODEL = "OpenMOSS-Team/MOSS-TTS-Nano" -_REPO_ROOT = Path(__file__).resolve().parents[4] -DEFAULT_STAGE_CONFIG = str(_REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "moss_tts_nano.yaml") - BATCH_SAMPLES = [ {"text": "Hello, this is a test of MOSS-TTS-Nano.", "voice": "Ava", "label": "en_hello"}, {"text": "你好,这是 MOSS-TTS-Nano 的语音合成测试。", "voice": "Junhao", "label": "zh_hello"}, @@ -98,7 +95,7 @@ def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = 48000) -> N def main(args) -> None: omni = Omni( model=MODEL, - stage_configs_path=args.stage_config, + deploy_config=args.deploy_config, stage_init_timeout=args.stage_init_timeout, ) @@ -170,7 +167,11 @@ def parse_args(): parser.add_argument("--text-temperature", type=float, default=1.0) parser.add_argument("--batch", action="store_true", help="Run built-in batch of diverse samples.") parser.add_argument("--output-dir", default="/tmp/moss_tts_nano_output", help="Directory for WAV outputs.") - parser.add_argument("--stage-config", default=DEFAULT_STAGE_CONFIG) + parser.add_argument( + "--deploy-config", + default=None, + help="Path to a deploy YAML; leave unset to auto-load vllm_omni/deploy/moss_tts_nano.yaml.", + ) parser.add_argument("--stage-init-timeout", type=int, default=120) return parser.parse_args() diff --git a/examples/online_serving/moss_tts_nano/README.md b/examples/online_serving/moss_tts_nano/README.md index 371bde789..3286f8ab7 100644 --- a/examples/online_serving/moss_tts_nano/README.md +++ b/examples/online_serving/moss_tts_nano/README.md @@ -29,14 +29,13 @@ Features: ## Launch the Server ```bash -vllm-omni serve OpenMOSS-Team/MOSS-TTS-Nano \ - --stage-configs-path vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml \ - --omni \ - --port 8091 \ - --trust-remote-code \ - --enforce-eager +vllm serve OpenMOSS-Team/MOSS-TTS-Nano --omni --port 8091 ``` +The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no +`--stage-configs-path`, `--trust-remote-code`, or `--enforce-eager` flags +are needed. + Or use the convenience script: ```bash diff --git a/examples/online_serving/moss_tts_nano/run_gradio_demo.sh b/examples/online_serving/moss_tts_nano/run_gradio_demo.sh index 1fb5d1914..cd96c6b4f 100755 --- a/examples/online_serving/moss_tts_nano/run_gradio_demo.sh +++ b/examples/online_serving/moss_tts_nano/run_gradio_demo.sh @@ -11,16 +11,12 @@ MODEL="${MODEL:-OpenMOSS-Team/MOSS-TTS-Nano}" PORT="${PORT:-8091}" GRADIO_PORT="${GRADIO_PORT:-7860}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" echo "Starting MOSS-TTS-Nano server (port $PORT)..." FLASHINFER_DISABLE_VERSION_CHECK=1 \ -vllm-omni serve "$MODEL" \ - --stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml" \ +vllm serve "$MODEL" \ --host 0.0.0.0 \ --port "$PORT" \ - --trust-remote-code \ - --enforce-eager \ --omni & SERVER_PID=$! diff --git a/examples/online_serving/moss_tts_nano/run_server.sh b/examples/online_serving/moss_tts_nano/run_server.sh index c5254c428..5ddbac09c 100755 --- a/examples/online_serving/moss_tts_nano/run_server.sh +++ b/examples/online_serving/moss_tts_nano/run_server.sh @@ -9,16 +9,11 @@ set -e MODEL="${MODEL:-OpenMOSS-Team/MOSS-TTS-Nano}" PORT="${PORT:-8091}" -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" echo "Starting MOSS-TTS-Nano server with model: $MODEL" FLASHINFER_DISABLE_VERSION_CHECK=1 \ -vllm-omni serve "$MODEL" \ - --stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml" \ +vllm serve "$MODEL" \ --host 0.0.0.0 \ --port "$PORT" \ - --trust-remote-code \ - --enforce-eager \ --omni diff --git a/tests/e2e/offline_inference/test_moss_tts_nano.py b/tests/e2e/offline_inference/test_moss_tts_nano.py index ea1236c62..5596654ef 100644 --- a/tests/e2e/offline_inference/test_moss_tts_nano.py +++ b/tests/e2e/offline_inference/test_moss_tts_nano.py @@ -12,7 +12,6 @@ from tests.utils import hardware_test from vllm_omni import Omni MODEL_NAME = "OpenMOSS-Team/MOSS-TTS-Nano" -STAGE_CONFIG = "vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml" SAMPLE_RATE = 48000 DEFAULT_SAMPLING = SamplingParams( @@ -61,7 +60,7 @@ def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]: @pytest.fixture(scope="module") def omni_engine(): """Module-scoped Omni engine to avoid re-loading the model for each test.""" - return Omni(model=MODEL_NAME, stage_configs_path=STAGE_CONFIG, stage_init_timeout=180) + return Omni(model=MODEL_NAME, stage_init_timeout=180) @pytest.mark.omni diff --git a/tests/e2e/online_serving/test_moss_tts_nano.py b/tests/e2e/online_serving/test_moss_tts_nano.py index 662114db3..ec30bd4da 100644 --- a/tests/e2e/online_serving/test_moss_tts_nano.py +++ b/tests/e2e/online_serving/test_moss_tts_nano.py @@ -12,21 +12,15 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" -from pathlib import Path - import pytest from tests.conftest import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path from tests.utils import hardware_test MODEL = "OpenMOSS-Team/MOSS-TTS-Nano" -def get_stage_config(name: str = "moss_tts_nano.yaml"): - """Get the stage config path from vllm_omni model_executor stage_configs.""" - return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name) - - def get_prompt(prompt_type="text"): """Text prompt for text-to-audio tests.""" prompts = { @@ -40,8 +34,8 @@ tts_server_params = [ pytest.param( OmniServerParams( model=MODEL, - stage_config_path=get_stage_config("moss_tts_nano.yaml"), - server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"], + stage_config_path=get_deploy_config_path("moss_tts_nano.yaml"), + server_args=["--disable-log-stats"], ), id="moss_tts_nano", ) diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index 69a004015..c06cc90e4 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -63,6 +63,10 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = { "vllm_omni.model_executor.models.fish_speech.pipeline", "FISH_SPEECH_PIPELINE", ), + "moss_tts_nano": ( + "vllm_omni.model_executor.models.moss_tts_nano.pipeline", + "MOSS_TTS_NANO_PIPELINE", + ), } # --- Single-stage diffusion pipelines (populated in PR 3/N) --- diff --git a/vllm_omni/deploy/moss_tts_nano.yaml b/vllm_omni/deploy/moss_tts_nano.yaml new file mode 100644 index 000000000..585e244ca --- /dev/null +++ b/vllm_omni/deploy/moss_tts_nano.yaml @@ -0,0 +1,31 @@ +# MOSS-TTS-Nano deploy: single-stage AR TTS (0.1B). +# Verified on 1x L4 24GB (and sufficient for H20 / A100 / 3090). +# +# Fields omitted from a stage fall back to StageDeployConfig defaults (see +# vllm_omni/config/stage_config.py). Notable choices: +# * enforce_eager=true — the model uses upstream inference_stream() with +# trust_remote_code; CUDA graph capture is not wired up. +# * gpu_memory_utilization=0.3 — the AR LM + codec fit in ~2 GiB. +# * max_num_seqs=4 — mirrors the legacy stage_configs contract; per-request +# streaming generators are stored in self._stream_gens. +# +# MOSS-TTS-Nano has no multimodal input preprocessing, so skip_mm_profiling +# bypasses the dummy-MM pass at profile time. +async_chunk: false +trust_remote_code: true + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.3 + enforce_eager: true + max_num_batched_tokens: 4096 + max_model_len: 4096 + devices: "0" + skip_mm_profiling: true + default_sampling_params: + temperature: 1.0 + top_p: 1.0 + top_k: 50 + max_tokens: 4096 + seed: 42 diff --git a/vllm_omni/model_executor/models/moss_tts_nano/__init__.py b/vllm_omni/model_executor/models/moss_tts_nano/__init__.py index 208f01a7c..896ead57d 100644 --- a/vllm_omni/model_executor/models/moss_tts_nano/__init__.py +++ b/vllm_omni/model_executor/models/moss_tts_nano/__init__.py @@ -1,2 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from .modeling_moss_tts_nano import MossTTSNanoForGeneration + +__all__ = ["MossTTSNanoForGeneration"] diff --git a/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py b/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py new file mode 100644 index 000000000..749c4f980 --- /dev/null +++ b/vllm_omni/model_executor/models/moss_tts_nano/pipeline.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MOSS-TTS-Nano pipeline topology (frozen). + +Single-stage AR TTS: text -> speech waveform in one pass. The 0.1B AR LM +and the MOSS-Audio-Tokenizer-Nano codec both run inside +``MossTTSNanoForGeneration.forward()``, which uses the VoxCPM-style +generator pattern (``inference_stream()`` stored per-request; one audio +chunk yielded per forward) to drive progressive streaming through the +AR scheduler. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +MOSS_TTS_NANO_PIPELINE = PipelineConfig( + model_type="moss_tts_nano", + model_arch="MossTTSNanoForCausalLM", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="moss_tts_nano", + execution_type=StageExecutionType.LLM_AR, + input_sources=(), + final_output=True, + final_output_type="audio", + owns_tokenizer=True, + engine_output_type="audio", + sampling_constraints={ + "detokenize": False, + # compute_logits() forces EOS (token id 2) when the last + # streaming chunk is yielded; keep a hard backstop here. + "stop_token_ids": [2], + }, + ), + ), +) diff --git a/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml b/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml deleted file mode 100644 index f45e63649..000000000 --- a/vllm_omni/model_executor/stage_configs/moss_tts_nano.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Stage config for MOSS-TTS-Nano (OpenMOSS-Team/MOSS-TTS-Nano). -# -# Single AR stage: the 0.1B AR LM and MOSS-Audio-Tokenizer-Nano codec -# both run inside MossTTSNanoForGeneration.forward(). The model uses the -# VoxCPM-style generator pattern for streaming: inference_stream() is -# stored per-request, and each forward() call yields one audio chunk. -# compute_logits() controls EOS to keep requests alive until done. -# -# Verified on: 1x GPU with ≥2 GB VRAM, or CPU (slower). -# Output: 48 kHz, 2-channel stereo audio. -# -# Request additional_information fields: -# text (str) – text to synthesize [required] -# voice (str) – built-in voice preset, default "Junhao" -# mode (str) – "continuation" (default) | "voice_clone" -# prompt_audio_path (str) – path to reference audio for custom voice clone -# prompt_text (str) – reference transcript (continuation mode) -# max_new_frames (int) – max AR frames, default 375 (~14 s) -# text_temperature (float) – default 1.0 -# audio_temperature (float) – default 0.8 -# audio_top_k (int) – default 25 -# audio_top_p (float) – default 0.95 -# seed (int) – optional RNG seed - -async_chunk: false - -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - engine_args: - model_stage: moss_tts_nano - model_arch: MossTTSNanoForCausalLM - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - trust_remote_code: true - enforce_eager: true - engine_output_type: audio - gpu_memory_utilization: 0.3 - enable_prefix_caching: false - skip_mm_profiling: true - max_num_seqs: 4 - max_model_len: 4096 - max_num_batched_tokens: 4096 - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 1.0 - top_p: 1.0 - top_k: 50 - max_tokens: 4096 - seed: 42 - detokenize: false - -runtime: - enabled: true - defaults: - window_size: -1 - max_inflight: 4