Files
vllm-omni/vllm_omni/model_executor/models/qwen3_tts/pipeline.py
T

119 lines
4.4 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen3-TTS pipeline: Talker (text → RVQ codec) → Code2Wav (codec → audio).
Chunked vs end-to-end mode is dispatched from ``deploy.async_chunk``.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_tts"
QWEN3_TTS_PIPELINE = PipelineConfig(
model_type="qwen3_tts",
default_deploy_config_name="qwen3_tts.yaml",
# Pipeline-level default; the code2wav stage overrides per-stage below.
model_arch="Qwen3TTSTalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="qwen3_tts",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [2150],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="Qwen3TTSCode2Wav",
# ``sync_process_input_func`` is the only input-proc override for
# this stage in sync (non-async-chunk) mode: a length-only
# ``_token_only`` placeholder. The bulk codec payload itself
# ships via the worker connector from stage 0's
# ``talker2code2wav_full_payload`` producer. Under async_chunk
# mode no pre-stage processing is needed -- chunks deliver
# directly to the consumer.
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
extras={"tts_args": {"max_instructions_length": 500}},
requires_full_payload_input=True,
),
),
)
# Talker-only pipeline: stage 0 is terminal and its codec frames (codes.audio)
# are published to the client for on-device decoding (codec-token streaming,
# see entrypoints/openai/codec_stream.py). No next-stage processors, so no
# omni connector is created; ``final_output_type="latent"`` keeps the AR
# output path (audio-specific terminal handling is not applied).
QWEN3_TTS_TALKER_ONLY_PIPELINE = PipelineConfig(
model_type="qwen3_tts_talker_only",
default_deploy_config_name="qwen3_tts_talker_only.yaml",
model_arch="Qwen3TTSTalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="qwen3_tts",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
final_output=True,
final_output_type="latent",
engine_output_type="latent",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [2150],
},
),
),
)
# Whole-session hybrid fallback for the pinned CustomVoice GGUF artifact.
# CrispASR owns talker, iterative code predictor and codec within one backend;
# graph is only the Omni complete-request admission and acknowledgement path.
QWEN3_TTS_CRISP_HYBRID_PIPELINE = PipelineConfig(
model_type="qwen3_tts_crisp_hybrid",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audio",
execution_type=StageExecutionType.GRAPH,
final_output=True,
final_output_type="audio",
),
),
)
# Native Windows CPU reference path with the official PyTorch wrapper in an
# isolated worker, preserving Omni's installed Transformers dependency.
QWEN3_TTS_CPU_WHOLE_PIPELINE = PipelineConfig(
model_type="qwen3_tts_cpu_whole",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audio",
execution_type=StageExecutionType.GRAPH,
final_output=True,
final_output_type="audio",
),
),
)