[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)

Signed-off-by: lishunyang <lishunyang12@163.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: reidliu41 <reid201711@gmail.com>
Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com>
Co-authored-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-04-19 14:36:00 +12:00
1 parent a683b1dd75
commit a3903810fe
104 files changed
+3959 -3814

No files matched your search

@@ -1,98 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0"
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
@@ -2,8 +2,6 @@
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
"""
from pathlib import Path
import pytest
from tests.conftest import (
@@ -12,36 +10,31 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen2.5-Omni-7B"]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section. NPU still uses the legacy per-platform YAML until it
# also migrates to the new schema.
_CI_DEPLOY = get_deploy_config_path("ci/qwen2_5_omni.yaml")
def get_cuda_graph_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
return modify_stage_config(
_CI_DEPLOY,
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
1: {"engine_args.enforce_eager": "true"},
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
},
},
)
return path
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
if current_omni_platform.is_npu():
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_xpu():
# Intel XPU stage config optimized for B60 GPU
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml")
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu() or current_omni_platform.is_npu():
stage_config = _CI_DEPLOY
else:
stage_config = get_cuda_graph_config()
+13 -17
View File
@@ -7,41 +7,37 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def get_cuda_graph_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
return modify_stage_config(
_CI_DEPLOY,
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
1: {"engine_args.enforce_eager": "true"},
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
},
},
)
return path
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
elif current_omni_platform.is_xpu():
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")]
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
stage_configs = [_CI_DEPLOY]
else:
stage_configs = [get_cuda_graph_config()]
@@ -13,12 +13,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import modify_stage_config
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
@@ -26,23 +24,31 @@ REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know wh
def get_cuda_graph_config():
path = modify_stage_config(
get_stage_config(),
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
Composes the synchronous (no-async-chunk) variant on top of the bundled
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
standalone variant yaml; same effective config, no checked-in file needed.
"""
return modify_stage_config(
get_deploy_config_path("qwen3_tts.yaml"),
updates={
"stage_args": {
"async_chunk": False,
"stages": {
0: {
"engine_args.enforce_eager": "true",
"max_num_seqs": 1,
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {"engine_args.enforce_eager": "true"},
},
},
)
return path
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
# Same structure as test_qwen3_omni: models, stage_configs, test_params
@@ -13,34 +13,40 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import modify_stage_config
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_cuda_graph_config():
path = modify_stage_config(
get_stage_config(),
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
Composes the synchronous (no-async-chunk) variant on top of the bundled
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
standalone variant yaml; same effective config, no checked-in file needed.
"""
return modify_stage_config(
get_deploy_config_path("qwen3_tts.yaml"),
updates={
"stage_args": {
"async_chunk": False,
"stages": {
0: {
"engine_args.enforce_eager": "true",
"max_num_seqs": 1,
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {"engine_args.enforce_eager": "true"},
},
},
)
return path
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
# Same structure as test_qwen3_omni: models, stage_configs, test_params
+4 -17
View File
@@ -3,7 +3,6 @@ E2E Online tests for Qwen2.5-Omni model with video input and audio output.
"""
import os
from pathlib import Path
import pytest
@@ -15,8 +14,7 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from vllm_omni.platforms import current_omni_platform
from tests.utils import get_deploy_config_path, hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -24,20 +22,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
models = ["Qwen/Qwen2.5-Omni-7B"]
def get_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
)
return path
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")]
else:
stage_configs = [get_config()]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section in vllm_omni/deploy/ci/qwen2_5_omni.yaml.
stage_configs = [modify_stage_config(get_deploy_config_path("ci/qwen2_5_omni.yaml"))]
# Create parameter combinations for model and stage config
test_params = [
+24 -38
View File
@@ -3,7 +3,6 @@ E2E Online tests for Qwen3-Omni model with video input and audio output.
"""
import os
from pathlib import Path
import pytest
@@ -15,7 +14,7 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -23,32 +22,24 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
QWEN3_OMNI_CONFIG_PATH = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
QWEN3_OMNI_XPU_CONFIG_PATH = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
_STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs"
_PD_SEP_CONFIG = str(_STAGE_CONFIGS_DIR / "qwen3_omni_moe_pd_ci.yaml")
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def get_chunk_config(config_path: str | None = None):
"""Load qwen3_omni_ci.yaml with async_chunk modifications for streaming mode."""
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
if config_path is None:
config_path = str(_STAGE_CONFIGS_DIR / "qwen3_omni_ci.yaml")
return modify_stage_config(
config_path,
updates={
"async_chunk": True,
"stage_args": {
0: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
},
1: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
},
},
},
deletes={"stage_args": {2: ["custom_process_input_func"]}},
)
config_path = _CI_DEPLOY
# TODO: remove this workaround once legacy `stage_args` path is deleted.
# The pipeline (qwen3_omni/pipeline.py) already wires
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
# so only async_chunk needs flipping. Writing nested `engine_args:` into
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
# drops flat fields (load_format, max_num_seqs, ...).
return modify_stage_config(config_path, updates={"async_chunk": True})
def get_prefix_caching_config(config_path: str):
@@ -64,21 +55,16 @@ def get_prefix_caching_config(config_path: str):
return path
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation, default tests async_chunk mode.
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
# Stage configs for H100/CUDA, ROCm MI325, and XPU platforms
if current_omni_platform.is_rocm():
rocm_config = str(_STAGE_CONFIGS_DIR / "rocm" / "qwen3_omni_ci.yaml")
stage_configs = [rocm_config]
prefix_caching_stage_configs = [get_prefix_caching_config(rocm_config)]
elif current_omni_platform.is_xpu():
xpu_config = str(_STAGE_CONFIGS_DIR / "xpu" / "qwen3_omni_ci.yaml")
stage_configs = [xpu_config]
prefix_caching_stage_configs = [get_prefix_caching_config(xpu_config)]
else:
stage_configs = [_PD_SEP_CONFIG if _USE_PD else get_chunk_config(QWEN3_OMNI_CONFIG_PATH)]
prefix_caching_stage_configs = [get_prefix_caching_config(QWEN3_OMNI_CONFIG_PATH)]
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
# overlay has been migrated to the new schema (previously used the deleted
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
if current_omni_platform.is_xpu():
stage_configs = [_CI_DEPLOY]
else: # CUDA + ROCm MI325 share the same deploy config
stage_configs = [get_chunk_config()]
prefix_caching_stage_configs = [get_prefix_caching_config(_CI_DEPLOY)]
# Create parameter combinations for model and stage config
test_params = [
@@ -6,10 +6,7 @@ E2E Online tests for Qwen3-Omni model.
import os
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from pathlib import Path
import pytest
@@ -21,7 +18,7 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
@@ -40,47 +37,56 @@ LARGE_IMAGE_HEIGHT = 1080
LONG_AUDIO_DURATION_SEC = 120
def get_chunk_config(default_path):
path = modify_stage_config(
def get_batch_token_config(default_path):
"""Override stage 1's max_num_batched_tokens to exercise small-batch paths.
Uses the new flat-stage schema (``stages.<id>.<field>``); the legacy
``stage_args.<id>.engine_args.<field>`` path no longer applies because
the deploy YAML doesn't nest engine fields under ``engine_args:``.
"""
return modify_stage_config(
default_path,
updates={
"stages": {1: {"max_num_batched_tokens": 64}},
},
)
def get_async_chunk_config(default_path):
"""Flip async_chunk on and bump stage 0 thinker output to 2048 tokens.
Pipeline registry (qwen3_omni/pipeline.py) already wires
thinker2talker_async_chunk / talker2code2wav_async_chunk on stages 0/1,
so no per-stage processor override is needed. Using only flat-schema
writes so _parse_stage_deploy stays in its flat branch (nested
``engine_args:`` would drop other overlay fields).
"""
return modify_stage_config(
default_path,
updates={
"async_chunk": True,
"stage_args": {
0: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk",
"default_sampling_params.max_tokens": 2048,
},
1: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
},
},
},
deletes={"stage_args": {2: ["custom_process_input_func"]}},
)
return path
def get_batch_token_config(default_path):
path = modify_stage_config(
default_path,
updates={
"stage_args": {1: {"engine_args.max_num_batched_tokens": 64}},
"stages": {0: {"default_sampling_params.max_tokens": 2048}},
},
)
return path
# CI stage config for 2*H100-80G GPUs
default_path = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
# CI deploy YAML (single file; xpu deltas applied via ``platforms:`` section).
# The overlay explicitly sets ``async_chunk: False``, so ``default`` tests the
# sync path and ``async_chunk`` tests the streaming path with a longer thinker
# output — two distinct scenarios, kept as separate parametrizations.
default_path = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
if current_omni_platform.is_xpu():
default_path = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
# Create parameter combinations for model and stage config
test_params = [
pytest.param(OmniServerParams(model=model, stage_config_path=default_path, use_stage_cli=True), id="default"),
pytest.param(
OmniServerParams(model=model, stage_config_path=get_chunk_config(default_path), use_stage_cli=True),
OmniServerParams(model=model, stage_config_path=default_path, use_stage_cli=True),
id="default",
),
pytest.param(
OmniServerParams(
model=model,
stage_config_path=get_async_chunk_config(default_path),
use_stage_cli=True,
),
id="async_chunk",
),
]
@@ -12,7 +12,6 @@ import io
import json
import os
import wave
from pathlib import Path
import pytest
import websockets
@@ -24,23 +23,22 @@ from tests.conftest import (
generate_synthetic_audio,
modify_stage_config,
)
from tests.utils import hardware_test
from vllm_omni.platforms import current_omni_platform
from tests.utils import get_deploy_config_path, hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
default_stage_config = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
if current_omni_platform.is_xpu():
default_stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
# via its ``platforms:`` section, so one path serves all three.
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def _realtime_stage_config_path() -> str:
"""CI omni layout without async_chunk; stage 0 thinker max_tokens=10."""
return modify_stage_config(
default_stage_config,
updates={"stage_args": {0: {"default_sampling_params.max_tokens": 10}}},
updates={"stages": {0: {"default_sampling_params.max_tokens": 10}}},
)
@@ -12,12 +12,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -25,11 +23,6 @@ REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="text"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -48,7 +41,7 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
@@ -12,12 +12,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -25,11 +23,6 @@ REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="text"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -48,16 +41,19 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
),
# Synchronous (no async-chunk) variant — ``--no-async-chunk`` alone
# flips the deploy yaml's bool and the pipeline dispatches to the
# end-to-end codec processor. No variant yaml / pipeline needed.
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"],
),
id="no_async_chunk",
),
@@ -27,14 +27,15 @@ from tests.conftest import (
convert_audio_file_to_text,
cosine_similarity_text,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
STAGE_INIT_TIMEOUT_S = 120
def get_stage_config(name: str = "qwen3_tts.yaml"):
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_stage_config(name: str = "qwen3_tts.yaml") -> str:
"""Resolve a deploy config path under vllm_omni/deploy/."""
return get_deploy_config_path(name)
@pytest.fixture(scope="module")
@@ -12,21 +12,14 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="text"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -45,7 +38,7 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
@@ -12,21 +12,14 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="english"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -46,16 +39,19 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
),
# Synchronous (no async-chunk) variant — ``--no-async-chunk`` alone
# flips the deploy yaml's bool and the pipeline dispatches to the
# end-to-end codec processor. No variant yaml / pipeline needed.
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"],
),
id="no_async_chunk",
),
@@ -13,13 +13,12 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import struct
from pathlib import Path
import httpx
import pytest
from tests.conftest import OmniServer
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
@@ -37,10 +36,8 @@ MIN_AUDIO_BYTES = 2000
MAX_NEW_TOKENS = 256
def get_stage_config():
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
)
def get_stage_config() -> str:
return get_deploy_config_path("qwen3_tts.yaml")
def _server_args():
@@ -7,13 +7,12 @@ E2E online tests for Qwen3-TTS WebSocket streaming speech.
import asyncio
import json
import os
from pathlib import Path
import pytest
import websockets
from tests.conftest import OmniServer
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -23,9 +22,7 @@ STAGE_INIT_TIMEOUT_S = 120
def get_stage_config() -> str:
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
)
return get_deploy_config_path("qwen3_tts.yaml")
@pytest.fixture(scope="module")
@@ -1,104 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
load_format: dummy
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.4
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "2" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.5 #increase the gpu memory utilization to enable the test on H800
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
max_num_batched_tokens: 8192
max_model_len: 8192
load_format: dummy
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 8192
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
@@ -1,31 +0,0 @@
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
-102
View File
@@ -1,102 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
engine_args:
model_stage: thinker
max_num_seqs: 5
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
max_num_batched_tokens: 32768
max_model_len: 32768
enable_prefix_caching: false
mm_processor_cache_gb: 0
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 150
seed: 42
ignore_eos: False
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
engine_args:
model_stage: talker
max_num_seqs: 5
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.5
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
enable_prefix_caching: false
max_num_batched_tokens: 32768
max_model_len: 32768
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 1000
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
engine_args:
model_stage: code2wav
max_num_seqs: 5
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 100000
hf_config_name: thinker_config
async_scheduling: false
load_format: dummy
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2000
seed: 42
detokenize: True
repetition_penalty: 1.1
@@ -1,101 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
max_num_batched_tokens: 4096
max_model_len: 4096
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
@@ -1,100 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
engine_args:
model_stage: thinker
max_num_seqs: 1
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
mm_processor_cache_gb: 0
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
engine_args:
model_stage: talker
max_num_seqs: 1
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 100
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
load_format: dummy
async_scheduling: false
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 200
seed: 42
detokenize: True
repetition_penalty: 1.1
@@ -1,102 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# The following config is verified with 2 * Intel Arc Pro B60 XPU.
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9 # thinker weight is around 16.74GB for Qwen2.5-Omni-7B
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.5 # talker weight is 6.03GB for Qwen2.5-Omni-7B
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true
devices: "2"
engine_args:
max_num_seqs: 1
model_stage: code2wav
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.3 # code2wav weight is around 1.46GB for Qwen2.5-Omni-7B
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2048
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
@@ -1,109 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config is verified with 8 * Intel Arc Pro B60 XPU.
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "0,1,2,3"
engine_args:
max_num_seqs: 1
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.85 # thinker weight is around 61.08GB for Qwen3-Omni-30B-A3B-Instruct
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
max_num_batched_tokens: 4096
max_model_len: 4096
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 4
max_cudagraph_capture_size: 0
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
ignore_eos: False
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "4"
engine_args:
max_num_seqs: 1
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6 # talker weight is around 8.5GB for Qwen3-Omni-30B-A3B-Instruct
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
enable_prefix_caching: false
max_num_batched_tokens: 4096
max_model_len: 4096
distributed_executor_backend: "mp"
hf_config_name: talker_config
max_cudagraph_capture_size: 0
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "5"
engine_args:
max_num_seqs: 1
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.3 # code2wav weight is around 0.4GB for Qwen3-Omni-30B-A3B-Instruct
skip_mm_profiling: true
distributed_executor_backend: "mp"
max_num_batched_tokens: 100000
hf_config_name: thinker_config
async_scheduling: false
max_cudagraph_capture_size: 0
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2000
seed: 42
detokenize: True
repetition_penalty: 1.1