[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)
Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: reidliu41 <reid201711@gmail.com> Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com> Co-authored-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
104 files changed
+3959
-3814
No files matched your search
@@ -1,98 +0,0 @@
|
||||
# stage config for running qwen2.5-omni for multi-stage omni runtime.
|
||||
|
||||
# This config is optimized for CI e2e tests.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
@@ -2,8 +2,6 @@
|
||||
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
@@ -12,36 +10,31 @@ from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
|
||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||
# platforms: section. NPU still uses the legacy per-platform YAML until it
|
||||
# also migrates to the new schema.
|
||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen2_5_omni.yaml")
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
|
||||
return modify_stage_config(
|
||||
_CI_DEPLOY,
|
||||
updates={
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
"stages": {
|
||||
0: {"enforce_eager": True},
|
||||
1: {"enforce_eager": True},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
|
||||
if current_omni_platform.is_npu():
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
|
||||
elif current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
|
||||
elif current_omni_platform.is_xpu():
|
||||
# Intel XPU stage config optimized for B60 GPU
|
||||
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml")
|
||||
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu() or current_omni_platform.is_npu():
|
||||
stage_config = _CI_DEPLOY
|
||||
else:
|
||||
stage_config = get_cuda_graph_config()
|
||||
|
||||
|
||||
@@ -7,41 +7,37 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
|
||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
|
||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
|
||||
return modify_stage_config(
|
||||
_CI_DEPLOY,
|
||||
updates={
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
"stages": {
|
||||
0: {"enforce_eager": True},
|
||||
1: {"enforce_eager": True},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
|
||||
if current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
elif current_omni_platform.is_xpu():
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")]
|
||||
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
|
||||
stage_configs = [_CI_DEPLOY]
|
||||
else:
|
||||
stage_configs = [get_cuda_graph_config()]
|
||||
|
||||
|
||||
@@ -13,12 +13,10 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
|
||||
@@ -26,23 +24,31 @@ REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know wh
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
get_stage_config(),
|
||||
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
|
||||
|
||||
Composes the synchronous (no-async-chunk) variant on top of the bundled
|
||||
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
|
||||
standalone variant yaml; same effective config, no checked-in file needed.
|
||||
"""
|
||||
return modify_stage_config(
|
||||
get_deploy_config_path("qwen3_tts.yaml"),
|
||||
updates={
|
||||
"stage_args": {
|
||||
"async_chunk": False,
|
||||
"stages": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
|
||||
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
# Same structure as test_qwen3_omni: models, stage_configs, test_params
|
||||
|
||||
@@ -13,34 +13,40 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
get_stage_config(),
|
||||
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
|
||||
|
||||
Composes the synchronous (no-async-chunk) variant on top of the bundled
|
||||
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
|
||||
standalone variant yaml; same effective config, no checked-in file needed.
|
||||
"""
|
||||
return modify_stage_config(
|
||||
get_deploy_config_path("qwen3_tts.yaml"),
|
||||
updates={
|
||||
"stage_args": {
|
||||
"async_chunk": False,
|
||||
"stages": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
|
||||
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
# Same structure as test_qwen3_omni: models, stage_configs, test_params
|
||||
|
||||
@@ -3,7 +3,6 @@ E2E Online tests for Qwen2.5-Omni model with video input and audio output.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -15,8 +14,7 @@ from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
@@ -24,20 +22,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
|
||||
|
||||
def get_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
|
||||
if current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")]
|
||||
else:
|
||||
stage_configs = [get_config()]
|
||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||
# platforms: section in vllm_omni/deploy/ci/qwen2_5_omni.yaml.
|
||||
stage_configs = [modify_stage_config(get_deploy_config_path("ci/qwen2_5_omni.yaml"))]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [
|
||||
|
||||
@@ -3,7 +3,6 @@ E2E Online tests for Qwen3-Omni model with video input and audio output.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -15,7 +14,7 @@ from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
@@ -23,32 +22,24 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
QWEN3_OMNI_CONFIG_PATH = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
QWEN3_OMNI_XPU_CONFIG_PATH = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
|
||||
|
||||
_STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs"
|
||||
_PD_SEP_CONFIG = str(_STAGE_CONFIGS_DIR / "qwen3_omni_moe_pd_ci.yaml")
|
||||
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
|
||||
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
||||
|
||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
|
||||
def get_chunk_config(config_path: str | None = None):
|
||||
"""Load qwen3_omni_ci.yaml with async_chunk modifications for streaming mode."""
|
||||
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
|
||||
if config_path is None:
|
||||
config_path = str(_STAGE_CONFIGS_DIR / "qwen3_omni_ci.yaml")
|
||||
return modify_stage_config(
|
||||
config_path,
|
||||
updates={
|
||||
"async_chunk": True,
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
|
||||
},
|
||||
1: {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
|
||||
},
|
||||
},
|
||||
},
|
||||
deletes={"stage_args": {2: ["custom_process_input_func"]}},
|
||||
)
|
||||
config_path = _CI_DEPLOY
|
||||
# TODO: remove this workaround once legacy `stage_args` path is deleted.
|
||||
# The pipeline (qwen3_omni/pipeline.py) already wires
|
||||
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
|
||||
# so only async_chunk needs flipping. Writing nested `engine_args:` into
|
||||
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
|
||||
# drops flat fields (load_format, max_num_seqs, ...).
|
||||
return modify_stage_config(config_path, updates={"async_chunk": True})
|
||||
|
||||
|
||||
def get_prefix_caching_config(config_path: str):
|
||||
@@ -64,21 +55,16 @@ def get_prefix_caching_config(config_path: str):
|
||||
return path
|
||||
|
||||
|
||||
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation, default tests async_chunk mode.
|
||||
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
|
||||
|
||||
# Stage configs for H100/CUDA, ROCm MI325, and XPU platforms
|
||||
if current_omni_platform.is_rocm():
|
||||
rocm_config = str(_STAGE_CONFIGS_DIR / "rocm" / "qwen3_omni_ci.yaml")
|
||||
stage_configs = [rocm_config]
|
||||
prefix_caching_stage_configs = [get_prefix_caching_config(rocm_config)]
|
||||
elif current_omni_platform.is_xpu():
|
||||
xpu_config = str(_STAGE_CONFIGS_DIR / "xpu" / "qwen3_omni_ci.yaml")
|
||||
stage_configs = [xpu_config]
|
||||
prefix_caching_stage_configs = [get_prefix_caching_config(xpu_config)]
|
||||
else:
|
||||
stage_configs = [_PD_SEP_CONFIG if _USE_PD else get_chunk_config(QWEN3_OMNI_CONFIG_PATH)]
|
||||
prefix_caching_stage_configs = [get_prefix_caching_config(QWEN3_OMNI_CONFIG_PATH)]
|
||||
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
|
||||
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
|
||||
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
|
||||
# overlay has been migrated to the new schema (previously used the deleted
|
||||
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
|
||||
if current_omni_platform.is_xpu():
|
||||
stage_configs = [_CI_DEPLOY]
|
||||
else: # CUDA + ROCm MI325 share the same deploy config
|
||||
stage_configs = [get_chunk_config()]
|
||||
prefix_caching_stage_configs = [get_prefix_caching_config(_CI_DEPLOY)]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [
|
||||
|
||||
@@ -6,10 +6,7 @@ E2E Online tests for Qwen3-Omni model.
|
||||
|
||||
import os
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -21,7 +18,7 @@ from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
@@ -40,47 +37,56 @@ LARGE_IMAGE_HEIGHT = 1080
|
||||
LONG_AUDIO_DURATION_SEC = 120
|
||||
|
||||
|
||||
def get_chunk_config(default_path):
|
||||
path = modify_stage_config(
|
||||
def get_batch_token_config(default_path):
|
||||
"""Override stage 1's max_num_batched_tokens to exercise small-batch paths.
|
||||
|
||||
Uses the new flat-stage schema (``stages.<id>.<field>``); the legacy
|
||||
``stage_args.<id>.engine_args.<field>`` path no longer applies because
|
||||
the deploy YAML doesn't nest engine fields under ``engine_args:``.
|
||||
"""
|
||||
return modify_stage_config(
|
||||
default_path,
|
||||
updates={
|
||||
"stages": {1: {"max_num_batched_tokens": 64}},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def get_async_chunk_config(default_path):
|
||||
"""Flip async_chunk on and bump stage 0 thinker output to 2048 tokens.
|
||||
|
||||
Pipeline registry (qwen3_omni/pipeline.py) already wires
|
||||
thinker2talker_async_chunk / talker2code2wav_async_chunk on stages 0/1,
|
||||
so no per-stage processor override is needed. Using only flat-schema
|
||||
writes so _parse_stage_deploy stays in its flat branch (nested
|
||||
``engine_args:`` would drop other overlay fields).
|
||||
"""
|
||||
return modify_stage_config(
|
||||
default_path,
|
||||
updates={
|
||||
"async_chunk": True,
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk",
|
||||
"default_sampling_params.max_tokens": 2048,
|
||||
},
|
||||
1: {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
|
||||
},
|
||||
},
|
||||
},
|
||||
deletes={"stage_args": {2: ["custom_process_input_func"]}},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def get_batch_token_config(default_path):
|
||||
path = modify_stage_config(
|
||||
default_path,
|
||||
updates={
|
||||
"stage_args": {1: {"engine_args.max_num_batched_tokens": 64}},
|
||||
"stages": {0: {"default_sampling_params.max_tokens": 2048}},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
# CI stage config for 2*H100-80G GPUs
|
||||
default_path = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
# CI deploy YAML (single file; xpu deltas applied via ``platforms:`` section).
|
||||
# The overlay explicitly sets ``async_chunk: False``, so ``default`` tests the
|
||||
# sync path and ``async_chunk`` tests the streaming path with a longer thinker
|
||||
# output — two distinct scenarios, kept as separate parametrizations.
|
||||
default_path = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
if current_omni_platform.is_xpu():
|
||||
default_path = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [
|
||||
pytest.param(OmniServerParams(model=model, stage_config_path=default_path, use_stage_cli=True), id="default"),
|
||||
pytest.param(
|
||||
OmniServerParams(model=model, stage_config_path=get_chunk_config(default_path), use_stage_cli=True),
|
||||
OmniServerParams(model=model, stage_config_path=default_path, use_stage_cli=True),
|
||||
id="default",
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=get_async_chunk_config(default_path),
|
||||
use_stage_cli=True,
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -12,7 +12,6 @@ import io
|
||||
import json
|
||||
import os
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
@@ -24,23 +23,22 @@ from tests.conftest import (
|
||||
generate_synthetic_audio,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
default_stage_config = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
if current_omni_platform.is_xpu():
|
||||
default_stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
|
||||
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
|
||||
# via its ``platforms:`` section, so one path serves all three.
|
||||
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
|
||||
def _realtime_stage_config_path() -> str:
|
||||
"""CI omni layout without async_chunk; stage 0 thinker max_tokens=10."""
|
||||
return modify_stage_config(
|
||||
default_stage_config,
|
||||
updates={"stage_args": {0: {"default_sampling_params.max_tokens": 10}}},
|
||||
updates={"stages": {0: {"default_sampling_params.max_tokens": 10}}},
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -12,12 +12,10 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServerParams
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
|
||||
@@ -25,11 +23,6 @@ REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/
|
||||
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml"):
|
||||
"""Get the stage config path from vllm_omni model_executor stage_configs."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text"):
|
||||
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
|
||||
prompts = {
|
||||
@@ -48,7 +41,7 @@ tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config("qwen3_tts.yaml"),
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="async_chunk",
|
||||
|
||||
@@ -12,12 +12,10 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServerParams
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
|
||||
@@ -25,11 +23,6 @@ REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/
|
||||
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml"):
|
||||
"""Get the stage config path from vllm_omni model_executor stage_configs."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text"):
|
||||
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
|
||||
prompts = {
|
||||
@@ -48,16 +41,19 @@ tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config("qwen3_tts.yaml"),
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
# Synchronous (no async-chunk) variant — ``--no-async-chunk`` alone
|
||||
# flips the deploy yaml's bool and the pipeline dispatches to the
|
||||
# end-to-end codec processor. No variant yaml / pipeline needed.
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"],
|
||||
),
|
||||
id="no_async_chunk",
|
||||
),
|
||||
|
||||
@@ -27,14 +27,15 @@ from tests.conftest import (
|
||||
convert_audio_file_to_text,
|
||||
cosine_similarity_text,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
||||
STAGE_INIT_TIMEOUT_S = 120
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml"):
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml") -> str:
|
||||
"""Resolve a deploy config path under vllm_omni/deploy/."""
|
||||
return get_deploy_config_path(name)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@@ -12,21 +12,14 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServerParams
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml"):
|
||||
"""Get the stage config path from vllm_omni model_executor stage_configs."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text"):
|
||||
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
|
||||
prompts = {
|
||||
@@ -45,7 +38,7 @@ tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config("qwen3_tts.yaml"),
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="async_chunk",
|
||||
|
||||
@@ -12,21 +12,14 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServerParams
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml"):
|
||||
"""Get the stage config path from vllm_omni model_executor stage_configs."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
def get_prompt(prompt_type="english"):
|
||||
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
|
||||
prompts = {
|
||||
@@ -46,16 +39,19 @@ tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config("qwen3_tts.yaml"),
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
# Synchronous (no async-chunk) variant — ``--no-async-chunk`` alone
|
||||
# flips the deploy yaml's bool and the pipeline dispatches to the
|
||||
# end-to-end codec processor. No variant yaml / pipeline needed.
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"],
|
||||
),
|
||||
id="no_async_chunk",
|
||||
),
|
||||
|
||||
@@ -13,13 +13,12 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServer
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
@@ -37,10 +36,8 @@ MIN_AUDIO_BYTES = 2000
|
||||
MAX_NEW_TOKENS = 256
|
||||
|
||||
|
||||
def get_stage_config():
|
||||
return str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
|
||||
)
|
||||
def get_stage_config() -> str:
|
||||
return get_deploy_config_path("qwen3_tts.yaml")
|
||||
|
||||
|
||||
def _server_args():
|
||||
|
||||
@@ -7,13 +7,12 @@ E2E online tests for Qwen3-TTS WebSocket streaming speech.
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.conftest import OmniServer
|
||||
from tests.utils import hardware_test
|
||||
from tests.utils import get_deploy_config_path, hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
@@ -23,9 +22,7 @@ STAGE_INIT_TIMEOUT_S = 120
|
||||
|
||||
|
||||
def get_stage_config() -> str:
|
||||
return str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
|
||||
)
|
||||
return get_deploy_config_path("qwen3_tts.yaml")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# stage config for running qwen2.5-omni for multi-stage omni runtime.
|
||||
|
||||
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
|
||||
# This config is optimized for CI e2e tests.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.9
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
load_format: dummy
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.4
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "2" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.5 #increase the gpu memory utilization to enable the test on H800
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 8192
|
||||
load_format: dummy
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 8192
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
@@ -1,31 +0,0 @@
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.9
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,102 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
max_num_seqs: 5
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 32768
|
||||
max_model_len: 32768
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 150
|
||||
seed: 42
|
||||
ignore_eos: False
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
max_num_seqs: 5
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.5
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
max_model_len: 32768
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 1000
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 5
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 100000
|
||||
hf_config_name: thinker_config
|
||||
async_scheduling: false
|
||||
load_format: dummy
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 2000
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,101 +0,0 @@
|
||||
# stage config for running qwen2.5-omni for multi-stage omni runtime.
|
||||
|
||||
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
|
||||
# This config is optimized for CI e2e tests.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
@@ -1,100 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
load_format: dummy
|
||||
async_scheduling: false
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 200
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,102 +0,0 @@
|
||||
# stage config for running qwen2.5-omni for multi-stage omni runtime.
|
||||
|
||||
# The following config is verified with 2 * Intel Arc Pro B60 XPU.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.9 # thinker weight is around 16.74GB for Qwen2.5-Omni-7B
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 16384
|
||||
max_num_batched_tokens: 16384
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.5 # talker weight is 6.03GB for Qwen2.5-Omni-7B
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
process: true
|
||||
devices: "2"
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.3 # code2wav weight is around 1.46GB for Qwen2.5-Omni-7B
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
@@ -1,109 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config is verified with 8 * Intel Arc Pro B60 XPU.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
devices: "0,1,2,3"
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.85 # thinker weight is around 61.08GB for Qwen3-Omni-30B-A3B-Instruct
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 4
|
||||
max_cudagraph_capture_size: 0
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
ignore_eos: False
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
devices: "4"
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6 # talker weight is around 8.5GB for Qwen3-Omni-30B-A3B-Instruct
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
max_cudagraph_capture_size: 0
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
devices: "5"
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.3 # code2wav weight is around 0.4GB for Qwen3-Omni-30B-A3B-Instruct
|
||||
skip_mm_profiling: true
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 100000
|
||||
hf_config_name: thinker_config
|
||||
async_scheduling: false
|
||||
max_cudagraph_capture_size: 0
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 2000
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
Reference in new issue
Block a user