[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)
Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: reidliu41 <reid201711@gmail.com> Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com> Co-authored-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
104 files changed
+3959
-3814
No files matched your search
@@ -35,7 +35,7 @@ from tests.dfx.conftest import (
|
||||
from tests.dfx.perf.scripts.run_benchmark import run_benchmark
|
||||
|
||||
STABILITY_DIR = Path(__file__).resolve().parent.parent
|
||||
STAGE_CONFIGS_DIR = STABILITY_DIR / "stage_configs"
|
||||
DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
|
||||
CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test.json")
|
||||
DEFAULT_NUM_PROMPTS_PER_BATCH = 20
|
||||
|
||||
@@ -45,7 +45,7 @@ try:
|
||||
except FileNotFoundError:
|
||||
BENCHMARK_CONFIGS = []
|
||||
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
|
||||
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
|
||||
|
||||
_omni_server_lock = threading.Lock()
|
||||
@@ -219,11 +219,20 @@ def omni_server(request):
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
test_name, model, stage_config_path = request.param
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args = request.param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
|
||||
server_args = ["--stage-init-timeout", "120"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
async_chunk: false
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 64
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 64
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 64
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -3,7 +3,11 @@
|
||||
"test_name": "test_qwen3_omni_stability",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_config_name": "qwen3_omni.yaml"
|
||||
"stage_overrides": {
|
||||
"2": {
|
||||
"max_num_batched_tokens": 1000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
@@ -36,25 +40,12 @@
|
||||
"test_name": "test_qwen3_omni_stability_async_chunk",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_config_name": "qwen3_omni.yaml",
|
||||
"update": {
|
||||
"async_chunk": true,
|
||||
"stage_args": {
|
||||
"0": {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
|
||||
},
|
||||
"1": {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
|
||||
}
|
||||
"stage_overrides": {
|
||||
"2": {
|
||||
"max_num_batched_tokens": 1000000
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"stage_args": {
|
||||
"2": [
|
||||
"custom_process_input_func"
|
||||
]
|
||||
}
|
||||
}
|
||||
"extra_cli_args": ["--async-chunk"]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user