[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)
Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: reidliu41 <reid201711@gmail.com> Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com> Co-authored-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
104 files changed
+3959
-3814
No files matched your search
+16
-6
@@ -40,22 +40,32 @@ def modify_stage(default_path, updates, deletes):
|
||||
def create_unique_server_params(
|
||||
configs: list[dict[str, Any]],
|
||||
stage_configs_dir: Path,
|
||||
) -> list[tuple[str, str, str]]:
|
||||
) -> list[tuple[str, str, str | None, str | None, tuple[str, ...]]]:
|
||||
unique_params = []
|
||||
seen = set()
|
||||
for config in configs:
|
||||
test_name = config["test_name"]
|
||||
model = config["server_params"]["model"]
|
||||
stage_config_name = config["server_params"].get("stage_config_name")
|
||||
server_params = config["server_params"]
|
||||
model = server_params["model"]
|
||||
stage_config_name = server_params.get("stage_config_name")
|
||||
if stage_config_name:
|
||||
stage_config_path = str(stage_configs_dir / stage_config_name)
|
||||
delete = config["server_params"].get("delete", None)
|
||||
update = config["server_params"].get("update", None)
|
||||
delete = server_params.get("delete", None)
|
||||
update = server_params.get("update", None)
|
||||
stage_config_path = modify_stage(stage_config_path, update, delete)
|
||||
else:
|
||||
stage_config_path = None
|
||||
|
||||
server_param = (test_name, model, stage_config_path)
|
||||
stage_overrides = server_params.get("stage_overrides")
|
||||
stage_overrides_json = json.dumps(stage_overrides) if stage_overrides else None
|
||||
|
||||
# ``extra_cli_args`` passes raw CLI flags straight through to
|
||||
# ``vllm_omni.entrypoints.cli.main serve`` — used for flags that
|
||||
# don't map to stage-level overrides, e.g. ``--async-chunk`` /
|
||||
# ``--no-async-chunk`` toggling the deploy-level async_chunk bool.
|
||||
extra_cli_args = tuple(server_params.get("extra_cli_args") or ())
|
||||
|
||||
server_param = (test_name, model, stage_config_path, stage_overrides_json, extra_cli_args)
|
||||
if server_param not in seen:
|
||||
seen.add(server_param)
|
||||
unique_params.append(server_param)
|
||||
|
||||
@@ -48,8 +48,8 @@ BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
|
||||
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
|
||||
|
||||
|
||||
STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs"
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR)
|
||||
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
|
||||
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
|
||||
|
||||
_omni_server_lock = threading.Lock()
|
||||
@@ -62,13 +62,19 @@ def omni_server(request):
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
test_name, model, stage_config_path = request.param
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args = request.param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
server_args = ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--stage-configs-path", stage_config_path] + server_args
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
async_chunk: false
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
max_num_seqs: 64
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
max_num_seqs: 64
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type for AR stages
|
||||
runtime:
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 64
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 100000
|
||||
hf_config_name: thinker_config
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,91 +0,0 @@
|
||||
# Stage config for running Qwen3-TTS with 2-stage architecture
|
||||
# Stage 0: Talker (text -> 8-layer RVQ codec codes)
|
||||
# Stage 1: Code2Wav (codec codes -> audio waveform)
|
||||
#
|
||||
# The following config has been verified on 1x H100-80G GPU.
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
max_num_seqs: 4
|
||||
model_stage: qwen3_tts
|
||||
model_arch: Qwen3TTSTalkerForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.3
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 512
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
max_num_seqs: 4
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3TTSCode2Wav
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.2
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 32768
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
tts_args:
|
||||
max_instructions_length: 500
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 72
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -2,8 +2,7 @@
|
||||
{
|
||||
"test_name": "test_qwen3_omni",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_config_name": "qwen3_omni.yaml"
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
@@ -109,25 +108,7 @@
|
||||
"test_name": "test_qwen3_omni_chunk",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_config_name": "qwen3_omni.yaml",
|
||||
"update": {
|
||||
"async_chunk": true,
|
||||
"stage_args": {
|
||||
"0": {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
|
||||
},
|
||||
"1": {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"stage_args": {
|
||||
"2": [
|
||||
"custom_process_input_func"
|
||||
]
|
||||
}
|
||||
}
|
||||
"extra_cli_args": ["--async-chunk"]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
|
||||
@@ -35,7 +35,7 @@ from tests.dfx.conftest import (
|
||||
from tests.dfx.perf.scripts.run_benchmark import run_benchmark
|
||||
|
||||
STABILITY_DIR = Path(__file__).resolve().parent.parent
|
||||
STAGE_CONFIGS_DIR = STABILITY_DIR / "stage_configs"
|
||||
DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
|
||||
CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test.json")
|
||||
DEFAULT_NUM_PROMPTS_PER_BATCH = 20
|
||||
|
||||
@@ -45,7 +45,7 @@ try:
|
||||
except FileNotFoundError:
|
||||
BENCHMARK_CONFIGS = []
|
||||
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
|
||||
test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
|
||||
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
|
||||
|
||||
_omni_server_lock = threading.Lock()
|
||||
@@ -219,11 +219,20 @@ def omni_server(request):
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
test_name, model, stage_config_path = request.param
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args = request.param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
|
||||
server_args = ["--stage-init-timeout", "120"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
async_chunk: false
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 64
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 64
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 64
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -3,7 +3,11 @@
|
||||
"test_name": "test_qwen3_omni_stability",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_config_name": "qwen3_omni.yaml"
|
||||
"stage_overrides": {
|
||||
"2": {
|
||||
"max_num_batched_tokens": 1000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
@@ -36,25 +40,12 @@
|
||||
"test_name": "test_qwen3_omni_stability_async_chunk",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"stage_config_name": "qwen3_omni.yaml",
|
||||
"update": {
|
||||
"async_chunk": true,
|
||||
"stage_args": {
|
||||
"0": {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
|
||||
},
|
||||
"1": {
|
||||
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
|
||||
}
|
||||
"stage_overrides": {
|
||||
"2": {
|
||||
"max_num_batched_tokens": 1000000
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"stage_args": {
|
||||
"2": [
|
||||
"custom_process_input_func"
|
||||
]
|
||||
}
|
||||
}
|
||||
"extra_cli_args": ["--async-chunk"]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user