[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)

Signed-off-by: lishunyang <lishunyang12@163.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: reidliu41 <reid201711@gmail.com>
Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com>
Co-authored-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-04-19 14:36:00 +12:00
1 parent a683b1dd75
commit a3903810fe
104 files changed
+3959 -3814

No files matched your search

@@ -320,14 +320,7 @@ def main(args):
query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
else:
query_result = query_func()
omni = Omni(
model=model_name,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
batch_timeout=args.batch_timeout,
init_timeout=args.init_timeout,
shm_threshold_bytes=args.shm_threshold_bytes,
)
omni = Omni.from_cli_args(args, model=model_name)
thinker_sampling_params = SamplingParams(
temperature=0.0, # Deterministic - no randomness
top_p=1.0, # Disable nucleus sampling
@@ -70,8 +70,8 @@ For true stage-level concurrency -- where downstream stages (Talker, Code2Wav)
start **before** the upstream stage (Thinker) finishes -- use the async_chunk
example. This requires:
1. A stage config YAML with ``async_chunk: true`` (e.g.
``qwen3_omni_moe_async_chunk.yaml``).
1. A deploy config YAML with ``async_chunk: true`` (e.g.
``qwen3_omni_moe.yaml``).
2. Hardware that matches the config (e.g. 2x H100 for the default 3-stage
config).
@@ -101,7 +101,7 @@ python end2end_async_chunk.py --query-type text --modalities text
```bash
python end2end_async_chunk.py \
--query-type use_audio \
--stage-configs-path /path/to/your_async_chunk.yaml
--deploy-config /path/to/your_deploy_config.yaml
```
> **Note**: The synchronous ``end2end.py`` (using ``Omni``) is still the
@@ -294,14 +294,7 @@ def main(args):
else:
query_result = query_func()
omni = Omni(
model=model_name,
dtype=args.dtype,
stage_configs_path=args.stage_configs_path,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
init_timeout=args.init_timeout,
)
omni = Omni.from_cli_args(args, model=model_name)
thinker_sampling_params = SamplingParams(
temperature=0.9,
@@ -14,7 +14,7 @@ actively processing while stage-0 is still generating.
Usage
-----
python end2end_async_chunk.py --query-type use_audio \
--stage-configs-path <path-to-async-chunk-yaml>
--deploy-config <path-to-deploy-config-yaml>
See ``--help`` for all options.
"""
@@ -179,20 +179,26 @@ def clone_prompt_for_request(template: dict) -> dict:
return cloned
def _default_async_chunk_stage_configs_path() -> str | None:
"""Best-effort default stage config for running Qwen3-Omni with async_chunk.
def _default_deploy_config_path() -> str | None:
"""Best-effort default deploy config for running Qwen3-Omni with async_chunk.
When this example is executed from within the repository, we resolve the
default YAML path relative to this file. When installed elsewhere, the
file may not exist and callers should pass --stage-configs-path explicitly.
The default ``vllm_omni/deploy/qwen3_omni_moe.yaml`` ships with
``async_chunk: true`` at the top level, so loading it is enough to
enable async-chunk semantics. To disable it, copy the YAML and set
``async_chunk: false`` (or pass ``--deploy-config`` to a YAML that
overrides the flag).
When this example is executed from within the repository, we resolve
the default YAML path relative to this file. When installed elsewhere,
the file may not exist and callers should pass ``--deploy-config``
explicitly.
"""
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
candidate = os.path.join(
repo_root,
"vllm_omni",
"model_executor",
"stage_configs",
"qwen3_omni_moe_async_chunk.yaml",
"deploy",
"qwen3_omni_moe.yaml",
)
return candidate if os.path.exists(candidate) else None
@@ -374,15 +380,16 @@ async def run_all(args):
prompt["modalities"] = output_modalities
# Create AsyncOmni
print(f"[Info] Creating AsyncOmni with stage_configs_path={args.stage_configs_path}")
print(f"[Info] Creating AsyncOmni with deploy_config={args.deploy_config}")
async_omni = None
try:
async_omni = AsyncOmni(
model=args.model,
stage_configs_path=args.stage_configs_path,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
)
# ``from_cli_args`` expands vars(args) into kwargs and auto-captures
# ``_cli_explicit_keys`` from ``sys.argv[1:]`` so argparse defaults
# do not silently override deploy YAML values. Mirrors the
# ``EngineArgs.from_cli_args`` pattern used throughout vllm /
# vllm-omni. ``deploy_config=None`` (the default) falls through to
# the bundled ``vllm_omni/deploy/qwen3_omni_moe.yaml``.
async_omni = AsyncOmni.from_cli_args(args)
# Use default sampling params from stage config (they are pre-configured
# in the YAML for each stage).
@@ -470,11 +477,11 @@ def parse_args():
help="Query type.",
)
parser.add_argument(
"--stage-configs-path",
"--deploy-config",
type=str,
default=_default_async_chunk_stage_configs_path(),
default=_default_deploy_config_path(),
help=(
"Path to an async_chunk stage config YAML. "
"Path to a deploy config YAML. "
"If not set, uses the model's default config "
"(make sure it has async_chunk: true)."
),
@@ -17,7 +17,7 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
python "${SCRIPT_DIR}/end2end_async_chunk.py" \
--query-type text \
--txt-prompts "${SCRIPT_DIR}/text_prompts_10.txt" \
--stage-configs-path "${REPO_ROOT}/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml" \
--deploy-config "${REPO_ROOT}/vllm_omni/deploy/qwen3_omni_moe.yaml" \
--output-dir output_audio_async_chunk \
--max-in-flight 2 \
"$@"
@@ -6,13 +6,13 @@
# achieving true stage-level concurrency via chunk-level streaming.
#
# Prerequisites:
# - An async_chunk stage config YAML (e.g. qwen3_omni_moe_async_chunk.yaml)
# - A deploy config YAML (e.g. qwen3_omni_moe.yaml)
# - Hardware matching the config (e.g. 2x H100 for the default 3-stage config)
#
# Usage:
# bash run_single_prompt_async_chunk.sh
# bash run_single_prompt_async_chunk.sh --query-type text --modalities text
# bash run_single_prompt_async_chunk.sh --stage-configs-path /path/to/custom.yaml
# bash run_single_prompt_async_chunk.sh --deploy-config /path/to/custom.yaml
set -euo pipefail
@@ -21,6 +21,6 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
python "${SCRIPT_DIR}/end2end_async_chunk.py" \
--query-type use_audio \
--stage-configs-path "${REPO_ROOT}/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml" \
--deploy-config "${REPO_ROOT}/vllm_omni/deploy/qwen3_omni_moe.yaml" \
--output-dir output_audio_async_chunk \
"$@"
@@ -104,13 +104,13 @@ completes. This demonstrates that audio data is available progressively rather t
## Batched Decoding
The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, provide a stage config with `max_num_seqs > 1` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
```
python end2end.py --query-type CustomVoice \
--txt-prompts benchmark_prompts.txt \
--batch-size 4 \
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml
--stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
```
**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
@@ -366,12 +366,7 @@ def main(args):
output_dir = args.output_dir
os.makedirs(output_dir, exist_ok=True)
omni = Omni(
model=model_name,
stage_configs_path=args.stage_configs_path,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
)
omni = Omni.from_cli_args(args, model=model_name)
batch_size = args.batch_size
for batch_start in range(0, len(inputs), batch_size):
@@ -387,12 +382,7 @@ async def main_streaming(args):
output_dir = args.output_dir
os.makedirs(output_dir, exist_ok=True)
omni = AsyncOmni(
model=model_name,
stage_configs_path=args.stage_configs_path,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
)
omni = AsyncOmni.from_cli_args(args, model=model_name)
for i, prompt in enumerate(inputs):
request_id = str(i)