[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)
Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: reidliu41 <reid201711@gmail.com> Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com> Co-authored-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
104 files changed
+3959
-3814
No files matched your search
@@ -320,14 +320,7 @@ def main(args):
|
||||
query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
|
||||
else:
|
||||
query_result = query_func()
|
||||
omni = Omni(
|
||||
model=model_name,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
batch_timeout=args.batch_timeout,
|
||||
init_timeout=args.init_timeout,
|
||||
shm_threshold_bytes=args.shm_threshold_bytes,
|
||||
)
|
||||
omni = Omni.from_cli_args(args, model=model_name)
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.0, # Deterministic - no randomness
|
||||
top_p=1.0, # Disable nucleus sampling
|
||||
|
||||
@@ -70,8 +70,8 @@ For true stage-level concurrency -- where downstream stages (Talker, Code2Wav)
|
||||
start **before** the upstream stage (Thinker) finishes -- use the async_chunk
|
||||
example. This requires:
|
||||
|
||||
1. A stage config YAML with ``async_chunk: true`` (e.g.
|
||||
``qwen3_omni_moe_async_chunk.yaml``).
|
||||
1. A deploy config YAML with ``async_chunk: true`` (e.g.
|
||||
``qwen3_omni_moe.yaml``).
|
||||
2. Hardware that matches the config (e.g. 2x H100 for the default 3-stage
|
||||
config).
|
||||
|
||||
@@ -101,7 +101,7 @@ python end2end_async_chunk.py --query-type text --modalities text
|
||||
```bash
|
||||
python end2end_async_chunk.py \
|
||||
--query-type use_audio \
|
||||
--stage-configs-path /path/to/your_async_chunk.yaml
|
||||
--deploy-config /path/to/your_deploy_config.yaml
|
||||
```
|
||||
|
||||
> **Note**: The synchronous ``end2end.py`` (using ``Omni``) is still the
|
||||
|
||||
@@ -294,14 +294,7 @@ def main(args):
|
||||
else:
|
||||
query_result = query_func()
|
||||
|
||||
omni = Omni(
|
||||
model=model_name,
|
||||
dtype=args.dtype,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
init_timeout=args.init_timeout,
|
||||
)
|
||||
omni = Omni.from_cli_args(args, model=model_name)
|
||||
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.9,
|
||||
|
||||
@@ -14,7 +14,7 @@ actively processing while stage-0 is still generating.
|
||||
Usage
|
||||
-----
|
||||
python end2end_async_chunk.py --query-type use_audio \
|
||||
--stage-configs-path <path-to-async-chunk-yaml>
|
||||
--deploy-config <path-to-deploy-config-yaml>
|
||||
|
||||
See ``--help`` for all options.
|
||||
"""
|
||||
@@ -179,20 +179,26 @@ def clone_prompt_for_request(template: dict) -> dict:
|
||||
return cloned
|
||||
|
||||
|
||||
def _default_async_chunk_stage_configs_path() -> str | None:
|
||||
"""Best-effort default stage config for running Qwen3-Omni with async_chunk.
|
||||
def _default_deploy_config_path() -> str | None:
|
||||
"""Best-effort default deploy config for running Qwen3-Omni with async_chunk.
|
||||
|
||||
When this example is executed from within the repository, we resolve the
|
||||
default YAML path relative to this file. When installed elsewhere, the
|
||||
file may not exist and callers should pass --stage-configs-path explicitly.
|
||||
The default ``vllm_omni/deploy/qwen3_omni_moe.yaml`` ships with
|
||||
``async_chunk: true`` at the top level, so loading it is enough to
|
||||
enable async-chunk semantics. To disable it, copy the YAML and set
|
||||
``async_chunk: false`` (or pass ``--deploy-config`` to a YAML that
|
||||
overrides the flag).
|
||||
|
||||
When this example is executed from within the repository, we resolve
|
||||
the default YAML path relative to this file. When installed elsewhere,
|
||||
the file may not exist and callers should pass ``--deploy-config``
|
||||
explicitly.
|
||||
"""
|
||||
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
|
||||
candidate = os.path.join(
|
||||
repo_root,
|
||||
"vllm_omni",
|
||||
"model_executor",
|
||||
"stage_configs",
|
||||
"qwen3_omni_moe_async_chunk.yaml",
|
||||
"deploy",
|
||||
"qwen3_omni_moe.yaml",
|
||||
)
|
||||
return candidate if os.path.exists(candidate) else None
|
||||
|
||||
@@ -374,15 +380,16 @@ async def run_all(args):
|
||||
prompt["modalities"] = output_modalities
|
||||
|
||||
# Create AsyncOmni
|
||||
print(f"[Info] Creating AsyncOmni with stage_configs_path={args.stage_configs_path}")
|
||||
print(f"[Info] Creating AsyncOmni with deploy_config={args.deploy_config}")
|
||||
async_omni = None
|
||||
try:
|
||||
async_omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
# ``from_cli_args`` expands vars(args) into kwargs and auto-captures
|
||||
# ``_cli_explicit_keys`` from ``sys.argv[1:]`` so argparse defaults
|
||||
# do not silently override deploy YAML values. Mirrors the
|
||||
# ``EngineArgs.from_cli_args`` pattern used throughout vllm /
|
||||
# vllm-omni. ``deploy_config=None`` (the default) falls through to
|
||||
# the bundled ``vllm_omni/deploy/qwen3_omni_moe.yaml``.
|
||||
async_omni = AsyncOmni.from_cli_args(args)
|
||||
|
||||
# Use default sampling params from stage config (they are pre-configured
|
||||
# in the YAML for each stage).
|
||||
@@ -470,11 +477,11 @@ def parse_args():
|
||||
help="Query type.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=_default_async_chunk_stage_configs_path(),
|
||||
default=_default_deploy_config_path(),
|
||||
help=(
|
||||
"Path to an async_chunk stage config YAML. "
|
||||
"Path to a deploy config YAML. "
|
||||
"If not set, uses the model's default config "
|
||||
"(make sure it has async_chunk: true)."
|
||||
),
|
||||
|
||||
@@ -17,7 +17,7 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
python "${SCRIPT_DIR}/end2end_async_chunk.py" \
|
||||
--query-type text \
|
||||
--txt-prompts "${SCRIPT_DIR}/text_prompts_10.txt" \
|
||||
--stage-configs-path "${REPO_ROOT}/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml" \
|
||||
--deploy-config "${REPO_ROOT}/vllm_omni/deploy/qwen3_omni_moe.yaml" \
|
||||
--output-dir output_audio_async_chunk \
|
||||
--max-in-flight 2 \
|
||||
"$@"
|
||||
@@ -6,13 +6,13 @@
|
||||
# achieving true stage-level concurrency via chunk-level streaming.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - An async_chunk stage config YAML (e.g. qwen3_omni_moe_async_chunk.yaml)
|
||||
# - A deploy config YAML (e.g. qwen3_omni_moe.yaml)
|
||||
# - Hardware matching the config (e.g. 2x H100 for the default 3-stage config)
|
||||
#
|
||||
# Usage:
|
||||
# bash run_single_prompt_async_chunk.sh
|
||||
# bash run_single_prompt_async_chunk.sh --query-type text --modalities text
|
||||
# bash run_single_prompt_async_chunk.sh --stage-configs-path /path/to/custom.yaml
|
||||
# bash run_single_prompt_async_chunk.sh --deploy-config /path/to/custom.yaml
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -21,6 +21,6 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
|
||||
python "${SCRIPT_DIR}/end2end_async_chunk.py" \
|
||||
--query-type use_audio \
|
||||
--stage-configs-path "${REPO_ROOT}/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml" \
|
||||
--deploy-config "${REPO_ROOT}/vllm_omni/deploy/qwen3_omni_moe.yaml" \
|
||||
--output-dir output_audio_async_chunk \
|
||||
"$@"
|
||||
@@ -104,13 +104,13 @@ completes. This demonstrates that audio data is available progressively rather t
|
||||
|
||||
## Batched Decoding
|
||||
|
||||
The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, provide a stage config with `max_num_seqs > 1` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
|
||||
The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
|
||||
|
||||
```
|
||||
python end2end.py --query-type CustomVoice \
|
||||
--txt-prompts benchmark_prompts.txt \
|
||||
--batch-size 4 \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml
|
||||
--stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
|
||||
```
|
||||
|
||||
**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
|
||||
|
||||
@@ -366,12 +366,7 @@ def main(args):
|
||||
output_dir = args.output_dir
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
omni = Omni(
|
||||
model=model_name,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
omni = Omni.from_cli_args(args, model=model_name)
|
||||
|
||||
batch_size = args.batch_size
|
||||
for batch_start in range(0, len(inputs), batch_size):
|
||||
@@ -387,12 +382,7 @@ async def main_streaming(args):
|
||||
output_dir = args.output_dir
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
omni = AsyncOmni(
|
||||
model=model_name,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
omni = AsyncOmni.from_cli_args(args, model=model_name)
|
||||
|
||||
for i, prompt in enumerate(inputs):
|
||||
request_id = str(i)
|
||||
|
||||
Reference in new issue
Block a user