anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Config Refactor][2/N] Pipeline + Deploy Config Schema (#2383)

Browse Source 
Signed-off-by: lishunyang <lishunyang12@163.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: reidliu41 <reid201711@gmail.com>
Co-authored-by: xiaohajiayou <75477391+xiaohajiayou@users.noreply.github.com>
Co-authored-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
SYLARreidliu41xiaohajiayouAlex BrooksHongsheng Liu
 authored and GitHub committed 2026-04-19 14:36:00 +12:00
1 parent a683b1dd75
commit a3903810fe
104 files changed
+3959 -3814
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/conftest.py
+108 -21
View File
@@ -47,6 +47,7 @@ from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm.logger import init_logger
from vllm.utils.network_utils import get_open_port
from vllm_omni.config.stage_config import resolve_deploy_yaml
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -1339,12 +1340,14 @@ def modify_stage_config(
else:
print(f"Path {path} does not exist")
_stage_key = "stages" if "stages" in config else "stage_args"
# Apply deletions first
if deletes:
for key, value in deletes.items():
if key == "stage_args":
if key in ("stage_args", "stages"):
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
stage_args = config.get(_stage_key, [])
if not stage_args:
raise ValueError("stage_args does not exist in config")
@@ -1377,9 +1380,9 @@ def modify_stage_config(
# Apply updates
if updates:
for key, value in updates.items():
if key == "stage_args":
if key in ("stage_args", "stages"):
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
stage_args = config.get(_stage_key, [])
if not stage_args:
raise ValueError("stage_args does not exist in config")
@@ -1585,32 +1588,46 @@ class OmniServerStageCli(OmniServer):
self.stage_config_path = stage_config_path
self.master_port = get_open_port()
self.visible_device_list = self._load_visible_device_list(env_dict)
self.stage_runtime_devices = self._load_stage_runtime_devices(stage_config_path)
self.stage_ids = stage_ids or self._load_stage_ids(stage_config_path)
resolved_cfg = resolve_deploy_yaml(stage_config_path)
# Dump the resolved deploy config so CI logs show each stage's
# gpu_memory_utilization / max_model_len / max_num_seqs after
# base_config inheritance and overlay merge — essential when
# diagnosing OOMs that depend on the merged values.
print(
f"[OmniServerStageCli] Resolved deploy config from {stage_config_path}:\n"
f"{yaml.safe_dump(resolved_cfg, sort_keys=False, default_flow_style=False)}",
flush=True,
)
self.stage_runtime_devices = self._load_stage_runtime_devices(resolved_cfg)
self.stage_ids = stage_ids or self._load_stage_ids(resolved_cfg)
if 0 not in self.stage_ids:
raise ValueError(f"Stage CLI test requires stage_id=0 in config: {stage_config_path}")
self.stage_procs: dict[int, subprocess.Popen] = {}
self.proc = None
@staticmethod
def _load_stage_ids(stage_config_path: str) -> list[int]:
with open(stage_config_path, encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
def _stage_entries(cfg: dict) -> list[dict]:
"""Return the list of stage entries from either legacy (``stage_args``)
or new-schema (``stages``) deploy YAMLs."""
return cfg.get("stage_args") or cfg.get("stages") or []
stage_ids = [stage["stage_id"] for stage in cfg.get("stage_args", []) if "stage_id" in stage]
@staticmethod
def _load_stage_ids(resolved_config: dict) -> list[int]:
stage_ids = [
stage["stage_id"] for stage in OmniServerStageCli._stage_entries(resolved_config) if "stage_id" in stage
]
if not stage_ids:
raise ValueError(f"No stage IDs found in config: {stage_config_path}")
raise ValueError("No stage IDs found in resolved config")
return stage_ids
@staticmethod
def _load_stage_runtime_devices(stage_config_path: str) -> dict[int, str]:
with open(stage_config_path, encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
def _load_stage_runtime_devices(resolved_config: dict) -> dict[int, str]:
runtime_devices: dict[int, str] = {}
for stage in cfg.get("stage_args", []):
for stage in OmniServerStageCli._stage_entries(resolved_config):
stage_id = stage.get("stage_id")
devices = stage.get("runtime", {}).get("devices")
# New schema: stage.devices is flat at stage level.
# Legacy schema: stage.runtime.devices is nested.
devices = stage.get("devices") or stage.get("runtime", {}).get("devices")
if stage_id is not None and devices:
runtime_devices[int(stage_id)] = str(devices)
return runtime_devices
@@ -1696,10 +1713,21 @@ class OmniServerStageCli(OmniServer):
cmd = self._build_stage_cmd(stage_id, headless=headless)
print(f"Launching OmniServerStageCli stage {stage_id}: {' '.join(cmd)}")
# Capture each subprocess's stdout+stderr to a per-stage log file so
# debugging "Stage N exited before API server ready" doesn't rely on
# guessing; the file is surfaced in the RuntimeError message.
log_path = Path(tempfile.gettempdir()) / f"omni_stage_{stage_id}_{self.master_port}.log"
self._stage_log_paths = getattr(self, "_stage_log_paths", {})
self._stage_log_paths[stage_id] = log_path
log_fh = open(log_path, "w", buffering=1) # noqa: SIM115 - closed in __exit__
self._stage_log_files = getattr(self, "_stage_log_files", {})
self._stage_log_files[stage_id] = log_fh
proc = subprocess.Popen(
cmd,
env=env,
cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
stdout=log_fh,
stderr=subprocess.STDOUT,
)
self.stage_procs[stage_id] = proc
if stage_id == 0:
@@ -1709,7 +1737,18 @@ class OmniServerStageCli(OmniServer):
for stage_id, proc in self.stage_procs.items():
ret = proc.poll()
if ret is not None:
raise RuntimeError(f"Stage {stage_id} exited with code {ret} before API server became ready.")
log_path = getattr(self, "_stage_log_paths", {}).get(stage_id)
tail = ""
if log_path and log_path.exists():
try:
with open(log_path, encoding="utf-8", errors="replace") as f:
lines = f.readlines()
tail = "\n=== Last 60 lines of stage {} log ({}) ===\n{}".format(
stage_id, log_path, "".join(lines[-60:]) or "<empty>"
)
except Exception as exc: # pragma: no cover - diagnostic only
tail = f"\n<failed to read stage log {log_path}: {exc}>"
raise RuntimeError(f"Stage {stage_id} exited with code {ret} before API server became ready.{tail}")
def _start_server(self) -> None:
ordered_stage_ids = [0, *[stage_id for stage_id in self.stage_ids if stage_id != 0]]
@@ -1735,7 +1774,46 @@ class OmniServerStageCli(OmniServer):
raise RuntimeError(f"OmniServerStageCli failed to start within {max_wait} seconds")
def _dump_stage_logs_for_debug(self, head_lines: int = 300, tail_lines: int = 500) -> None:
"""Tail each stage's subprocess log back to stdout on teardown.
Stage subprocesses redirect stdout/stderr to ``/tmp/omni_stage_*.log``
so we don't spam the main CI stream while tests run; but that also
hides engine init (KV cache size, Available KV cache memory, vLLM
engine config) when things go wrong. Dump them here so buildkite
captures them post-run. Head covers engine init; tail covers
whatever state the stage was in when it was torn down.
"""
log_paths = getattr(self, "_stage_log_paths", {}) or {}
for stage_id in sorted(log_paths):
log_path = log_paths[stage_id]
if not log_path or not log_path.exists():
continue
try:
with open(log_path, encoding="utf-8", errors="replace") as f:
lines = f.readlines()
except Exception as exc: # pragma: no cover - diagnostic only
print(f"[OmniServerStageCli] stage {stage_id} log read failed: {exc}", flush=True)
continue
total = len(lines)
if total <= head_lines + tail_lines:
head_chunk = lines
tail_chunk = []
elided = 0
else:
head_chunk = lines[:head_lines]
tail_chunk = lines[-tail_lines:]
elided = total - head_lines - tail_lines
print(f"\n=== stage {stage_id} log HEAD ({log_path}) ===", flush=True)
print("".join(head_chunk).rstrip("\n"), flush=True)
if tail_chunk:
print(f"\n... [{elided} lines elided] ...", flush=True)
print(f"\n=== stage {stage_id} log TAIL ({log_path}) ===", flush=True)
print("".join(tail_chunk).rstrip("\n"), flush=True)
print(f"=== end stage {stage_id} log ===\n", flush=True)
def __exit__(self, exc_type, exc_val, exc_tb):
self._dump_stage_logs_for_debug()
for stage_id in sorted(self.stage_procs, reverse=True):
proc = self.stage_procs[stage_id]
if proc.poll() is None:
@@ -1781,10 +1859,18 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st
if run_level == "advanced_model" and stage_config_path is not None:
with open(stage_config_path, encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
stage_ids = [stage["stage_id"] for stage in cfg.get("stage_args", []) if "stage_id" in stage]
# Strip ``load_format: dummy`` (CI overlay default) so advanced_model
# tests use real weights. New schema (``stages:``) writes the field
# flat at stage level; legacy schema (``stage_args:``) nests it as
# ``engine_args.load_format``. Handle both.
new_schema_stages = cfg.get("stages")
stage_key = "stages" if new_schema_stages is not None else "stage_args"
delete_path = "load_format" if new_schema_stages is not None else "engine_args.load_format"
stage_entries = cfg.get(stage_key, [])
stage_ids = [stage["stage_id"] for stage in stage_entries if "stage_id" in stage]
stage_config_path = modify_stage_config(
stage_config_path,
deletes={"stage_args": {stage_id: ["engine_args.load_format"] for stage_id in stage_ids}},
deletes={stage_key: {stage_id: [delete_path] for stage_id in stage_ids}},
)
server_args = params.server_args or []
@@ -1801,6 +1887,7 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st
raise ValueError("omni_server with use_stage_cli=True requires use_omni=True")
if stage_config_path is None:
raise ValueError("omni_server with use_stage_cli=True requires a stage_config_path")
server_args += ["--stage-configs-path", stage_config_path]
with OmniServerStageCli(
model,
@@ -3291,7 +3378,7 @@ def omni_runner(request, model_prefix):
with _omni_server_lock:
model, stage_config_path = request.param
model = model_prefix + model
with OmniRunner(model, seed=42, stage_configs_path=stage_config_path) as runner:
with OmniRunner(model, seed=42, stage_configs_path=stage_config_path, stage_init_timeout=300) as runner:
print("OmniRunner started successfully")
yield runner
print("OmniRunner stopping...")
tests/dfx/conftest.py
+16 -6
View File
@@ -40,22 +40,32 @@ def modify_stage(default_path, updates, deletes):
def create_unique_server_params(
configs: list[dict[str, Any]],
stage_configs_dir: Path,
) -> list[tuple[str, str, str]]:
) -> list[tuple[str, str, str | None, str | None, tuple[str, ...]]]:
unique_params = []
seen = set()
for config in configs:
test_name = config["test_name"]
model = config["server_params"]["model"]
stage_config_name = config["server_params"].get("stage_config_name")
server_params = config["server_params"]
model = server_params["model"]
stage_config_name = server_params.get("stage_config_name")
if stage_config_name:
stage_config_path = str(stage_configs_dir / stage_config_name)
delete = config["server_params"].get("delete", None)
update = config["server_params"].get("update", None)
delete = server_params.get("delete", None)
update = server_params.get("update", None)
stage_config_path = modify_stage(stage_config_path, update, delete)
else:
stage_config_path = None
server_param = (test_name, model, stage_config_path)
stage_overrides = server_params.get("stage_overrides")
stage_overrides_json = json.dumps(stage_overrides) if stage_overrides else None
# ``extra_cli_args`` passes raw CLI flags straight through to
# ``vllm_omni.entrypoints.cli.main serve`` — used for flags that
# don't map to stage-level overrides, e.g. ``--async-chunk`` /
# ``--no-async-chunk`` toggling the deploy-level async_chunk bool.
extra_cli_args = tuple(server_params.get("extra_cli_args") or ())
server_param = (test_name, model, stage_config_path, stage_overrides_json, extra_cli_args)
if server_param not in seen:
seen.add(server_param)
unique_params.append(server_param)
tests/dfx/perf/scripts/run_benchmark.py
+10 -4
View File
@@ -48,8 +48,8 @@ BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs"
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR)
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
_omni_server_lock = threading.Lock()
@@ -62,13 +62,19 @@ def omni_server(request):
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
test_name, model, stage_config_path = request.param
test_name, model, stage_config_path, stage_overrides, extra_cli_args = request.param
print(f"Starting OmniServer with test: {test_name}, model: {model}")
server_args = ["--stage-init-timeout", "600", "--init-timeout", "900"]
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
if stage_config_path:
server_args = ["--stage-configs-path", stage_config_path] + server_args
server_args = ["--deploy-config", stage_config_path] + server_args
if stage_overrides:
server_args = ["--stage-overrides", stage_overrides] + server_args
if extra_cli_args:
server_args = list(extra_cli_args) + server_args
with OmniServer(model, server_args) as server:
server.test_name = test_name
print("OmniServer started successfully")
tests/dfx/perf/stage_configs/qwen3_omni.yaml
-101
View File
@@ -1,101 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
async_chunk: false
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "0"
engine_args:
model_stage: thinker
max_num_seqs: 64
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
max_num_batched_tokens: 32768
hf_config_name: thinker_config
tensor_parallel_size: 1
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 2048
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "1"
engine_args:
model_stage: talker
max_num_seqs: 64
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
enable_prefix_caching: false
max_num_batched_tokens: 32768
distributed_executor_backend: "mp"
hf_config_name: talker_config
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "1"
engine_args:
model_stage: code2wav
max_num_seqs: 64
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 100000
hf_config_name: thinker_config
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
detokenize: True
repetition_penalty: 1.1
tests/dfx/perf/stage_configs/qwen3_tts.yaml
-91
View File
@@ -1,91 +0,0 @@
# Stage config for running Qwen3-TTS with 2-stage architecture
# Stage 0: Talker (text -> 8-layer RVQ codec codes)
# Stage 1: Code2Wav (codec codes -> audio waveform)
#
# The following config has been verified on 1x H100-80G GPU.
async_chunk: true
stage_args:
- stage_id: 0
stage_type: llm
is_comprehension: true
runtime:
devices: "0"
engine_args:
max_num_seqs: 4
model_stage: qwen3_tts
model_arch: Qwen3TTSTalkerForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
enforce_eager: false
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: latent
gpu_memory_utilization: 0.3
distributed_executor_backend: "mp"
max_num_batched_tokens: 512
max_model_len: 4096
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
output_connectors:
to_stage_1: connector_of_shared_memory
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: false
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 1
stage_type: llm
runtime:
devices: "0"
engine_args:
max_num_seqs: 4
model_stage: code2wav
model_arch: Qwen3TTSCode2Wav
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: audio
gpu_memory_utilization: 0.2
distributed_executor_backend: "mp"
max_num_batched_tokens: 8192
max_model_len: 32768
engine_input_source: [0]
final_output: true
final_output_type: audio
input_connectors:
from_stage_0: connector_of_shared_memory
tts_args:
max_instructions_length: 500
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
detokenize: true
repetition_penalty: 1.0
runtime:
enabled: true
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
shm_threshold_bytes: 65536
codec_streaming: true
connector_get_sleep_s: 0.01
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
codec_chunk_frames: 25
codec_left_context_frames: 72
edges:
- from: 0
to: 1
tests/dfx/perf/tests/test_qwen_omni.json
+2 -21
View File
@@ -2,8 +2,7 @@
{
"test_name": "test_qwen3_omni",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"stage_config_name": "qwen3_omni.yaml"
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct"
},
"benchmark_params": [
{
@@ -109,25 +108,7 @@
"test_name": "test_qwen3_omni_chunk",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"stage_config_name": "qwen3_omni.yaml",
"update": {
"async_chunk": true,
"stage_args": {
"0": {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
},
"1": {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
}
}
},
"delete": {
"stage_args": {
"2": [
"custom_process_input_func"
]
}
}
"extra_cli_args": ["--async-chunk"]
},
"benchmark_params": [
{
tests/dfx/stability/scripts/test_benchmark_stability.py
+13 -4
View File
@@ -35,7 +35,7 @@ from tests.dfx.conftest import (
from tests.dfx.perf.scripts.run_benchmark import run_benchmark
STABILITY_DIR = Path(__file__).resolve().parent.parent
STAGE_CONFIGS_DIR = STABILITY_DIR / "stage_configs"
DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test.json")
DEFAULT_NUM_PROMPTS_PER_BATCH = 20
@@ -45,7 +45,7 @@ try:
except FileNotFoundError:
BENCHMARK_CONFIGS = []
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
_omni_server_lock = threading.Lock()
@@ -219,11 +219,20 @@ def omni_server(request):
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
test_name, model, stage_config_path = request.param
test_name, model, stage_config_path, stage_overrides, extra_cli_args = request.param
print(f"Starting OmniServer with test: {test_name}, model: {model}")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
server_args = ["--stage-init-timeout", "120"]
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
if stage_config_path:
server_args = ["--deploy-config", stage_config_path] + server_args
if stage_overrides:
server_args = ["--stage-overrides", stage_overrides] + server_args
if extra_cli_args:
server_args = list(extra_cli_args) + server_args
with OmniServer(model, server_args) as server:
server.test_name = test_name
print("OmniServer started successfully")
yield server
tests/dfx/stability/stage_configs/qwen3_omni.yaml
-101
View File
@@ -1,101 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
async_chunk: false
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type to launch OmniLLM
runtime:
devices: "0"
max_batch_size: 64
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
max_num_batched_tokens: 32768
hf_config_name: thinker_config
tensor_parallel_size: 1
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 2048
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
stage_type: llm # Use llm stage type to launch OmniLLM
runtime:
devices: "1"
max_batch_size: 64
engine_args:
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
enable_prefix_caching: false
max_num_batched_tokens: 32768
distributed_executor_backend: "mp"
hf_config_name: talker_config
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
stage_type: llm # Use llm stage type to launch OmniLLM
runtime:
devices: "1"
max_batch_size: 64
engine_args:
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
detokenize: True
repetition_penalty: 1.1
tests/dfx/stability/tests/test.json
+9 -18
View File
@@ -3,7 +3,11 @@
"test_name": "test_qwen3_omni_stability",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"stage_config_name": "qwen3_omni.yaml"
"stage_overrides": {
"2": {
"max_num_batched_tokens": 1000000
}
}
},
"benchmark_params": [
{
@@ -36,25 +40,12 @@
"test_name": "test_qwen3_omni_stability_async_chunk",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"stage_config_name": "qwen3_omni.yaml",
"update": {
"async_chunk": true,
"stage_args": {
"0": {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
},
"1": {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
}
"stage_overrides": {
"2": {
"max_num_batched_tokens": 1000000
}
},
"delete": {
"stage_args": {
"2": [
"custom_process_input_func"
]
}
}
"extra_cli_args": ["--async-chunk"]
},
"benchmark_params": [
{
tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml
-98
View File
@@ -1,98 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0"
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
tests/e2e/offline_inference/test_qwen2_5_omni.py
+13 -20
View File
@@ -2,8 +2,6 @@
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
"""
from pathlib import Path
import pytest
from tests.conftest import (
@@ -12,36 +10,31 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen2.5-Omni-7B"]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section. NPU still uses the legacy per-platform YAML until it
# also migrates to the new schema.
_CI_DEPLOY = get_deploy_config_path("ci/qwen2_5_omni.yaml")
def get_cuda_graph_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
return modify_stage_config(
_CI_DEPLOY,
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
1: {"engine_args.enforce_eager": "true"},
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
},
},
)
return path
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
if current_omni_platform.is_npu():
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_xpu():
# Intel XPU stage config optimized for B60 GPU
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml")
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu() or current_omni_platform.is_npu():
stage_config = _CI_DEPLOY
else:
stage_config = get_cuda_graph_config()
tests/e2e/offline_inference/test_qwen3_omni.py
+13 -17
View File
@@ -7,41 +7,37 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def get_cuda_graph_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
return modify_stage_config(
_CI_DEPLOY,
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
1: {"engine_args.enforce_eager": "true"},
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
},
},
)
return path
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
elif current_omni_platform.is_xpu():
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")]
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
stage_configs = [_CI_DEPLOY]
else:
stage_configs = [get_cuda_graph_config()]
tests/e2e/offline_inference/test_qwen3_tts_base.py
+20 -14
View File
@@ -13,12 +13,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import modify_stage_config
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
@@ -26,23 +24,31 @@ REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know wh
def get_cuda_graph_config():
path = modify_stage_config(
get_stage_config(),
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
Composes the synchronous (no-async-chunk) variant on top of the bundled
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
standalone variant yaml; same effective config, no checked-in file needed.
"""
return modify_stage_config(
get_deploy_config_path("qwen3_tts.yaml"),
updates={
"stage_args": {
"async_chunk": False,
"stages": {
0: {
"engine_args.enforce_eager": "true",
"max_num_seqs": 1,
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {"engine_args.enforce_eager": "true"},
},
},
)
return path
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
# Same structure as test_qwen3_omni: models, stage_configs, test_params
tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+20 -14
View File
@@ -13,34 +13,40 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import modify_stage_config
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_cuda_graph_config():
path = modify_stage_config(
get_stage_config(),
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
Composes the synchronous (no-async-chunk) variant on top of the bundled
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
standalone variant yaml; same effective config, no checked-in file needed.
"""
return modify_stage_config(
get_deploy_config_path("qwen3_tts.yaml"),
updates={
"stage_args": {
"async_chunk": False,
"stages": {
0: {
"engine_args.enforce_eager": "true",
"max_num_seqs": 1,
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {"engine_args.enforce_eager": "true"},
},
},
)
return path
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
# Same structure as test_qwen3_omni: models, stage_configs, test_params
tests/e2e/online_serving/test_qwen2_5_omni.py
+4 -17
View File
@@ -3,7 +3,6 @@ E2E Online tests for Qwen2.5-Omni model with video input and audio output.
"""
import os
from pathlib import Path
import pytest
@@ -15,8 +14,7 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from vllm_omni.platforms import current_omni_platform
from tests.utils import get_deploy_config_path, hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -24,20 +22,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
models = ["Qwen/Qwen2.5-Omni-7B"]
def get_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
)
return path
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")]
else:
stage_configs = [get_config()]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section in vllm_omni/deploy/ci/qwen2_5_omni.yaml.
stage_configs = [modify_stage_config(get_deploy_config_path("ci/qwen2_5_omni.yaml"))]
# Create parameter combinations for model and stage config
test_params = [
tests/e2e/online_serving/test_qwen3_omni.py
+24 -38
View File
@@ -3,7 +3,6 @@ E2E Online tests for Qwen3-Omni model with video input and audio output.
"""
import os
from pathlib import Path
import pytest
@@ -15,7 +14,7 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -23,32 +22,24 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
QWEN3_OMNI_CONFIG_PATH = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
QWEN3_OMNI_XPU_CONFIG_PATH = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
_STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs"
_PD_SEP_CONFIG = str(_STAGE_CONFIGS_DIR / "qwen3_omni_moe_pd_ci.yaml")
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation (follow-up — deploy overlay not yet migrated).
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def get_chunk_config(config_path: str | None = None):
"""Load qwen3_omni_ci.yaml with async_chunk modifications for streaming mode."""
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
if config_path is None:
config_path = str(_STAGE_CONFIGS_DIR / "qwen3_omni_ci.yaml")
return modify_stage_config(
config_path,
updates={
"async_chunk": True,
"stage_args": {
0: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
},
1: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
},
},
},
deletes={"stage_args": {2: ["custom_process_input_func"]}},
)
config_path = _CI_DEPLOY
# TODO: remove this workaround once legacy `stage_args` path is deleted.
# The pipeline (qwen3_omni/pipeline.py) already wires
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
# so only async_chunk needs flipping. Writing nested `engine_args:` into
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
# drops flat fields (load_format, max_num_seqs, ...).
return modify_stage_config(config_path, updates={"async_chunk": True})
def get_prefix_caching_config(config_path: str):
@@ -64,21 +55,16 @@ def get_prefix_caching_config(config_path: str):
return path
# Set VLLM_TEST_PD_MODE=1 to test PD disaggregation, default tests async_chunk mode.
_USE_PD = os.environ.get("VLLM_TEST_PD_MODE", "0") == "1"
# Stage configs for H100/CUDA, ROCm MI325, and XPU platforms
if current_omni_platform.is_rocm():
rocm_config = str(_STAGE_CONFIGS_DIR / "rocm" / "qwen3_omni_ci.yaml")
stage_configs = [rocm_config]
prefix_caching_stage_configs = [get_prefix_caching_config(rocm_config)]
elif current_omni_platform.is_xpu():
xpu_config = str(_STAGE_CONFIGS_DIR / "xpu" / "qwen3_omni_ci.yaml")
stage_configs = [xpu_config]
prefix_caching_stage_configs = [get_prefix_caching_config(xpu_config)]
else:
stage_configs = [_PD_SEP_CONFIG if _USE_PD else get_chunk_config(QWEN3_OMNI_CONFIG_PATH)]
prefix_caching_stage_configs = [get_prefix_caching_config(QWEN3_OMNI_CONFIG_PATH)]
# Platform-specific overrides live inside the new deploy yaml's ``platforms:``
# section, so a single ``_CI_DEPLOY`` path serves CUDA, ROCm, and XPU.
# TODO: re-add VLLM_TEST_PD_MODE branch once the PD-disaggregation deploy
# overlay has been migrated to the new schema (previously used the deleted
# ``qwen3_omni_moe_pd_ci.yaml`` stage-configs file).
if current_omni_platform.is_xpu():
stage_configs = [_CI_DEPLOY]
else: # CUDA + ROCm MI325 share the same deploy config
stage_configs = [get_chunk_config()]
prefix_caching_stage_configs = [get_prefix_caching_config(_CI_DEPLOY)]
# Create parameter combinations for model and stage config
test_params = [
tests/e2e/online_serving/test_qwen3_omni_expansion.py
+41 -35
View File
@@ -6,10 +6,7 @@ E2E Online tests for Qwen3-Omni model.
import os
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from pathlib import Path
import pytest
@@ -21,7 +18,7 @@ from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
@@ -40,47 +37,56 @@ LARGE_IMAGE_HEIGHT = 1080
LONG_AUDIO_DURATION_SEC = 120
def get_chunk_config(default_path):
path = modify_stage_config(
def get_batch_token_config(default_path):
"""Override stage 1's max_num_batched_tokens to exercise small-batch paths.
Uses the new flat-stage schema (``stages.<id>.<field>``); the legacy
``stage_args.<id>.engine_args.<field>`` path no longer applies because
the deploy YAML doesn't nest engine fields under ``engine_args:``.
"""
return modify_stage_config(
default_path,
updates={
"stages": {1: {"max_num_batched_tokens": 64}},
},
)
def get_async_chunk_config(default_path):
"""Flip async_chunk on and bump stage 0 thinker output to 2048 tokens.
Pipeline registry (qwen3_omni/pipeline.py) already wires
thinker2talker_async_chunk / talker2code2wav_async_chunk on stages 0/1,
so no per-stage processor override is needed. Using only flat-schema
writes so _parse_stage_deploy stays in its flat branch (nested
``engine_args:`` would drop other overlay fields).
"""
return modify_stage_config(
default_path,
updates={
"async_chunk": True,
"stage_args": {
0: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk",
"default_sampling_params.max_tokens": 2048,
},
1: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
},
},
},
deletes={"stage_args": {2: ["custom_process_input_func"]}},
)
return path
def get_batch_token_config(default_path):
path = modify_stage_config(
default_path,
updates={
"stage_args": {1: {"engine_args.max_num_batched_tokens": 64}},
"stages": {0: {"default_sampling_params.max_tokens": 2048}},
},
)
return path
# CI stage config for 2*H100-80G GPUs
default_path = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
# CI deploy YAML (single file; xpu deltas applied via ``platforms:`` section).
# The overlay explicitly sets ``async_chunk: False``, so ``default`` tests the
# sync path and ``async_chunk`` tests the streaming path with a longer thinker
# output — two distinct scenarios, kept as separate parametrizations.
default_path = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
if current_omni_platform.is_xpu():
default_path = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
# Create parameter combinations for model and stage config
test_params = [
pytest.param(OmniServerParams(model=model, stage_config_path=default_path, use_stage_cli=True), id="default"),
pytest.param(
OmniServerParams(model=model, stage_config_path=get_chunk_config(default_path), use_stage_cli=True),
OmniServerParams(model=model, stage_config_path=default_path, use_stage_cli=True),
id="default",
),
pytest.param(
OmniServerParams(
model=model,
stage_config_path=get_async_chunk_config(default_path),
use_stage_cli=True,
),
id="async_chunk",
),
]
tests/e2e/online_serving/test_qwen3_omni_realtime_websocket.py
+5 -7
View File
@@ -12,7 +12,6 @@ import io
import json
import os
import wave
from pathlib import Path
import pytest
import websockets
@@ -24,23 +23,22 @@ from tests.conftest import (
generate_synthetic_audio,
modify_stage_config,
)
from tests.utils import hardware_test
from vllm_omni.platforms import current_omni_platform
from tests.utils import get_deploy_config_path, hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
default_stage_config = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
if current_omni_platform.is_xpu():
default_stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")
# The new-schema CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU
# via its ``platforms:`` section, so one path serves all three.
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def _realtime_stage_config_path() -> str:
"""CI omni layout without async_chunk; stage 0 thinker max_tokens=10."""
return modify_stage_config(
default_stage_config,
updates={"stage_args": {0: {"default_sampling_params.max_tokens": 10}}},
updates={"stages": {0: {"default_sampling_params.max_tokens": 10}}},
)
tests/e2e/online_serving/test_qwen3_tts_base.py
+2 -9
View File
@@ -12,12 +12,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -25,11 +23,6 @@ REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="text"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -48,7 +41,7 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
tests/e2e/online_serving/test_qwen3_tts_base_expansion.py
+7 -11
View File
@@ -12,12 +12,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -25,11 +23,6 @@ REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="text"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -48,16 +41,19 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
),
# Synchronous (no async-chunk) variant — ``--no-async-chunk`` alone
# flips the deploy yaml's bool and the pipeline dispatches to the
# end-to-end codec processor. No variant yaml / pipeline needed.
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"],
),
id="no_async_chunk",
),
tests/e2e/online_serving/test_qwen3_tts_batch.py
+4 -3
View File
@@ -27,14 +27,15 @@ from tests.conftest import (
convert_audio_file_to_text,
cosine_similarity_text,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
STAGE_INIT_TIMEOUT_S = 120
def get_stage_config(name: str = "qwen3_tts.yaml"):
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_stage_config(name: str = "qwen3_tts.yaml") -> str:
"""Resolve a deploy config path under vllm_omni/deploy/."""
return get_deploy_config_path(name)
@pytest.fixture(scope="module")
tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+2 -9
View File
@@ -12,21 +12,14 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="text"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -45,7 +38,7 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py
+7 -11
View File
@@ -12,21 +12,14 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_stage_config(name: str = "qwen3_tts.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="english"):
"""Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
prompts = {
@@ -46,16 +39,19 @@ tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts.yaml"),
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="async_chunk",
),
# Synchronous (no async-chunk) variant — ``--no-async-chunk`` alone
# flips the deploy yaml's bool and the pipeline dispatches to the
# end-to-end codec processor. No variant yaml / pipeline needed.
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats", "--no-async-chunk"],
),
id="no_async_chunk",
),
tests/e2e/online_serving/test_qwen3_tts_speaker_embedding.py
+3 -6
View File
@@ -13,13 +13,12 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import struct
from pathlib import Path
import httpx
import pytest
from tests.conftest import OmniServer
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
@@ -37,10 +36,8 @@ MIN_AUDIO_BYTES = 2000
MAX_NEW_TOKENS = 256
def get_stage_config():
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
)
def get_stage_config() -> str:
return get_deploy_config_path("qwen3_tts.yaml")
def _server_args():
tests/e2e/online_serving/test_qwen3_tts_websocket.py
+2 -5
View File
@@ -7,13 +7,12 @@ E2E online tests for Qwen3-TTS WebSocket streaming speech.
import asyncio
import json
import os
from pathlib import Path
import pytest
import websockets
from tests.conftest import OmniServer
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -23,9 +22,7 @@ STAGE_INIT_TIMEOUT_S = 120
def get_stage_config() -> str:
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
)
return get_deploy_config_path("qwen3_tts.yaml")
@pytest.fixture(scope="module")
tests/e2e/stage_configs/qwen2_5_omni_ci.yaml
-104
View File
@@ -1,104 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
load_format: dummy
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.4
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "2" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.5 #increase the gpu memory utilization to enable the test on H800
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
max_num_batched_tokens: 8192
max_model_len: 8192
load_format: dummy
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 8192
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
tests/e2e/stage_configs/qwen2_5_omni_thinker_ci.yaml
-31
View File
@@ -1,31 +0,0 @@
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
tests/e2e/stage_configs/qwen3_omni_ci.yaml
-102
View File
@@ -1,102 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
engine_args:
model_stage: thinker
max_num_seqs: 5
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
max_num_batched_tokens: 32768
max_model_len: 32768
enable_prefix_caching: false
mm_processor_cache_gb: 0
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 150
seed: 42
ignore_eos: False
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
engine_args:
model_stage: talker
max_num_seqs: 5
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.5
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
enable_prefix_caching: false
max_num_batched_tokens: 32768
max_model_len: 32768
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 1000
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
engine_args:
model_stage: code2wav
max_num_seqs: 5
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 100000
hf_config_name: thinker_config
async_scheduling: false
load_format: dummy
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2000
seed: 42
detokenize: True
repetition_penalty: 1.1
tests/e2e/stage_configs/rocm/qwen2_5_omni_ci.yaml
-101
View File
@@ -1,101 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
max_num_batched_tokens: 4096
max_model_len: 4096
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
tests/e2e/stage_configs/rocm/qwen3_omni_ci.yaml
-100
View File
@@ -1,100 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
engine_args:
model_stage: thinker
max_num_seqs: 1
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
mm_processor_cache_gb: 0
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
engine_args:
model_stage: talker
max_num_seqs: 1
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 100
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
load_format: dummy
async_scheduling: false
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 200
seed: 42
detokenize: True
repetition_penalty: 1.1
tests/e2e/stage_configs/xpu/qwen2_5_omni_ci.yaml
-102
View File
@@ -1,102 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# The following config is verified with 2 * Intel Arc Pro B60 XPU.
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9 # thinker weight is around 16.74GB for Qwen2.5-Omni-7B
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.5 # talker weight is 6.03GB for Qwen2.5-Omni-7B
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true
devices: "2"
engine_args:
max_num_seqs: 1
model_stage: code2wav
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.3 # code2wav weight is around 1.46GB for Qwen2.5-Omni-7B
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2048
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
tests/e2e/stage_configs/xpu/qwen3_omni_ci.yaml
-109
View File
@@ -1,109 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config is verified with 8 * Intel Arc Pro B60 XPU.
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "0,1,2,3"
engine_args:
max_num_seqs: 1
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.85 # thinker weight is around 61.08GB for Qwen3-Omni-30B-A3B-Instruct
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
max_num_batched_tokens: 4096
max_model_len: 4096
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 4
max_cudagraph_capture_size: 0
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
ignore_eos: False
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "4"
engine_args:
max_num_seqs: 1
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6 # talker weight is around 8.5GB for Qwen3-Omni-30B-A3B-Instruct
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
enable_prefix_caching: false
max_num_batched_tokens: 4096
max_model_len: 4096
distributed_executor_backend: "mp"
hf_config_name: talker_config
max_cudagraph_capture_size: 0
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
stage_type: llm # Use llm stage type for AR stages
runtime:
devices: "5"
engine_args:
max_num_seqs: 1
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.3 # code2wav weight is around 0.4GB for Qwen3-Omni-30B-A3B-Instruct
skip_mm_profiling: true
distributed_executor_backend: "mp"
max_num_batched_tokens: 100000
hf_config_name: thinker_config
async_scheduling: false
max_cudagraph_capture_size: 0
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2000
seed: 42
detokenize: True
repetition_penalty: 1.1
tests/engine/test_arg_utils.py
+12 -5
View File
@@ -39,21 +39,28 @@ def test_default_stage_id_is_concrete_int():
assert cfg.stage_id == 0
def test_multimodal_kwarg_overrides():
def test_multimodal_kwarg_overrides(mocker):
"""Ensure that overrides in the multimodal config are preserved."""
# Get a different value than the default for a multimodal field
sig = inspect.signature(OmniEngineArgs)
default_mm_cache = sig.parameters["mm_processor_cache_gb"].default
override_val = default_mm_cache + 1
# NOTE: This needs to be a model that resolves to supports_multimodal=True
# in vLLM, otherwise we won't have an MM config
fake_model_config = SimpleNamespace(
multimodal_config=SimpleNamespace(mm_processor_cache_gb=override_val),
)
def _fake_parent_create_model_config(self):
assert self.mm_processor_cache_gb == override_val
return fake_model_config
mocker.patch.object(EngineArgs, "create_model_config", _fake_parent_create_model_config)
mocker.patch.object(OmniModelConfig, "from_vllm_model_config", side_effect=lambda model_config, **_: model_config)
cfg = OmniEngineArgs(
model="Qwen/Qwen2-VL-2B-Instruct",
mm_processor_cache_gb=override_val,
).create_model_config()
# Ensure that the override was applied correctly
assert cfg.multimodal_config is not None
assert cfg.multimodal_config.mm_processor_cache_gb == override_val
tests/engine/test_async_omni_engine_abort.py
+7 -3
View File
@@ -2,20 +2,24 @@ import asyncio
import os
import sys
from contextlib import ExitStack
from pathlib import Path
import pytest
from vllm import SamplingParams
from vllm.inputs import PromptType
from tests.utils import hardware_test
# Side-effect import: registers QWEN2_5_OMNI_THINKER_ONLY_PIPELINE in the
# pipeline registry so the materialized deploy overlay below can select it
# via its top-level ``pipeline:`` field.
import vllm_omni.model_executor.models.qwen2_5_omni.pipeline # noqa: F401, E402
from tests.utils import get_deploy_config_path, hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
SEED = 42
stage_config = str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen2_5_omni_thinker_ci.yaml")
# Single-stage thinker-only deploy, materialized from tests.utils._CI_OVERLAYS.
stage_config = get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml")
model = "Qwen/Qwen2.5-Omni-7B"
tests/examples/online_serving/test_qwen2_5_omni.py
+4 -10
View File
@@ -5,8 +5,6 @@ See examples/online_serving/qwen2_5_omni/README.md
import os
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from pathlib import Path
@@ -19,19 +17,15 @@ from tests.examples.conftest import (
run_cmd,
strip_trailing_audio_saved_line,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
pytestmark = [pytest.mark.advanced_model, pytest.mark.example]
models = ["Qwen/Qwen2.5-Omni-7B"]
stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_configs" / "qwen2_5_omni_ci.yaml")]
if current_omni_platform.is_xpu():
stage_configs = [
str(Path(__file__).parent.parent.parent / "e2e" / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml")
]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section in vllm_omni/deploy/ci/qwen2_5_omni.yaml.
stage_configs = [get_deploy_config_path("ci/qwen2_5_omni.yaml")]
example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving")
# Create parameter combinations for model and stage config
tests/examples/online_serving/test_qwen3_omni.py
+2 -7
View File
@@ -5,8 +5,6 @@ See examples/online_serving/qwen3_omni/README.md
import os
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from pathlib import Path
@@ -19,17 +17,14 @@ from tests.examples.conftest import (
run_cmd,
strip_trailing_audio_saved_line,
)
from tests.utils import hardware_test
from tests.utils import get_deploy_config_path, hardware_test
pytestmark = [pytest.mark.advanced_model, pytest.mark.example]
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_configs" / "qwen3_omni_ci.yaml")]
if current_omni_platform.is_xpu():
stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")]
stage_configs = [get_deploy_config_path("ci/qwen3_omni_moe.yaml")]
example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving")
tests/test_arg_utils.py
+353
View File
@@ -0,0 +1,353 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for vllm_omni.engine.arg_utils — invariants that must
hold for the orchestrator/engine/server CLI flag partition."""
from __future__ import annotations
import logging
from dataclasses import dataclass, fields
import pytest
from vllm_omni.engine.arg_utils import (
SHARED_FIELDS,
derive_server_dests_from_vllm_parser,
internal_blacklist_keys,
orchestrator_args_from_argparse,
orchestrator_field_names,
split_kwargs,
)
# ---------------------------------------------------------------------------
# Fake engine class for unit testing — avoids pulling in the full vllm
# EngineArgs and its heavy __post_init__ at test time.
# ---------------------------------------------------------------------------
@dataclass
class _FakeEngineArgs:
"""Stand-in for OmniEngineArgs with a representative subset of fields."""
model: str = ""
stage_id: int = 0
max_num_seqs: int = 64
gpu_memory_utilization: float = 0.9
async_chunk: bool = False # also in OrchestratorArgs → shared
log_stats: bool = False # also in OrchestratorArgs → shared
stage_configs_path: str | None = None
# ============================================================================
# Invariant 1 — OrchestratorArgs and engine must not ambiguously overlap.
# ============================================================================
def test_no_ambiguous_overlap_with_fake_engine():
"""OrchestratorArgs ∩ engine fields must be ⊆ SHARED_FIELDS."""
orch = orchestrator_field_names()
engine = {f.name for f in fields(_FakeEngineArgs)}
overlap = orch & engine
unexpected = overlap - SHARED_FIELDS
assert not unexpected, (
f"Fields declared in both OrchestratorArgs and the engine class "
f"but not in SHARED_FIELDS: {sorted(unexpected)}. These cause "
f"double-routing — either remove the duplicate declaration or add "
f"to SHARED_FIELDS if sharing is intentional."
)
def test_no_ambiguous_overlap_with_real_engine():
"""Same check, but against the real OmniEngineArgs."""
try:
from vllm_omni.engine.arg_utils import OmniEngineArgs
except Exception as exc:
pytest.skip(f"OmniEngineArgs not importable: {exc}")
orch = orchestrator_field_names()
engine = {f.name for f in fields(OmniEngineArgs)}
overlap = orch & engine
unexpected = overlap - SHARED_FIELDS
assert not unexpected, (
f"Real OmniEngineArgs has ambiguous overlap with OrchestratorArgs: "
f"{sorted(unexpected)}. Update SHARED_FIELDS or remove duplication."
)
# ============================================================================
# Invariant 2 — split_kwargs partitions correctly.
# ============================================================================
def test_split_orchestrator_only():
"""Pure orchestrator fields go to OrchestratorArgs, not engine_kwargs."""
raw = {"stage_init_timeout": 500, "worker_backend": "ray"}
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
assert orch.stage_init_timeout == 500
assert orch.worker_backend == "ray"
assert "stage_init_timeout" not in engine
assert "worker_backend" not in engine
def test_split_engine_only():
"""Pure engine fields go to engine_kwargs, not OrchestratorArgs."""
raw = {"max_num_seqs": 128, "gpu_memory_utilization": 0.85}
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
assert engine["max_num_seqs"] == 128
assert engine["gpu_memory_utilization"] == 0.85
# These fields don't exist on OrchestratorArgs at all.
def test_split_shared_fields_go_to_both():
"""Fields in SHARED_FIELDS are copied to both buckets."""
raw = {"model": "Qwen/Qwen2.5-Omni-7B", "log_stats": True}
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
assert orch.log_stats is True
assert engine["model"] == "Qwen/Qwen2.5-Omni-7B"
assert engine["log_stats"] is True
def test_split_drops_unclassified():
"""Unclassified fields (uvicorn/server) are dropped silently."""
raw = {
"max_num_seqs": 64, # engine
"host": "0.0.0.0", # unclassified (server)
"port": 8091, # unclassified (server)
"ssl_keyfile": "key.pem", # unclassified (server)
}
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
assert engine == {"max_num_seqs": 64}
assert "host" not in engine
assert "port" not in engine
assert "ssl_keyfile" not in engine
def test_split_mixed_real_world():
"""End-to-end: raw CLI kwargs with all three classes present."""
raw = {
# orchestrator
"stage_init_timeout": 400,
"deploy_config": "/tmp/deploy.yaml",
"worker_backend": "multi_process",
"async_chunk": True,
# engine
"max_num_seqs": 32,
"gpu_memory_utilization": 0.8,
# shared
"model": "Qwen/Qwen3-Omni",
"log_stats": False,
# server / unclassified
"host": "0.0.0.0",
"port": 8091,
"api_key": "secret",
# None values
"ray_address": None,
}
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
# Orchestrator side
assert orch.stage_init_timeout == 400
assert orch.deploy_config == "/tmp/deploy.yaml"
assert orch.worker_backend == "multi_process"
assert orch.async_chunk is True
assert orch.log_stats is False # shared, read from raw
assert orch.ray_address is None # default preserved
# Engine side
assert engine["max_num_seqs"] == 32
assert engine["gpu_memory_utilization"] == 0.8
assert engine["model"] == "Qwen/Qwen3-Omni"
assert engine["log_stats"] is False
assert "host" not in engine
assert "port" not in engine
assert "api_key" not in engine
# orchestrator-only keys never reach engine
assert "stage_init_timeout" not in engine
assert "deploy_config" not in engine
assert "async_chunk" not in engine
# ============================================================================
# Invariant 3 — user-typed unclassifiable flags warn (don't fail silently).
# ============================================================================
def test_user_typed_unclassified_warns(caplog):
"""If the user types a flag we can't route, warn — don't silently drop."""
raw = {"bogus_flag": "value", "max_num_seqs": 64}
with caplog.at_level(logging.WARNING, logger="vllm_omni.engine.arg_utils"):
split_kwargs(raw, engine_cls=_FakeEngineArgs, user_typed={"bogus_flag"})
assert any("bogus_flag" in rec.message for rec in caplog.records), (
f"Expected warning mentioning 'bogus_flag', got: {[rec.message for rec in caplog.records]}"
)
def test_unclassified_without_user_typed_silent(caplog):
"""Without user_typed, unclassified keys drop silently (argparse defaults
for server flags shouldn't spam logs on every launch)."""
raw = {"host": "0.0.0.0", "port": 8091, "max_num_seqs": 64}
with caplog.at_level(logging.WARNING, logger="vllm_omni.engine.arg_utils"):
split_kwargs(raw, engine_cls=_FakeEngineArgs, user_typed=None)
# No warnings because we don't know these were user-typed.
assert not any("host" in rec.message or "port" in rec.message for rec in caplog.records)
# ============================================================================
# Invariant 4 — CLI flag classification completeness.
# Catches new flags added without updating OrchestratorArgs or OmniEngineArgs.
# ============================================================================
def test_all_omni_cli_flags_classified():
"""Every vllm-omni-added CLI flag must be classifiable.
Runs ``OmniServeCommand.subparser_init`` and checks that every new
argument (compared to vllm's base parser) is either:
- a field on OrchestratorArgs, OR
- a field on OmniEngineArgs, OR
- in SHARED_FIELDS
"""
try:
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.engine.arg_utils import OmniEngineArgs
from vllm_omni.entrypoints.cli.serve import OmniServeCommand
except Exception as exc:
pytest.skip(f"Cannot build parser in this environment: {exc}")
# Build the serve parser
root = FlexibleArgumentParser()
subparsers = root.add_subparsers()
cmd = OmniServeCommand()
try:
parser = cmd.subparser_init(subparsers)
except Exception as exc:
pytest.skip(f"subparser_init failed (dev env issue): {exc}")
all_dests = {a.dest for a in parser._actions if a.dest and a.dest not in {"help", "model_tag"}}
orch = orchestrator_field_names()
engine = {f.name for f in fields(OmniEngineArgs)}
server_derived = derive_server_dests_from_vllm_parser()
unclassified = all_dests - orch - engine - SHARED_FIELDS - server_derived
# Some argparse-internal dests (suppressed, private) may not match —
# filter those out.
unclassified = {d for d in unclassified if not d.startswith("_")}
assert not unclassified, (
f"These CLI flags are not classified as "
f"orchestrator/engine/shared/server: {sorted(unclassified)}. "
f"Add them to OrchestratorArgs (if consumed by orchestrator), "
f"OmniEngineArgs (if consumed by per-stage engine), or the known-server "
f"allowlist (if they're vllm frontend flags). "
f"If intentional (e.g. a new CLI-only flag that doesn't map to either "
f"dataclass), add it to a KNOWN_UNROUTED allowlist."
)
# ============================================================================
# argparse interop (Phase 3).
# ============================================================================
def test_orchestrator_args_from_argparse():
"""Can build OrchestratorArgs from an argparse.Namespace."""
import argparse
ns = argparse.Namespace(
stage_init_timeout=500,
deploy_config="/tmp/x.yaml",
max_num_seqs=64, # engine field — ignored
host="0.0.0.0", # server field — ignored
)
orch = orchestrator_args_from_argparse(ns)
assert orch.stage_init_timeout == 500
assert orch.deploy_config == "/tmp/x.yaml"
assert orch.worker_backend == "multi_process" # default
def test_derive_server_dests_returns_frozenset():
"""Server-dest derivation returns a frozenset (possibly empty)."""
result = derive_server_dests_from_vllm_parser()
assert isinstance(result, frozenset)
# ============================================================================
# internal_blacklist_keys — single source of truth for per-stage forwarding.
# ============================================================================
def test_internal_blacklist_keys_derived_from_orchestrator():
"""Blacklist is exactly OrchestratorArgs fields minus SHARED_FIELDS.
This function replaces the old hardcoded INTERNAL_STAGE_OVERRIDE_KEYS
frozenset. Asserts the contract so future changes to OrchestratorArgs
automatically propagate to the blacklist.
"""
blacklist = internal_blacklist_keys()
assert blacklist == orchestrator_field_names() - SHARED_FIELDS
# Spot-check expected entries
assert "stage_init_timeout" in blacklist
assert "deploy_config" in blacklist
assert "async_chunk" in blacklist
# Shared fields must NOT appear — they flow to both orchestrator and engine
assert "model" not in blacklist
assert "log_stats" not in blacklist
# ============================================================================
# Boundary value analysis — edge cases around split_kwargs.
# ============================================================================
def test_split_empty_kwargs():
"""Empty kwargs yields default OrchestratorArgs and empty engine dict."""
orch, engine = split_kwargs({}, engine_cls=_FakeEngineArgs)
assert orch.stage_init_timeout == 300 # dataclass default
assert orch.worker_backend == "multi_process" # dataclass default
assert engine == {}
def test_split_all_none_values_preserved_on_orchestrator():
"""None values for orchestrator fields are kept (represents 'not set')."""
raw = {"ray_address": None, "deploy_config": None, "max_num_seqs": None}
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
assert orch.ray_address is None
assert orch.deploy_config is None
# Engine-side None still passes through; caller decides semantics downstream.
assert engine.get("max_num_seqs") is None
def test_split_user_typed_with_empty_kwargs_no_warn(caplog):
"""user_typed non-empty but kwargs empty — no warnings emitted."""
with caplog.at_level(logging.WARNING, logger="vllm_omni.engine.arg_utils"):
split_kwargs({}, engine_cls=_FakeEngineArgs, user_typed={"nothing"})
assert not caplog.records
def test_ambiguous_field_strict_raises():
"""strict=True raises ValueError on overlap outside SHARED_FIELDS."""
# deploy_config is on OrchestratorArgs; declaring it on the engine class
# too (without adding to SHARED_FIELDS) creates an ambiguous route.
@dataclass
class _AmbiguousEngine:
deploy_config: str | None = None
with pytest.raises(ValueError, match="both OrchestratorArgs and"):
split_kwargs({"deploy_config": "x"}, engine_cls=_AmbiguousEngine, strict=True)
def test_ambiguous_field_non_strict_routes_to_orchestrator(caplog):
"""strict=False logs ERROR but routes the ambiguous field to orchestrator."""
@dataclass
class _AmbiguousEngine:
deploy_config: str | None = None
with caplog.at_level(logging.ERROR, logger="vllm_omni.engine.arg_utils"):
orch, engine = split_kwargs({"deploy_config": "x"}, engine_cls=_AmbiguousEngine, strict=False)
assert orch.deploy_config == "x"
assert "deploy_config" not in engine
assert any("both OrchestratorArgs" in r.message for r in caplog.records)
tests/test_config_factory.py
+665 -13
View File
@@ -4,12 +4,26 @@
Unit tests for StageConfigFactory and related classes.
"""
from dataclasses import dataclass
from pathlib import Path
import pytest
from vllm_omni.config.stage_config import (
_EXECUTION_TYPE_TO_SCHEDULER,
_PIPELINE_REGISTRY,
ModelPipeline,
PipelineConfig,
StageConfig,
StageConfigFactory,
StageExecutionType,
StagePipelineConfig,
StageType,
build_stage_runtime_overrides,
register_pipeline,
strip_parent_engine_args,
)
from vllm_omni.engine.arg_utils import internal_blacklist_keys
class TestStageType:
@@ -241,8 +255,9 @@ class TestStageConfigFactory:
def test_default_diffusion_with_parallel_config(self):
"""Test diffusion config calculates devices from parallel_config."""
@dataclass
class MockParallelConfig:
world_size = 4
world_size: int = 4
kwargs = {
"parallel_config": MockParallelConfig(),
@@ -270,7 +285,7 @@ class TestStageConfigFactory:
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
cli_overrides = {
"gpu_memory_utilization": 0.9, # Well-known param
"custom_engine_flag": True, # Not in _INTERNAL_KEYS, so forwarded
"custom_engine_flag": True, # Not orchestrator-owned, so forwarded
}
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
@@ -311,6 +326,53 @@ class TestStageConfigFactory:
assert "batch_timeout" not in overrides
class TestStageResolutionHelpers:
"""Tests for shared stage override / filtering helpers."""
def test_build_stage_runtime_overrides_ignores_other_stage_and_internal_keys(self):
overrides = build_stage_runtime_overrides(
0,
{
"gpu_memory_utilization": 0.5,
"stage_0_gpu_memory_utilization": 0.9,
"stage_1_gpu_memory_utilization": 0.1,
"stage_0_model": "should_be_ignored",
"parallel_config": {"world_size": 2},
},
internal_keys=internal_blacklist_keys(),
)
assert overrides["gpu_memory_utilization"] == 0.9
assert "model" not in overrides
assert "parallel_config" not in overrides
def test_strip_parent_engine_args_reports_only_surprising_parent_overrides(self):
from dataclasses import fields as dc_fields
from vllm.engine.arg_utils import EngineArgs
parent_fields = {f.name: f for f in dc_fields(EngineArgs)}
filtered, overridden = strip_parent_engine_args(
{
"model": "some/model",
"stage_configs_path": "/tmp/stages.yaml",
"tensor_parallel_size": 4,
"worker_extension_cls": "some.Extension",
"custom_pipeline_args": {"pipeline_class": "demo.Pipeline"},
},
parent_fields=parent_fields,
keep_keys={"worker_extension_cls"},
strip_keys={"stage_configs_path"},
no_warn_keys={"model"},
)
assert filtered == {
"worker_extension_cls": "some.Extension",
"custom_pipeline_args": {"pipeline_class": "demo.Pipeline"},
}
assert overridden == ["tensor_parallel_size"]
class TestPipelineYamlParsing:
"""Tests for pipeline YAML file parsing (@ZJY0516)."""
@@ -609,16 +671,606 @@ stages:
assert pipeline.async_chunk is False
class TestArchitectureFallback:
"""Tests for architecture-based model detection fallback."""
class TestPipelineDiscovery:
"""Tests for auto-discovery of pipelines from models/*/pipeline.py."""
def test_architecture_models_mapping_exists(self):
"""Test that _ARCHITECTURE_MODELS contains expected entries."""
assert "MiMoAudioForConditionalGeneration" in StageConfigFactory._ARCHITECTURE_MODELS
assert StageConfigFactory._ARCHITECTURE_MODELS["MiMoAudioForConditionalGeneration"] == "mimo_audio"
assert "HunyuanImage3ForCausalMM" in StageConfigFactory._ARCHITECTURE_MODELS
assert StageConfigFactory._ARCHITECTURE_MODELS["HunyuanImage3ForCausalMM"] == "hunyuan_image3"
def test_discover_populates_registry_with_known_models(self):
"""``_discover_all_pipelines`` imports every pipeline.py so the
registry is populated with the built-in models after one call."""
from vllm_omni.config.stage_config import _discover_all_pipelines
def test_mimo_audio_in_pipeline_models(self):
"""Test that mimo_audio is registered in PIPELINE_MODELS."""
assert "mimo_audio" in StageConfigFactory.PIPELINE_MODELS
_discover_all_pipelines()
# These models have a pipeline.py in-tree and must be registered.
assert "qwen2_5_omni" in _PIPELINE_REGISTRY
assert "qwen3_omni_moe" in _PIPELINE_REGISTRY
assert "qwen3_tts" in _PIPELINE_REGISTRY
def test_pipeline_config_supports_hf_architectures(self):
"""PipelineConfig accepts hf_architectures for HF-arch fallback
(replaces the old _ARCHITECTURE_MODELS dict)."""
p = PipelineConfig(
model_type="custom_collide",
hf_architectures=("SomeCollidingArch",),
)
assert p.hf_architectures == ("SomeCollidingArch",)
class TestStagePipelineConfig:
def test_frozen(self):
s = StagePipelineConfig(stage_id=0, model_stage="a")
with pytest.raises(AttributeError):
s.model_stage = "changed"
def test_defaults(self):
s = StagePipelineConfig(stage_id=0, model_stage="a")
assert s.execution_type == StageExecutionType.LLM_AR
assert s.input_sources == ()
assert s.final_output is False
assert s.sampling_constraints == {}
assert s.engine_output_type is None
class TestPipelineConfigNew:
def test_frozen(self):
p = PipelineConfig(model_type="t", model_arch="A")
with pytest.raises(AttributeError):
p.model_type = "changed"
def test_validate_valid(self):
p = PipelineConfig(
model_type="t",
model_arch="A",
stages=(
StagePipelineConfig(stage_id=0, model_stage="a"),
StagePipelineConfig(stage_id=1, model_stage="b", input_sources=(0,)),
),
)
assert p.validate() == []
def test_validate_no_stages(self):
p = PipelineConfig(model_type="t", model_arch="A")
assert any("no stages" in e.lower() for e in p.validate())
def test_get_scheduler_cls(self):
p = PipelineConfig(
model_type="t",
model_arch="A",
stages=(
StagePipelineConfig(stage_id=0, model_stage="a", execution_type=StageExecutionType.LLM_AR),
StagePipelineConfig(
stage_id=1, model_stage="b", execution_type=StageExecutionType.LLM_GENERATION, input_sources=(0,)
),
),
)
assert "OmniARScheduler" in p.get_scheduler_cls(0)
assert "OmniGenerationScheduler" in p.get_scheduler_cls(1)
class TestExecutionTypeToScheduler:
def test_all_types_mapped(self):
for et in StageExecutionType:
assert et in _EXECUTION_TYPE_TO_SCHEDULER
class TestPipelineRegistry:
def test_register_and_lookup(self):
p = PipelineConfig(
model_type="__test_only__",
model_arch="A",
stages=(StagePipelineConfig(stage_id=0, model_stage="a"),),
)
register_pipeline(p)
assert _PIPELINE_REGISTRY["__test_only__"] is p
del _PIPELINE_REGISTRY["__test_only__"]
class TestDeployConfigLoading:
def test_load_deploy_config(self):
from pathlib import Path
from vllm_omni.config.stage_config import load_deploy_config
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
assert len(deploy.stages) == 3
assert deploy.async_chunk is True
assert deploy.connectors is not None
assert deploy.platforms is not None
def test_merge_pipeline_deploy(self):
from pathlib import Path
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
stages = merge_pipeline_deploy(pipeline, deploy)
assert len(stages) == 3
s0 = stages[0]
assert s0.model_stage == "thinker"
assert s0.yaml_engine_args["model_arch"] == "Qwen3OmniMoeForConditionalGeneration"
assert s0.yaml_engine_args["engine_output_type"] == "latent"
assert s0.yaml_extras["default_sampling_params"]["detokenize"] is True
class TestQwen3OmniPipeline:
def test_registered(self):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
p = _PIPELINE_REGISTRY.get("qwen3_omni_moe")
assert p is not None
assert p.model_arch == "Qwen3OmniMoeForConditionalGeneration"
assert len(p.stages) == 3
assert p.validate() == []
def test_thinker(self):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen3_omni_moe"].get_stage(0)
assert s.model_stage == "thinker"
assert s.execution_type == StageExecutionType.LLM_AR
assert s.owns_tokenizer is True
assert s.engine_output_type == "latent"
assert s.sampling_constraints["detokenize"] is True
def test_talker(self):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen3_omni_moe"].get_stage(1)
assert s.input_sources == (0,)
assert s.sampling_constraints["stop_token_ids"] == [2150]
assert s.custom_process_input_func is not None
assert s.custom_process_next_stage_input_func is not None
def test_code2wav(self):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen3_omni_moe"].get_stage(2)
assert s.execution_type == StageExecutionType.LLM_GENERATION
assert s.final_output_type == "audio"
assert s.custom_process_input_func is not None
class TestQwen2_5OmniPipeline:
def test_registered(self):
import vllm_omni.model_executor.models.qwen2_5_omni.pipeline # noqa: F401
p = _PIPELINE_REGISTRY.get("qwen2_5_omni")
assert p is not None
assert p.model_arch == "Qwen2_5OmniForConditionalGeneration"
assert len(p.stages) == 3
assert p.validate() == []
def test_thinker(self):
import vllm_omni.model_executor.models.qwen2_5_omni.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen2_5_omni"].get_stage(0)
assert s.model_stage == "thinker"
assert s.execution_type == StageExecutionType.LLM_AR
assert s.owns_tokenizer is True
assert s.engine_output_type == "latent"
assert s.requires_multimodal_data is True
def test_talker(self):
import vllm_omni.model_executor.models.qwen2_5_omni.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen2_5_omni"].get_stage(1)
assert s.input_sources == (0,)
assert s.sampling_constraints["stop_token_ids"] == [8294]
assert s.custom_process_input_func is not None
def test_code2wav(self):
import vllm_omni.model_executor.models.qwen2_5_omni.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen2_5_omni"].get_stage(2)
assert s.execution_type == StageExecutionType.LLM_GENERATION
assert s.final_output_type == "audio"
assert s.engine_output_type == "audio"
class TestQwen3TTSPipeline:
def test_registered(self):
import vllm_omni.model_executor.models.qwen3_tts.pipeline # noqa: F401
p = _PIPELINE_REGISTRY.get("qwen3_tts")
assert p is not None
assert p.model_arch == "Qwen3TTSTalkerForConditionalGeneration"
assert len(p.stages) == 2
assert p.validate() == []
def test_talker_stage(self):
import vllm_omni.model_executor.models.qwen3_tts.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen3_tts"].get_stage(0)
assert s.model_stage == "qwen3_tts"
assert s.execution_type == StageExecutionType.LLM_AR
assert s.owns_tokenizer is True
assert s.engine_output_type == "latent"
assert s.sampling_constraints["stop_token_ids"] == [2150]
# Stage 0 inherits the pipeline-level model_arch
assert s.model_arch is None
def test_code2wav_stage_has_per_stage_model_arch(self):
import vllm_omni.model_executor.models.qwen3_tts.pipeline # noqa: F401
s = _PIPELINE_REGISTRY["qwen3_tts"].get_stage(1)
assert s.execution_type == StageExecutionType.LLM_GENERATION
assert s.final_output_type == "audio"
assert s.engine_output_type == "audio"
# Per-stage model_arch override (different from pipeline-level talker)
assert s.model_arch == "Qwen3TTSCode2Wav"
# tts_args is passed through via extras
assert s.extras["tts_args"]["max_instructions_length"] == 500
def test_per_stage_model_arch_flows_through_merge(self, tmp_path):
"""Verify the new ps.model_arch override survives merge_pipeline_deploy."""
import vllm_omni.model_executor.models.qwen3_tts.pipeline # noqa: F401
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_tts.yaml"
if not deploy_path.exists():
pytest.skip("qwen3_tts deploy yaml not found")
deploy = load_deploy_config(deploy_path)
pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
stages = merge_pipeline_deploy(pipeline, deploy)
# Stage 0 inherits pipeline-level model_arch
assert stages[0].yaml_engine_args["model_arch"] == "Qwen3TTSTalkerForConditionalGeneration"
# Stage 1 uses its per-stage override
assert stages[1].yaml_engine_args["model_arch"] == "Qwen3TTSCode2Wav"
class TestBaseConfigInheritance:
"""Test deploy YAML base_config inheritance."""
def test_ci_inherits_from_main(self):
from tests.utils import get_deploy_config_path
from vllm_omni.config.stage_config import load_deploy_config
ci_path = Path(get_deploy_config_path("ci/qwen3_omni_moe.yaml"))
if not ci_path.exists():
pytest.skip("CI deploy config not found")
deploy = load_deploy_config(ci_path)
assert len(deploy.stages) == 3
# CI overrides
assert deploy.stages[0].engine_extras.get("load_format") == "dummy"
assert deploy.stages[0].max_num_seqs == 5
# Inherited from base
assert deploy.stages[0].gpu_memory_utilization == 0.9
assert deploy.connectors is not None
assert "connector_of_shared_memory" in deploy.connectors
assert deploy.async_chunk is True
def test_ci_sampling_merge(self):
from tests.utils import get_deploy_config_path
from vllm_omni.config.stage_config import load_deploy_config
ci_path = Path(get_deploy_config_path("ci/qwen3_omni_moe.yaml"))
if not ci_path.exists():
pytest.skip("CI deploy config not found")
deploy = load_deploy_config(ci_path)
s0 = deploy.stages[0].default_sampling_params
# CI overrides max_tokens
assert s0["max_tokens"] == 150
# Inherited from base
assert s0["temperature"] == 0.4
assert s0["seed"] == 42
def test_pure_inheritance_overlay(self, tmp_path):
"""An overlay with only ``base_config`` inherits everything."""
from vllm_omni.config.stage_config import load_deploy_config
base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not base.exists():
pytest.skip("Base deploy config not found")
overlay = tmp_path / "overlay.yaml"
overlay.write_text(f"base_config: {base}\n")
deploy = load_deploy_config(overlay)
assert len(deploy.stages) == 3
assert deploy.stages[0].gpu_memory_utilization == 0.9
def test_single_field_overlay(self, tmp_path):
"""An overlay overriding one stage field merges with the base."""
from vllm_omni.config.stage_config import load_deploy_config
base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not base.exists():
pytest.skip("Base deploy config not found")
overlay = tmp_path / "overlay.yaml"
overlay.write_text(f"base_config: {base}\nstages:\n - stage_id: 2\n max_num_batched_tokens: 1000000\n")
deploy = load_deploy_config(overlay)
assert deploy.stages[2].max_num_batched_tokens == 1000000
# Rest inherited
assert deploy.stages[0].gpu_memory_utilization == 0.9
class TestPlatformOverrides:
"""Test platform-specific deploy config overrides."""
def test_npu_overrides(self):
from pathlib import Path
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
deploy = _apply_platform_overrides(deploy, platform="npu")
assert deploy.stages[0].gpu_memory_utilization == 0.6
assert deploy.stages[0].tensor_parallel_size == 2
assert deploy.stages[0].devices == "0,1"
# Stage 2 unaffected fields stay at base
assert deploy.stages[2].enforce_eager is True
def test_xpu_overrides(self):
from pathlib import Path
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
deploy = _apply_platform_overrides(deploy, platform="xpu")
assert deploy.stages[0].tensor_parallel_size == 4
assert deploy.stages[0].devices == "0,1,2,3"
assert deploy.stages[0].engine_extras.get("max_cudagraph_capture_size") == 0
def test_unknown_platform_noop(self):
from pathlib import Path
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
original_mem = deploy.stages[0].gpu_memory_utilization
deploy = _apply_platform_overrides(deploy, platform="unknown_hw")
assert deploy.stages[0].gpu_memory_utilization == original_mem
def test_platforms_deep_merge_inheritance(self, tmp_path):
"""Overlay's platforms: block layers onto base's, per-stage."""
from vllm_omni.config.stage_config import _apply_platform_overrides, load_deploy_config
base = tmp_path / "base.yaml"
base.write_text(
"stages:\n"
" - stage_id: 0\n"
" gpu_memory_utilization: 0.9\n"
"platforms:\n"
" rocm:\n"
" stages:\n"
" - stage_id: 0\n"
" enforce_eager: true\n"
)
overlay = tmp_path / "overlay.yaml"
overlay.write_text(
f"base_config: {base.name}\n"
"platforms:\n"
" rocm:\n"
" stages:\n"
" - stage_id: 0\n"
" max_num_seqs: 1\n"
)
deploy = load_deploy_config(overlay)
deploy = _apply_platform_overrides(deploy, platform="rocm")
# Both base's enforce_eager and overlay's max_num_seqs should apply.
assert deploy.stages[0].enforce_eager is True
assert deploy.stages[0].max_num_seqs == 1
# Inherited stage default not touched by overlay platforms section.
assert deploy.stages[0].gpu_memory_utilization == 0.9
class TestCLIOverrideFlow:
"""Test --stage-overrides JSON merge into StageConfig."""
def test_stage_overrides_merge(self):
from pathlib import Path
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
stages = merge_pipeline_deploy(pipeline, deploy)
# Simulate --stage-overrides '{"0": {"gpu_memory_utilization": 0.5}}'
overrides = {"stage_0_gpu_memory_utilization": 0.5}
stages[0].runtime_overrides = StageConfigFactory._merge_cli_overrides(stages[0], overrides)
assert stages[0].runtime_overrides["gpu_memory_utilization"] == 0.5
def test_global_override_applies_to_all(self):
from pathlib import Path
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
stages = merge_pipeline_deploy(pipeline, deploy)
overrides = {"enforce_eager": True}
for s in stages:
s.runtime_overrides = StageConfigFactory._merge_cli_overrides(s, overrides)
assert s.runtime_overrides["enforce_eager"] is True
class TestCLIExplicitPrecedence:
"""Verify YAML > argparse defaults; explicit CLI args > YAML."""
def _stages(self, cli_overrides, cli_explicit_keys):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
return StageConfigFactory._create_from_registry(
"qwen3_omni_moe",
cli_overrides=cli_overrides,
cli_explicit_keys=cli_explicit_keys,
)
def test_explicit_cli_overrides_yaml(self):
"""User-typed --max-num-seqs wins over the deploy YAML value."""
stages = self._stages(
cli_overrides={"max_num_seqs": 999},
cli_explicit_keys={"max_num_seqs"},
)
# Stage 2 yaml has max_num_seqs=1; explicit CLI must beat it.
assert stages[2].runtime_overrides.get("max_num_seqs") == 999
def test_default_cli_does_not_override_yaml(self):
"""Argparse defaults must NOT clobber values that are present in YAML."""
stages = self._stages(
cli_overrides={"max_num_seqs": 256},
cli_explicit_keys=set(), # user typed nothing
)
# Stage 2's YAML value (1) should win because the user didn't type --max-num-seqs.
assert stages[2].runtime_overrides.get("max_num_seqs") != 256
def test_default_cli_fills_missing_yaml_field(self):
"""Argparse defaults still fill fields the YAML doesn't set."""
stages = self._stages(
cli_overrides={"some_unrelated_knob": "fallback"},
cli_explicit_keys=set(),
)
# Field absent from YAML → CLI default flows through as a fallback.
assert stages[0].runtime_overrides.get("some_unrelated_knob") == "fallback"
def test_per_stage_overrides_always_explicit(self):
"""``stage_<id>_*`` keys are always treated as explicit."""
stages = self._stages(
cli_overrides={"stage_0_gpu_memory_utilization": 0.42},
cli_explicit_keys=set(), # not in the explicit set, but per-stage
)
assert stages[0].runtime_overrides.get("gpu_memory_utilization") == 0.42
def test_none_explicit_set_treats_all_as_explicit(self):
"""Programmatic Omni() callers (cli_explicit_keys=None) keep current behavior."""
stages = self._stages(
cli_overrides={"max_num_seqs": 999},
cli_explicit_keys=None,
)
assert stages[2].runtime_overrides.get("max_num_seqs") == 999
def test_explicit_async_chunk_false_overrides_yaml(self):
"""``--no-async-chunk`` flips the deploy-level async_chunk to False even
when the YAML sets it to True. Verifies that the per-stage
``async_chunk: True`` injection in ``merge_pipeline_deploy`` is skipped
and that ``async_chunk`` does not leak through ``_merge_cli_overrides``.
"""
stages = self._stages(
cli_overrides={"async_chunk": False},
cli_explicit_keys={"async_chunk"},
)
# qwen3_omni_moe.yaml has `async_chunk: true`, so by default every
# stage's engine_args would carry it. With the explicit override, it
# must NOT show up.
for stage in stages:
assert stage.yaml_engine_args.get("async_chunk") is not True
assert stage.runtime_overrides.get("async_chunk") is None
def test_default_async_chunk_leaves_yaml_alone(self):
"""An unset ``--async-chunk`` (default None) must leave the YAML's True
in force on every stage."""
stages = self._stages(
cli_overrides={"async_chunk": None},
cli_explicit_keys=set(),
)
# qwen3_omni_moe.yaml: `async_chunk: true` → injected on every stage.
for stage in stages:
assert stage.yaml_engine_args.get("async_chunk") is True
def test_explicit_enable_prefix_caching_overrides_yaml(self):
"""``--enable-prefix-caching`` (global) flips every stage's
``enable_prefix_caching`` to True regardless of the YAML default."""
stages = self._stages(
cli_overrides={"enable_prefix_caching": True},
cli_explicit_keys={"enable_prefix_caching"},
)
for stage in stages:
assert stage.runtime_overrides.get("enable_prefix_caching") is True
def test_async_chunk_dispatches_processors(self):
"""A single ``qwen3_tts`` pipeline picks per-chunk vs end-to-end
processors based on ``deploy.async_chunk``, without needing a
separate variant pipeline registration."""
import vllm_omni.model_executor.models.qwen3_tts.pipeline # noqa: F401
from vllm_omni.config.stage_config import (
_PIPELINE_REGISTRY,
DeployConfig,
merge_pipeline_deploy,
)
pipeline = _PIPELINE_REGISTRY["qwen3_tts"]
# async_chunk=True → stage 0's per-chunk processor wires up, stage 1
# has no sync input processor.
async_stages = merge_pipeline_deploy(pipeline, DeployConfig(async_chunk=True))
assert (
async_stages[0]
.yaml_engine_args.get("custom_process_next_stage_input_func", "")
.endswith("talker2code2wav_async_chunk")
)
assert async_stages[1].custom_process_input_func is None
# async_chunk=False → stage 0 has no streaming processor, stage 1's
# batch-end processor wires up.
sync_stages = merge_pipeline_deploy(pipeline, DeployConfig(async_chunk=False))
assert "custom_process_next_stage_input_func" not in sync_stages[0].yaml_engine_args
assert sync_stages[1].custom_process_input_func is not None
assert sync_stages[1].custom_process_input_func.endswith("talker2code2wav")
class TestSamplingConstraintsPrecedence:
"""Test that pipeline sampling_constraints override deploy defaults."""
def test_constraints_win(self):
from pathlib import Path
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
deploy_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not deploy_path.exists():
pytest.skip("Deploy config not found")
deploy = load_deploy_config(deploy_path)
stages = merge_pipeline_deploy(pipeline, deploy)
# Pipeline says detokenize=True for thinker, deploy can't override
assert stages[0].yaml_extras["default_sampling_params"]["detokenize"] is True
# Pipeline says stop_token_ids=[2150] for talker
assert stages[1].yaml_extras["default_sampling_params"]["stop_token_ids"] == [2150]
# Deploy temperature still flows through
assert stages[0].yaml_extras["default_sampling_params"]["temperature"] == 0.4
tests/utils.py
+216
View File
@@ -11,6 +11,7 @@ import threading
import time
from collections.abc import Callable
from contextlib import ExitStack, contextmanager, suppress
from pathlib import Path
from typing import Any, Literal
import cloudpickle
@@ -24,6 +25,221 @@ from vllm_omni.platforms import current_omni_platform
_P = ParamSpec("_P")
_REPO_ROOT = Path(__file__).resolve().parent.parent
_DEPLOY_DIR = _REPO_ROOT / "vllm_omni" / "deploy"
_CI_GENERATED_DIR = _REPO_ROOT / "tests" / ".ci_generated"
# CI overlays as Python dicts (LSP-friendly). Materialized on demand to
# tests/.ci_generated/<model>.yaml via get_deploy_config_path("ci/<name>.yaml").
_CI_OVERLAYS: dict[str, dict[str, Any]] = {
"qwen2_5_omni": {
"base_config": "qwen2_5_omni.yaml",
"async_chunk": False,
"stages": [
{
"stage_id": 0,
"max_model_len": 16384,
"max_num_batched_tokens": 16384,
"max_num_seqs": 1,
"gpu_memory_utilization": 0.9,
"skip_mm_profiling": True,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 128},
},
{
"stage_id": 1,
"max_model_len": 16384,
"max_num_batched_tokens": 16384,
"max_num_seqs": 1,
"gpu_memory_utilization": 0.4,
"skip_mm_profiling": True,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 4096},
},
{
"stage_id": 2,
"max_num_seqs": 1,
"gpu_memory_utilization": 0.5,
"max_num_batched_tokens": 8192,
"max_model_len": 8192,
"load_format": "dummy",
"devices": "2",
"default_sampling_params": {"max_tokens": 8192},
},
],
"platforms": {
"rocm": {
"stages": [
{"stage_id": 0, "gpu_memory_utilization": 0.9},
{"stage_id": 1, "gpu_memory_utilization": 0.4},
{"stage_id": 2, "gpu_memory_utilization": 0.5, "devices": "2"},
],
},
"xpu": {
"stages": [
{
"stage_id": 0,
"gpu_memory_utilization": 0.9,
"max_num_batched_tokens": 16384,
"max_model_len": 16384,
},
{"stage_id": 1, "gpu_memory_utilization": 0.5},
{
"stage_id": 2,
"gpu_memory_utilization": 0.3,
"max_num_batched_tokens": 4096,
"max_model_len": 4096,
"devices": "2",
},
],
},
},
},
"qwen3_omni_moe": {
"base_config": "qwen3_omni_moe.yaml",
"async_chunk": False,
"stages": [
{
"stage_id": 0,
"max_num_seqs": 5,
"max_model_len": 32768,
"mm_processor_cache_gb": 0,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 150, "ignore_eos": False},
},
{
"stage_id": 1,
"gpu_memory_utilization": 0.5,
"max_num_seqs": 5,
"max_model_len": 32768,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 1000},
},
{
"stage_id": 2,
"max_num_seqs": 5,
"max_num_batched_tokens": 100000,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 2000},
},
],
"platforms": {
"rocm": {
"stages": [
{"stage_id": 0, "max_num_seqs": 1, "default_sampling_params": {"max_tokens": 100}},
{
"stage_id": 1,
"max_num_seqs": 1,
"enforce_eager": True,
"default_sampling_params": {"max_tokens": 100},
},
{
"stage_id": 2,
"max_num_seqs": 1,
"max_num_batched_tokens": 1000000,
"default_sampling_params": {"max_tokens": 200},
},
],
},
"xpu": {
"stages": [
{
"stage_id": 0,
"gpu_memory_utilization": 0.85,
"max_num_seqs": 1,
"tensor_parallel_size": 4,
"enforce_eager": True,
"max_num_batched_tokens": 4096,
"max_model_len": 4096,
"max_cudagraph_capture_size": 0,
"skip_mm_profiling": True,
"devices": "0,1,2,3",
"default_sampling_params": {"max_tokens": 100, "ignore_eos": False},
},
{
"stage_id": 1,
"gpu_memory_utilization": 0.6,
"max_num_seqs": 1,
"enforce_eager": True,
"max_num_batched_tokens": 4096,
"max_model_len": 4096,
"max_cudagraph_capture_size": 0,
"skip_mm_profiling": True,
"devices": "4",
},
{
"stage_id": 2,
"gpu_memory_utilization": 0.3,
"max_num_seqs": 1,
"max_num_batched_tokens": 100000,
"max_cudagraph_capture_size": 0,
"skip_mm_profiling": True,
"devices": "5",
"default_sampling_params": {"max_tokens": 2000},
},
],
},
},
},
# Single-stage thinker-only topology for the abort test.
"qwen2_5_omni_thinker_only": {
"async_chunk": False,
"pipeline": "qwen2_5_omni_thinker_only",
"stages": [
{
"stage_id": 0,
"max_num_seqs": 1,
"gpu_memory_utilization": 0.9,
"enforce_eager": True,
"max_num_batched_tokens": 16384,
"max_model_len": 16384,
"skip_mm_profiling": True,
"mm_processor_cache_gb": 0,
"load_format": "dummy",
"devices": "0",
"default_sampling_params": {
"temperature": 0.0,
"top_p": 1.0,
"top_k": -1,
"max_tokens": 128,
"seed": 42,
"repetition_penalty": 1.1,
},
},
],
},
}
def _materialize_ci_overlay(model_type: str) -> Path:
import yaml
if model_type not in _CI_OVERLAYS:
raise KeyError(f"No CI overlay registered for {model_type!r}. Available: {sorted(_CI_OVERLAYS)}")
_CI_GENERATED_DIR.mkdir(parents=True, exist_ok=True)
out = _CI_GENERATED_DIR / f"{model_type}.yaml"
overlay = {**_CI_OVERLAYS[model_type]}
base = overlay.get("base_config")
if base:
overlay["base_config"] = str(_DEPLOY_DIR / base)
with open(out, "w", encoding="utf-8") as f:
yaml.safe_dump(overlay, f, sort_keys=False)
return out
def get_deploy_config_path(rel_path: str) -> str:
"""Resolve a deploy yaml; ``ci/<model>.yaml`` materializes from ``_CI_OVERLAYS``."""
if rel_path.startswith("ci/") and rel_path.endswith(".yaml"):
model_type = rel_path[len("ci/") : -len(".yaml")]
if model_type in _CI_OVERLAYS:
return str(_materialize_ci_overlay(model_type))
return str(_DEPLOY_DIR / rel_path)
if current_platform.is_rocm():
from amdsmi import (
amdsmi_get_gpu_vram_usage,