Merge branch 'main' into feat/moss-tts-nano

Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
2026-04-21 17:50:21 -04:00
commit 3bd77d3f8f
@@ -173,8 +173,6 @@ Example configuration for TP=2 on GPUs 0 and 1:

 | Parameter             | Value   | Description                      |
 | :-------------------- | :------ | :------------------------------- |
-| `window_size`         | `-1`    | Window size (-1 means unlimited) |
-| `max_inflight`        | `1`     | Maximum inflight requests        |
 | `shm_threshold_bytes` | `65536` | Shared memory threshold (64KB)   |

 ## Using Mooncake Connector
@@ -58,7 +58,14 @@ Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py
    - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features.
    - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform.

-Stage wiring is configured in `vllm_omni/model_executor/stage_configs/cosyvoice3.yaml`.
+Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`;
+runtime tunables (batch size, memory limits, sampling) live in
+`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by
+HF `model_type` and defaults to `async_chunk: true` (shared-memory
+streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the
+legacy sync path where stage 1 runs `text2flow` over the full
+speech-token sequence.

 - Stage 0 emits latent speech tokens.
- Stage 1 consumes them via `custom_process_input_func` and outputs audio.
+- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the
+  shared-memory connector (async-chunk mode) and outputs audio.
@@ -24,7 +24,13 @@ def run_e2e():
        required=True,
        help="Path to CosyVoice3 model directory (e.g., pretrained_models/Fun-CosyVoice3-0.5B/).",
    )
-    parser.add_argument("--stage-config", type=str, default="vllm_omni/model_executor/stage_configs/cosyvoice3.yaml")
+    parser.add_argument(
+        "--deploy-config",
+        type=str,
+        default=None,
+        help="Override the deploy config path. If unset, auto-loads "
+        "vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.",
+    )
    parser.add_argument("--prompt", type=str, default="Hello, this is a test of the CosyVoice system capability.")
    parser.add_argument(
        "--prompt-text",
@@ -43,18 +49,14 @@ def run_e2e():
    if not os.path.exists(args.tokenizer):
        raise FileNotFoundError(f"{args.tokenizer} does not exist!")

-    # Ensure stage config exists
-    if not os.path.exists(args.stage_config):
-        raise FileNotFoundError(f"{args.stage_config} does not exist!")
+    if args.deploy_config is not None and not os.path.exists(args.deploy_config):
+        raise FileNotFoundError(f"{args.deploy_config} does not exist!")

    print(f"Initializing cosyvoice E2E with model={args.model}")

-    # Initialize Omni
-    # This spins up the engine(s) based on the stage config
-    # We pass trust_remote_code=True same as Qwen examples
    omni = Omni(
        model=args.model,
-        stage_configs_path=args.stage_config,
+        deploy_config=args.deploy_config,
        trust_remote_code=True,
        tokenizer=args.tokenizer,
        log_stats=True,
@@ -57,22 +57,22 @@ GLM_IMAGE_EOS_TOKEN_ID = 16385  # eos_token_id from generation_config.json
 GLM_IMAGE_VISION_VOCAB_SIZE = 16512  # top_k should be vision_vocab_size


-def compute_max_tokens(height: int, width: int, factor: int = 32) -> int:
+def compute_max_tokens(height: int, width: int, factor: int = 32, is_i2i: bool = False) -> int:
    """
    Compute max_new_tokens for GLM-Image AR generation.

-    GLM-Image generates tokens in this order for text-to-image:
-    1. Small preview image (half resolution in each dimension)
-    2. Large target image (full resolution)
-    3. EOS token
+    GLM-Image generation differs by mode:
+    - text-to-image (t2i): small preview + large target + EOS
+    - image-to-image (i2i): large target + EOS

    Args:
        height: Target image height in pixels
        width: Target image width in pixels
        factor: Downsampling factor (32 for GLM-Image AR output)
+        is_i2i: Whether the request is image-to-image mode

    Returns:
-        Total number of tokens to generate (small + large + EOS)
+        Total number of tokens to generate for the specified mode
    """
    # Large image tokens (target resolution)
    token_h = height // factor
@@ -80,11 +80,15 @@ def compute_max_tokens(height: int, width: int, factor: int = 32) -> int:
    large_tokens = token_h * token_w

    # Small preview tokens (half resolution in each dimension)
-    small_h = token_h // 2
-    small_w = token_w // 2
-    small_tokens = small_h * small_w
+    import math

-    # Total: small + large + EOS
+    ratio = token_h / token_w if token_w > 0 else 1.0
+    small_token_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
+    small_token_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
+    small_tokens = small_token_h * small_token_w
+
+    if is_i2i:
+        return large_tokens + 1
    return small_tokens + large_tokens + 1


@@ -282,14 +286,18 @@ def main(args: argparse.Namespace) -> None:
    # Compute max_tokens dynamically based on target image size
    target_height = prompt_dict.get("height", 1024)
    target_width = prompt_dict.get("width", 1024)
-    calculated_max_tokens = compute_max_tokens(target_height, target_width)
+    is_i2i = source_image is not None
+    calculated_max_tokens = compute_max_tokens(target_height, target_width, is_i2i=is_i2i)

    # Use calculated value unless user explicitly specified a different value
    # Default args.max_tokens is 16384 (very large), so prefer calculated value
    effective_max_tokens = calculated_max_tokens if args.max_tokens == 16384 else args.max_tokens

    if args.verbose:
-        print(f"AR max_tokens: {effective_max_tokens} (calculated: {calculated_max_tokens}, arg: {args.max_tokens})")
+        print(
+            f"AR max_tokens: {effective_max_tokens} "
+            f"(calculated: {calculated_max_tokens}, arg: {args.max_tokens}, mode: {'i2i' if is_i2i else 't2i'})"
+        )

    # IMPORTANT: GLM-Image AR model requires these exact sampling parameters
    # from generation_config.json for proper image token generation.
@@ -303,6 +311,12 @@ def main(args: argparse.Namespace) -> None:
        stop_token_ids=[GLM_IMAGE_EOS_TOKEN_ID],  # 16385, CRITICAL for stopping
        seed=args.seed,
        detokenize=False,
+        # Keep target size available in runner/model for deterministic M-RoPE
+        # decode grids in t2i (no mm_features available in this path).
+        extra_args={
+            "target_h": int(target_height),
+            "target_w": int(target_width),
+        },
    )

    # For diffusion stage, sampling_params contains diffusion-specific parameters
@@ -1,25 +1,161 @@
-# HunyuanImage-3.0 Image-to-Text Inference
+# HunyuanImage-3.0-Instruct

-This example demonstrates how to run HunyuanImage-3.0 Image-to-Text with the vLLM-Omni.
+## Set up

-## Local CLI Usage
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.

-Download the example image:
+## Run examples
+
+**Note**: These examples work with the default configuration on **8x NVIDIA L40S (48GB)**. For different GPU setups, modify the stage configuration to adjust device allocation and memory utilization.
+
+Get into the hunyuan_image3 folder:

 ```bash
-wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg
+cd examples/offline_inference/hunyuan_image3
 ```

-Run example:
+### Modality Control
+
+HunyuanImage-3.0-Instruct supports multiple modality modes. You can control the mode using the `--modality` argument:
+
+#### Text to Image (text2img)
+
+- **Pipeline**: Text → AR (CoT + latent tokens) → DiT (denoise) → VAE Decode → Image
+- **Stages Used**: Stage 0 (AR) + Stage 1 (DiT)
+- **KV Transfer**: AR sends KV cache to DiT for conditioned generation
+- **Default Config**: `hunyuan_image3_t2i.yaml`

 ```bash
-python image_to_text.py \
-  --image cherry_blossom.jpg \
-  --prompt "<|startoftext|>You are an assistant that understands images and outputs text.<img>Describe the content of the picture."
+python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
+                  --modality text2img \
+                  --prompts "A cute cat sitting on a windowsill watching the sunset"
 ```

-Key arguments:
+#### Image to Image (img2img)

- `--model`: Model used. Default is: tencent/HunyuanImage-3.0-Instruct (Optional).
- `--image`: Path to input image (required).
- `--prompt`: Text description used to guide image understanding (required).
+- **Pipeline**: Image + Text → AR (CoT + recaption + latent) → DiT → Edited Image
+- **Stages Used**: Stage 0 (AR) + Stage 1 (DiT)
+- **KV Transfer**: AR sends KV cache to DiT
+- **Default Config**: `hunyuan_image3_it2i.yaml`
+
+```bash
+python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
+                  --modality img2img \
+                  --image-path /path/to/image.png \
+                  --prompts "Make the petals neon pink"
+```
+
+#### Image to Text (img2text)
+
+- **Pipeline**: Image + Question → AR → Text description
+- **Stages Used**: Stage 0 (AR) only
+- **Default Config**: `hunyuan_image3_i2t.yaml`
+
+```bash
+python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
+                  --modality img2text \
+                  --image-path /path/to/image.jpg \
+                  --prompts "Describe the content of the picture."
+```
+
+#### Text to Text (text2text)
+
+- **Pipeline**: Text → AR → Text
+- **Stages Used**: Stage 0 (AR) only
+- **Default Config**: `hunyuan_image3_t2t.yaml`
+
+```bash
+python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
+                  --modality text2text \
+                  --prompts "What is the capital of France?"
+```
+
+### Inference Steps & Guidance
+
+Control generation quality for image modalities:
+
+```bash
+python end2end.py --modality text2img \
+                  --steps 50 \
+                  --guidance-scale 5.0 \
+                  --height 1024 --width 1024 \
+                  --prompts "A photo-realistic sunset over the ocean"
+```
+
+### Key Arguments
+
+#### 📌 Command Line Arguments (end2end.py)
+
+| Argument               | Type   | Default                              | Description                                                  |
+| :--------------------- | :----- | :----------------------------------- | :----------------------------------------------------------- |
+| `--model`              | string | `tencent/HunyuanImage-3.0-Instruct` | Model path or name                                           |
+| `--modality`           | choice | `text2img`                           | Modality: `text2img`, `img2img`, `img2text`, `text2text`     |
+| `--prompts`            | list   | `None`                               | Input text prompts                                           |
+| `--image-path`         | string | `None`                               | Input image path (for `img2img`/`img2text`)                  |
+| `--output`             | string | `.`                                  | Output directory for saved images                            |
+| `--steps`              | int    | `50`                                 | Number of inference steps                                    |
+| `--guidance-scale`     | float  | `5.0`                                | Classifier-free guidance scale                               |
+| `--seed`               | int    | `42`                                 | Random seed                                                  |
+| `--height`             | int    | `1024`                               | Output image height                                          |
+| `--width`              | int    | `1024`                               | Output image width                                           |
+| `--bot-task`           | string | auto                                 | Override prompt task (e.g. `it2i_think`, `t2i_recaption`)    |
+| `--sys-type`           | string | auto                                 | Override system prompt type (e.g. `en_unified`, `en_vanilla`) |
+| `--stage-configs-path` | string | auto                                 | Custom stage config YAML path                                |
+| `--enforce-eager`      | flag   | `False`                              | Disable torch.compile                                        |
+| `--init-timeout`       | int    | `300`                                | Initialization timeout (seconds)                             |
+
+------
+
+#### ⚙️ Stage Configurations
+
+| Config YAML                         | Modality  | Stages | GPUs   | Description                           |
+| :---------------------------------- | :-------- | :----- | :----- | :------------------------------------ |
+| `hunyuan_image3_t2i.yaml`           | text2img  | 2      | 8      | T2I with AR→DiT, 4 GPU each          |
+| `hunyuan_image3_it2i.yaml`          | img2img   | 2      | 8      | IT2I with AR→DiT, 4 GPU each         |
+| `hunyuan_image3_i2t.yaml`           | img2text  | 1      | 4      | I2T (AR only)                         |
+| `hunyuan_image3_t2t.yaml`           | text2text | 1      | 4      | T2T (AR only)                         |
+| `hunyuan_image3_t2i_2gpu.yaml`      | text2img  | 2      | 2      | T2I for 2-GPU setups                  |
+| `hunyuan_image3_moe.yaml`           | text2img  | 2      | 8      | T2I with MoE AR→DiT KV reuse          |
+| `hunyuan_image3_moe_dit_2gpu_fp8.yaml` | text2img | 2   | 2      | T2I with FP8 quantization             |
+
+------
+
+## Using MoE Config
+
+The `hunyuan_image3_moe.yaml` config enables AR→DiT KV cache reuse with 8 GPUs (4 for AR + 4 for DiT).
+
+```bash
+python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
+                  --modality text2img \
+                  --stage-configs-path hunyuan_image3_moe.yaml \
+                  --prompts "A cute cat"
+```
+
+------
+
+## Prompt Format
+
+HunyuanImage-3.0 uses a pretrain template format:
+
+```
+<|startoftext|>{system_prompt}{<img>}{trigger_tag}{user_prompt}
+```
+
+- `<img>`: Placeholder for each input image (auto-inserted by `prompt_utils.py`)
+- Trigger tags: `<think>` (CoT), `<recaption>` (recaptioning)
+- System prompt: Auto-selected based on task
+
+The `prompt_utils.build_prompt()` handles this formatting automatically.
+
+------
+
+## FAQ
+
+- **OOM errors**: Decrease `gpu_memory_utilization` in the YAML stage config, or use a smaller `max_num_batched_tokens`.
+- **Custom image sizes**: Use `--height` and `--width` flags (multiples of 16 recommended).
+
+| Stage             | VRAM (approx)        |
+| :---------------- | :------------------- |
+| Stage 0 (AR)      | ~15 GiB + KV Cache   |
+| Stage 1 (DiT)     | ~30 GiB              |
+| Total (8-GPU)     | ~45 GiB + KV Cache   |
@@ -0,0 +1,262 @@
+"""
+HunyuanImage-3.0-Instruct unified end-to-end inference script.
+
+Supports all modalities through a single entry point:
+  - text2img:  Text → AR → DiT → Image
+  - img2img:   Text+Image → AR → DiT → Edited Image (IT2I)
+  - img2text:  Image+Text → AR → Text description (I2T)
+  - text2text: Text → AR → Text (comprehension, no image)
+
+Usage:
+    python end2end.py --modality text2img --prompts "A cute cat"
+    python end2end.py --modality img2img --image-path input.png --prompts "Make it snowy"
+    python end2end.py --modality img2text --image-path input.png --prompts "Describe this image"
+"""
+
+import argparse
+import os
+
+from vllm_omni.diffusion.models.hunyuan_image3.system_prompt import (
+    get_system_prompt,
+)
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniPromptType
+
+# task → (sys_type, bot_task, trigger_tag)
+_TASK_PRESETS: dict[str, tuple[str, str | None, str | None]] = {
+    "t2t": ("en_unified", None, None),
+    "i2t": ("en_unified", None, None),
+    "it2i_think": ("en_unified", "think", "<think>"),
+    "it2i_recaption": ("en_unified", "recaption", "<recaption>"),
+    "t2i_think": ("en_unified", "think", "<think>"),
+    "t2i_recaption": ("en_unified", "recaption", "<recaption>"),
+    "t2i_vanilla": ("en_vanilla", "image", None),
+}
+
+# Modality → prompt_utils task mapping
+_MODALITY_TASK_MAP = {
+    "text2img": "t2i_think",
+    "img2img": "it2i_think",
+    "img2text": "i2t",
+    "text2text": "t2t",
+}
+
+
+def build_prompt(
+    user_prompt: str,
+    task: str = "it2i_think",
+    sys_type: str | None = None,
+    custom_system_prompt: str | None = None,
+) -> str:
+    """Build a HunyuanImage-3.0 prompt using pretrain template format."""
+    if task not in _TASK_PRESETS:
+        raise ValueError(f"Unknown task {task!r}. Choose from: {sorted(_TASK_PRESETS)}")
+
+    preset_sys_type, preset_bot_task, trigger_tag = _TASK_PRESETS[task]
+    effective_sys_type = sys_type or preset_sys_type
+
+    system_prompt = get_system_prompt(effective_sys_type, preset_bot_task, custom_system_prompt)
+    sys_text = system_prompt.strip() if system_prompt else ""
+
+    has_image_input = task.startswith("i2t") or task.startswith("it2i")
+
+    parts = ["<|startoftext|>"]
+    if sys_text:
+        parts.append(sys_text)
+    if has_image_input:
+        parts.append("<img>")
+    if trigger_tag:
+        parts.append(trigger_tag)
+    parts.append(user_prompt)
+
+    return "".join(parts)
+
+
+# Modality → default stage config
+_MODALITY_DEFAULT_CONFIG = {
+    "text2img": "hunyuan_image3_t2i.yaml",
+    "img2img": "hunyuan_image3_it2i.yaml",
+    "img2text": "hunyuan_image3_i2t.yaml",
+    "text2text": "hunyuan_image3_t2t.yaml",
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="HunyuanImage-3.0-Instruct end-to-end inference.")
+    parser.add_argument(
+        "--model",
+        default="tencent/HunyuanImage-3.0-Instruct",
+        help="Model name or local path.",
+    )
+    parser.add_argument(
+        "--modality",
+        default="text2img",
+        choices=["text2img", "img2img", "img2text", "text2text"],
+        help="Modality mode to control stage execution.",
+    )
+    parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
+    parser.add_argument(
+        "--image-path",
+        type=str,
+        default=None,
+        help="Path to input image (for img2img/img2text).",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=".",
+        help="Output directory to save results.",
+    )
+
+    # Generation parameters
+    parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
+    parser.add_argument("--guidance-scale", type=float, default=5.0, help="Classifier-free guidance scale.")
+    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
+    parser.add_argument("--height", type=int, default=1024, help="Output image height.")
+    parser.add_argument("--width", type=int, default=1024, help="Output image width.")
+
+    # Prompt configuration
+    parser.add_argument(
+        "--bot-task",
+        type=str,
+        default=None,
+        help="Override prompt task (e.g. it2i_think, t2i_recaption). Default: auto from modality.",
+    )
+    parser.add_argument(
+        "--sys-type",
+        type=str,
+        default=None,
+        help="Override system prompt type (e.g. en_unified, en_vanilla).",
+    )
+
+    # Omni init args
+    parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom stage config YAML path.")
+    parser.add_argument("--log-stats", action="store_true", default=False)
+    parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
+    parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    os.makedirs(args.output, exist_ok=True)
+
+    # Determine task for prompt formatting
+    task = args.bot_task or _MODALITY_TASK_MAP[args.modality]
+
+    # Determine stage config
+    stage_configs_path = args.stage_configs_path or _MODALITY_DEFAULT_CONFIG[args.modality]
+
+    # Build Omni
+    omni_kwargs = {
+        "model": args.model,
+        "stage_configs_path": stage_configs_path,
+        "log_stats": args.log_stats,
+        "init_timeout": args.init_timeout,
+        "enforce_eager": args.enforce_eager,
+    }
+    if args.modality in ("text2img", "img2img"):
+        omni_kwargs["mode"] = "text-to-image"
+
+    omni = Omni(**omni_kwargs)
+
+    # Prepare prompts
+    prompts = args.prompts or ["A cute cat"]
+    if not prompts:
+        print("[Info] No prompts provided, using default.")
+        prompts = ["A cute cat"]
+
+    # Load image if needed
+    input_image = None
+    if args.modality in ("img2img", "img2text"):
+        if not args.image_path or not os.path.exists(args.image_path):
+            raise ValueError(f"--image-path required for {args.modality}, got: {args.image_path}")
+        from PIL import Image
+
+        input_image = Image.open(args.image_path).convert("RGB")
+
+    # Format prompts
+    formatted_prompts: list[OmniPromptType] = []
+    for p in prompts:
+        formatted_text = build_prompt(p, task=task, sys_type=args.sys_type)
+
+        prompt_dict: dict = {"prompt": formatted_text}
+
+        if args.modality == "text2img":
+            prompt_dict["modalities"] = ["image"]
+        elif args.modality == "img2img":
+            prompt_dict["modalities"] = ["image"]
+            prompt_dict["multi_modal_data"] = {"image": input_image}
+            prompt_dict["height"] = input_image.height
+            prompt_dict["width"] = input_image.width
+        elif args.modality == "img2text":
+            prompt_dict["modalities"] = ["text"]
+            prompt_dict["multi_modal_data"] = {"image": input_image}
+        elif args.modality == "text2text":
+            prompt_dict["modalities"] = ["text"]
+
+        formatted_prompts.append(prompt_dict)
+
+    # Build sampling params from defaults
+    params_list = list(omni.default_sampling_params_list)
+
+    # Override diffusion params if applicable
+    from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+    for i, sp in enumerate(params_list):
+        if isinstance(sp, OmniDiffusionSamplingParams):
+            sp.num_inference_steps = args.steps
+            sp.guidance_scale = args.guidance_scale
+            if args.seed is not None:
+                sp.seed = args.seed
+            if args.modality in ("text2img",):
+                sp.height = args.height
+                sp.width = args.width
+
+    # Print configuration
+    print(f"\n{'=' * 60}")
+    print("HunyuanImage-3.0 Generation Configuration:")
+    print(f"  Model: {args.model}")
+    print(f"  Modality: {args.modality}")
+    print(f"  Stage config: {stage_configs_path}")
+    print(f"  Num stages: {omni.num_stages}")
+    if args.modality in ("text2img", "img2img"):
+        print(f"  Inference steps: {args.steps}")
+        print(f"  Guidance scale: {args.guidance_scale}")
+        print(f"  Seed: {args.seed}")
+    if args.modality == "text2img":
+        print(f"  Output size: {args.width}x{args.height}")
+    if args.image_path:
+        print(f"  Input image: {args.image_path}")
+    print(f"  Prompts: {prompts}")
+    print(f"{'=' * 60}\n")
+
+    # Generate
+    omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
+
+    # Process outputs
+    img_idx = 0
+    for req_output in omni_outputs:
+        # Text output (AR stage or text-only)
+        ro = getattr(req_output, "request_output", None)
+        if ro and getattr(ro, "outputs", None):
+            txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
+            if txt:
+                print(f"[Output] Text:\n{txt}")
+
+        # Image output (DiT stage)
+        images = getattr(req_output, "images", None)
+        if not images and ro and hasattr(ro, "images"):
+            images = ro.images
+
+        if images:
+            for j, img in enumerate(images):
+                save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
+                img.save(save_path)
+                print(f"[Output] Saved image to {save_path}")
+            img_idx += 1
+
+
+if __name__ == "__main__":
+    main()
@@ -1,92 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import argparse
-import os
-
-from PIL import Image
-
-from vllm_omni.entrypoints.omni import Omni
-
-"""
-The tencent/HunyuanImage-3.0-Instruct base model uses the tencent/Hunyuan-A13B-Instruct backbone. It utilizes two tokenizer delimiter templates:
-
-1) Pretrained template (default for gen_text mode), which concatenates system, image
-   tokens, and user question WITHOUT role delimiters:
-"<|startoftext|>{system_prompt}{image_tokens}{user_question}"
-
-   Example (before image token expansion):
-"<|startoftext|>You are an assistant that understands images and outputs text.<img>Describe the content of the picture."
-
-2) Instruct template, which uses explicit role prefixes and separators.
-"""
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Generate text from image using HunyuanImage-3.0-Instruct.")
-    parser.add_argument(
-        "--model",
-        default="tencent/HunyuanImage-3.0-Instruct",
-        help="Model name or local path.",
-    )
-    parser.add_argument(
-        "--image",
-        type=str,
-        required=True,
-        help="Path to input image file (PNG, JPG, etc.).",
-    )
-    parser.add_argument(
-        "--prompt",
-        type=str,
-        required=True,
-        help="Pretrain template prompt: <|startoftext|>{system}<img>{question}",
-    )
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler to display stage durations.",
-    )
-    return parser.parse_args()
-
-
-def load_image(image_path: str) -> Image.Image:
-    """Load an image from file path."""
-    if not os.path.exists(image_path):
-        raise FileNotFoundError(f"Image file not found: {image_path}")
-    return Image.open(image_path).convert("RGB")
-
-
-def main(args: argparse.Namespace) -> None:
-    omni = Omni(
-        model=args.model,
-        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
-        mode="image-to-text",
-    )
-
-    prompt = "<|startoftext|>You are an assistant that understands images and outputs text.<img>" + args.prompt
-
-    prompt_dict = {
-        "prompt": prompt,
-        "modalities": ["text"],
-    }
-
-    # Add image input if provided
-    if args.image:
-        if not os.path.exists(args.image):
-            raise FileNotFoundError(f"Input image not found: {args.image}")
-
-        input_image = load_image(args.image)
-        prompt_dict["multi_modal_data"] = {"image": input_image}
-
-    prompts = [prompt_dict]
-    omni_outputs = omni.generate(prompts=prompts)
-
-    prompt_text = omni_outputs[0].request_output.prompt
-    generated_text = omni_outputs[0].request_output.outputs[0].text
-    print(f"Prompt: {prompt_text}")
-    print(f"Text: {generated_text}")
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    main(args)
@@ -39,7 +39,6 @@ Run a single sample for basic TTS:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type tts_sft
 ```
@@ -48,7 +47,6 @@ Run batch samples for basic TTS:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type tts_sft \
  --num-prompts {batch_size}
@@ -66,7 +64,6 @@ Generate speech from text input:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type tts_sft \
  --text "The weather is so nice today."
@@ -78,7 +75,6 @@ Generate speech with explicit voice style instructions:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type tts_sft_with_instruct \
  --text "The weather is so nice today." \
@@ -91,7 +87,6 @@ Generate speech using an audio reference for voice cloning:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type tts_sft_with_audio \
  --text "The weather is so nice today." \
@@ -104,7 +99,6 @@ Generate speech from text containing natural voice descriptions:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type tts_sft_with_natural_instruction \
  --text "In a panting young male voice, he said: I can't run anymore, wait for me!"
@@ -116,7 +110,6 @@ Transcribe audio to text:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type audio_trancribing_sft \
  --audio-path "./spoken_dialogue_assistant_turn_1.wav"
@@ -128,7 +121,6 @@ Understand and analyze audio content with text queries:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type audio_understanding_sft \
  --text "Summarize the audio." \
@@ -141,7 +133,6 @@ Audio understanding with reasoning chain:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type audio_understanding_sft_with_thinking \
  --text "Summarize the audio." \
@@ -154,7 +145,6 @@ Multi-turn dialogue with audio input and output:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type spoken_dialogue_sft_multiturn \
  --audio-path "./prompt_speech_zh_m.wav"
@@ -168,7 +158,6 @@ Multi-turn dialogue converting speech to text:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type speech2text_dialogue_sft_multiturn
 ```
@@ -181,7 +170,6 @@ Multi-turn text-only dialogue:

 ```bash
 python3 -u end2end.py \
-  --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
  --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \
  --query-type text_dialogue_sft_multiturn
 ```
@@ -182,7 +182,7 @@ def main(args):

    omni = Omni(
        model=model_name,
-        stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
        log_stats=args.enable_stats,
        log_file=("omni_pipeline.log" if args.enable_stats else None),
        init_sleep_seconds=args.init_sleep_seconds,
@@ -428,10 +428,11 @@ def parse_args():
        help="Sampling rate for audio.",
    )
    parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
        type=str,
-        default="../../../model_executor/stage_configs/mimo_audio.yaml",
-        help="Path to a stage configs file.",
+        default=None,
+        help="Override the deploy config path. If unset, auto-loads "
+        "vllm_omni/deploy/mimo_audio.yaml based on the HF model_type.",
    )

    return parser.parse_args()
@@ -0,0 +1,76 @@
+# Ming-flash-omni 2.0
+
+[Ming-flash-omni-2.0](https://github.com/inclusionAI/Ming) is an omni-modal model supporting text, image, video, and audio understanding, with outputs in text, image, and audio. For now, Ming-flash-omni-2.0 in vLLM-Omni is supported with thinker stage (multi-modal understanding).
+
+## Setup
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+
+## Run examples
+
+### Text-only
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type text
+```
+
+#### Reasoning (Thinking Mode)
+
+Reasoning (Thinking) mode is enabled via applying "detailed thinking on" when building the system prompt template (in `apply_chat_template`).
+
+In the end2end example, a default problem for thinking mode is provided, as referred to the example usage of Ming's cookbook;
+To utilize it, you have to download the example figure from https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/figures/cases/3_0.png
+
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py -q reasoning --image-path ./3_0.png
+```
+
+### Image understanding
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_image
+
+# With a local image
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_image --image-path /path/to/image.jpg
+```
+
+### Audio understanding
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_audio
+
+# With a local audio file
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_audio --audio-path /path/to/audio.wav
+```
+
+### Video understanding
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_video
+
+# With a local video and custom frame count
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_video --video-path /path/to/video.mp4 --num-frames 16
+```
+
+### Mixed modalities (image + audio)
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_mixed_modalities \
+    --image-path /path/to/image.jpg \
+    --audio-path /path/to/audio.wav
+```
+
+If media file paths are not provided, the script uses built-in default assets.
+
+### Modality control
+To control output modalities (e.g. text-only output):
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_audio --modalities text
+```
+
+*For now, only text output is supported*
+
+### Custom stage config
+```bash
+python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_image \
+    --stage-configs-path /path/to/your_config.yaml
+```
+
+## Online serving
+
+For online serving via the OpenAI-compatible API, see [examples/online_serving/ming_flash_omni/README.md](../../online_serving/ming_flash_omni/README.md).
@@ -0,0 +1,485 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Partial example cases are referred from
+# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/cookbook.ipynb
+import os
+import time
+from typing import NamedTuple
+
+import numpy as np
+import vllm
+from PIL import Image
+from transformers import AutoProcessor
+from vllm import SamplingParams
+from vllm.assets.audio import AudioAsset
+from vllm.assets.image import ImageAsset
+from vllm.assets.video import VideoAsset, video_to_ndarrays
+from vllm.multimodal.image import convert_image_mode
+from vllm.multimodal.media.audio import load_audio
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+import vllm_omni
+from vllm_omni.entrypoints.omni import Omni
+
+# Imports the processor also registers itself
+from vllm_omni.transformers_utils.processors.ming import MingFlashOmniProcessor  # noqa: F401
+
+SEED = 42
+MODEL_NAME = "Jonathan1909/Ming-flash-omni-2.0"
+
+
+class QueryResult(NamedTuple):
+    inputs: dict
+    limit_mm_per_prompt: dict[str, int]
+
+
+def get_text_query(processor: MingFlashOmniProcessor, question: str | None = None) -> QueryResult:
+    if question is None:
+        question = "请详细介绍鹦鹉的生活习性。"
+    conversation = [{"role": "HUMAN", "content": question}]
+    prompt = processor.apply_chat_template(conversation, tokenize=False)
+    return QueryResult(
+        inputs={"prompt": prompt},
+        limit_mm_per_prompt={},
+    )
+
+
+def get_image_query(
+    processor: MingFlashOmniProcessor,
+    question: str | None = None,
+    image_path: str | None = None,
+) -> QueryResult:
+    if question is None:
+        question = "Describe this image in detail."
+
+    if image_path:
+        if not os.path.exists(image_path):
+            raise FileNotFoundError(f"Image file not found: {image_path}")
+        image_data = convert_image_mode(Image.open(image_path), "RGB")
+    else:
+        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
+    conversation = [
+        {
+            "role": "HUMAN",
+            "content": [
+                {"type": "image", "image": image_data},
+                {"type": "text", "text": question},
+            ],
+        }
+    ]
+    prompt = processor.apply_chat_template(conversation, tokenize=False)
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {"image": image_data},
+        },
+        limit_mm_per_prompt={"image": 1},
+    )
+
+
+def get_audio_query(
+    processor: MingFlashOmniProcessor,
+    question: str | None = None,
+    audio_path: str | None = None,
+    sampling_rate: int = 16000,
+) -> QueryResult:
+    if question is None:
+        question = "Please recognize the language of this speech and transcribe it. Format: oral."
+
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+    else:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
+    # Use a string for "audio" so the processor counts it as 1 audio input
+    conversation = [
+        {
+            "role": "HUMAN",
+            "content": [
+                {"type": "audio", "audio": "input"},
+                {"type": "text", "text": question},
+            ],
+        }
+    ]
+    prompt = processor.apply_chat_template(conversation, tokenize=False)
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {"audio": audio_data},
+        },
+        limit_mm_per_prompt={"audio": 1},
+    )
+
+
+def get_video_query(
+    processor: MingFlashOmniProcessor,
+    question: str | None = None,
+    video_path: str | None = None,
+    num_frames: int = 16,
+) -> QueryResult:
+    if question is None:
+        question = "Describe what is happening in this video."
+
+    if video_path:
+        if not os.path.exists(video_path):
+            raise FileNotFoundError(f"Video file not found: {video_path}")
+        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+    else:
+        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+
+    conversation = [
+        {
+            "role": "HUMAN",
+            "content": [
+                {"type": "video"},
+                {"type": "text", "text": question},
+            ],
+        }
+    ]
+    prompt = processor.apply_chat_template(conversation, tokenize=False)
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {"video": video_frames},
+        },
+        limit_mm_per_prompt={"video": 1},
+    )
+
+
+def get_mixed_modalities_query(
+    processor: MingFlashOmniProcessor,
+    image_path: str | None = None,
+    audio_path: str | None = None,
+    sampling_rate: int = 16000,
+) -> QueryResult:
+    """Mixed image + audio understanding."""
+    question = "Describe the image, and recognize the language of this speech and transcribe it. Format: oral"
+
+    if image_path:
+        if not os.path.exists(image_path):
+            raise FileNotFoundError(f"Image file not found: {image_path}")
+        image_data = convert_image_mode(Image.open(image_path), "RGB")
+    else:
+        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        sig, sr = load_audio(audio_path, sr=sampling_rate)
+        audio_data = (sig.astype(np.float32), sr)
+    else:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
+    conversation = [
+        {
+            "role": "HUMAN",
+            "content": [
+                {"type": "image", "image": image_data},
+                {"type": "audio", "audio": "input"},
+                {"type": "text", "text": question},
+            ],
+        }
+    ]
+    prompt = processor.apply_chat_template(conversation, tokenize=False)
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {"image": image_data, "audio": audio_data},
+        },
+        limit_mm_per_prompt={"image": 1, "audio": 1},
+    )
+
+
+def get_reasoning_query(
+    processor: MingFlashOmniProcessor,
+    question: str | None = None,
+    image_path: str | None = None,
+) -> QueryResult:
+    if question is None:
+        # NOTE: To use the following default question, input with example figure provided by Ming
+        # https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/figures/cases/3_0.png
+        # E.g.,
+        # python examples/offline_inference/ming_flash_omni/end2end.py -q reasoning --image-path ./3_0.png
+        # Otherwise, the problem solving might be false.
+        question = (
+            "Based on the following rules:\n•\tYou control the smiley face character\n"
+            "•\tYou can move up, down, left, and right, and only a single square at a time\n"
+            "•\tWalls are dark grey and cannot be moved into\n•\tThe brown square is a box\n•"
+            "\tThe box can be pushed by moving into it (i.e., if you are in the square "
+            "adjacent to the box to the left, and move onto the square with the box, "
+            "the box will move one square to the right).\n"
+            "•\tThe box cannot be pushed into walls\n"
+            "•\tThe blue door at the bottom is locked and cannot be passed through, "
+            "unless the box is placed on the blue square\n"
+            "•\tThe square beneath the blue door is the exit\n"
+            "•\tMoving from one square to another\n\n"
+            "Let's assume a coordinate system where the smiley face is "
+            "on the top left at (1,1) and the square below it is (1,2). "
+            "The smiley face performs the following moves: {down, right, right, right}, "
+            "such that the smiley face is at square (4,2) and the box is in square (5,2). "
+            "What are the next sequence of moves that must be done to move the box down to (5,3)? "
+            "Give your answer as a comma separated list."
+        )
+
+    if image_path:
+        if not os.path.exists(image_path):
+            raise FileNotFoundError(f"Image file not found: {image_path}")
+        image_data = convert_image_mode(Image.open(image_path), "RGB")
+        conversation = [
+            {
+                "role": "HUMAN",
+                "content": [
+                    {"type": "image", "image": image_data},
+                    {"type": "text", "text": question},
+                ],
+            }
+        ]
+        prompt = processor.apply_chat_template(conversation, tokenize=False, use_cot_system_prompt=True)
+        return QueryResult(
+            inputs={
+                "prompt": prompt,
+                "multi_modal_data": {"image": image_data},
+            },
+            limit_mm_per_prompt={"image": 1},
+        )
+
+    conversation = [{"role": "HUMAN", "content": question}]
+    prompt = processor.apply_chat_template(conversation, tokenize=False, use_cot_system_prompt=True)
+    return QueryResult(
+        inputs={"prompt": prompt},
+        limit_mm_per_prompt={},
+    )
+
+
+query_map = {
+    "text": get_text_query,
+    "use_audio": get_audio_query,
+    "use_image": get_image_query,
+    "use_video": get_video_query,
+    "use_mixed_modalities": get_mixed_modalities_query,
+    "reasoning": get_reasoning_query,
+}
+
+
+def main(args):
+    print(
+        "=" * 20,
+        "\n",
+        f"vllm version: {vllm.__version__}\n",
+        f"vllm-omni version: {vllm_omni.__version__}\n",
+        "=" * 20,
+        sep="",
+    )
+
+    processor = AutoProcessor.from_pretrained(MODEL_NAME, trust_remote_code=True)
+    assert isinstance(processor, MingFlashOmniProcessor), f"Wrong processor type being used: {type(processor)}"
+
+    query_func = query_map[args.query_type]
+    if args.query_type == "use_image":
+        query_result = query_func(processor, image_path=args.image_path)
+    elif args.query_type == "use_audio":
+        query_result = query_func(processor, audio_path=args.audio_path, sampling_rate=args.sampling_rate)
+    elif args.query_type == "use_video":
+        query_result = query_func(processor, video_path=args.video_path, num_frames=args.num_frames)
+    elif args.query_type == "use_mixed_modalities":
+        query_result = query_func(
+            processor,
+            image_path=args.image_path,
+            audio_path=args.audio_path,
+            sampling_rate=args.sampling_rate,
+        )
+    elif args.query_type == "reasoning":
+        query_result = query_func(processor, image_path=args.image_path)
+    else:
+        query_result = query_func(processor)
+
+    # Initialize Omni (with thinker-only stage config)
+    omni = Omni(
+        model=MODEL_NAME,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        init_timeout=args.init_timeout,
+        stage_init_timeout=args.stage_init_timeout,
+    )
+
+    # Thinker sampling params
+    thinker_sampling_params = SamplingParams(
+        temperature=0.4,
+        top_p=0.9,
+        max_tokens=args.max_tokens,
+        repetition_penalty=1.05,
+        seed=SEED,
+        detokenize=True,
+    )
+    sampling_params_list = [thinker_sampling_params]
+
+    prompts = [query_result.inputs for _ in range(args.num_prompts)]
+
+    if args.modalities is not None:
+        output_modalities = args.modalities.split(",")
+        for prompt in prompts:
+            prompt["modalities"] = output_modalities
+
+    total_requests = len(prompts)
+    processed_count = 0
+    print(f"Query type: {args.query_type}")
+    print(f"Number of prompts: {total_requests}")
+
+    output_dir = args.output_dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    profiler_enabled = args.enable_profiler
+    if profiler_enabled:
+        omni.start_profile(stages=args.profiler_stages)
+
+    for stage_outputs in omni.generate(prompts, sampling_params_list):
+        output = stage_outputs.request_output
+        if stage_outputs.final_output_type == "text":
+            request_id = output.request_id
+            text_output = output.outputs[0].text
+            lines = []
+            lines.append("Prompt:\n")
+            lines.append(str(output.prompt) + "\n")
+            lines.append("Text Output:\n")
+            lines.append(str(text_output).strip() + "\n")
+            print(*lines, sep="")
+
+            # Save to file
+            out_txt = os.path.join(output_dir, f"{request_id}.txt")
+            try:
+                with open(out_txt, "w", encoding="utf-8") as f:
+                    f.writelines(lines)
+                print(f"Request ID: {request_id}, text saved to {out_txt}")
+            except Exception as e:
+                print(f"Failed to write output file {out_txt}: {e}")
+
+        elif stage_outputs.final_output_type == "audio":
+            raise NotImplementedError("Add audio example after talker supported.")
+
+        processed_count += 1
+        if profiler_enabled and processed_count >= total_requests:
+            print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
+            # Stop the profiler while workers are still alive
+            omni.stop_profile(stages=args.profiler_stages)
+
+            print("[Info] Waiting 30s for workers to write trace files to disk...")
+            time.sleep(30)
+            print("[Info] Trace export wait time finished.")
+
+    omni.close()
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Ming-flash-omni 2.0 offline inference example")
+    parser.add_argument(
+        "--query-type",
+        "-q",
+        type=str,
+        default="text",
+        choices=query_map.keys(),
+        help="Query type.",
+    )
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=None,
+        help="Path to a stage configs YAML file.",
+    )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        default=False,
+        help="Enable detailed statistics logging.",
+    )
+    parser.add_argument("--init-timeout", type=int, default=2000, help="Timeout for initializing in seconds.")
+    parser.add_argument(
+        "--stage-init-timeout",
+        type=int,
+        default=2000,
+        help="Timeout for initializing a single stage in seconds.",
+    )
+    parser.add_argument(
+        "--enable-profiler",
+        action="store_true",
+        default=False,
+        help="Enables profiling when set.",
+    )
+    parser.add_argument(
+        "--profiler-stages",
+        type=int,
+        nargs="*",
+        default=[0],
+        help="List of stage IDs to profile. If not set, profiles all stages.",
+    )
+    parser.add_argument(
+        "--image-path",
+        "-i",
+        type=str,
+        default=None,
+        help="Path to local image file. Uses default asset if not provided.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file. Uses default asset if not provided.",
+    )
+    parser.add_argument(
+        "--video-path",
+        "-v",
+        type=str,
+        default=None,
+        help="Path to local video file. Uses default asset if not provided.",
+    )
+    parser.add_argument(
+        "--num-frames",
+        type=int,
+        default=16,
+        help="Number of frames to extract from video.",
+    )
+    parser.add_argument(
+        "--sampling-rate",
+        type=int,
+        default=16000,
+        help="Sampling rate for audio loading.",
+    )
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=16384,
+        help="Maximum tokens to generate.",
+    )
+    parser.add_argument(
+        "--num-prompts",
+        type=int,
+        default=1,
+        help="Number of prompts to generate.",
+    )
+    parser.add_argument(
+        "--modalities",
+        type=str,
+        default=None,
+        help="Output modalities (comma-separated).",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="output_ming",
+        help="Output directory for results.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)
@@ -320,14 +320,7 @@ def main(args):
        query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
    else:
        query_result = query_func()
-    omni = Omni(
-        model=model_name,
-        log_stats=args.log_stats,
-        stage_init_timeout=args.stage_init_timeout,
-        batch_timeout=args.batch_timeout,
-        init_timeout=args.init_timeout,
-        shm_threshold_bytes=args.shm_threshold_bytes,
-    )
+    omni = Omni.from_cli_args(args, model=model_name)
    thinker_sampling_params = SamplingParams(
        temperature=0.0,  # Deterministic - no randomness
        top_p=1.0,  # Disable nucleus sampling
@@ -70,8 +70,8 @@ For true stage-level concurrency -- where downstream stages (Talker, Code2Wav)
 start **before** the upstream stage (Thinker) finishes -- use the async_chunk
 example. This requires:

-1. A stage config YAML with ``async_chunk: true`` (e.g.
-   ``qwen3_omni_moe_async_chunk.yaml``).
+1. A deploy config YAML with ``async_chunk: true`` (e.g.
+   ``qwen3_omni_moe.yaml``).
 2. Hardware that matches the config (e.g. 2x H100 for the default 3-stage
   config).

@@ -101,7 +101,7 @@ python end2end_async_chunk.py --query-type text --modalities text
 ```bash
 python end2end_async_chunk.py \
    --query-type use_audio \
-    --stage-configs-path /path/to/your_async_chunk.yaml
+    --deploy-config /path/to/your_deploy_config.yaml
 ```

 > **Note**: The synchronous ``end2end.py`` (using ``Omni``) is still the
@@ -294,14 +294,7 @@ def main(args):
    else:
        query_result = query_func()

-    omni = Omni(
-        model=model_name,
-        dtype=args.dtype,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.log_stats,
-        stage_init_timeout=args.stage_init_timeout,
-        init_timeout=args.init_timeout,
-    )
+    omni = Omni.from_cli_args(args, model=model_name)

    thinker_sampling_params = SamplingParams(
        temperature=0.9,
@@ -14,7 +14,7 @@ actively processing while stage-0 is still generating.
 Usage
 -----
    python end2end_async_chunk.py --query-type use_audio \
-        --stage-configs-path <path-to-async-chunk-yaml>
+        --deploy-config <path-to-deploy-config-yaml>

 See ``--help`` for all options.
 """
@@ -179,20 +179,26 @@ def clone_prompt_for_request(template: dict) -> dict:
    return cloned


-def _default_async_chunk_stage_configs_path() -> str | None:
-    """Best-effort default stage config for running Qwen3-Omni with async_chunk.
+def _default_deploy_config_path() -> str | None:
+    """Best-effort default deploy config for running Qwen3-Omni with async_chunk.

-    When this example is executed from within the repository, we resolve the
-    default YAML path relative to this file. When installed elsewhere, the
-    file may not exist and callers should pass --stage-configs-path explicitly.
+    The default ``vllm_omni/deploy/qwen3_omni_moe.yaml`` ships with
+    ``async_chunk: true`` at the top level, so loading it is enough to
+    enable async-chunk semantics. To disable it, copy the YAML and set
+    ``async_chunk: false`` (or pass ``--deploy-config`` to a YAML that
+    overrides the flag).
+
+    When this example is executed from within the repository, we resolve
+    the default YAML path relative to this file. When installed elsewhere,
+    the file may not exist and callers should pass ``--deploy-config``
+    explicitly.
    """
    repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
    candidate = os.path.join(
        repo_root,
        "vllm_omni",
-        "model_executor",
-        "stage_configs",
-        "qwen3_omni_moe_async_chunk.yaml",
+        "deploy",
+        "qwen3_omni_moe.yaml",
    )
    return candidate if os.path.exists(candidate) else None

@@ -374,15 +380,16 @@ async def run_all(args):
            prompt["modalities"] = output_modalities

    # Create AsyncOmni
-    print(f"[Info] Creating AsyncOmni with stage_configs_path={args.stage_configs_path}")
+    print(f"[Info] Creating AsyncOmni with deploy_config={args.deploy_config}")
    async_omni = None
    try:
-        async_omni = AsyncOmni(
-            model=args.model,
-            stage_configs_path=args.stage_configs_path,
-            log_stats=args.log_stats,
-            stage_init_timeout=args.stage_init_timeout,
-        )
+        # ``from_cli_args`` expands vars(args) into kwargs and auto-captures
+        # ``_cli_explicit_keys`` from ``sys.argv[1:]`` so argparse defaults
+        # do not silently override deploy YAML values. Mirrors the
+        # ``EngineArgs.from_cli_args`` pattern used throughout vllm /
+        # vllm-omni. ``deploy_config=None`` (the default) falls through to
+        # the bundled ``vllm_omni/deploy/qwen3_omni_moe.yaml``.
+        async_omni = AsyncOmni.from_cli_args(args)

        # Use default sampling params from stage config (they are pre-configured
        # in the YAML for each stage).
@@ -470,11 +477,11 @@ def parse_args():
        help="Query type.",
    )
    parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
        type=str,
-        default=_default_async_chunk_stage_configs_path(),
+        default=_default_deploy_config_path(),
        help=(
-            "Path to an async_chunk stage config YAML. "
+            "Path to a deploy config YAML. "
            "If not set, uses the model's default config "
            "(make sure it has async_chunk: true)."
        ),
@@ -17,7 +17,7 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
 python "${SCRIPT_DIR}/end2end_async_chunk.py" \
    --query-type text \
    --txt-prompts "${SCRIPT_DIR}/text_prompts_10.txt" \
-    --stage-configs-path "${REPO_ROOT}/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml" \
+    --deploy-config "${REPO_ROOT}/vllm_omni/deploy/qwen3_omni_moe.yaml" \
    --output-dir output_audio_async_chunk \
    --max-in-flight 2 \
    "$@"
@@ -6,13 +6,13 @@
 # achieving true stage-level concurrency via chunk-level streaming.
 #
 # Prerequisites:
-#   - An async_chunk stage config YAML (e.g. qwen3_omni_moe_async_chunk.yaml)
+#   - A deploy config YAML (e.g. qwen3_omni_moe.yaml)
 #   - Hardware matching the config (e.g. 2x H100 for the default 3-stage config)
 #
 # Usage:
 #   bash run_single_prompt_async_chunk.sh
 #   bash run_single_prompt_async_chunk.sh --query-type text --modalities text
-#   bash run_single_prompt_async_chunk.sh --stage-configs-path /path/to/custom.yaml
+#   bash run_single_prompt_async_chunk.sh --deploy-config /path/to/custom.yaml

 set -euo pipefail

@@ -21,6 +21,6 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"

 python "${SCRIPT_DIR}/end2end_async_chunk.py" \
    --query-type use_audio \
-    --stage-configs-path "${REPO_ROOT}/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml" \
+    --deploy-config "${REPO_ROOT}/vllm_omni/deploy/qwen3_omni_moe.yaml" \
    --output-dir output_audio_async_chunk \
    "$@"
@@ -15,11 +15,11 @@ Please refer to the [stage configuration documentation](https://docs.vllm.ai/pro

 ### ROCm Dependencies

-You will need to install these two dependencies `onnxruntime-rocm` and `sox`.
+You will need to install the dependency `onnxruntime-rocm`.

 ```
 pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm sox
+pip install onnxruntime-rocm
 ```

 ## Quick Start
@@ -104,13 +104,13 @@ completes. This demonstrates that audio data is available progressively rather t

 ## Batched Decoding

-The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, provide a stage config with `max_num_seqs > 1` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
+The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.

 ```
 python end2end.py --query-type CustomVoice \
    --txt-prompts benchmark_prompts.txt \
    --batch-size 4 \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml
+    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
 ```

 **Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
@@ -366,12 +366,7 @@ def main(args):
    output_dir = args.output_dir
    os.makedirs(output_dir, exist_ok=True)

-    omni = Omni(
-        model=model_name,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.log_stats,
-        stage_init_timeout=args.stage_init_timeout,
-    )
+    omni = Omni.from_cli_args(args, model=model_name)

    batch_size = args.batch_size
    for batch_start in range(0, len(inputs), batch_size):
@@ -387,12 +382,7 @@ async def main_streaming(args):
    output_dir = args.output_dir
    os.makedirs(output_dir, exist_ok=True)

-    omni = AsyncOmni(
-        model=model_name,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.log_stats,
-        stage_init_timeout=args.stage_init_timeout,
-    )
+    omni = AsyncOmni.from_cli_args(args, model=model_name)

    for i, prompt in enumerate(inputs):
        request_id = str(i)
@@ -27,6 +27,22 @@ python text_to_audio.py \
  --output stable_audio_output.wav
 ```

+To reduce per-GPU memory for multi-GPU inference, launch with HSDP:
+
+```bash
+python text_to_audio.py \
+  --model stabilityai/stable-audio-open-1.0 \
+  --prompt "The sound of a hammer hitting a wooden surface" \
+  --negative-prompt "Low quality" \
+  --seed 42 \
+  --guidance-scale 7.0 \
+  --audio-length 10.0 \
+  --num-inference-steps 100 \
+  --use-hsdp \
+  --hsdp-shard-size 2 \
+  --output stable_audio_output.wav
+```
+
 Key arguments:

 - `--prompt`: text description (string).
@@ -35,5 +51,8 @@ Key arguments:
 - `--guidance-scale`: classifier-free guidance scale.
 - `--audio-length`: audio duration in seconds.
 - `--num-inference-steps`: diffusion sampling steps.(more steps = higher quality, slower).
+- `--use-hsdp`: enable HSDP weight sharding for the Stable Audio DiT.
+- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
+- `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--cache-backend`: cache acceleration backend. Stable Audio currently supports `tea_cache`.
 - `--output`: path to save the generated WAV file.
@@ -21,6 +21,7 @@ from pathlib import Path
 import numpy as np
 import torch

+from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.platforms import current_omni_platform
@@ -113,6 +114,23 @@ def parse_args() -> argparse.Namespace:
        action="store_true",
        help="Enable diffusion pipeline profiler to display stage durations.",
    )
+    parser.add_argument(
+        "--use-hsdp",
+        action="store_true",
+        help="Enable HSDP for Stable Audio DiT weight sharding.",
+    )
+    parser.add_argument(
+        "--hsdp-shard-size",
+        type=int,
+        default=1,
+        help="Number of GPUs to shard Stable Audio DiT weights across when HSDP is enabled.",
+    )
+    parser.add_argument(
+        "--hsdp-replicate-size",
+        type=int,
+        default=1,
+        help="Number of HSDP replica groups. Default 1 means pure sharding.",
+    )
    return parser.parse_args()


@@ -158,12 +176,23 @@ def main():
    print(f"  Inference steps: {args.num_inference_steps}")
    print(f"  Guidance scale: {args.guidance_scale}")
    print(f"  Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
+    if args.use_hsdp:
+        print(f"  HSDP: enabled (shard_size={args.hsdp_shard_size}, replicate_size={args.hsdp_replicate_size})")
+    else:
+        print("  HSDP: disabled")
    print(f"  Seed: {args.seed}")
    print(f"{'=' * 60}\n")

+    parallel_config = DiffusionParallelConfig(
+        use_hsdp=args.use_hsdp,
+        hsdp_shard_size=args.hsdp_shard_size,
+        hsdp_replicate_size=args.hsdp_replicate_size,
+    )
+
    # Initialize Omni with Stable Audio model
    omni = Omni(
        model=args.model,
+        parallel_config=parallel_config,
        cache_backend=args.cache_backend,
        cache_config=cache_config,
        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
@@ -29,7 +29,8 @@ This folder provides several entrypoints for experimenting with text-to-image di
 | `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 |
 | `OmniGen2/OmniGen2` |  1024 x 1024 | 20.1 | 14.7 |
 | `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 |
-| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 77.6 | 31.4 |
+| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 33.9 | 31.4 |
+| `black-forest-labs/FLUX.1-schnell` | 1024 x 1024 | 33.9 | 31.4 |
 | `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
 | `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
 | `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
@@ -0,0 +1,123 @@
+# VoxCPM Offline Example
+
+This directory contains the minimal offline VoxCPM example for vLLM Omni.
+
+`end2end.py` is intentionally small and only covers:
+
+- single text-to-speech
+- single voice cloning with `ref_audio` + `ref_text`
+- non-streaming with `vllm_omni/model_executor/stage_configs/voxcpm.yaml`
+- streaming with `vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml`
+
+Advanced workflows were moved out of the getting-started example:
+
+- `benchmarks/voxcpm/vllm_omni/bench_tts_offline.py`: warmup, batch prompts, profiler, offline TTFP / RTF
+- `benchmarks/voxcpm/vllm_omni/run_offline_matrix.py`: fixed offline smoke matrix
+- `benchmarks/voxcpm/`: benchmark scripts and benchmark docs
+
+## Prerequisites
+
+Install VoxCPM in one of these ways:
+
+```bash
+pip install voxcpm
+```
+
+or point vLLM Omni to the local VoxCPM source tree:
+
+```bash
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+The example writes WAV files with `soundfile`:
+
+```bash
+pip install soundfile
+```
+
+## Model Path
+
+Pass the native VoxCPM model directory directly:
+
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+```
+
+If the native VoxCPM `config.json` does not contain HuggingFace metadata such as
+`model_type`, prepare a persistent HF-compatible config directory and point the
+stage configs to it with `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
+
+```bash
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+If the model directory itself already has `model_type`, this extra directory is
+not required.
+
+## Quick Start
+
+Single text-to-speech, non-streaming:
+
+```bash
+python examples/offline_inference/voxcpm/end2end.py \
+  --model "$VOXCPM_MODEL" \
+  --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
+```
+
+Single voice cloning, non-streaming:
+
+```bash
+python examples/offline_inference/voxcpm/end2end.py \
+  --model "$VOXCPM_MODEL" \
+  --text "This sentence is synthesized with a cloned voice." \
+  --ref-audio /path/to/reference.wav \
+  --ref-text "The exact transcript spoken in reference.wav."
+```
+
+Streaming:
+
+```bash
+python examples/offline_inference/voxcpm/end2end.py \
+  --model "$VOXCPM_MODEL" \
+  --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+  --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
+```
+
+By default, `end2end.py` writes to `output_audio/` for non-streaming and
+`output_audio_streaming/` for streaming.
+
+## Advanced Workflows
+
+Use `benchmarks/voxcpm/vllm_omni/bench_tts_offline.py` when you need:
+
+- warmup runs
+- prompt files
+- batch JSONL inputs
+- profiler injection
+- offline TTFP / RTF emission
+
+Use `benchmarks/voxcpm/vllm_omni/run_offline_matrix.py` when you need the fixed offline smoke matrix that previously lived in `test.py`.
+
+Full matrix benchmark example:
+
+```bash
+python benchmarks/voxcpm/vllm_omni/run_offline_matrix.py \
+  --model "$VOXCPM_MODEL" \
+  --ref-audio /path/to/reference.wav \
+  --ref-text "The exact transcript spoken in reference.wav."
+```
+
+For online serving examples, see [examples/online_serving/voxcpm](../../online_serving/voxcpm/README.md).
+
+For benchmark reporting, see [benchmarks/voxcpm](../../../benchmarks/voxcpm/README.md).
+
+## Notes
+
+- `voxcpm.yaml` is the default non-streaming stage config.
+- `voxcpm_async_chunk.yaml` is the streaming stage config.
+- Streaming is currently single-request oriented; the fixed smoke matrix now lives in `benchmarks/voxcpm/vllm_omni/run_offline_matrix.py`.
+- `ref_text` must be the real transcript of the reference audio. Mismatched text usually causes obvious quality degradation.
@@ -0,0 +1,206 @@
+"""Minimal offline VoxCPM example for vLLM Omni."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from pathlib import Path
+from typing import Any
+
+import soundfile as sf
+import torch
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni import AsyncOmni, Omni
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+DEFAULT_SYNC_STAGE_CONFIG = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
+
+
+def _build_prompt(args) -> dict[str, Any]:
+    additional_information: dict[str, list[Any]] = {
+        "text": [args.text],
+        "cfg_value": [args.cfg_value],
+        "inference_timesteps": [args.inference_timesteps],
+        "min_len": [args.min_len],
+        "max_new_tokens": [args.max_new_tokens],
+    }
+    if args.streaming_prefix_len is not None:
+        additional_information["streaming_prefix_len"] = [args.streaming_prefix_len]
+    if args.ref_audio is not None:
+        additional_information["ref_audio"] = [args.ref_audio]
+    if args.ref_text is not None:
+        additional_information["ref_text"] = [args.ref_text]
+    return {
+        "prompt_token_ids": [1],
+        "additional_information": additional_information,
+    }
+
+
+def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor:
+    audio = mm.get("audio", mm.get("model_outputs"))
+    if audio is None:
+        raise ValueError("No audio output found in multimodal output.")
+    if isinstance(audio, list):
+        parts = [torch.as_tensor(item).float().cpu().reshape(-1) for item in audio]
+        audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0)
+    if not isinstance(audio, torch.Tensor):
+        audio = torch.as_tensor(audio)
+    return audio.float().cpu().reshape(-1)
+
+
+def _extract_sample_rate(mm: dict[str, Any]) -> int:
+    sr_raw = mm.get("sr", 24000)
+    if isinstance(sr_raw, list) and sr_raw:
+        sr_raw = sr_raw[-1]
+    if hasattr(sr_raw, "item"):
+        return int(sr_raw.item())
+    return int(sr_raw)
+
+
+def _is_streaming_stage_config(stage_config_path: str) -> bool:
+    return "async_chunk" in Path(stage_config_path).stem
+
+
+def _save_audio(audio: torch.Tensor, sample_rate: int, output_dir: Path, request_id: str) -> Path:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path = output_dir / f"output_{request_id}.wav"
+    sf.write(
+        output_path,
+        audio.float().cpu().clamp(-1.0, 1.0).numpy(),
+        sample_rate,
+        format="WAV",
+        subtype="PCM_16",
+    )
+    return output_path
+
+
+async def _run_streaming(args) -> Path:
+    prompt = _build_prompt(args)
+    output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio_streaming")
+    request_id = "streaming_example"
+    sample_rate = 24000
+    buffered_samples = 0
+    chunks: list[torch.Tensor] = []
+    started = time.perf_counter()
+    omni = AsyncOmni(
+        model=args.model,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+    )
+    try:
+        async for stage_output in omni.generate(prompt, request_id=request_id):
+            mm = getattr(stage_output, "multimodal_output", None)
+            if not isinstance(mm, dict):
+                request_output = getattr(stage_output, "request_output", None)
+                if request_output is None:
+                    continue
+                mm = getattr(request_output, "multimodal_output", None)
+                if not isinstance(mm, dict) and getattr(request_output, "outputs", None):
+                    mm = getattr(request_output.outputs[0], "multimodal_output", None)
+            if not isinstance(mm, dict):
+                continue
+            audio = _extract_audio_tensor(mm)
+            if audio.numel() == 0:
+                continue
+            sample_rate = _extract_sample_rate(mm)
+            if audio.numel() > buffered_samples:
+                delta = audio[buffered_samples:]
+                buffered_samples = int(audio.numel())
+            else:
+                delta = audio
+                buffered_samples += int(delta.numel())
+            if delta.numel() > 0:
+                chunks.append(delta)
+        if not chunks:
+            raise RuntimeError("No streaming audio chunks received from VoxCPM.")
+        output_audio = torch.cat(chunks, dim=0)
+        output_path = _save_audio(output_audio, sample_rate, output_dir, request_id)
+        print(f"Saved streaming audio to: {output_path} ({time.perf_counter() - started:.2f}s)")
+        return output_path
+    finally:
+        omni.shutdown()
+
+
+def _run_sync(args) -> Path:
+    prompt = _build_prompt(args)
+    output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio")
+    request_id = "sync_example"
+    started = time.perf_counter()
+    last_mm: dict[str, Any] | None = None
+    omni = Omni(
+        model=args.model,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+    )
+    for stage_outputs in omni.generate(prompt):
+        request_output = getattr(stage_outputs, "request_output", None)
+        if request_output is None:
+            continue
+        outputs = getattr(request_output, "outputs", None)
+        if outputs:
+            for output in outputs:
+                mm = getattr(output, "multimodal_output", None)
+                if isinstance(mm, dict):
+                    last_mm = mm
+        mm = getattr(request_output, "multimodal_output", None)
+        if isinstance(mm, dict):
+            last_mm = mm
+    if last_mm is None:
+        raise RuntimeError("No audio output received from VoxCPM.")
+    output_path = _save_audio(
+        _extract_audio_tensor(last_mm),
+        _extract_sample_rate(last_mm),
+        output_dir,
+        request_id,
+    )
+    print(f"Saved audio to: {output_path} ({time.perf_counter() - started:.2f}s)")
+    return output_path
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Minimal offline VoxCPM example for vLLM Omni.")
+    parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=str(DEFAULT_SYNC_STAGE_CONFIG),
+        help=("Stage config path. Use voxcpm.yaml for non-streaming or voxcpm_async_chunk.yaml for streaming."),
+    )
+    parser.add_argument("--text", type=str, required=True, help="Input text for synthesis.")
+    parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
+    parser.add_argument("--ref-text", type=str, default=None, help="Transcript of the reference audio.")
+    parser.add_argument("--output-dir", type=str, default=None, help="Output directory for generated wav files.")
+    parser.add_argument("--cfg-value", type=float, default=2.0, help="Guidance value passed to VoxCPM.")
+    parser.add_argument("--inference-timesteps", type=int, default=10, help="Number of diffusion timesteps.")
+    parser.add_argument("--min-len", type=int, default=2, help="Minimum latent length.")
+    parser.add_argument("--max-new-tokens", type=int, default=4096, help="Maximum latent length.")
+    parser.add_argument(
+        "--streaming-prefix-len",
+        type=int,
+        default=3,
+        help="Streaming prefix length used by voxcpm_async_chunk.yaml.",
+    )
+    parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.")
+    parser.add_argument("--log-stats", action="store_true", help="Enable vLLM Omni stats logging.")
+    args = parser.parse_args()
+    if (args.ref_audio is None) != (args.ref_text is None):
+        raise ValueError("Voice cloning requires --ref-audio and --ref-text together.")
+    return args
+
+
+def main(args) -> None:
+    route = "streaming" if _is_streaming_stage_config(args.stage_configs_path) else "sync"
+    print(f"Model: {args.model}")
+    print(f"Stage config: {args.stage_configs_path}")
+    print(f"Route: {route}")
+    if route == "streaming":
+        asyncio.run(_run_streaming(args))
+    else:
+        _run_sync(args)
+
+
+if __name__ == "__main__":
+    main(parse_args())
@@ -65,6 +65,12 @@ def parse_args():
        default=None,
        help="Text matching --prompt-audio for continuation mode.",
    )
+    parser.add_argument(
+        "--ref-text",
+        type=str,
+        default=None,
+        help="Optional transcript of --reference-audio (enables ref_continuation mode).",
+    )
    return parser.parse_args()


@@ -74,16 +80,20 @@ def extract_audio(multimodal_output: dict) -> torch.Tensor:
    The output processor concatenates per-step delta tensors under
    ``model_outputs``.  Falls back to ``audio`` for backwards compat.
    """
-    audio = multimodal_output.get("model_outputs") or multimodal_output.get("audio")
+    audio = multimodal_output.get("model_outputs")
+    if audio is None:
+        audio = multimodal_output.get("audio")
    if audio is None:
        raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")

    if isinstance(audio, list):
-        # Take the last valid tensor (most complete audio)
+        # Defensive: usually the output processor consolidates into a single
+        # tensor at request completion, but concatenate here too in case the
+        # caller consumes intermediate (pre-consolidation) outputs.
        valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
        if not valid:
            raise ValueError("Audio list is empty or all elements are None.")
-        return valid[-1]
+        return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]

    return torch.as_tensor(audio).float().cpu().reshape(-1)

@@ -99,24 +109,40 @@ def main():
        stage_configs_path=args.stage_configs_path,
    )

-    additional: dict = {}
-    if args.reference_audio:
-        additional["reference_audio"] = args.reference_audio
-    if args.prompt_audio and args.prompt_text:
-        additional["prompt_audio"] = args.prompt_audio
-        additional["prompt_text"] = args.prompt_text
+    from transformers import AutoTokenizer

-    prompt: dict = {"prompt": args.text}
-    if additional:
-        prompt["additional_information"] = additional
+    from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import (
+        build_cjk_split_map,
+        build_voxcpm2_prompt,
+    )
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    split_map = build_cjk_split_map(tokenizer)
+    hf_config = engine.engine.stage_vllm_configs[0].model_config.hf_config
+
+    ref_audio_arg = args.reference_audio or args.prompt_audio
+    ref_text_arg = args.ref_text or args.prompt_text
+    ref_wav, ref_sr = (None, None)
+    if ref_audio_arg:
+        ref_wav_arr, ref_sr = sf.read(ref_audio_arg)
+        ref_wav = ref_wav_arr.mean(axis=-1).tolist() if ref_wav_arr.ndim > 1 else ref_wav_arr.tolist()
+
+    prompt = build_voxcpm2_prompt(
+        hf_config=hf_config,
+        tokenizer=tokenizer,
+        split_map=split_map,
+        text=args.text,
+        ref_audio=ref_wav,
+        ref_sr=ref_sr,
+        ref_text=ref_text_arg,
+    )

    print(f"Model       : {args.model}")
    print(f"Text        : {args.text}")
-    if args.reference_audio:
-        print(f"Ref audio   : {args.reference_audio}")
-    if args.prompt_audio:
-        print(f"Prompt audio: {args.prompt_audio}")
-        print(f"Prompt text : {args.prompt_text}")
+    if ref_audio_arg:
+        print(f"Ref audio   : {ref_audio_arg}")
+    if ref_text_arg:
+        print(f"Ref text    : {ref_text_arg}")
    print(f"Output dir  : {output_dir}")

    t_start = time.perf_counter()
@@ -10,28 +10,24 @@ When `mistral_common` has `SpeechRequest` support, prompt token IDs are built vi
 ```bash
 # Basic single-prompt with cheerful_female voice preset
 python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \
    --write-audio --voice cheerful_female \
    --model mistralai/Voxtral-4B-TTS-2603 \
    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"

 # 32 replicate prompts with cheerful_female voice preset
 python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \
    --num-prompts 32 --write-audio --voice cheerful_female \
    --model mistralai/Voxtral-4B-TTS-2603 \
    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"

 # Streaming with neutral_female voice preset
 python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \
    --streaming --write-audio --voice neutral_female \
    --model mistralai/Voxtral-4B-TTS-2603 \
    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"

 # 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice
 python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \
    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
    --model mistralai/Voxtral-4B-TTS-2603 \
    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
@@ -39,7 +35,6 @@ python3 examples/offline_inference/voxtral_tts/end2end.py \
 # Short debug prompt with reference audio
 # Note: Reference audio capability is not yet released.
 python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \
    --write-audio \
    --model mistralai/Voxtral-4B-TTS-2603 \
    --text "This is a test message." \
@@ -54,7 +49,7 @@ python3 examples/offline_inference/voxtral_tts/end2end.py \
 | `--text TEXT` | Text to synthesize (default: `"This is a test message."`) |
 | `--audio-path PATH` | Path to reference audio file for voice cloning |
 | `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) |
-| `--stage-configs-path PATH` | Path to stage configs YAML (currently it must be set for VoxtralTTS) |
+| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. |
 | `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) |
 | `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) |
 | `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) |
@@ -39,7 +39,7 @@ logger = logging.getLogger(__name__)
 async def run_streaming(inputs, sampling_params_list, model_name, args, output_dir):
    async_omni = AsyncOmni(
        model=model_name,
-        stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
        log_stats=args.log_stats,
    )

@@ -192,7 +192,7 @@ def run_non_streaming(inputs, sampling_params_list, model_name, args, output_dir
    llm = Omni(
        model=model_name,
        log_stats=args.log_stats,
-        stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
    )

    if args.profiling_mode:
@@ -253,10 +253,11 @@ def parse_args() -> Namespace:
        help="Directory to write output wav files.",
    )
    parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
        type=str,
        default=None,
-        help="Path to stage configs YAML. Auto-resolved from model if not set.",
+        help="Override the deploy config path. If unset, auto-loads "
+        "vllm_omni/deploy/voxtral_tts.yaml based on the HF model_type.",
    )
    parser.add_argument(
        "--num-prompts", type=int, default=1, help="Number of replicate prompts to run for measuring performance"
@@ -30,9 +30,9 @@ dreamid_omni/
 ```

 ### Run the Inference
-```
+```python
 python x_to_video_audio.py \
-  --model /xx/dreamid_omni \
+  --model /path/to/dreamid_omni \
  --prompt "Two people walking together and singing happily" \
  --image-path ./example0.png ./example1.png \
  --audio-path ./example0.wav ./example1.wav \
@@ -42,11 +42,33 @@ python x_to_video_audio.py \
  --num-inference-steps 45 \
  --height 704 \
  --width 1280 \
-  --output dreamid_omni.mp4
+  --output out_dreamid_omni_twoip.mp4
 ```
 In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
 The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.

+
+You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
+
+For example, single IP ref resources can be found under https://github.com/Guoxu1233/DreamID-Omni/tree/main/test_case/oneip, you could download them correspondingly to your local and use them for testing.
+
+```python
+# Example usage for oneip, ref media from the official repo DreamID-Omni
+python x_to_video_audio.py \
+  --model /path/to/dreamid_omni \
+  --prompt "<img1>: In the frame, a woman with black long hair is identified as <sub1>.\n**Overall Environment/Scene**: A lively open-kitchen café at night; stove flames flare, steam rises, and warm pendant lights swing slightly as staff move behind her. The shot is an upper-body close-up.\n**Main Characters/Subjects Appearance**: <sub1> is a young woman with thick dark wavy hair and a side part. She wears a fitted black top under a light apron, a thin gold chain necklace, and small stud earrings.\n**Main Characters/Subjects Actions**: <sub1> tastes the sauce with a spoon, then turns her face toward the camera while still holding the spoon, her expression shifting from focused to conflicted.\n<sub1> maintains eye contact, swallows as if choosing her words, and says, <S>I keep telling myself I’m fine,but some nights it feels like I’m just performing calm.<E>" \
+  --image-path 9.png \
+  --audio-path 9.wav \
+  --video-negative-prompt "jitter, bad hands, blur, distortion" \
+  --audio-negative-prompt "robotic, muffled, echo, distorted" \
+  --cfg-parallel-size 2 \
+  --num-inference-steps 45 \
+  --height 704 \
+  --width 1280 \
+  --output out_dreamid_omni_oneip.mp4
+```
+
+
 Key arguments:
 - `--prompt`: text description (string).
 - `--model`: path to the model local directory.
@@ -5,10 +5,12 @@ import argparse
 import re
 import time

+import numpy as np
 from PIL import Image
 from vllm.multimodal.media.audio import load_audio

 from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams

@@ -56,6 +58,11 @@ def parse_args() -> argparse.Namespace:
        default=False,
        help="Enable CPU offloading for diffusion models.",
    )
+    parser.add_argument(
+        "--enable-layerwise-offload",
+        action="store_true",
+        help="Enable layerwise (blockwise) offloading on DiT modules.",
+    )
    return parser.parse_args()


@@ -124,6 +131,7 @@ def main() -> None:
        parallel_config=parallel_config,
        model_type=args.model_type,
        enable_cpu_offload=args.enable_cpu_offload,
+        enable_layerwise_offload=args.enable_layerwise_offload,
    )
    start = time.perf_counter()
    outputs = omni.generate(prompt, sampling_params)
@@ -131,15 +139,35 @@ def main() -> None:

    if not outputs:
        raise RuntimeError("No output returned from DreamID-Omni.")
-    output = outputs[0].request_output
-    generated_video = output.images[0][0]
-    generated_audio = output.images[0][1]
-    try:
-        from dreamid_omni.utils.io_utils import save_video
-    except Exception as e:
-        raise RuntimeError(f"Failed to extract video and audio from DreamID-Omni output. Error: {e}")
+    result = outputs[0]
+    if not result.images:
+        raise RuntimeError("No video frames found in DreamID-Omni output.")
+    generated_video = result.images[0]
+    mm = result.multimodal_output or {}
+    generated_audio = mm.get("audio")
+    fps = int(mm.get("fps", 24))
+    sample_rate = int(mm.get("audio_sample_rate", 16000))
+
+    # DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1].
+    # mux_video_audio_bytes expects (F, H, W, C) uint8.
+    if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4:
+        raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}")
+    frames = generated_video.transpose(1, 2, 3, 0)
+    frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8)
+
+    audio_np = None
+    if generated_audio is not None:
+        audio_np = np.squeeze(np.asarray(generated_audio)).astype(np.float32)
+
    output_path = args.output
-    save_video(output_path, generated_video, generated_audio, fps=24, sample_rate=16000)
+    video_bytes = mux_video_audio_bytes(
+        frames,
+        audio_np,
+        fps=float(fps),
+        audio_sample_rate=sample_rate,
+    )
+    with open(output_path, "wb") as f:
+        f.write(video_bytes)
    print(f"Saved generated video to {output_path}")
    print(f"Total time: {elapsed:.2f}s")

@@ -29,15 +29,12 @@ Features:
 ## Launch the Server

 ```bash
-vllm-omni serve fishaudio/s2-pro \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager \
-    --gpu-memory-utilization 0.9
+vllm serve fishaudio/s2-pro --omni --port 8091
 ```

+The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`
+(the HF `model_type` on the fishaudio checkpoints is `fish_qwen3_omni`).
+
 Or use the convenience script:

 ```bash
@@ -11,18 +11,13 @@ MODEL="${MODEL:-fishaudio/s2-pro}"
 PORT="${PORT:-8091}"
 GRADIO_PORT="${GRADIO_PORT:-7860}"
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"

 echo "Starting Fish Speech S2 Pro server (port $PORT)..."
 FLASHINFER_DISABLE_VERSION_CHECK=1 \
-vllm-omni serve "$MODEL" \
-    --stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml" \
+vllm serve "$MODEL" \
+    --omni \
    --host 0.0.0.0 \
-    --port "$PORT" \
-    --gpu-memory-utilization 0.9 \
-    --trust-remote-code \
-    --enforce-eager \
-    --omni &
+    --port "$PORT" &
 SERVER_PID=$!

 cleanup() {
@@ -13,11 +13,7 @@ PORT="${PORT:-8091}"
 echo "Starting Fish Speech S2 Pro server with model: $MODEL"

 FLASHINFER_DISABLE_VERSION_CHECK=1 \
-vllm-omni serve "$MODEL" \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \
+vllm serve "$MODEL" \
+    --omni \
    --host 0.0.0.0 \
-    --port "$PORT" \
-    --gpu-memory-utilization 0.9 \
-    --trust-remote-code \
-    --enforce-eager \
-    --omni
+    --port "$PORT"
@@ -13,10 +13,10 @@ Please refer to [README.md](../../../README.md)
 ```bash
 export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer"

-vllm-omni serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \
--served-model-name "MiMo-Audio-7B-Instruct"  \
--port 18091 --stage-configs-path ./vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
--chat-template ./examples/online_serving/mimo_audio/chat_template.jinja
+vllm serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \
+    --served-model-name "MiMo-Audio-7B-Instruct" \
+    --port 18091 \
+    --chat-template ./examples/online_serving/mimo_audio/chat_template.jinja
 ```
 > ⚠️ **Important**  
 > **MiMo-Audio is not compatible with the default chat template.**  
@@ -0,0 +1,204 @@
+# Ming-flash-omni 2.0
+
+## Installation
+
+Please refer to [README.md](../../../README.md)
+
+## Run examples (Ming-flash-omni 2.0)
+
+### Launch the Server
+
+```bash
+vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
+```
+
+If you have custom stage configs file, launch the server with command below
+```bash
+vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+```
+
+### Send Multi-modal Request
+
+#### Send request via python
+
+```bash
+python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py --model Jonathan1909/Ming-flash-omni-2.0 --query-type use_mixed_modalities --port 8091 --host "localhost" --modalities text
+```
+
+The Python client supports the following command-line arguments:
+
+- `--query-type` (or `-q`): Query type. Options: `text`, `use_audio`, `use_image`, `use_video`, `use_mixed_modalities`
+- `--video-path` (or `-v`): Path to local video file or URL. If not provided and query-type uses video, uses default video URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs. Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
+- `--image-path` (or `-i`): Path to local image file or URL. If not provided and query-type uses image, uses default image URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common image formats: JPEG, PNG, GIF, WebP. Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
+- `--audio-path` (or `-a`): Path to local audio file or URL. If not provided and query-type uses audio, uses default audio URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common audio formats: MP3, WAV, OGG, FLAC, M4A. Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
+- `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"`
+- `--modalities`: Output modalities. For now, only `text` is supported. Example: `--modalities text`
+
+
+#### Send request via curl
+
+```bash
+bash run_curl_multimodal_generation.sh text
+bash run_curl_multimodal_generation.sh use_image
+bash run_curl_multimodal_generation.sh use_audio
+bash run_curl_multimodal_generation.sh use_video
+bash run_curl_multimodal_generation.sh use_mixed_modalities
+```
+
+## Modality control
+
+Ming-flash-omni 2.0 currently supports text output only (thinker stage).
+
+| Modalities | Output |
+|------------|--------|
+| `["text"]` | Text only |
+| Not specified | Text only (default) |
+
+### Using curl
+
+```bash
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Jonathan1909/Ming-flash-omni-2.0",
+    "messages": [
+      {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
+      {"role": "user", "content": "请详细介绍鹦鹉的生活习性。"}
+    ],
+    "modalities": ["text"]
+  }'
+```
+
+### Using OpenAI Python SDK
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
+
+response = client.chat.completions.create(
+    model="Jonathan1909/Ming-flash-omni-2.0",
+    messages=[
+        {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
+        {"role": "user", "content": "请详细介绍鹦鹉的生活习性。"},
+    ],
+    modalities=["text"],
+)
+print(response.choices[0].message.content)
+```
+
+### Multi-modal input with OpenAI Python SDK
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
+
+response = client.chat.completions.create(
+    model="Jonathan1909/Ming-flash-omni-2.0",
+    messages=[
+        {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
+        {
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"}},
+                {"type": "text", "text": "Describe this image in detail."},
+            ],
+        },
+    ],
+    modalities=["text"],
+)
+print(response.choices[0].message.content)
+```
+
+## Streaming Output
+
+To enable streaming output:
+
+```bash
+python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py \
+    --query-type use_image \
+    --model Jonathan1909/Ming-flash-omni-2.0 \
+    --modalities text \
+    --stream
+```
+
+Or with the OpenAI Python SDK:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
+
+response = client.chat.completions.create(
+    model="Jonathan1909/Ming-flash-omni-2.0",
+    messages=[
+        {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
+        {"role": "user", "content": "请详细介绍鹦鹉的生活习性。"},
+    ],
+    modalities=["text"],
+    stream=True,
+)
+for chunk in response:
+    for choice in chunk.choices:
+        if hasattr(choice, "delta") and choice.delta.content:
+            print(choice.delta.content, end="", flush=True)
+print()
+```
+
+Or using curl:
+
+```bash
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Jonathan1909/Ming-flash-omni-2.0",
+    "messages": [
+      {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
+      {"role": "user", "content": "请详细介绍鹦鹉的生活习性。"}
+    ],
+    "modalities": ["text"],
+    "stream": true,
+  }'
+```
+
+
+## Reasoning (Thinking Mode)
+
+To enable reasoning/thinking mode, change `detailed thinking off` to `detailed thinking on` in the system prompt:
+
+### Using curl
+
+```bash
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Jonathan1909/Ming-flash-omni-2.0",
+    "messages": [
+      {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking on"}]},
+      {"role": "user", "content": [
+        {"type": "image_url", "image_url": {"url": "https://example.com/math_problem.png"}},
+        {"type": "text", "text": "Solve this math problem step by step."}
+      ]}
+    ],
+    "modalities": ["text"]
+  }'
+```
+
+### Using OpenAI Python SDK
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
+
+response = client.chat.completions.create(
+    model="Jonathan1909/Ming-flash-omni-2.0",
+    messages=[
+        {"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking on"}]},
+        {"role": "user", "content": "If a train travels 120 km in 2 hours, what is its average speed?"},
+    ],
+    modalities=["text"],
+)
+print(response.choices[0].message.content)
+```
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Server port
+PORT="${PORT:-8091}"
+# Default query type
+QUERY_TYPE="${1:-text}"
+
+# Validate query type
+if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modalities)$ ]]; then
+    echo "Error: Invalid query type '$QUERY_TYPE'"
+    echo "Usage: $0 [text|use_audio|use_image|use_video|use_mixed_modalities]"
+    echo "  text: Text-only query"
+    echo "  use_audio: Audio + Text query"
+    echo "  use_image: Image + Text query"
+    echo "  use_video: Video + Text query"
+    echo "  use_mixed_modalities: Audio + Image + Video + Text query"
+    exit 1
+fi
+
+thinker_sampling_params='{
+  "temperature": 0.4,
+  "top_p": 0.9,
+  "top_k": -1,
+  "max_tokens": 16384,
+  "seed": 42,
+  "detokenize": true,
+  "repetition_penalty": 1.05
+}'
+# Above is optional, it has a default setting in stage_configs of the corresponding model.
+
+# Define URLs for assets
+MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
+CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
+SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
+
+# Build user content based on query type
+case "$QUERY_TYPE" in
+  text)
+    user_content='[
+      {
+        "type": "text",
+        "text": "请详细介绍鹦鹉的生活习性。"
+      }
+    ]'
+    ;;
+  use_image)
+    user_content='[
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "Describe this image in detail."
+        }
+      ]'
+    ;;
+  use_audio)
+    user_content='[
+        {
+          "type": "audio_url",
+          "audio_url": {
+            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "Please recognize the language of this speech and transcribe it. Format: oral."
+        }
+      ]'
+    ;;
+  use_video)
+    user_content='[
+        {
+          "type": "video_url",
+          "video_url": {
+            "url": "'"$SAMPLE_VIDEO_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "Describe what is happening in this video."
+        }
+      ]'
+    ;;
+  use_mixed_modalities)
+    user_content='[
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
+          }
+        },
+        {
+          "type": "audio_url",
+          "audio_url": {
+            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "Describe the image, and recognize the language of this speech and transcribe it. Format: oral"
+        }
+      ]'
+    ;;
+esac
+
+echo "Running query type: $QUERY_TYPE"
+echo ""
+
+request_body=$(cat <<EOF
+{
+  "model": "Jonathan1909/Ming-flash-omni-2.0",
+  "sampling_params_list": [
+    $thinker_sampling_params
+  ],
+  "modalities": ["text"],
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "type": "text",
+          "text": "你是一个友好的AI助手。\n\ndetailed thinking off"
+        }
+      ]
+    },
+    {
+      "role": "user",
+      "content": $user_content
+    }
+  ]
+}
+EOF
+)
+
+output=$(curl -sS --retry 3 --retry-delay 3 --retry-connrefused \
+    -X POST http://localhost:${PORT}/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -d "$request_body")
+
+echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"
@@ -12,17 +12,159 @@ Please refer to [README.md](../../../README.md)
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```

-If you want to open async chunking for qwen3-omni, launch the server with command below
+The default deploy config at `vllm_omni/deploy/qwen3_omni_moe.yaml` is loaded
+automatically by the model registry — no `--deploy-config` flag needed for the
+common case. Async-chunk streaming is **enabled by default** in the bundled config.
+NPU / ROCm / XPU per-platform deltas are merged in automatically from the
+`platforms:` section of the same YAML.
+
+**Note:** The OpenAI-style **`/v1/realtime`** WebSocket (streaming PCM audio in, audio + transcription out) is **not supported** when `async_chunk` is enabled. Use the default omni layout or a stage config with `async_chunk: false` for realtime sessions.
+
+If you have a custom deploy YAML, point at it explicitly:

 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --deploy-config /path/to/your_deploy_config.yaml
 ```

-If you have custom stage configs file, launch the server with command below
+### Tuning deployment parameters
+
+Most engine knobs (`max_num_batched_tokens`, `max_model_len`, `enforce_eager`,
+`gpu_memory_utilization`, `tensor_parallel_size`, …) can be tuned without
+editing the YAML. There are three layers, in increasing specificity:
+
+#### 1. Global CLI flags (apply to every stage)
+
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+# Tighter memory budget on a smaller GPU
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --gpu-memory-utilization 0.85
+
+# Disable cudagraphs (e.g. for debugging)
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --enforce-eager
+
+# Reduce context length
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --max-model-len 32768
+
+# Toggle prefix caching on every stage (yaml default: off)
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --enable-prefix-caching
+# ...or force it off if the yaml turned it on
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --no-enable-prefix-caching
+
+# Toggle pipeline-wide async chunked streaming between stages
+# (yaml default for qwen3_omni_moe: on)
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --no-async-chunk
 ```

+For the TTS counterpart (synchronous codec variant), see
+[qwen3_tts README](../qwen3_tts/README.md#sync-vs-async-chunk-mode).
+
+Explicit CLI flags **override** the deploy YAML (which itself overrides the
+parser defaults). If you don't pass a flag, the YAML value wins.
+
+> **Note on `--no-async-chunk`**: Flips the deploy yaml's `async_chunk:`
+> bool. Pipelines that implement alternate processor functions for
+> chunked vs end-to-end modes (e.g. qwen3_tts code2wav) dispatch
+> automatically based on that bool — no extra flag or variant yaml is
+> needed.
+
+> ⚠️ **For multi-stage models that share GPUs (qwen3_omni_moe by default
+> shares cuda:1 between stages 1 and 2), avoid using global memory flags.**
+> A global `--gpu-memory-utilization 0.85` would apply to every stage and
+> oversubscribe the shared device. Use per-stage overrides instead — see
+> below.
+
+#### 2. Per-stage overrides via `--stage-overrides` (recommended for memory)
+
+```bash
+# Lower stage 1's memory budget; leave others at the YAML default
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+    --stage-overrides '{
+        "1": {"gpu_memory_utilization": 0.5},
+        "2": {"max_num_batched_tokens": 65536}
+    }'
+```
+
+Per-stage values are always treated as explicit and beat YAML defaults for
+the named stage. Other stages keep their YAML values.
+
+#### 3. Custom deploy YAML
+
+When per-stage overrides get long, write a small overlay YAML that inherits
+from the bundled default:
+
+```yaml
+# my_qwen3_omni_overrides.yaml
+base_config: /path/to/vllm_omni/deploy/qwen3_omni_moe.yaml
+
+stages:
+  - stage_id: 0
+    max_num_batched_tokens: 65536
+    enforce_eager: true
+  - stage_id: 1
+    gpu_memory_utilization: 0.5
+  - stage_id: 2
+    max_model_len: 8192
+```
+
+Then start the server with `--deploy-config my_qwen3_omni_overrides.yaml`.
+The `base_config:` line tells the loader to inherit everything else (stages,
+connectors, edges, platforms section) from the bundled production YAML, so
+you only need to spell out the deltas.
+
+#### 4. Multi-node deployment (cross-host transfer connector)
+
+The bundled `qwen3_omni_moe.yaml` uses `SharedMemoryConnector` between stages,
+which only works when all stages run on the same physical host. For
+**cross-node** deployments, write a small overlay YAML that swaps in a
+network-capable connector (e.g. `MooncakeStoreConnector`) and re-points each
+stage's connector wiring at it. The connector spec carries your own server
+addresses — there is no checked-in default because every cluster is
+different.
+
+```yaml
+# my_qwen3_omni_multinode.yaml
+base_config: /path/to/vllm_omni/deploy/qwen3_omni_moe.yaml
+
+connectors:
+  mooncake_connector:
+    name: MooncakeStoreConnector
+    extra:
+      host: "127.0.0.1"
+      metadata_server: "http://YOUR_METADATA_HOST:8080/metadata"
+      master: "YOUR_MASTER_HOST:50051"
+      segment: 512000000    # 512 MB transfer segment
+      localbuf: 64000000    # 64 MB local buffer
+      proto: "tcp"
+
+stages:
+  - stage_id: 0
+    output_connectors:
+      to_stage_1: mooncake_connector
+  - stage_id: 1
+    input_connectors:
+      from_stage_0: mooncake_connector
+    output_connectors:
+      to_stage_2: mooncake_connector
+  - stage_id: 2
+    input_connectors:
+      from_stage_1: mooncake_connector
+```
+
+Then launch with `--deploy-config my_qwen3_omni_multinode.yaml`. Same
+pattern works for Qwen2.5-Omni — replace `base_config:` with the path to
+`vllm_omni/deploy/qwen2_5_omni.yaml`.
+
+> ⚠️ Replace `YOUR_METADATA_HOST` / `YOUR_MASTER_HOST` with the actual
+> mooncake server addresses for your cluster. The `base_config:` overlay
+> inherits all stage budgets, devices, and edges from the bundled prod
+> YAML — you only need to spell out the connector swap.
+
 ### Send Multi-modal Request

 Get into the example folder
@@ -38,36 +180,43 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene

 #### Realtime WebSocket client (`openai_realtime_client.py`)

-[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, uploads a local audio file as **PCM16 mono @ 16 kHz** chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and prints **streaming transcription** (`transcription.delta` / `transcription.done`).
+[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and receives **`response.audio.delta`** (incremental PCM for the reply) plus **`transcription.*`** events. By default it concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
+
+Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.

 **Dependencies:**

 ```bash
-pip install websockets numpy
+pip install websockets
 ```

 **From this directory** (`examples/online_serving/qwen3_omni`):

 ```bash
 python openai_realtime_client.py \
-  --host localhost \
-  --port 8091 \
+  --url ws://localhost:8091/v1/realtime \
  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --audio_path /path/to/your.wav
+  --input-wav /path/to/input_16k_mono.wav \
+  --output-wav realtime_output.wav \
+  --delta-dump-dir ./rt_delta_wavs
 ```

-If `--audio_path` is omitted, the script uses a bundled default clip (`mary_had_lamb` via vLLM assets).
-
 **Arguments:**

 | Flag | Default | Description |
 |------|---------|-------------|
-| `--host` | `localhost` | API server host |
-| `--port` | `8000` | API server port (match your `vllm serve` port, e.g. `8091`) |
-| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (also sent in `session.update`) |
-| `--audio_path` | *(optional)* | Path to input audio; resampled to 16 kHz mono inside the client |
+| `--url` | `ws://localhost:8091/v1/realtime` | Full WebSocket URL including path |
+| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (sent in `session.update`) |
+| `--input-wav` | *(required)* | Input WAV: mono, 16-bit PCM, **16 kHz** |
+| `--output-wav` | `realtime_output.wav` | Output path for concatenated reply audio |
+| `--output-text` | *(optional)* | If set, write final transcription text to this path |
+| `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
+| `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
+| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files |
+| `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
+| `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |

-Ensure the vLLM-Omni server is running with realtime support for this endpoint, for example:
+Ensure the server is running **without** `async_chunk` if you use `/v1/realtime`, for example:

 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
@@ -276,7 +425,7 @@ The script supports the following arguments:
 - `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--deploy-config`: Path to custom deploy config YAML file (optional)
 - `--server-host`: Host for vLLM server (default: 0.0.0.0)
 - `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
 - `--share`: Share Gradio demo publicly (creates a public link)
@@ -291,7 +440,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091

 If you have custom stage configs file:
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file
 ```

 **Step 2: Run the Gradio demo**
@@ -1,81 +1,118 @@
+"""Realtime client for vLLM-Omni /v1/realtime (audio + text events).
+
+This client:
+1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
+2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
+3) Receives response.audio.* and transcription.* events,
+4) Saves synthesized audio to an output WAV file and optional text file.
+
+By default each ``response.audio.delta`` is treated as an **incremental PCM**
+chunk and all chunks are concatenated into the final ``--output-wav``.
+
+Optional debugging: pass ``--delta-dump-dir DIR`` to write every
+``response.audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, …
+
+Usage:
+  python openai_realtime_client.py \
+      --url ws://localhost:8091/v1/realtime \
+      --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+      --input-wav input_16k_mono.wav \
+      --output-wav realtime_output.wav \
+      --delta-dump-dir ./rt_delta_wavs
+
+Dependencies:
+  pip install websockets
 """
-This script demonstrates how to use the vLLM-Omni Realtime WebSocket API to perform
-audio transcription by uploading an audio file.

-Before running this script, you must start the vLLM-Omni server with a realtime-capable
-model, for example:
-
-    vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni
-
-Requirements:
- vllm with audio support
- websockets
- soundfile
- numpy
-
-The script:
-1. Connects to the Realtime WebSocket endpoint
-2. Converts an audio file to PCM16 @ 16kHz
-3. Sends audio chunks to the server
-4. Receives and prints transcription as it streams
-"""
+from __future__ import annotations

 import argparse
 import asyncio
 import base64
 import json
+import wave
+from pathlib import Path

-import numpy as np
-import websockets
-from vllm.assets.audio import AudioAsset
-from vllm.multimodal.media.audio import load_audio
+try:
+    import websockets
+except ImportError:
+    print("Please install websockets: pip install websockets")
+    raise SystemExit(1)


-def audio_to_pcm16_base64(audio_path: str) -> str:
-    """
-    Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
-    """
-    # Load audio and resample to 16kHz mono
-    audio, _ = load_audio(audio_path, sr=16000, mono=True)
-    # Convert to PCM16
-    pcm16 = (audio * 32767).astype(np.int16)
-    # Encode as base64
-    return base64.b64encode(pcm16.tobytes()).decode("utf-8")
+def _read_wav_pcm16(path: Path) -> bytes:
+    with wave.open(str(path), "rb") as wf:
+        nchannels = wf.getnchannels()
+        sampwidth = wf.getsampwidth()
+        framerate = wf.getframerate()
+        comptype = wf.getcomptype()
+        nframes = wf.getnframes()
+
+        if nchannels != 1:
+            raise ValueError(f"Input WAV must be mono (got {nchannels} channels).")
+        if sampwidth != 2:
+            raise ValueError(f"Input WAV must be 16-bit PCM (got sample width={sampwidth}).")
+        if framerate != 16000:
+            raise ValueError(f"Input WAV must be 16kHz (got {framerate} Hz).")
+        if comptype != "NONE":
+            raise ValueError(f"Input WAV must be uncompressed PCM (got comptype={comptype}).")
+        if nframes <= 0:
+            raise ValueError("Input WAV has no audio frames.")
+
+        return wf.readframes(nframes)


-async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
-    """
-    Connect to the Realtime API and transcribe an audio file.
-    """
-    uri = f"ws://{host}:{port}/v1/realtime"
+def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> None:
+    with wave.open(str(path), "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate_hz)
+        wf.writeframes(pcm16_bytes)

-    async with websockets.connect(uri) as ws:
-        # Wait for session.created
-        response = json.loads(await ws.recv())
-        if response["type"] == "session.created":
-            print(f"Session created: {response['id']}")
-        else:
-            print(f"Unexpected response: {response}")
-            return

-        # Validate model
-        await ws.send(json.dumps({"type": "session.update", "model": model}))
+async def run_client(
+    url: str,
+    model: str,
+    input_wav: Path,
+    output_wav: Path,
+    output_text: Path | None,
+    chunk_ms: int,
+    send_delay_ms: int,
+    delta_dump_dir: Path | None,
+    request_idx: int = 1,
+    total_requests: int = 1,
+) -> None:
+    log_prefix = f"[req {request_idx:02d}/{total_requests:02d}] " if total_requests > 1 else ""
+    pcm16 = _read_wav_pcm16(input_wav)
+    bytes_per_ms = 16000 * 2 // 1000  # mono PCM16 at 16kHz
+    chunk_bytes = max(bytes_per_ms * chunk_ms, 2)

-        # Signal ready to start
-        await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
+    incremental_pcm_parts: list[bytes] = []
+    output_sample_rate = 24000
+    delta_index = 0
+    text_chunks: list[str] = []
+    final_text: str = ""

-        # Convert audio file to base64 PCM16
-        print(f"Loading audio from: {audio_path}")
-        audio_base64 = audio_to_pcm16_base64(audio_path)
+    if delta_dump_dir is not None:
+        delta_dump_dir.mkdir(parents=True, exist_ok=True)

-        # Send audio in chunks (4KB of raw audio = ~8KB base64)
-        chunk_size = 4096
-        audio_bytes = base64.b64decode(audio_base64)
-        total_chunks = (len(audio_bytes) + chunk_size - 1) // chunk_size
+    async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
+        # 1) Validate model.
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "session.update",
+                    "model": model,
+                }
+            )
+        )

-        print(f"Sending {total_chunks} audio chunks...")
-        for i in range(0, len(audio_bytes), chunk_size):
-            chunk = audio_bytes[i : i + chunk_size]
+        # 2) Start generation once (non-final commit).
+        await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
+
+        # 3) Stream audio chunks.
+        for i in range(0, len(pcm16), chunk_bytes):
+            chunk = pcm16[i : i + chunk_bytes]
            await ws.send(
                json.dumps(
                    {
@@ -84,63 +121,212 @@ async def realtime_transcribe(audio_path: str, host: str, port: int, model: str)
                    }
                )
            )
+            if send_delay_ms > 0:
+                await asyncio.sleep(send_delay_ms / 1000.0)

-        # Signal all audio is sent
+        # 4) Final commit closes input stream.
        await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
-        print("Audio sent. Waiting for transcription...\n")

-        # Receive transcription
-        print("Transcription: ", end="", flush=True)
+        # 5) Receive server events until audio done.
        while True:
-            response = json.loads(await ws.recv())
-            if response["type"] == "transcription.delta":
-                print(response["delta"], end="", flush=True)
-            elif response["type"] == "transcription.done":
-                print(f"\n\nFinal transcription: {response['text']}")
-                if response.get("usage"):
-                    print(f"Usage: {response['usage']}")
-                break
-            elif response["type"] == "error":
-                print(f"\nError: {response['error']}")
+            message = await ws.recv()
+            if isinstance(message, bytes):
+                # We only expect JSON text frames.
+                continue
+
+            event = json.loads(message)
+            event_type = event.get("type")
+
+            if event_type == "session.created":
+                continue
+
+            if event_type == "response.audio.delta":
+                sr = event.get("sample_rate_hz")
+                if isinstance(sr, int) and sr > 0:
+                    output_sample_rate = sr
+                audio_b64 = event.get("audio", "")
+                if audio_b64:
+                    pcm_delta = base64.b64decode(audio_b64)
+                    incremental_pcm_parts.append(pcm_delta)
+                    if delta_dump_dir is not None and pcm_delta:
+                        delta_index += 1
+                        dump_path = delta_dump_dir / f"delta_{delta_index:06d}.wav"
+                        _write_wav_pcm16(dump_path, pcm_delta, output_sample_rate)
+                        print(
+                            f"{log_prefix}delta dump #{delta_index}: {dump_path} "
+                            f"(pcm bytes={len(pcm_delta)}, sr={output_sample_rate})"
+                        )
+                continue
+
+            if event_type == "transcription.delta":
+                delta = event.get("delta", "")
+                if delta:
+                    text_chunks.append(delta)
+                    print(delta, end="", flush=True)
+                continue
+
+            if event_type == "transcription.done":
+                final_text = event.get("text", "") or "".join(text_chunks)
+                usage = event.get("usage")
+                final_text_with_tag = f"Final transcription: {final_text}"
+                if text_chunks:
+                    print()
+                print(f"{log_prefix}{final_text_with_tag}")
+                if usage:
+                    print(f"{log_prefix}text usage: {usage}")
+                continue
+
+            if event_type == "response.audio.done":
                break

+            if event_type == "error":
+                raise RuntimeError(f"Server error: {event}")

-def main(args):
-    if args.audio_path:
-        audio_path = args.audio_path
+        all_pcm16 = b"".join(incremental_pcm_parts)
+        if not all_pcm16:
+            raise RuntimeError("No audio received from server.")
+
+        output_wav.parent.mkdir(parents=True, exist_ok=True)
+        _write_wav_pcm16(output_wav, all_pcm16, output_sample_rate)
+        print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")
+
+        if output_text is not None:
+            text_to_save = final_text if final_text else "".join(text_chunks)
+            output_text.parent.mkdir(parents=True, exist_ok=True)
+            output_text.write_text(text_to_save, encoding="utf-8")
+            print(f"{log_prefix}Saved realtime text to: {output_text}")
+
+
+def _indexed_output_path(path: Path | None, index: int, total: int) -> Path | None:
+    if path is None or total <= 1:
+        return path
+    return path.with_name(f"{path.stem}_{index:02d}{path.suffix}")
+
+
+async def run_clients_concurrent(
+    *,
+    url: str,
+    model: str,
+    input_wav: Path,
+    output_wav: Path,
+    output_text: Path | None,
+    chunk_ms: int,
+    send_delay_ms: int,
+    delta_dump_dir: Path | None,
+    num_requests: int,
+    concurrency: int,
+) -> None:
+    sem = asyncio.Semaphore(concurrency)
+
+    async def _run_one(index: int) -> tuple[int, bool, str | None]:
+        per_output_wav = _indexed_output_path(output_wav, index, num_requests)
+        per_output_text = _indexed_output_path(output_text, index, num_requests)
+        per_delta_dir = None
+        if delta_dump_dir is not None:
+            per_delta_dir = delta_dump_dir / f"req_{index:02d}"
+        async with sem:
+            try:
+                await run_client(
+                    url=url,
+                    model=model,
+                    input_wav=input_wav,
+                    output_wav=per_output_wav,
+                    output_text=per_output_text,
+                    chunk_ms=chunk_ms,
+                    send_delay_ms=send_delay_ms,
+                    delta_dump_dir=per_delta_dir,
+                    request_idx=index,
+                    total_requests=num_requests,
+                )
+                return index, True, None
+            except Exception as exc:
+                return index, False, str(exc)
+
+    tasks = [asyncio.create_task(_run_one(i), name=f"rt-client-{i}") for i in range(1, num_requests + 1)]
+    results = await asyncio.gather(*tasks)
+
+    failed = [(idx, err) for idx, ok, err in results if not ok]
+    succeeded = num_requests - len(failed)
+    print(f"[summary] succeeded={succeeded}, failed={len(failed)}, total={num_requests}")
+    if failed:
+        for idx, err in failed:
+            print(f"[summary] req {idx:02d} failed: {err}")
+        raise RuntimeError(f"{len(failed)} concurrent request(s) failed")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Realtime audio/text client for vLLM-Omni")
+    parser.add_argument("--url", default="ws://localhost:8091/v1/realtime", help="WebSocket URL")
+    parser.add_argument(
+        "--model",
+        default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
+        help="Model name for session.update",
+    )
+    parser.add_argument("--input-wav", required=True, type=Path, help="Input WAV (mono, PCM16, 16kHz)")
+    parser.add_argument("--output-wav", default=Path("realtime_output.wav"), type=Path, help="Output WAV path")
+    parser.add_argument(
+        "--output-text",
+        default=None,
+        type=Path,
+        help="Optional output text path for final transcription",
+    )
+    parser.add_argument("--chunk-ms", type=int, default=200, help="Input chunk size in milliseconds")
+    parser.add_argument(
+        "--send-delay-ms",
+        type=int,
+        default=0,
+        help="Delay between chunk sends; set >0 to simulate realtime upload",
+    )
+    parser.add_argument(
+        "--delta-dump-dir",
+        type=Path,
+        default=None,
+        help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory",
+    )
+    parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
+    parser.add_argument(
+        "--concurrency",
+        type=int,
+        default=1,
+        help="Maximum number of concurrent websocket requests",
+    )
+    args = parser.parse_args()
+
+    if args.num_requests <= 0:
+        raise ValueError("--num-requests must be >= 1")
+    if args.concurrency <= 0:
+        raise ValueError("--concurrency must be >= 1")
+    concurrency = min(args.concurrency, args.num_requests)
+
+    if args.num_requests == 1:
+        asyncio.run(
+            run_client(
+                url=args.url,
+                model=args.model,
+                input_wav=args.input_wav,
+                output_wav=args.output_wav,
+                output_text=args.output_text,
+                chunk_ms=args.chunk_ms,
+                send_delay_ms=args.send_delay_ms,
+                delta_dump_dir=args.delta_dump_dir,
+            )
+        )
    else:
-        # Use default audio asset
-        audio_path = str(AudioAsset("mary_had_lamb").get_local_path())
-        print(f"No audio path provided, using default: {audio_path}")
-
-    asyncio.run(realtime_transcribe(audio_path, args.host, args.port, args.model))
+        asyncio.run(
+            run_clients_concurrent(
+                url=args.url,
+                model=args.model,
+                input_wav=args.input_wav,
+                output_wav=args.output_wav,
+                output_text=args.output_text,
+                chunk_ms=args.chunk_ms,
+                send_delay_ms=args.send_delay_ms,
+                delta_dump_dir=args.delta_dump_dir,
+                num_requests=args.num_requests,
+                concurrency=concurrency,
+            )
+        )


 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Realtime WebSocket Transcription Client")
-    parser.add_argument(
-        "--model",
-        type=str,
-        default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        help="Model that is served and should be pinged.",
-    )
-    parser.add_argument(
-        "--audio_path",
-        type=str,
-        default=None,
-        help="Path to the audio file to transcribe.",
-    )
-    parser.add_argument(
-        "--host",
-        type=str,
-        default="localhost",
-        help="vLLM-Omni server host (default: localhost)",
-    )
-    parser.add_argument(
-        "--port",
-        type=int,
-        default=8000,
-        help="vLLM-Omni server port (default: 8000)",
-    )
-    args = parser.parse_args()
-    main(args)
+    main()
@@ -43,7 +43,7 @@ Then open http://localhost:7860 in your browser.

 ### Launch the Server

-The default stage config is located at `vllm_omni/model_executor/stage_configs/qwen3_tts.yaml`. For other platforms (e.g., NPU), refer to `vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml`.
+The default deploy config is located at `vllm_omni/deploy/qwen3_tts.yaml` and is loaded automatically by the model registry — no `--deploy-config` flag needed for default use. Platform-specific deltas (NPU, ROCm, XPU) are merged in automatically from the `platforms:` block of the same YAML based on the detected runtime.

 ```bash
 # CustomVoice model (predefined speakers)
@@ -70,6 +70,22 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
    --port 8091
 ```

+#### Sync vs async-chunk mode
+
+Qwen3-TTS supports both **chunked streaming** (default, lower latency) and
+**synchronous end-to-end** modes from the same deploy YAML. The bundled
+`qwen3_tts.yaml` ships with `async_chunk: true`; flip with `--no-async-chunk`
+and the pipeline automatically dispatches to the end-to-end codec processor:
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091 \
+    --no-async-chunk
+```
+
+No variant YAML or extra flag is needed — the `StagePipelineConfig` on each
+stage declares both processor functions and the runtime picks based on the
+`async_chunk:` bool.
+
 Alternatively, use the convenience script:
 ```bash
 ./run_server.sh                  # Default: CustomVoice model
@@ -5,11 +5,13 @@ A particularly useful scenario is voice cloning: set ref_audio once at the
 batch level and generate many utterances in the cloned voice without repeating
 the reference for each item.

-Start the server (with batch-optimized config for best throughput):
+Start the server (with batch-optimized stage settings for best throughput):

    vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-        --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml \
-        --trust-remote-code
+        --omni \
+        --trust-remote-code \
+        --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},
+                            "1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'

 Examples:
    # Batch with a predefined voice
@@ -127,7 +127,7 @@ echo "Starting vLLM server..."
 LOG_FILE="/tmp/vllm_tts_server_${SERVER_PORT}.log"

 vllm-omni serve "$MODEL" \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
+    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
    --host "$SERVER_HOST" \
    --port "$SERVER_PORT" \
    --gpu-memory-utilization 0.9 \
@@ -31,7 +31,7 @@ esac
 echo "Starting Qwen3-TTS server with model: $MODEL"

 vllm-omni serve "$MODEL" \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
+    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
    --host 0.0.0.0 \
    --port 8091 \
    --gpu-memory-utilization 0.9 \
@@ -5,7 +5,7 @@ speaker encoder from a Qwen3-TTS checkpoint, then interpolates between them
 using SLERP and sends the result to the /v1/audio/speech API.

 Requirements:
-    pip install torch resampy soundfile numpy httpx
+    pip install torch soundfile numpy httpx

 Examples:
    # Extract and save an embedding
@@ -143,11 +143,12 @@ def _load_speaker_encoder_weights(encoder: torch.nn.Module, model_path: str) ->

 def compute_mel_spectrogram(audio: np.ndarray, sr: int = 24000) -> torch.Tensor:
    """Compute 128-bin mel spectrogram matching Qwen3-TTS's extraction pipeline."""
-    from vllm.multimodal.audio import resample_audio_resampy
+    from vllm.multimodal.audio import AudioResampler

    # Resample to 24kHz if needed
    if sr != 24000:
-        audio = resample_audio_resampy(audio.astype(np.float32), orig_sr=sr, target_sr=24000)
+        resampler = AudioResampler(target_sr=24000)
+        audio = resampler.resample(audio.astype(np.float32), orig_sr=sr)

    y = torch.from_numpy(audio).unsqueeze(0).float()

@@ -0,0 +1,166 @@
+# VoxCPM
+
+## Prerequisites
+
+Install VoxCPM in one of these ways:
+
+```bash
+pip install voxcpm
+```
+
+or point vLLM-Omni to a local VoxCPM source tree:
+
+```bash
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` lacks HF metadata such as `model_type`,
+prepare a persistent HF-compatible config directory and export:
+
+```bash
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+The VoxCPM stage configs read `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` directly. The `python3 -c` form above avoids heredoc/indentation issues in interactive shells.
+
+## Launch the Server
+
+Use the async-chunk stage config by default:
+
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+cd examples/online_serving/voxcpm
+./run_server.sh
+```
+
+Use the non-streaming stage config:
+
+```bash
+./run_server.sh sync
+```
+
+You can also launch the server directly:
+
+```bash
+vllm serve "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni \
+    --port 8091
+```
+
+## Send Requests
+
+### Basic text-to-speech
+
+```bash
+python openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a VoxCPM online text-to-speech example."
+```
+
+### Voice cloning
+
+```bash
+python openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "The exact transcript spoken in reference.wav."
+```
+
+`ref_text` must be the real transcript of the reference audio. Placeholder text or mismatched text will usually degrade quality badly.
+
+### Streaming PCM output
+
+```bash
+python openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a streaming VoxCPM request." \
+    --stream \
+    --output voxcpm_stream.pcm
+```
+
+### Using curl
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "OpenBMB/VoxCPM1.5",
+        "input": "Hello from VoxCPM online serving.",
+        "response_format": "wav"
+    }' --output output.wav
+```
+
+Voice cloning:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "OpenBMB/VoxCPM1.5",
+        "input": "This sentence uses a cloned voice.",
+        "ref_audio": "https://example.com/reference.wav",
+        "ref_text": "The exact transcript spoken in the reference audio.",
+        "response_format": "wav"
+    }' --output cloned.wav
+```
+
+Streaming PCM:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "OpenBMB/VoxCPM1.5",
+        "input": "This is a streaming VoxCPM request.",
+        "stream": true,
+        "response_format": "pcm"
+    }' --output output.pcm
+```
+
+## Supported Request Shape
+
+VoxCPM online serving currently supports:
+
+- plain text-to-speech
+- voice cloning with `ref_audio` + `ref_text`
+- `stream=true` with `response_format=pcm` or `wav`
+
+VoxCPM online serving does not use these generic TTS fields:
+
+- `voice`
+- `instructions`
+- `language`
+- `speaker_embedding`
+- `x_vector_only_mode`
+
+## Streaming vs Non-Streaming
+
+- `voxcpm_async_chunk.yaml` enables async-chunk streaming and is best for single-request streaming latency.
+- `voxcpm.yaml` performs one-shot latent generation then VAE decode.
+
+Like native VoxCPM, the async streaming path should be treated as single-request. If you need stable throughput benchmarking, prefer `voxcpm.yaml`.
+
+Do not use `voxcpm_async_chunk.yaml` for concurrent online streaming or `/v1/audio/speech/batch`. For multiple requests, prefer `voxcpm.yaml`.
+
+## Benchmark
+
+The serving benchmark reports TTFP and RTF:
+
+```bash
+python benchmarks/voxcpm/vllm_omni/bench_tts_serve.py \
+    --host 127.0.0.1 \
+    --port 8091 \
+    --num-prompts 10 \
+    --max-concurrency 1 \
+    --result-dir /tmp/voxcpm_bench
+```
+
+For the async-chunk server, keep `--max-concurrency 1`.
@@ -0,0 +1,155 @@
+"""OpenAI-compatible client for VoxCPM via /v1/audio/speech.
+
+Examples:
+    # Basic text-to-speech
+    python openai_speech_client.py --text "Hello from VoxCPM"
+
+    # Voice cloning
+    python openai_speech_client.py \
+        --text "This sentence uses the cloned voice." \
+        --ref-audio /path/to/reference.wav \
+        --ref-text "The exact transcript spoken in the reference audio."
+
+    # Streaming PCM output
+    python openai_speech_client.py \
+        --text "This is a streaming VoxCPM request." \
+        --stream \
+        --output output.pcm
+"""
+
+import argparse
+import base64
+import os
+
+import httpx
+
+DEFAULT_API_BASE = "http://localhost:8091"
+DEFAULT_API_KEY = "EMPTY"
+DEFAULT_MODEL = "OpenBMB/VoxCPM1.5"
+
+
+def encode_audio_to_base64(audio_path: str) -> str:
+    """Encode a local audio file to base64 data URL."""
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+    ext = audio_path.lower().rsplit(".", 1)[-1]
+    mime_map = {
+        "wav": "audio/wav",
+        "mp3": "audio/mpeg",
+        "flac": "audio/flac",
+        "ogg": "audio/ogg",
+    }
+    mime_type = mime_map.get(ext, "audio/wav")
+
+    with open(audio_path, "rb") as f:
+        audio_b64 = base64.b64encode(f.read()).decode("utf-8")
+    return f"data:{mime_type};base64,{audio_b64}"
+
+
+def build_payload(args) -> dict[str, object]:
+    payload: dict[str, object] = {
+        "model": args.model,
+        "input": args.text,
+        "response_format": "pcm" if args.stream else args.response_format,
+    }
+
+    if args.ref_audio:
+        if args.ref_audio.startswith(("http://", "https://", "data:")):
+            payload["ref_audio"] = args.ref_audio
+        else:
+            payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
+    if args.ref_text:
+        payload["ref_text"] = args.ref_text
+    if args.max_new_tokens is not None:
+        payload["max_new_tokens"] = args.max_new_tokens
+    if args.stream:
+        payload["stream"] = True
+
+    return payload
+
+
+def run_tts(args) -> None:
+    payload = build_payload(args)
+    api_url = f"{args.api_base}/v1/audio/speech"
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {args.api_key}",
+    }
+
+    print(f"Model: {args.model}")
+    print(f"Text: {args.text}")
+    if args.ref_audio:
+        print("Mode: voice cloning")
+        print(f"Reference audio: {args.ref_audio}")
+    else:
+        print("Mode: text-to-speech")
+
+    if args.stream:
+        output_path = args.output or "voxcpm_output.pcm"
+        with httpx.Client(timeout=300.0) as client:
+            with client.stream("POST", api_url, json=payload, headers=headers) as response:
+                if response.status_code != 200:
+                    print(f"Error: {response.status_code}")
+                    print(response.read().decode("utf-8", errors="ignore"))
+                    return
+
+                total_bytes = 0
+                with open(output_path, "wb") as f:
+                    for chunk in response.iter_bytes():
+                        if not chunk:
+                            continue
+                        f.write(chunk)
+                        total_bytes += len(chunk)
+        print(f"Streamed {total_bytes} bytes to: {output_path}")
+        return
+
+    with httpx.Client(timeout=300.0) as client:
+        response = client.post(api_url, json=payload, headers=headers)
+
+    if response.status_code != 200:
+        print(f"Error: {response.status_code}")
+        print(response.text)
+        return
+
+    try:
+        text = response.content.decode("utf-8")
+        if text.startswith('{"error"'):
+            print(f"Error: {text}")
+            return
+    except UnicodeDecodeError:
+        pass
+
+    output_path = args.output or "voxcpm_output.wav"
+    with open(output_path, "wb") as f:
+        f.write(response.content)
+    print(f"Audio saved to: {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="VoxCPM OpenAI-compatible speech client")
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
+    parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
+    parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name or path")
+    parser.add_argument("--text", required=True, help="Text to synthesize")
+    parser.add_argument("--ref-audio", default=None, help="Reference audio path, URL, or data URL")
+    parser.add_argument(
+        "--ref-text",
+        default=None,
+        help="The exact transcript spoken in the reference audio",
+    )
+    parser.add_argument("--stream", action="store_true", help="Enable streaming PCM output")
+    parser.add_argument(
+        "--response-format",
+        default="wav",
+        choices=["wav", "pcm", "flac", "mp3", "aac", "opus"],
+        help="Audio format for non-streaming mode (default: wav)",
+    )
+    parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate")
+    parser.add_argument("--output", "-o", default=None, help="Output file path")
+    args = parser.parse_args()
+    run_tts(args)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Launch vLLM-Omni server for VoxCPM online speech serving.
+#
+# Usage:
+#   ./run_server.sh                 # default: async_chunk stage config
+#   ./run_server.sh async           # async_chunk stage config
+#   ./run_server.sh sync            # no-async-chunk stage config
+#   VOXCPM_MODEL=/path/to/model ./run_server.sh
+
+set -e
+
+MODE="${1:-async}"
+MODEL="${VOXCPM_MODEL:-OpenBMB/VoxCPM1.5}"
+
+case "$MODE" in
+    async)
+        STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml"
+        ;;
+    sync)
+        STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm.yaml"
+        ;;
+    *)
+        echo "Unknown mode: $MODE"
+        echo "Supported: async, sync"
+        exit 1
+        ;;
+esac
+
+echo "Starting VoxCPM server with model: $MODEL"
+echo "Stage config: $STAGE_CONFIG"
+
+vllm serve "$MODEL" \
+    --stage-configs-path "$STAGE_CONFIG" \
+    --host 0.0.0.0 \
+    --port 8091 \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni
@@ -5,12 +5,13 @@ Serve VoxCPM2 TTS via the OpenAI-compatible `/v1/audio/speech` endpoint.
 ## Start the Server

 ```bash
-python -m vllm_omni.entrypoints.openai.api_server \
-    --model openbmb/VoxCPM2 \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \
-    --host 0.0.0.0 --port 8000
+vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
 ```

+The deploy config is auto-loaded from `vllm_omni/deploy/voxcpm2.yaml`. Pass
+`--deploy-config <path>` to override, or `--stage-N-<field> <value>` (e.g.
+`--stage-0-max-num-seqs 8`) for per-stage runtime overrides.
+
 ## Zero-shot Synthesis

 ```bash
@@ -0,0 +1,599 @@
+"""Gradio demo for VoxCPM2 TTS with gapless streaming audio playback.
+
+Uses a custom AudioWorklet-based player for gap-free streaming
+(adapted from the Qwen3-TTS demo). Audio is streamed from the vLLM
+server through a same-origin proxy and played via the Web Audio API's
+AudioWorklet, which maintains a FIFO buffer queue and plays samples at
+the audio clock rate.
+
+Usage:
+    # Start the vLLM server first:
+    vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
+
+    # Then launch the demo:
+    python gradio_demo.py --api-base http://localhost:8000
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import io
+import json
+import logging
+
+import gradio as gr
+import httpx
+import numpy as np
+import soundfile as sf
+from fastapi import FastAPI, Request
+from fastapi.responses import Response, StreamingResponse
+
+logger = logging.getLogger(__name__)
+
+SAMPLE_RATE = 48000
+
+# ── AudioWorklet processor (loaded in browser via Blob URL) ──────────
+WORKLET_JS = r"""
+class TTSPlaybackProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+        this.queue = [];
+        this.buf = null;
+        this.pos = 0;
+        this.playing = false;
+        this.played = 0;
+        this.port.onmessage = (e) => {
+            if (e.data && e.data.type === 'clear') {
+                this.queue = []; this.buf = null; this.pos = 0; this.played = 0;
+                if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped'}); }
+                return;
+            }
+            this.queue.push(e.data);
+        };
+    }
+    process(inputs, outputs) {
+        const out = outputs[0][0];
+        for (let i = 0; i < out.length; i++) {
+            if (!this.buf || this.pos >= this.buf.length) {
+                if (this.queue.length > 0) {
+                    this.buf = this.queue.shift(); this.pos = 0;
+                } else {
+                    for (let j = i; j < out.length; j++) out[j] = 0;
+                    if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped', played:this.played}); }
+                    return true;
+                }
+            }
+            out[i] = this.buf[this.pos++] / 32768;
+            this.played++;
+        }
+        if (!this.playing) { this.playing = true; this.port.postMessage({type:'started'}); }
+        return true;
+    }
+}
+registerProcessor('tts-playback-processor', TTSPlaybackProcessor);
+"""
+
+PLAYER_HTML = """
+<div id="tts-player">
+  <div style="display:flex; align-items:center; gap:10px;">
+    <div id="tts-status-dot" style="width:10px;height:10px;border-radius:50%;background:#ccc;flex-shrink:0;"></div>
+    <span id="tts-status" style="font-weight:600;font-size:1.05em;">Ready</span>
+    <button id="tts-stop-btn" onclick="window.ttsStop()"
+      style="display:none; margin-left:auto; padding:5px 16px; border-radius:6px; border:1px solid #EF5552;
+             background:#fff; color:#EF5552; cursor:pointer; font-size:0.85em;">Stop</button>
+  </div>
+  <div id="tts-metrics" style="display:none; grid-template-columns:repeat(4,1fr); gap:10px; margin-top:12px;">
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">TTFP</div>
+      <div id="tts-m-ttfp" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">RTF</div>
+      <div id="tts-m-rtf" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">Audio</div>
+      <div id="tts-m-dur" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">Speed</div>
+      <div id="tts-m-speed" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+  </div>
+  <div id="tts-rtf-bar-wrap" style="display:none; background:#e8ecf1; border-radius:4px; height:20px; overflow:hidden; position:relative; margin-top:10px;">
+    <div id="tts-rtf-bar" style="height:100%; border-radius:4px; transition:width 0.3s ease, background 0.3s ease; width:0%;"></div>
+    <span id="tts-rtf-label" style="position:absolute;top:50%;left:50%;transform:translate(-50%,-50%);font-size:0.75em;font-weight:600;color:#444;"></span>
+  </div>
+  <div id="tts-elapsed" style="display:none; margin-top:6px; font-size:0.8em; color:#999; text-align:right;"></div>
+</div>
+"""
+
+
+def _build_player_js() -> str:
+    return f"""
+    <script>
+    const SR = {SAMPLE_RATE};
+    const WC = {json.dumps(WORKLET_JS)};
+    let ctx = null, node = null, abort = null, gen = false, st = {{}};
+
+    async function init() {{
+        if (ctx) return;
+        ctx = new AudioContext({{ sampleRate: SR }});
+        const b = new Blob([WC], {{ type: 'application/javascript' }});
+        const u = URL.createObjectURL(b);
+        await ctx.audioWorklet.addModule(u);
+        URL.revokeObjectURL(u);
+        node = new AudioWorkletNode(ctx, 'tts-playback-processor');
+        node.connect(ctx.destination);
+        node.port.onmessage = (e) => {{
+            if (e.data.type === 'started') setStatus('Playing...', '#64dd17');
+            else if (e.data.type === 'stopped' && !gen) {{
+                setStatus('Done', '#64dd17'); showStats(true);
+                const btn = document.getElementById('tts-stop-btn');
+                if (btn) btn.style.display = 'none';
+            }}
+        }};
+    }}
+
+    function setStatus(text, color) {{
+        const s = document.getElementById('tts-status');
+        const d = document.getElementById('tts-status-dot');
+        if (s) s.textContent = text;
+        if (d) d.style.background = color || '#ccc';
+    }}
+
+    function showStats(fin) {{
+        if (!st.t0) return;
+        const elapsed = (fin && st.streamEnd ? (st.streamEnd - st.t0) : (performance.now() - st.t0)) / 1000;
+        const dur = st.samples / SR;
+        const mTtfp = document.getElementById('tts-m-ttfp');
+        const mRtf = document.getElementById('tts-m-rtf');
+        const mDur = document.getElementById('tts-m-dur');
+        const mSpeed = document.getElementById('tts-m-speed');
+        const bar = document.getElementById('tts-rtf-bar');
+        const barLabel = document.getElementById('tts-rtf-label');
+        const elapsedEl = document.getElementById('tts-elapsed');
+        if (mTtfp && st.ttfp != null) mTtfp.textContent = st.ttfp.toFixed(0) + 'ms';
+        if (mDur) mDur.textContent = dur.toFixed(1) + 's';
+        if (dur > 0 && elapsed > 0) {{
+            const rtf = elapsed / dur;
+            const speed = 1 / rtf;
+            if (mRtf) {{
+                mRtf.textContent = rtf.toFixed(2) + 'x';
+                mRtf.style.color = rtf < 1 ? '#64dd17' : rtf < 1.5 ? '#e8a317' : '#EF5552';
+            }}
+            if (mSpeed) {{
+                mSpeed.textContent = speed.toFixed(1) + 'x';
+                mSpeed.style.color = speed > 1 ? '#64dd17' : speed > 0.7 ? '#e8a317' : '#EF5552';
+            }}
+            if (bar) {{
+                const pct = Math.min(speed / 10 * 100, 100);
+                bar.style.width = pct + '%';
+                bar.style.background = speed > 1 ? 'linear-gradient(90deg,#4A90D9,#64dd17)' : 'linear-gradient(90deg,#EF5552,#f87171)';
+            }}
+            if (barLabel) barLabel.textContent = speed.toFixed(1) + 'x realtime';
+        }}
+        if (elapsedEl) {{
+            elapsedEl.style.display = 'block';
+            elapsedEl.textContent = fin ? 'Completed in ' + elapsed.toFixed(1) + 's  (' + st.chunks + ' chunks)' : elapsed.toFixed(1) + 's elapsed  (' + st.chunks + ' chunks)';
+        }}
+    }}
+
+    window.ttsStop = function() {{
+        if (abort) abort.abort();
+        if (node) node.port.postMessage({{ type: 'clear' }});
+        gen = false;
+        setStatus('Stopped', '#999');
+        const btn = document.getElementById('tts-stop-btn');
+        if (btn) btn.style.display = 'none';
+    }};
+
+    window.ttsGenerate = async function(payload) {{
+        try {{ await init(); if (ctx.state === 'suspended') await ctx.resume(); }}
+        catch (e) {{ setStatus('Audio init error: ' + e.message, '#EF5552'); return; }}
+        if (abort) abort.abort();
+        node.port.postMessage({{ type: 'clear' }});
+        await new Promise(r => setTimeout(r, 50));
+        node.port.postMessage({{ type: 'clear' }});
+
+        gen = true;
+        st = {{ t0: null, chunks: 0, samples: 0, ttfp: null }};
+        setStatus('Connecting...', '#4A90D9');
+        const bEl = document.getElementById('tts-stop-btn');
+        if (bEl) bEl.style.display = 'inline-block';
+        const mp = document.getElementById('tts-metrics');
+        if (mp) {{ mp.style.display = 'grid'; ['tts-m-ttfp','tts-m-rtf','tts-m-dur','tts-m-speed'].forEach(id => {{ const e = document.getElementById(id); if(e) {{ e.textContent = '\\u2014'; e.style.color = '#333'; }} }}); }}
+        const bw = document.getElementById('tts-rtf-bar-wrap');
+        if (bw) bw.style.display = 'block';
+        const bar = document.getElementById('tts-rtf-bar');
+        if (bar) bar.style.width = '0%';
+        const bl = document.getElementById('tts-rtf-label');
+        if (bl) bl.textContent = '';
+        const ee = document.getElementById('tts-elapsed');
+        if (ee) {{ ee.style.display = 'none'; ee.textContent = ''; }}
+        abort = new AbortController();
+
+        try {{
+            st.t0 = performance.now();
+            const r = await fetch('/proxy/v1/audio/speech', {{
+                method: 'POST',
+                headers: {{ 'Content-Type': 'application/json' }},
+                body: JSON.stringify(payload),
+                signal: abort.signal,
+            }});
+            if (!r.ok) {{ const t = await r.text(); throw new Error('Server ' + r.status + ': ' + t.slice(0, 200)); }}
+            setStatus('Streaming...', '#4A90D9');
+            const reader = r.body.getReader();
+            let left = new Uint8Array(0);
+            while (true) {{
+                const {{ done, value }} = await reader.read();
+                if (done) break;
+                let raw;
+                if (left.length > 0) {{
+                    raw = new Uint8Array(left.length + value.length);
+                    raw.set(left); raw.set(value, left.length);
+                }} else {{ raw = value; }}
+                const usable = raw.length - (raw.length % 2);
+                left = usable < raw.length ? raw.slice(usable) : new Uint8Array(0);
+                if (usable > 0) {{
+                    const ab = new ArrayBuffer(usable);
+                    new Uint8Array(ab).set(raw.subarray(0, usable));
+                    const pcm = new Int16Array(ab);
+                    node.port.postMessage(pcm);
+                    st.chunks++;
+                    st.samples += pcm.length;
+                    if (st.ttfp == null) st.ttfp = performance.now() - st.t0;
+                    showStats(false);
+                }}
+            }}
+        }} catch (e) {{
+            if (e.name !== 'AbortError') {{
+                setStatus('Error: ' + e.message, '#EF5552');
+                console.error('TTS error:', e);
+            }}
+        }} finally {{
+            st.streamEnd = performance.now();
+            showStats(true);
+            gen = false;
+            if (st.samples > 0) setStatus('Finishing playback...', '#64dd17');
+            else {{
+                setStatus('No audio received', '#999');
+                if (bEl) bEl.style.display = 'none';
+            }}
+        }}
+    }};
+    </script>
+"""
+
+
+def _encode_audio(audio_data: tuple) -> str:
+    sr, audio_np = audio_data
+    if audio_np.dtype in (np.float32, np.float64):
+        audio_np = np.clip(audio_np, -1.0, 1.0)
+        audio_np = (audio_np * 32767).astype(np.int16)
+    elif audio_np.dtype != np.int16:
+        audio_np = audio_np.astype(np.int16)
+    buf = io.BytesIO()
+    sf.write(buf, audio_np, sr, format="WAV")
+    return f"data:audio/wav;base64,{base64.b64encode(buf.getvalue()).decode()}"
+
+
+def create_app(api_base: str):
+    app = FastAPI()
+    _pending: dict[str, dict] = {}
+
+    @app.post("/proxy/v1/audio/speech")
+    async def proxy_speech(request: Request):
+        body = await request.json()
+        req_id = body.get("_req_id")
+        if req_id and req_id in _pending:
+            body = _pending.pop(req_id)
+        logger.info("Proxy: %s", {k: (f"<{len(str(v))} chars>" if k == "ref_audio" else v) for k, v in body.items()})
+        try:
+            client = httpx.AsyncClient(timeout=300)
+            resp = await client.send(
+                client.build_request(
+                    "POST",
+                    f"{api_base}/v1/audio/speech",
+                    json=body,
+                    headers={"Authorization": "Bearer EMPTY", "Content-Type": "application/json"},
+                ),
+                stream=True,
+            )
+        except Exception as exc:
+            logger.exception("Proxy connection error")
+            await client.aclose()
+            return Response(content=str(exc), status_code=502)
+        if resp.status_code != 200:
+            content = await resp.aread()
+            await resp.aclose()
+            await client.aclose()
+            return Response(content=content, status_code=resp.status_code)
+
+        async def relay():
+            try:
+                async for chunk in resp.aiter_bytes():
+                    yield chunk
+            finally:
+                await resp.aclose()
+                await client.aclose()
+
+        return StreamingResponse(relay(), media_type="application/octet-stream")
+
+    css = """
+    #generate-btn button { width: 100%; }
+    #streaming-player { border: 1px solid var(--border-color-primary) !important; border-radius: var(--block-radius) !important; padding: var(--block-padding) !important; }
+    """
+    theme = gr.themes.Default(
+        primary_hue=gr.themes.Color(
+            c50="#f0f5ff",
+            c100="#dce6f9",
+            c200="#b8cef3",
+            c300="#8eb2eb",
+            c400="#6496e0",
+            c500="#4A90D9",
+            c600="#3a7bc8",
+            c700="#2d66b0",
+            c800="#1f4f8f",
+            c900="#163a6e",
+            c950="#0e2650",
+        ),
+    )
+
+    with gr.Blocks(title="VoxCPM2 TTS Demo") as demo:
+        gr.HTML(f"""
+        <div style="display:flex; align-items:center; gap:16px; margin-bottom:8px;">
+          <img src="https://raw.githubusercontent.com/vllm-project/vllm-omni/main/docs/source/logos/vllm-omni-logo.png"
+               alt="vLLM-Omni" style="height:42px;">
+          <div>
+            <h1 style="margin:0; font-size:1.5em;">VoxCPM2 Streaming Demo</h1>
+            <span style="font-size:0.85em; color:#666;">
+              Served by <a href="https://github.com/vllm-project/vllm-omni" target="_blank"
+              style="color:#4A90D9; text-decoration:none; font-weight:600;">vLLM-Omni</a>
+              &middot; <code style="background:#eef2f7; padding:2px 6px; border-radius:4px; font-size:0.9em;">{api_base}</code>
+              &middot; 48 kHz
+            </span>
+          </div>
+        </div>
+        """)
+
+        gr.Markdown(
+            "**Three modes:** "
+            "**Voice Design** (control instruction only) &middot; "
+            "**Controllable Cloning** (ref audio + optional style control) &middot; "
+            "**Ultimate Cloning** (ref audio + transcript for audio continuation)"
+        )
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                text_input = gr.Textbox(
+                    label="Target Text",
+                    placeholder="Enter text to synthesize...",
+                    lines=4,
+                )
+                control_instruction = gr.Textbox(
+                    label="Control Instruction (optional)",
+                    placeholder="e.g. A warm young woman / Excited and fast-paced",
+                    lines=2,
+                    info="Describe voice style, emotion, pace. Works for both Voice Design and Controllable Cloning.",
+                )
+
+                with gr.Accordion("Voice Cloning", open=False):
+                    ref_audio = gr.Audio(
+                        label="Reference Audio (upload for cloning)",
+                        type="numpy",
+                        sources=["upload", "microphone"],
+                    )
+                    ref_audio_url = gr.Textbox(
+                        label="or Reference Audio URL",
+                        placeholder="https://example.com/reference.wav",
+                    )
+                    ultimate_clone = gr.Checkbox(
+                        label="Ultimate Cloning Mode",
+                        value=False,
+                        info="Provide transcript of ref audio for audio continuation (disables control instruction)",
+                    )
+                    prompt_text = gr.Textbox(
+                        label="Reference Audio Transcript",
+                        placeholder="Transcript of your reference audio (for ultimate cloning)",
+                        lines=2,
+                        visible=False,
+                    )
+
+                with gr.Row():
+                    stream_checkbox = gr.Checkbox(
+                        label="Stream (gapless)",
+                        value=True,
+                        info="AudioWorklet streaming",
+                    )
+                with gr.Row():
+                    generate_btn = gr.Button(
+                        "Generate Speech",
+                        variant="primary",
+                        size="lg",
+                        elem_id="generate-btn",
+                        scale=3,
+                    )
+                    reset_btn = gr.Button("Reset", variant="secondary", size="lg", scale=1)
+
+            with gr.Column(scale=2):
+                player_html = gr.HTML(
+                    value=PLAYER_HTML,
+                    visible=True,
+                    label="streaming player",
+                    elem_id="streaming-player",
+                )
+                audio_output = gr.Audio(
+                    label="generated audio",
+                    interactive=False,
+                    autoplay=True,
+                    visible=False,
+                )
+                gr.Examples(
+                    examples=[
+                        ["Hello, this is a VoxCPM2 demo running on vLLM-Omni.", ""],
+                        [
+                            "I have a dream that my four little children will one day live in a nation "
+                            "where they will not be judged by the color of their skin but by the content "
+                            "of their character.",
+                            "",
+                        ],
+                        [
+                            "I never asked you to stay. It's not like I care or anything. "
+                            "But why does it still hurt so much now that you're gone?",
+                            "A young girl with a soft, sweet voice. Speaks slowly with a melancholic tone.",
+                        ],
+                    ],
+                    inputs=[text_input, control_instruction],
+                    label="examples",
+                )
+                gr.HTML("""
+                <div style="text-align:center; padding:8px 0; margin-top:4px;">
+                  <a href="https://github.com/vllm-project/vllm-omni" target="_blank">
+                    <img src="https://raw.githubusercontent.com/vllm-project/vllm-omni/main/docs/source/logos/vllm-omni-logo.png"
+                         alt="vLLM-Omni" style="height:28px; opacity:0.7;">
+                  </a>
+                </div>
+                """)
+
+        hidden_payload = gr.Textbox(visible=False, elem_id="tts-payload")
+
+        def on_ultimate_toggle(checked):
+            return (
+                gr.update(visible=checked),  # prompt_text
+                gr.update(interactive=not checked),  # control_instruction
+            )
+
+        ultimate_clone.change(
+            fn=on_ultimate_toggle,
+            inputs=[ultimate_clone],
+            outputs=[prompt_text, control_instruction],
+        )
+
+        def on_stream_change(stream: bool):
+            if stream:
+                return gr.update(visible=True), gr.update(visible=False)
+            return gr.update(visible=False), gr.update(visible=True)
+
+        stream_checkbox.change(
+            fn=on_stream_change,
+            inputs=[stream_checkbox],
+            outputs=[player_html, audio_output],
+        )
+
+        def on_reset():
+            return "", "", None, "", False, "", PLAYER_HTML
+
+        reset_btn.click(
+            fn=on_reset,
+            outputs=[
+                text_input,
+                control_instruction,
+                audio_output,
+                hidden_payload,
+                ultimate_clone,
+                prompt_text,
+                player_html,
+            ],
+            js="() => { if (window.ttsStop) window.ttsStop(); }",
+        )
+
+        def on_generate(stream_enabled, text, ctrl_instr, ref_a, ref_url, ult_clone, p_text):
+            import time as _time
+
+            if not text or not text.strip():
+                raise gr.Error("Please enter text to synthesize.")
+
+            # VoxCPM2 uses "(instruction)text" format for control
+            ctrl = ctrl_instr.strip() if ctrl_instr and not ult_clone else ""
+            final_text = f"({ctrl}){text.strip()}" if ctrl else text.strip()
+
+            payload: dict = {
+                "input": final_text,
+                "voice": "default",
+                "response_format": "pcm" if stream_enabled else "wav",
+                "stream": stream_enabled,
+            }
+
+            # Reference audio for cloning
+            ref_url_s = ref_url.strip() if ref_url else ""
+            if ref_url_s:
+                payload["ref_audio"] = ref_url_s
+            elif ref_a is not None:
+                payload["ref_audio"] = _encode_audio(ref_a)
+
+            # Ultimate cloning: prompt_audio + prompt_text for continuation
+            if ult_clone and p_text and p_text.strip():
+                if ref_url_s:
+                    payload["prompt_audio"] = ref_url_s
+                elif ref_a is not None:
+                    payload["prompt_audio"] = payload.get("ref_audio", "")
+                payload["prompt_text"] = p_text.strip()
+
+            if stream_enabled:
+                if ref_a is not None and not ref_url_s:
+                    req_id = f"req-{int(_time.time() * 1000)}"
+                    _pending[req_id] = payload
+                    browser_payload = {"_req_id": req_id, "_nonce": int(_time.time() * 1000)}
+                    return json.dumps(browser_payload), gr.update()
+                payload["_nonce"] = int(_time.time() * 1000)
+                return json.dumps(payload), gr.update()
+            else:
+                try:
+                    with httpx.Client(timeout=300.0) as client:
+                        resp = client.post(
+                            f"{api_base}/v1/audio/speech",
+                            json=payload,
+                            headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
+                        )
+                except httpx.ConnectError:
+                    raise gr.Error(f"Cannot connect to server at {api_base}.")
+                if resp.status_code != 200:
+                    raise gr.Error(f"Server error ({resp.status_code}): {resp.text[:200]}")
+                audio_np, sr = sf.read(io.BytesIO(resp.content))
+                if audio_np.ndim > 1:
+                    audio_np = audio_np[:, 0]
+                return "", (sr, audio_np.astype(np.float32))
+
+        generate_btn.click(
+            fn=on_generate,
+            inputs=[
+                stream_checkbox,
+                text_input,
+                control_instruction,
+                ref_audio,
+                ref_audio_url,
+                ultimate_clone,
+                prompt_text,
+            ],
+            outputs=[hidden_payload, audio_output],
+        ).then(
+            fn=lambda p: p,
+            inputs=[hidden_payload],
+            outputs=[hidden_payload],
+            js="(p) => { if (p && p.trim()) { const d = JSON.parse(p); delete d._nonce; window.ttsGenerate(d); } return p; }",
+        )
+
+        demo.queue()
+
+    return gr.mount_gradio_app(app, demo, path="/", css=css, theme=theme, head=_build_player_js())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="VoxCPM2 streaming Gradio demo")
+    parser.add_argument("--api-base", default="http://localhost:8000", help="vLLM API server URL")
+    parser.add_argument("--host", default="0.0.0.0", help="Gradio server host")
+    parser.add_argument("--port", type=int, default=7860, help="Gradio server port")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+    print(f"Connecting to vLLM server at: {args.api_base}")
+
+    import uvicorn
+
+    uvicorn.run(create_app(args.api_base), host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()
@@ -13,10 +13,7 @@ Examples:
        --ref-audio "https://example.com/reference.wav"

 Server setup:
-    python -m vllm_omni.entrypoints.openai.api_server \
-        --model openbmb/VoxCPM2 \
-        --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \
-        --host 0.0.0.0 --port 8000
+    vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
 """

 from __future__ import annotations