Merge remote-tracking branch 'origin/main' into qwen-image-2.1

# Conflicts:
#	tests/entrypoints/openai_api/test_image_server.py
#	vllm_omni/entrypoints/openai/api_server.py

.buildkite/common/ci_source_file_dependencies.yml

@@ -453,6 +453,7 @@ source_file_dependencies:
     - *lingbot_video
     - *lingbot_world
     - tests/e2e/online_serving/test_lingbot_video.py
+    - tests/e2e/online_serving/test_lingbot_video_expansion.py
     - tests/e2e/online_serving/test_lingbot_video_moe.py
     - tests/e2e/offline_inference/test_lingbot_world_v2.py
 

.buildkite/cuda/test-merge.yml

@@ -175,6 +175,16 @@ steps:
             "
         mirror_hardwares: h100_1
 
+      - label: "Diffusion · LingBot-Video Test"
+        source_file_dependencies: diffusion_lingbot_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
       - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
         source_file_dependencies: diffusion_minimax_h3_function
         # Keep four server lifecycles in one Buildkite job, but never in one

.buildkite/cuda/test-nightly.yml

@@ -349,7 +349,7 @@ steps:
         commands:
           - >-
             pytest -s -v
-            tests/e2e/online_serving/test_lingbot_video.py
+            tests/e2e/online_serving/test_lingbot_video_expansion.py
             tests/e2e/online_serving/test_lingbot_video_moe.py
             tests/e2e/offline_inference/test_lingbot_world_v2.py
             -m "full_model and diffusion and H100 and B200 and cards_1"

.buildkite/cuda/test-ready.yml

@@ -260,3 +260,13 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
+
+      - label: "Diffusion · LingBot-Video Test"
+        source_file_dependencies: diffusion_lingbot_function
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'core_model and cuda' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor:
 
 ```bash
 bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
-  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+  pytest -s -v tests/dfx/stability/scripts/run_stability_<model>.py -m slow
 ```
 
 Check the target revision's monitor backends. Do not claim CPU/NPU monitoring

apps/ComfyUI-vLLM-Omni/README.md

@@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C
 
 - **Generate Image** for text-to-image and image-to-image tasks
 - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks)
 - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
@@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it
 | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
 | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
 | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
-| Teal | Video References | Reference media |
+| Teal | Video References, Latent Mask Editing | Reference media |
 
 Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
 
@@ -263,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then:
 
 The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
 
+#### Latent-mask editing (MiniMax-H3)
+
+The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations.
+
+Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
+
+- `source_video` / `source_audio` — the media to edit.
+- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
+- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
+
+A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
+
 ### TTS (e.g., Qwen TTS series)
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import (
     VLLMOmniGenerateImage,
     VLLMOmniGenerateMusic,
     VLLMOmniGenerateVideo,
+    VLLMOmniLatentMaskEditing,
     VLLMOmniMiniMaxH3Params,
+    VLLMOmniMiniMaxH3TemporalMask,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
@@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import (
 
 # A dictionary that contains all nodes you want to export with their names
 NODE_CLASS_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
     # === Generation ===
     "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
     "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
@@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
     "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
+    "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = {
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
 NODE_DISPLAY_NAME_MAPPINGS = {
+    "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
     # === Generation ===
     "VLLMOmniGenerateImage": "Generate Image",
     "VLLMOmniGenerateVideo": "Generate Video",
@@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniGenerateMusic": "Generate Music",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
     "VLLMOmniVideoReferences": "Video References",
+    "VLLMOmniLatentMaskEditing": "Latent Mask Editing",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,12 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+import math
 from typing import Literal
 
 import torch
 from comfy_api.input import AudioInput, VideoInput
 
 from .utils.api_client import VLLMOmniClient
+from .utils.latent_mask import _align_frame_count, _video_latent_t
 from .utils.logger import get_logger
 from .utils.models import lookup_model_spec
 from .utils.types import (
@@ -17,6 +19,7 @@ from .utils.types import (
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
     FastH3Deployment,
+    LatentMaskEditing,
     MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
     VideoReferences,
@@ -210,6 +213,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
                 "fast_h3": ("FASTH3_DEPLOYMENT",),
+                "latent_edit": ("LATENT_MASK_EDITING",),
             },
         }
 
@@ -262,6 +266,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         model_params: dict | None = None,
         lora: dict | None = None,
         fast_h3: dict | None = None,
+        latent_edit: dict | None = None,
         **kwargs,
     ):
         if kwargs:
@@ -350,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
             sampling_params=sampling_params,
             lora=lora,
             model_params=model_params,
+            latent_edit=latent_edit,
         )
         return (output,)
 
@@ -1051,3 +1057,89 @@ class VLLMOmniVideoReferences:
                 if value is not None:
                     refs[f"{kind}_{index}"] = value
         return (refs,)
+
+
+class VLLMOmniLatentMaskEditing:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "source_video": ("VIDEO",),
+                "source_audio": ("AUDIO",),
+                "video_mask": ("MASK",),
+                "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}),
+            },
+        }
+
+    RETURN_TYPES = ("LATENT_MASK_EDITING",)
+    RETURN_NAMES = ("latent_edit",)
+    FUNCTION = "get_latent_edit"
+    CATEGORY = "vLLM-Omni"
+
+    def get_latent_edit(
+        self,
+        source_video: VideoInput | None = None,
+        source_audio: AudioInput | None = None,
+        video_mask: torch.Tensor | None = None,
+        audio_mask: float = -1.0,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info("Uncaught kwargs: %s", kwargs)
+        edit = LatentMaskEditing()
+        if source_video is not None:
+            edit["source_video"] = source_video
+        if source_audio is not None:
+            edit["source_audio"] = source_audio
+        if video_mask is not None:
+            edit["video_mask"] = video_mask
+        if audio_mask >= 0.0:
+            edit["audio_mask"] = audio_mask
+        return (edit,)
+
+
+class VLLMOmniMiniMaxH3TemporalMask:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}),
+                "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}),
+                "mode": (["continuation", "extension"],),
+                "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}),
+            }
+        }
+
+    RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK")
+    RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask")
+    FUNCTION = "build"
+    CATEGORY = "vLLM-Omni"
+
+    def build(self, images, source_fps, duration, mode, preserve_fraction=0.5):
+        if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0:
+            raise ValueError("Source must contain frames and have a positive finite FPS.")
+        if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1:
+            raise ValueError("Invalid duration or preserve_fraction.")
+        frames = _align_frame_count(max(1, round(duration * 24)))
+        source_seconds = images.shape[0] / source_fps
+        if mode == "extension":
+            if frames / 24 <= source_seconds:
+                raise ValueError("Extension output must be longer than the source; increase duration.")
+            boundary = source_seconds
+        elif mode == "continuation":
+            boundary = min(source_seconds, frames / 24) * preserve_fraction
+        else:
+            raise ValueError(f"Unknown temporal mask mode: {mode}")
+        available = min(frames, math.floor(boundary * 24 + 1e-8))
+        prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
+        preserved = _video_latent_t(prefix) if prefix else 0
+        total = _video_latent_t(frames)
+        mask = torch.ones(total, 1, 1)
+        mask[:preserved] = 0
+        indices = torch.arange(frames, device=images.device)
+        source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
+        preview_images = images.index_select(0, source_indices)
+        preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
+        return mask, 24.0, preview_images, preview_mask

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -30,6 +30,7 @@ from .format import (
     video_to_base64,
     video_to_bytes,
 )
+from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
 from .types import (
@@ -283,6 +284,7 @@ class VLLMOmniClient:
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
+        latent_edit: dict | None = None,
         spec_model: str | None = None,
         **extra_params,
     ) -> VideoInput:
@@ -386,6 +388,53 @@ class VLLMOmniClient:
                 content_type="image/png",
             )
 
+        # === latent-mask editing (MiniMax H3) ===
+        if latent_edit is not None:
+            source_video = latent_edit.get("source_video")
+            source_audio = latent_edit.get("source_audio")
+            video_mask = latent_edit.get("video_mask")
+            audio_mask = latent_edit.get("audio_mask")
+
+            if video_mask is None and audio_mask is None:
+                raise ValueError("Latent-mask editing requires at least one mask.")
+
+            video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
+            audio_mask_trivial = audio_mask is None or audio_mask == 1.0
+            if not video_mask_trivial and source_video is None:
+                raise ValueError("A non-trivial video mask requires a source video.")
+            if not audio_mask_trivial and source_audio is None and source_video is None:
+                raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
+
+            if source_video is not None:
+                form.add_field(
+                    "source_video",
+                    video_to_bytes(source_video, "source.mp4"),
+                    filename="source.mp4",
+                    content_type="video/mp4",
+                )
+            if source_audio is not None:
+                form.add_field(
+                    "source_audio",
+                    audio_to_bytes(source_audio, "source_audio.mp3"),
+                    filename="source_audio.mp3",
+                    content_type="audio/mpeg",
+                )
+            if video_mask is not None:
+                mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
+                form.add_field(
+                    "video_noise_mask",
+                    mask_json.encode("utf-8"),
+                    filename="video-mask.json",
+                    content_type="application/json",
+                )
+            if audio_mask is not None:
+                form.add_field(
+                    "audio_noise_mask",
+                    scalar_mask_to_json(audio_mask).encode("utf-8"),
+                    filename="audio-mask.json",
+                    content_type="application/json",
+                )
+
         if len(keyframe_images) == 2:
             for image_filename, image in keyframe_images:
                 form.add_field(

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import json
+
+import torch
+import torch.nn.functional as F
+
+
+def scalar_mask_to_json(value: float) -> str:
+    if not 0.0 <= value <= 1.0:
+        raise ValueError(f"mask value must be in [0, 1], got {value}")
+    return str(value)
+
+
+def _align_frame_count(frame_count: int) -> int:
+    if frame_count <= 0:
+        return 1
+    current = int(frame_count)
+    while current % 17 != 5:
+        current += 1
+    return current
+
+
+def _video_latent_t(frame_count: int) -> int:
+    if frame_count <= 5:
+        return 2
+    return ((int(frame_count) - 5) // 17) * 5 + 2
+
+
+def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
+    tv = _video_latent_t(_align_frame_count(num_frames))
+    height = int(height) // 32 * 32
+    width = int(width) // 32 * 32
+    gh, gw = height // 16, width // 16
+
+    if mask.ndim == 2:
+        mask = mask.unsqueeze(0)
+    elif mask.ndim != 3:
+        raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
+
+    grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
+    if grid.shape[0] == 1:
+        grid = grid.expand(tv, gh, gw)
+    elif grid.shape[0] != tv:
+        grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
+    return grid
+
+
+def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
+    return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -40,6 +40,10 @@ class VideoReferences(dict):
     pass
 
 
+class LatentMaskEditing(dict):
+    pass
+
+
 class FastH3Deployment(dict):
     """Descriptor for a server that fused FastH3 at startup."""
 

apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md

@@ -0,0 +1,17 @@
+# WF-05: MiniMax-H3 latent-mask editing
+
+Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case.
+
+## Requirements
+
+- ComfyUI with ComfyUI-vLLM-Omni installed.
+- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
+
+## Use
+
+Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
+
+1. Upload your source video in Load Video.
+2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
+3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
+4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json

@@ -0,0 +1,499 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadImageMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "STRING",
+          "widget": {
+            "name": "image"
+          },
+          "link": null
+        },
+        {
+          "name": "channel",
+          "type": "STRING",
+          "widget": {
+            "name": "channel"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImageMask"
+      },
+      "widgets_values": [
+        "",
+        "red"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json

@@ -0,0 +1,560 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        100
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "file",
+          "type": "STRING",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        0.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        500
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 5,
+      "type": "BatchMasksNode",
+      "pos": [
+        250,
+        380
+      ],
+      "size": [
+        200,
+        100
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "masks.mask0",
+          "type": "MASK",
+          "link": 2,
+          "slot_index": 0
+        },
+        {
+          "name": "masks.mask1",
+          "type": "MASK",
+          "link": 3,
+          "slot_index": 1
+        }
+      ],
+      "outputs": [
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "BatchMasksNode"
+      },
+      "widgets_values": []
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        450,
+        180
+      ],
+      "size": [
+        400,
+        120
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1
+        },
+        {
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 4
+        },
+        {
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        -1.0
+      ]
+    },
+    {
+      "id": 7,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        700
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            6
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 8,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        700,
+        180
+      ],
+      "size": [
+        450,
+        380
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null
+        },
+        {
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null
+        },
+        {
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "name": "lora",
+          "type": "LORA",
+          "link": null
+        },
+        {
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 6
+        },
+        {
+          "name": "fast_h3",
+          "type": "FAST_H3",
+          "link": null
+        },
+        {
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 5
+        },
+        {
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            7
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "temporal mask: regenerate the ending",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 9,
+      "type": "SaveVideo",
+      "pos": [
+        900,
+        180
+      ],
+      "size": [
+        420,
+        200
+      ],
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 7
+        },
+        {
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "name": "format",
+          "type": "STRING",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "temporal_mask",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      6,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      5,
+      0,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      5,
+      1,
+      "MASK"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      2,
+      "MASK"
+    ],
+    [
+      5,
+      6,
+      0,
+      8,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      6,
+      7,
+      0,
+      8,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      7,
+      8,
+      0,
+      9,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json

@@ -0,0 +1,511 @@
+{
+  "id": "latent-mask-editing-7380",
+  "revision": 0,
+  "last_node_id": 7,
+  "last_link_id": 5,
+  "nodes": [
+    {
+      "id": 1,
+      "type": "MarkdownNote",
+      "pos": [
+        40,
+        20
+      ],
+      "size": [
+        420,
+        210
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Latent-mask editing server prerequisite",
+      "properties": {},
+      "widgets_values": [
+        "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
+      ],
+      "color": "#432",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 2,
+      "type": "LoadVideo",
+      "pos": [
+        40,
+        280
+      ],
+      "size": [
+        420,
+        120
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "file",
+          "name": "file",
+          "type": "COMBO",
+          "widget": {
+            "name": "file"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            1
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        ""
+      ]
+    },
+    {
+      "id": 3,
+      "type": "SolidMask",
+      "pos": [
+        40,
+        460
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "value",
+          "name": "value",
+          "type": "FLOAT",
+          "widget": {
+            "name": "value"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "mask",
+          "name": "mask",
+          "type": "MASK",
+          "links": [
+            2
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SolidMask"
+      },
+      "widgets_values": [
+        1.0,
+        160,
+        120
+      ]
+    },
+    {
+      "id": 4,
+      "type": "VLLMOmniLatentMaskEditing",
+      "pos": [
+        560,
+        260
+      ],
+      "size": [
+        420,
+        300
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "source_video",
+          "name": "source_video",
+          "type": "VIDEO",
+          "link": 1,
+          "shape": 7
+        },
+        {
+          "localized_name": "source_audio",
+          "name": "source_audio",
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "localized_name": "video_mask",
+          "name": "video_mask",
+          "type": "MASK",
+          "link": 2,
+          "shape": 7
+        },
+        {
+          "localized_name": "audio_mask",
+          "name": "audio_mask",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_mask"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "links": [
+            3
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniLatentMaskEditing"
+      },
+      "widgets_values": [
+        0.5
+      ]
+    },
+    {
+      "id": 5,
+      "type": "VLLMOmniMiniMaxH3Params",
+      "pos": [
+        40,
+        680
+      ],
+      "size": [
+        420,
+        140
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "audio_flow_shift",
+          "name": "audio_flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "audio_flow_shift"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "flow_shift",
+          "name": "flow_shift",
+          "type": "FLOAT",
+          "widget": {
+            "name": "flow_shift"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video params",
+          "name": "video params",
+          "type": "VIDEO_PARAMS",
+          "links": [
+            4
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniMiniMaxH3Params"
+      },
+      "widgets_values": [
+        3.0,
+        12.0
+      ]
+    },
+    {
+      "id": 6,
+      "type": "VLLMOmniGenerateVideo",
+      "pos": [
+        1080,
+        200
+      ],
+      "size": [
+        540,
+        680
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "frame",
+          "name": "frame",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "references",
+          "name": "references",
+          "type": "VIDEO_REFERENCES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "localized_name": "sampling_params",
+          "name": "sampling_params",
+          "type": "SAMPLING_PARAMS",
+          "link": null
+        },
+        {
+          "localized_name": "lora",
+          "name": "lora",
+          "type": "REMOTE_LORA",
+          "link": null
+        },
+        {
+          "localized_name": "model_params",
+          "name": "model_params",
+          "type": "VIDEO_PARAMS",
+          "link": 4,
+          "shape": 7
+        },
+        {
+          "localized_name": "fast_h3",
+          "name": "fast_h3",
+          "type": "FASTH3_DEPLOYMENT",
+          "link": null
+        },
+        {
+          "localized_name": "latent_edit",
+          "name": "latent_edit",
+          "type": "LATENT_MASK_EDITING",
+          "link": 3,
+          "shape": 7
+        },
+        {
+          "localized_name": "url",
+          "name": "url",
+          "type": "STRING",
+          "widget": {
+            "name": "url"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "model",
+          "name": "model",
+          "type": "STRING",
+          "widget": {
+            "name": "model"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "prompt",
+          "name": "prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "negative_prompt",
+          "name": "negative_prompt",
+          "type": "STRING",
+          "widget": {
+            "name": "negative_prompt"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "width",
+          "name": "width",
+          "type": "INT",
+          "widget": {
+            "name": "width"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "height",
+          "name": "height",
+          "type": "INT",
+          "widget": {
+            "name": "height"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "fps",
+          "name": "fps",
+          "type": "INT",
+          "widget": {
+            "name": "fps"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "duration",
+          "name": "duration",
+          "type": "FLOAT",
+          "widget": {
+            "name": "duration"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": [
+            5
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VLLMOmniGenerateVideo"
+      },
+      "widgets_values": [
+        "http://127.0.0.1:8000/v1",
+        "MiniMaxAI/MiniMax-H3",
+        "Partially restyle the clip and soundtrack",
+        "",
+        160,
+        120,
+        24,
+        4.458
+      ]
+    },
+    {
+      "id": 7,
+      "type": "SaveVideo",
+      "pos": [
+        1700,
+        300
+      ],
+      "size": [
+        420,
+        160
+      ],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "link": 5,
+          "shape": 7
+        },
+        {
+          "localized_name": "filename_prefix",
+          "name": "filename_prefix",
+          "type": "STRING",
+          "widget": {
+            "name": "filename_prefix"
+          },
+          "link": null
+        },
+        {
+          "localized_name": "format",
+          "name": "format",
+          "type": "COMFY_DYNAMICCOMBO_V3",
+          "widget": {
+            "name": "format"
+          },
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "localized_name": "video",
+          "name": "video",
+          "type": "VIDEO",
+          "links": []
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SaveVideo"
+      },
+      "widgets_values": [
+        "vllm-omni/latent-edit",
+        "mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      1,
+      2,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ],
+    [
+      2,
+      3,
+      0,
+      4,
+      2,
+      "MASK"
+    ],
+    [
+      3,
+      4,
+      0,
+      6,
+      6,
+      "LATENT_MASK_EDITING"
+    ],
+    [
+      4,
+      5,
+      0,
+      6,
+      4,
+      "VIDEO_PARAMS"
+    ],
+    [
+      5,
+      6,
+      0,
+      7,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {},
+  "version": 0.4
+}

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = {
     REMOTE_LORA: "deployment",
     FASTH3_DEPLOYMENT: "deployment",
     VIDEO_REFERENCES: "media",
+    LATENT_MASK_EDITING: "media",
 };
 
 function familyOf(nodeData) {

artifacts/pr7759-review/e2e_slurm.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#SBATCH --partition=batch
+#SBATCH --nodes=1
+#SBATCH --time=90
+#SBATCH --job-name=qi21-e2e
+#SBATCH --output=/home/mozf/vllm-omni/artifacts/pr7759-review/e2e_slurm_%j.out
+
+set -euo pipefail
+cd /home/mozf/vllm-omni
+
+export HF_HOME=/mnt/lustre/hf-models
+export QWEN_IMAGE_21_TEST_MODEL=Qwen/Qwen-Image-2.1
+export VLLM_LOGGING_LEVEL=INFO
+
+echo "node: $(hostname)"
+nvidia-smi -L | head -4
+.venv/bin/python -c 'import vllm; print("vllm", vllm.__version__)'
+
+.venv/bin/python -m pytest tests/e2e/online_serving/test_qwen_image_21.py \
+  -o addopts="" -v -s 2>&1

artifacts/pr7759-review/e2e_slurm_30092.out

@@ -0,0 +1,742 @@
+node: gb200-rack1-01
+GPU 0: NVIDIA GB200 (UUID: GPU-56e80c56-4d0c-e81e-db5f-70f33aab4e18)
+GPU 1: NVIDIA GB200 (UUID: GPU-2ce1a9bc-dbf1-3bbd-116d-6b4e61022280)
+GPU 2: NVIDIA GB200 (UUID: GPU-f7cdefd9-3d67-384f-dbc1-53bc44d93985)
+GPU 3: NVIDIA GB200 (UUID: GPU-7c9bf698-f3f3-4634-9309-dc1e32f62fba)
+vllm 0.29.0
+INFO 09-21 08:43:18 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-21 08:43:21 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL).
+INFO 09-21 08:43:21 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM.
+INFO 09-21 08:43:21 [patch.py:503] inductor factorable-divisibility patch: installed.
+INFO 09-21 08:43:21 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+============================= test session starts ==============================
+platform linux -- Python 3.12.3, pytest-9.1.1, pluggy-1.6.0 -- /home/mozf/vllm-omni/.venv/bin/python
+cachedir: .pytest_cache
+rootdir: /home/mozf/vllm-omni
+configfile: pyproject.toml
+plugins: asyncio-1.4.0, mock-3.15.1, anyio-4.14.2
+asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function
+collecting ... INFO 09-21 08:43:27 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell
+collected 4 items
+
+tests/e2e/online_serving/test_qwen_image_21.py::test_text_to_image[request] INFO 09-21 08:43:32 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
+INFO 09-21 08:43:32 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+/home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.26.1.dev1+g76de642c5
+ --> vLLM version 0.29.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+INFO 09-21 08:43:50 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-21 08:43:50 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL).
+INFO 09-21 08:43:50 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM.
+INFO 09-21 08:43:50 [patch.py:503] inductor factorable-divisibility patch: installed.
+INFO 09-21 08:43:50 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-21 08:43:55 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell
+INFO 09-21 08:43:57 [serve.py:313] Detected diffusion model: Qwen/Qwen-Image-2.1
+INFO 09-21 08:43:57 [logo.py:52]        █     █     █▄   ▄█       ▄▀▀▀▀▄ █▄   ▄█ █▄    █ ▀█▀ 
+INFO 09-21 08:43:57 [logo.py:52]  ▄▄ ▄█ █     █     █ ▀▄▀ █  ▄▄▄  █    █ █ ▀▄▀ █ █ ▀▄  █  █  
+INFO 09-21 08:43:57 [logo.py:52]   █▄█▀ █     █     █     █       █    █ █     █ █   ▀▄█  █  
+INFO 09-21 08:43:57 [logo.py:52]    ▀▀  ▀▀▀▀▀ ▀▀▀▀▀ ▀     ▀        ▀▀▀▀  ▀     ▀ ▀     ▀ ▀▀▀ 
+INFO 09-21 08:43:57 [logo.py:52] 
+(APIServer pid=2856381) INFO 09-21 08:43:57 [api_utils.py:347] vLLM server version 0.29.0, serving model Qwen/Qwen-Image-2.1
+(APIServer pid=2856381) INFO 09-21 08:43:57 [api_utils.py:286] non-default args: {'model_tag': 'Qwen/Qwen-Image-2.1', 'host': '127.0.0.1', 'port': 45115, 'model': 'Qwen/Qwen-Image-2.1', 'max_num_seqs': 2}
+(APIServer pid=2856381) INFO 09-21 08:43:58 [weight_utils.py:58] Using model weights format ['*']
+(APIServer pid=2856381) INFO 09-21 08:45:02 [weight_utils.py:90] Time spent downloading weights for Qwen/Qwen-Image-2.1: 64.568430 seconds
+(APIServer pid=2856381) INFO 09-21 08:45:02 [omni_base.py:208] [AsyncOmni] Initializing with model Qwen/Qwen-Image-2.1
+(APIServer pid=2856381) INFO 09-21 08:45:02 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1
+(APIServer pid=2856381) INFO 09-21 08:45:05 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'.
+(APIServer pid=2856381) INFO 09-21 08:45:05 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages
+(APIServer pid=2856381) INFO 09-21 08:45:05 [weight_utils.py:58] Using model weights format ['*']
+(APIServer pid=2856381) INFO 09-21 08:45:05 [weight_utils.py:58] Using model weights format ['*']
+(APIServer pid=2856381) INFO 09-21 08:45:06 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(APIServer pid=2856381) INFO 09-21 08:45:06 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
+(APIServer pid=2856381) INFO 09-21 08:45:06 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment.
+(APIServer pid=2856381) INFO 09-21 08:45:06 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True).
+(APIServer pid=2856381) INFO 09-21 08:45:06 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:07 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor']
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:07 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock
+(DiffusionWorker pid=2856381) ␍Fetching 22 files:   0%|          | 0/22 [00:00<?, ?it/s]␍Fetching 22 files: 100%|██████████| 22/22 [00:00<00:00, 1912.94it/s]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:07 [hub_prefetch.py:344] Prefetch complete for Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2856381) /home/mozf/vllm-omni/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py:205: UserWarning: The `local_dir_use_symlinks` argument is deprecated and ignored in `hf_hub_download`. Downloading to a local directory does not use symlinks anymore.
+(DiffusionWorker pid=2856381)   warnings.warn(
+(DiffusionWorker pid=2856381) ␍Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]␍Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 1459.90it/s]
+(DiffusionWorker pid=2856381) ␍Loading weights:   0%|          | 0/750 [00:00<?, ?it/s]␍Loading weights:   0%|          | 2/750 [00:00<02:10,  5.73it/s]␍Loading weights:   2%|▏         | 17/750 [00:00<00:15, 46.47it/s]␍Loading weights:   4%|▍         | 32/750 [00:00<00:09, 73.11it/s]␍Loading weights:   6%|▋         | 48/750 [00:00<00:07, 92.32it/s]␍Loading weights:   8%|▊         | 61/750 [00:00<00:07, 96.35it/s]␍Loading weights:  10%|▉         | 73/750 [00:00<00:06, 101.61it/s]␍Loading weights:  12%|█▏        | 89/750 [00:01<00:05, 115.39it/s]␍Loading weights:  14%|█▎        | 103/750 [00:01<00:05, 120.74it/s]␍Loading weights:  15%|█▌        | 116/750 [00:01<00:05, 107.27it/s]␍Loading weights:  17%|█▋        | 131/750 [00:01<00:05, 116.83it/s]␍Loading weights:  20%|█▉        | 147/750 [00:01<00:04, 121.08it/s]␍Loading weights:  21%|██▏       | 160/750 [00:01<00:05, 116.44it/s]␍Loading weights:  24%|██▎       | 178/750 [00:01<00:04, 132.80it/s]␍Loading weights:  26%|██▌       | 192/750 [00:01<00:04, 121.63it/s]␍Loading weights:  27%|██▋       | 205/750 [00:01<00:04, 121.55it/s]␍Loading weights:  29%|██▉       | 219/750 [00:02<00:04, 125.35it/s]␍Loading weights:  31%|███       | 234/750 [00:02<00:04, 127.29it/s]␍Loading weights:  33%|███▎      | 248/750 [00:02<00:03, 128.37it/s]␍Loading weights:  35%|███▌      | 265/750 [00:02<00:03, 138.24it/s]␍Loading weights:  37%|███▋      | 279/750 [00:02<00:03, 128.84it/s]␍Loading weights:  39%|███▉      | 293/750 [00:02<00:03, 130.26it/s]␍Loading weights:  41%|████      | 307/750 [00:02<00:03, 128.54it/s]␍Loading weights:  43%|████▎     | 323/750 [00:02<00:03, 132.60it/s]␍Loading weights:  45%|████▍     | 337/750 [00:02<00:03, 132.95it/s]␍Loading weights:  47%|████▋     | 353/750 [00:03<00:02, 140.36it/s]␍Loading weights:  49%|████▉     | 368/750 [00:03<00:02, 133.40it/s]␍Loading weights:  51%|█████     | 382/750 [00:03<00:02, 133.67it/s]␍Loading weights:  56%|█████▌    | 419/750 [00:03<00:01, 199.39it/s]␍Loading weights:  59%|█████▉    | 443/750 [00:03<00:01, 209.10it/s]␍Loading weights:  81%|████████  | 607/750 [00:03<00:00, 616.28it/s]␍Loading weights:  97%|█████████▋| 727/750 [00:03<00:00, 768.60it/s]␍Loading weights: 100%|██████████| 750/750 [00:03<00:00, 199.34it/s]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:13 [platform.py:338] Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_100, cuDNN 92000, head_dim 128)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:13 [selector.py:93] Resolved diffusion attention backend 'CUDNN_ATTN' for role='self' via platform default
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:15 [weight_utils.py:58] Using model weights format ['transformer/diffusion_pytorch_model-00001-of-00002.safetensors', 'transformer/diffusion_pytorch_model-00002-of-00002.safetensors']
+(DiffusionWorker pid=2856381) ␍Multi-thread loading shards:   0% Completed | 0/2 [00:00<?, ?it/s]
+(DiffusionWorker pid=2856381) ␍Multi-thread loading shards:  50% Completed | 1/2 [00:00<00:00,  1.35it/s]
+(DiffusionWorker pid=2856381) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:02<00:00,  1.11s/it]
+(DiffusionWorker pid=2856381) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:02<00:00,  1.05s/it]
+(DiffusionWorker pid=2856381) 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:17 [diffusers_loader.py:1098] Loading weights took 2.17 seconds
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_model_runner.py:351] Model loading took 30.2190 GiB and 11.749674 seconds
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_model_runner.py:357] Model runner: Model loaded successfully.
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_model_runner.py:243] Model runner: transformer uses CUDA graph decode instead of torch.compile.
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_model_runner.py:448] Model runner: Initialization complete.
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_worker.py:425] Worker 0: Process-scoped GPU memory after model loading: 31.01 GiB.
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [manager.py:112] Initializing DiffusionLoRAManager: device=cuda:0, dtype=torch.bfloat16, max_cached_adapters=1, static_lora_path=None
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_worker.py:302] Worker 0: Initialization complete.
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [uniproc_executor.py:74] Diffusion worker initialized in-process (uniproc executor)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_engine.py:402] [RequestBatch] engine init max_num_seqs=2 max_wait_ms=0.0
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:18 [diffusion_engine.py:1289] dummy run to warm up the model
+(DiffusionWorker pid=2856381) ␍  0%|          | 0/2 [00:00<?, ?it/s]␍ 50%|█████     | 1/2 [00:02<00:02,  2.16s/it](DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 10, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+␍100%|██████████| 2/2 [00:02<00:00,  1.01s/it]␍100%|██████████| 2/2 [00:02<00:00,  1.18s/it]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [inline_stage_diffusion_client.py:82] [InlineStageDiffusionClient] stage-0 [rep-0] initialized inline (max_num_seqs=2)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [stage_runtime.py:1226] [StageRuntime] Stage 0 replica 0 initialized (diffusion)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [orchestrator.py:420] [Orchestrator] Starting event loop
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [omni_engine_base.py:344] [OmniEngine] Orchestrator ready with 1 stages
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [omni_base.py:227] [AsyncOmni] AsyncOmniEngine initialized in 19.83 seconds
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [omni_base.py:246] [AsyncOmni] Initialized with 1 stages for model Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [api_server.py:767] Detected pure diffusion mode (single diffusion stage)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [serving_video.py:170] Video response frame conversion pool configured: workers=8
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [speaker_cache.py:239] Speaker cache ready (max_bytes=536870912)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [serving_speech.py:264] Speaker storage: dir=/home/mozf/.cache/vllm-omni/speakers, max_speakers=1000, restored=0
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [serving.py:91] Robot OpenPI serving disabled for model Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [api_server.py:855] Pure diffusion API server initialized for model: Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [api_server.py:367] Starting vLLM API server (pure diffusion mode) on http://127.0.0.1:45115
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:61] Available routes are:
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /openapi.json, Methods: HEAD, GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /docs, Methods: HEAD, GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /redoc, Methods: HEAD, GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /load, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /version, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /metrics, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /tokenize, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /detokenize, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /ping, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /ping, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /invocations, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/responses, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/responses/{response_id}, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/responses/{response_id}/cancel, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/completions, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/messages, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/messages/count_tokens, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /generative_scoring, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /scale_elastic_ep, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /is_scaling_elastic_ep, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/chat/completions/render, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/messages/render, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/completions/render, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/chat/completions/derender, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/completions/derender, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /inference/v1/generate, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/chat/completions, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/chat/completions/batch, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/audio/speech, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/audio/speech/batch, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/audio/generate, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/audio/voices, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/audio/voices, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/audio/voices/{name}, Methods: DELETE
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/realtime/sessions, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/step, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/reset, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/close, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/status, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /health, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/models, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/images/generations, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/images/edits, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/videos, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/videos/sync, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/videos, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/videos/{video_id}, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/videos/{video_id}, Methods: DELETE
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/videos/{video_id}/content, Methods: GET
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/omni/sleep, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:70] Route: /v1/omni/wakeup, Methods: POST
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:81] Route: /v1/audio/speech/stream, Endpoint: streaming_speech
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:81] Route: /v1/video/chat/stream, Endpoint: streaming_video_chat
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:81] Route: /v1/realtime/video, Endpoint: streaming_video_output
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:81] Route: /v1/realtime, Endpoint: realtime_websocket
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:81] Route: /v1/duplex, Endpoint: duplex_websocket
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:22 [launcher.py:81] Route: /v1/realtime/robot/openpi, Endpoint: realtime_robot_openpi
+(DiffusionWorker pid=2856381) INFO:     Started server process [2856381]
+(DiffusionWorker pid=2856381) INFO:     Waiting for application startup.
+(DiffusionWorker pid=2856381) INFO:     Application startup complete.
+Pre-test device status:
+[Device Memory Monitor] Waiting for device(s) 0, 1, 2, 3 to free memory, Condition: Memory usage ratio ≤ 5.0%
+[Device Memory Status] Current usage:
+  Device 0: 0.7GiB/184.3GiB (0.4%)
+  Device 1: 0.7GiB/184.3GiB (0.4%)
+  Device 2: 0.7GiB/184.3GiB (0.4%)
+  Device 3: 0.7GiB/184.3GiB (0.4%)
+[Device Memory Freed] Device(s) 0, 1, 2, 3 meet memory condition
+   Condition: Memory usage ratio ≤ 5.0%
+   Wait time: 0.6 seconds (0.0 minutes)
+Post-test device status:
+
+================================================================================
+NVIDIA GPU Information (nvidia-smi)
+================================================================================
+Mon Sep 21 08:43:32 2026       
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 580.126.09             Driver Version: 580.126.09     CUDA Version: 13.0     |
++-----------------------------------------+------------------------+----------------------+
+| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
+|                                         |                        |               MIG M. |
+|=========================================+========================+======================|
+|   0  NVIDIA GB200                   Off |   00000000:89:00.0 Off |                    0 |
+| N/A   26C    P0            212W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   1  NVIDIA GB200                   Off |   00000000:8A:00.0 Off |                    0 |
+| N/A   25C    P0            213W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   2  NVIDIA GB200                   Off |   00000000:C1:00.0 Off |                    0 |
+| N/A   26C    P0            176W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+... (showing first 20 of 35 lines)
+
+================================================================================
+Detailed GPU Processes (nvidia-smi pmon)
+================================================================================
+# gpu         pid   type     sm    mem    enc    dec    jpg    ofa    command 
+# Idx           #    C/G      %      %      %      %      %      %    name 
+    0    2853490     C      0      0      -      -      -      -    python         
+    1    2853490     C      0      0      -      -      -      -    python         
+    2    2853490     C      0      0      -      -      -      -    python         
+    3    2853490     C      0      0      -      -      -      -    python
+Launching OmniServer with: /home/mozf/vllm-omni/.venv/bin/python -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen-Image-2.1 --host 127.0.0.1 --port 45115 --omni --vae-use-tiling --max-num-seqs 2 --stage-init-timeout 600 --init-timeout 900 --log-stats
+Server ready on 127.0.0.1:45115 (OmniServer startup took 112.093s)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:28 [serving_chat.py:3651] Diffusion chat request chatcmpl-a2e0b3f285fd47cf: ref_images=0, params={'num_inference_steps': 4, 'true_cfg_scale': 4.0, 'width': 512, 'height': 512, 'negative_prompt': 'blurry, low quality', 'seed': 42}
+(DiffusionWorker pid=2856381) ␍  0%|          | 0/4 [00:00<?, ?it/s]␍ 25%|██▌       | 1/4 [00:00<00:02,  1.50it/s](DiffusionWorker pid=2856381) INFO 09-21 08:45:29 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 18, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('uncond', 1, 13, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+␍ 50%|█████     | 2/4 [00:01<00:01,  1.89it/s]␍100%|██████████| 4/4 [00:01<00:00,  3.39it/s]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] [Overall Summary]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] | Field                       |     Value |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] | e2e_requests                |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] | e2e_wall_time_ms            | 1,390.033 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] | e2e_avg_time_per_request_ms | 1,390.033 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] | e2e_stage_0_wall_time_ms    | 1,389.915 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] [RequestE2EStats [request_id=chatcmpl-a2e0b3f285fd47cf-940070ef]]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] | Field        |     Value |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] | e2e_total_ms | 1,389.915 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:878] [OmniTiming] req=chatcmpl-a2e0b3f285fd47cf-940070ef total=1.39s engine=1.39s stages=[0:1.39s]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] [StageRequestStats [request_id=chatcmpl-a2e0b3f285fd47cf-940070ef]]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | Field                           |         0 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | batch_id                        |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | batch_size                      |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | denoise_step_latency_ms         |   347.310 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | diffusion_engine_exec_time_s    |     1.382 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | height                          |   512.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | image_num                       |     1.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | image_pixels                    |   262,144 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | image_time_to_first_output_ms   | 1,389.218 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | inter_output_latencies_ms       |           |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | kv_recv_time_s                  |     0.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | num_inference_steps             |     4.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | output_unit_count               |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | output_unit_type                |     image |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | postprocess_time_s              |     0.007 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | preprocess_time_s               |     0.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | resolution                      |   640.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | scheduler_queue_wait_s          |     0.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | serving_time_to_first_output_ms | 1,389.586 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | stage_gen_time_ms               | 1,389.240 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | vllm_itls_ms                    |           |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] | width                           |   512.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [serving_chat.py:3943] Diffusion chat completed for request chatcmpl-a2e0b3f285fd47cf: output_type=image, image_count=1
+(DiffusionWorker pid=2856381) INFO:     127.0.0.1:58082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+OmniServer started successfully
+--- Running test: test_text_to_image[request]
+[diffusion] request#1 success in 5.256s
+PASSED
+tests/e2e/online_serving/test_qwen_image_21.py::test_image_edit[request] (DiffusionWorker pid=2856381) INFO 09-21 08:45:30 [serving_chat.py:3651] Diffusion chat request chatcmpl-b801b6b9b3664cfd: ref_images=1, params={'num_inference_steps': 4, 'true_cfg_scale': 4.0, 'width': 512, 'height': 512, 'negative_prompt': 'blurry, low quality', 'seed': 42}
+(DiffusionWorker pid=2856381) ␍  0%|          | 0/4 [00:00<?, ?it/s]␍ 25%|██▌       | 1/4 [00:01<00:04,  1.39s/it](DiffusionWorker pid=2856381) INFO 09-21 08:45:32 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 4115, ((1, 64, 64), (1, 32, 32)), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('uncond', 1, 4115, ((1, 64, 64), (1, 32, 32)), torch.bfloat16, 0, 'CUDNN_ATTN')
+␍ 50%|█████     | 2/4 [00:01<00:01,  1.25it/s]␍100%|██████████| 4/4 [00:01<00:00,  3.02it/s]␍100%|██████████| 4/4 [00:01<00:00,  2.13it/s]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] [Overall Summary]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] | Field                       |     Value |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] | e2e_requests                |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] | e2e_wall_time_ms            | 2,587.491 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] | e2e_avg_time_per_request_ms | 2,587.491 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] | e2e_stage_0_wall_time_ms    | 2,587.400 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] [RequestE2EStats [request_id=chatcmpl-b801b6b9b3664cfd-9f06e779]]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] | Field        |     Value |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] | e2e_total_ms | 2,587.400 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:878] [OmniTiming] req=chatcmpl-b801b6b9b3664cfd-9f06e779 total=2.59s engine=2.59s stages=[0:2.59s]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] 
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] [StageRequestStats [request_id=chatcmpl-b801b6b9b3664cfd-9f06e779]]
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | Field                           |         0 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | batch_id                        |         2 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | batch_size                      |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | denoise_step_latency_ms         |   646.529 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | diffusion_engine_exec_time_s    |     2.530 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | height                          |   512.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | image_num                       |     1.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | image_pixels                    |   262,144 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | image_time_to_first_output_ms   | 2,586.083 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | inter_output_latencies_ms       |           |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | kv_recv_time_s                  |     0.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | num_inference_steps             |     4.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | output_unit_count               |         1 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | output_unit_type                |     image |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | postprocess_time_s              |     0.013 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | preprocess_time_s               |     0.042 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | resolution                      |   640.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | scheduler_queue_wait_s          |     0.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | serving_time_to_first_output_ms | 2,586.323 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | stage_gen_time_ms               | 2,586.114 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | vllm_itls_ms                    |           |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] | width                           |   512.000 |
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [serving_chat.py:3943] Diffusion chat completed for request chatcmpl-b801b6b9b3664cfd: output_type=image, image_count=1
+(DiffusionWorker pid=2856381) INFO:     127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+--- Running test: test_image_edit[request]
+[diffusion] request#1 success in 2.645s
+PASSED
+tests/e2e/online_serving/test_qwen_image_21.py::test_text_to_image[step] (DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [launcher.py:124] [shutdown] API server: shutdown triggered
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [launcher.py:140] [shutdown] API server: stopping engine client mode=abort timeout=0.0s
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [omni_base.py:771] [AsyncOmni] Shutting down
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [omni_engine_base.py:1144] [OmniEngine] Shutting down Orchestrator
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [orchestrator.py:512] [Orchestrator] Received shutdown signal
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [orchestrator.py:2720] [Orchestrator] Shutting down all 1 client(s)
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [stage_pool.py:1362] [StagePool] Stage 0 replica 0 shut down
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [launcher.py:149] [shutdown] API server: engine client stopped
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [launcher.py:152] [shutdown] API server: signalling HTTP server shutdown
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [launcher.py:173] [shutdown] API server: shutting down FastAPI HTTP server
+(DiffusionWorker pid=2856381) INFO:     Shutting down
+(DiffusionWorker pid=2856381) INFO:     Waiting for application shutdown.
+(DiffusionWorker pid=2856381) INFO:     Application shutdown complete.
+(DiffusionWorker pid=2856381) INFO 09-21 08:45:33 [omni_base.py:771] [AsyncOmni] Shutting down
+/home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.26.1.dev1+g76de642c5
+ --> vLLM version 0.29.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+INFO 09-21 08:45:52 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-21 08:45:53 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL).
+INFO 09-21 08:45:53 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM.
+INFO 09-21 08:45:53 [patch.py:503] inductor factorable-divisibility patch: installed.
+INFO 09-21 08:45:53 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-21 08:45:57 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell
+INFO 09-21 08:45:58 [serve.py:313] Detected diffusion model: Qwen/Qwen-Image-2.1
+INFO 09-21 08:45:58 [logo.py:52]        █     █     █▄   ▄█       ▄▀▀▀▀▄ █▄   ▄█ █▄    █ ▀█▀ 
+INFO 09-21 08:45:58 [logo.py:52]  ▄▄ ▄█ █     █     █ ▀▄▀ █  ▄▄▄  █    █ █ ▀▄▀ █ █ ▀▄  █  █  
+INFO 09-21 08:45:58 [logo.py:52]   █▄█▀ █     █     █     █       █    █ █     █ █   ▀▄█  █  
+INFO 09-21 08:45:58 [logo.py:52]    ▀▀  ▀▀▀▀▀ ▀▀▀▀▀ ▀     ▀        ▀▀▀▀  ▀     ▀ ▀     ▀ ▀▀▀ 
+INFO 09-21 08:45:58 [logo.py:52] 
+(APIServer pid=2864057) INFO 09-21 08:45:58 [api_utils.py:347] vLLM server version 0.29.0, serving model Qwen/Qwen-Image-2.1
+(APIServer pid=2864057) INFO 09-21 08:45:58 [api_utils.py:286] non-default args: {'model_tag': 'Qwen/Qwen-Image-2.1', 'host': '127.0.0.1', 'port': 44729, 'model': 'Qwen/Qwen-Image-2.1', 'max_num_seqs': 2}
+(APIServer pid=2864057) INFO 09-21 08:45:58 [weight_utils.py:58] Using model weights format ['*']
+(APIServer pid=2864057) INFO 09-21 08:45:58 [omni_base.py:208] [AsyncOmni] Initializing with model Qwen/Qwen-Image-2.1
+(APIServer pid=2864057) INFO 09-21 08:45:58 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1
+(APIServer pid=2864057) INFO 09-21 08:46:00 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'.
+(APIServer pid=2864057) INFO 09-21 08:46:00 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages
+(APIServer pid=2864057) INFO 09-21 08:46:00 [weight_utils.py:58] Using model weights format ['*']
+(APIServer pid=2864057) INFO 09-21 08:46:00 [weight_utils.py:58] Using model weights format ['*']
+(APIServer pid=2864057) INFO 09-21 08:46:01 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+(APIServer pid=2864057) INFO 09-21 08:46:01 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
+(APIServer pid=2864057) INFO 09-21 08:46:01 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment.
+(APIServer pid=2864057) INFO 09-21 08:46:01 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True).
+(APIServer pid=2864057) INFO 09-21 08:46:01 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:01 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor']
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:01 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock
+(DiffusionWorker pid=2864057) ␍Fetching 22 files:   0%|          | 0/22 [00:00<?, ?it/s]␍Fetching 22 files: 100%|██████████| 22/22 [00:00<00:00, 2099.82it/s]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:01 [hub_prefetch.py:344] Prefetch complete for Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2864057) /home/mozf/vllm-omni/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py:205: UserWarning: The `local_dir_use_symlinks` argument is deprecated and ignored in `hf_hub_download`. Downloading to a local directory does not use symlinks anymore.
+(DiffusionWorker pid=2864057)   warnings.warn(
+(DiffusionWorker pid=2864057) ␍Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]␍Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 1604.55it/s]
+(DiffusionWorker pid=2864057) ␍Loading weights:   0%|          | 0/750 [00:00<?, ?it/s]␍Loading weights:   3%|▎         | 26/750 [00:00<00:03, 241.16it/s]␍Loading weights:   7%|▋         | 51/750 [00:00<00:03, 232.24it/s]␍Loading weights:  11%|█         | 81/750 [00:00<00:02, 239.92it/s]␍Loading weights:  15%|█▍        | 109/750 [00:00<00:02, 251.29it/s]␍Loading weights:  18%|█▊        | 136/750 [00:00<00:02, 255.29it/s]␍Loading weights:  22%|██▏       | 162/750 [00:00<00:02, 250.35it/s]␍Loading weights:  25%|██▌       | 191/750 [00:00<00:02, 259.13it/s]␍Loading weights:  29%|██▉       | 217/750 [00:00<00:02, 254.52it/s]␍Loading weights:  33%|███▎      | 246/750 [00:00<00:01, 263.13it/s]␍Loading weights:  36%|███▋      | 273/750 [00:01<00:01, 261.43it/s]␍Loading weights:  40%|████      | 301/750 [00:01<00:01, 266.52it/s]␍Loading weights:  44%|████▎     | 328/750 [00:01<00:01, 265.10it/s]␍Loading weights:  48%|████▊     | 357/750 [00:01<00:01, 264.28it/s]␍Loading weights:  51%|█████▏    | 385/750 [00:01<00:01, 268.44it/s]␍Loading weights:  72%|███████▏  | 539/750 [00:01<00:00, 640.17it/s]␍Loading weights:  99%|█████████▊| 739/750 [00:01<00:00, 1035.77it/s]␍Loading weights: 100%|██████████| 750/750 [00:01<00:00, 441.58it/s] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:04 [platform.py:338] Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_100, cuDNN 92000, head_dim 128)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:04 [selector.py:93] Resolved diffusion attention backend 'CUDNN_ATTN' for role='self' via platform default
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:06 [weight_utils.py:58] Using model weights format ['transformer/diffusion_pytorch_model-00001-of-00002.safetensors', 'transformer/diffusion_pytorch_model-00002-of-00002.safetensors']
+(DiffusionWorker pid=2864057) ␍Multi-thread loading shards:   0% Completed | 0/2 [00:00<?, ?it/s]
+(DiffusionWorker pid=2864057) ␍Multi-thread loading shards:  50% Completed | 1/2 [00:00<00:00,  1.88it/s]
+(DiffusionWorker pid=2864057) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:01<00:00,  1.06it/s]
+(DiffusionWorker pid=2864057) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:01<00:00,  1.14it/s]
+(DiffusionWorker pid=2864057) 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:08 [diffusers_loader.py:1098] Loading weights took 1.83 seconds
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_model_runner.py:351] Model loading took 30.2190 GiB and 7.928883 seconds
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_model_runner.py:357] Model runner: Model loaded successfully.
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_model_runner.py:243] Model runner: transformer uses CUDA graph decode instead of torch.compile.
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_model_runner.py:448] Model runner: Initialization complete.
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_worker.py:425] Worker 0: Process-scoped GPU memory after model loading: 31.01 GiB.
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [manager.py:112] Initializing DiffusionLoRAManager: device=cuda:0, dtype=torch.bfloat16, max_cached_adapters=1, static_lora_path=None
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_worker.py:302] Worker 0: Initialization complete.
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [uniproc_executor.py:74] Diffusion worker initialized in-process (uniproc executor)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:09 [diffusion_engine.py:1289] dummy run to warm up the model
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 10, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [inline_stage_diffusion_client.py:82] [InlineStageDiffusionClient] stage-0 [rep-0] initialized inline (max_num_seqs=2)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [stage_runtime.py:1226] [StageRuntime] Stage 0 replica 0 initialized (diffusion)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [orchestrator.py:420] [Orchestrator] Starting event loop
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [omni_engine_base.py:344] [OmniEngine] Orchestrator ready with 1 stages
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [omni_base.py:227] [AsyncOmni] AsyncOmniEngine initialized in 11.77 seconds
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [omni_base.py:246] [AsyncOmni] Initialized with 1 stages for model Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [api_server.py:767] Detected pure diffusion mode (single diffusion stage)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [serving_video.py:170] Video response frame conversion pool configured: workers=8
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [speaker_cache.py:239] Speaker cache ready (max_bytes=536870912)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [serving_speech.py:264] Speaker storage: dir=/home/mozf/.cache/vllm-omni/speakers, max_speakers=1000, restored=0
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [serving.py:91] Robot OpenPI serving disabled for model Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [api_server.py:855] Pure diffusion API server initialized for model: Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [api_server.py:367] Starting vLLM API server (pure diffusion mode) on http://127.0.0.1:44729
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:61] Available routes are:
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /openapi.json, Methods: HEAD, GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /docs, Methods: HEAD, GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /redoc, Methods: HEAD, GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /load, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /version, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /metrics, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /tokenize, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /detokenize, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /ping, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /ping, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /invocations, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/responses, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/responses/{response_id}, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/responses/{response_id}/cancel, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/completions, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/messages, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/messages/count_tokens, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /generative_scoring, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /scale_elastic_ep, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /is_scaling_elastic_ep, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/chat/completions/render, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/messages/render, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/completions/render, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/chat/completions/derender, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/completions/derender, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /inference/v1/generate, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/chat/completions, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/chat/completions/batch, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/audio/speech, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/audio/speech/batch, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/audio/generate, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/audio/voices, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/audio/voices, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/audio/voices/{name}, Methods: DELETE
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/realtime/sessions, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/step, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/reset, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/close, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/realtime/sessions/{session_id}/status, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /health, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/models, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/images/generations, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/images/edits, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/videos, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/videos/sync, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/videos, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/videos/{video_id}, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/videos/{video_id}, Methods: DELETE
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/videos/{video_id}/content, Methods: GET
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/omni/sleep, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:70] Route: /v1/omni/wakeup, Methods: POST
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:81] Route: /v1/audio/speech/stream, Endpoint: streaming_speech
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:81] Route: /v1/video/chat/stream, Endpoint: streaming_video_chat
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:81] Route: /v1/realtime/video, Endpoint: streaming_video_output
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:81] Route: /v1/realtime, Endpoint: realtime_websocket
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:81] Route: /v1/duplex, Endpoint: duplex_websocket
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:10 [launcher.py:81] Route: /v1/realtime/robot/openpi, Endpoint: realtime_robot_openpi
+(DiffusionWorker pid=2864057) INFO:     Started server process [2864057]
+(DiffusionWorker pid=2864057) INFO:     Waiting for application startup.
+(DiffusionWorker pid=2864057) INFO:     Application startup complete.
+OmniServer stopping...
+Pre-test device status:
+[Device Memory Monitor] Waiting for device(s) 0, 1, 2, 3 to free memory, Condition: Memory usage ratio ≤ 5.0%
+[Device Memory Status] Current usage:
+  Device 0: 0.7GiB/184.3GiB (0.4%)
+  Device 1: 0.7GiB/184.3GiB (0.4%)
+  Device 2: 0.7GiB/184.3GiB (0.4%)
+  Device 3: 0.7GiB/184.3GiB (0.4%)
+[Device Memory Freed] Device(s) 0, 1, 2, 3 meet memory condition
+   Condition: Memory usage ratio ≤ 5.0%
+   Wait time: 0.0 seconds (0.0 minutes)
+Post-test device status:
+
+================================================================================
+NVIDIA GPU Information (nvidia-smi)
+================================================================================
+Mon Sep 21 08:45:35 2026       
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 580.126.09             Driver Version: 580.126.09     CUDA Version: 13.0     |
++-----------------------------------------+------------------------+----------------------+
+| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
+|                                         |                        |               MIG M. |
+|=========================================+========================+======================|
+|   0  NVIDIA GB200                   Off |   00000000:89:00.0 Off |                    0 |
+| N/A   27C    P0            219W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   1  NVIDIA GB200                   Off |   00000000:8A:00.0 Off |                    0 |
+| N/A   26C    P0            229W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   2  NVIDIA GB200                   Off |   00000000:C1:00.0 Off |                    0 |
+| N/A   26C    P0            201W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+... (showing first 20 of 35 lines)
+
+================================================================================
+Detailed GPU Processes (nvidia-smi pmon)
+================================================================================
+# gpu         pid   type     sm    mem    enc    dec    jpg    ofa    command 
+# Idx           #    C/G      %      %      %      %      %      %    name 
+    0    2853490     C      -      -      -      -      -      -    python         
+    1    2853490     C      0      0      -      -      -      -    python         
+    2    2853490     C      0      0      -      -      -      -    python         
+    3    2853490     C      0      0      -      -      -      -    python
+OmniServer stopped
+Pre-test device status:
+[Device Memory Monitor] Waiting for device(s) 0, 1, 2, 3 to free memory, Condition: Memory usage ratio ≤ 5.0%
+[Device Memory Status] Current usage:
+  Device 0: 0.7GiB/184.3GiB (0.4%)
+  Device 1: 0.7GiB/184.3GiB (0.4%)
+  Device 2: 0.7GiB/184.3GiB (0.4%)
+  Device 3: 0.7GiB/184.3GiB (0.4%)
+[Device Memory Freed] Device(s) 0, 1, 2, 3 meet memory condition
+   Condition: Memory usage ratio ≤ 5.0%
+   Wait time: 0.0 seconds (0.0 minutes)
+Post-test device status:
+
+================================================================================
+NVIDIA GPU Information (nvidia-smi)
+================================================================================
+Mon Sep 21 08:45:36 2026       
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 580.126.09             Driver Version: 580.126.09     CUDA Version: 13.0     |
++-----------------------------------------+------------------------+----------------------+
+| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
+|                                         |                        |               MIG M. |
+|=========================================+========================+======================|
+|   0  NVIDIA GB200                   Off |   00000000:89:00.0 Off |                    0 |
+| N/A   27C    P0            219W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   1  NVIDIA GB200                   Off |   00000000:8A:00.0 Off |                    0 |
+| N/A   26C    P0            229W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   2  NVIDIA GB200                   Off |   00000000:C1:00.0 Off |                    0 |
+| N/A   26C    P0            201W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+... (showing first 20 of 35 lines)
+
+================================================================================
+Detailed GPU Processes (nvidia-smi pmon)
+================================================================================
+# gpu         pid   type     sm    mem    enc    dec    jpg    ofa    command 
+# Idx           #    C/G      %      %      %      %      %      %    name 
+    0    2853490     C      -      -      -      -      -      -    python         
+    1    2853490     C      0      0      -      -      -      -    python         
+    2    2853490     C      0      0      -      -      -      -    python         
+    3    2853490     C      0      0      -      -      -      -    python
+Launching OmniServer with: /home/mozf/vllm-omni/.venv/bin/python -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen-Image-2.1 --host 127.0.0.1 --port 44729 --omni --vae-use-tiling --max-num-seqs 2 --step-execution --stage-init-timeout 600 --init-timeout 900 --log-stats
+Server ready on 127.0.0.1:44729 (OmniServer startup took 36.070s)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:12 [serving_chat.py:3651] Diffusion chat request chatcmpl-298a1e96b0c44416: ref_images=0, params={'negative_prompt': 'blurry, low quality', 'height': 512, 'seed': 42, 'true_cfg_scale': 4.0, 'width': 512, 'num_inference_steps': 4}
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:12 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 18, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('uncond', 1, 13, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] [Overall Summary]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] +-----------------------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] | Field                       |   Value |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] +-----------------------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] | e2e_requests                |       1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] | e2e_wall_time_ms            | 964.303 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] | e2e_avg_time_per_request_ms | 964.303 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] | e2e_stage_0_wall_time_ms    | 964.184 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:811] +-----------------------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] [RequestE2EStats [request_id=chatcmpl-298a1e96b0c44416-a113efb5]]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] +--------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] | Field        |   Value |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] +--------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] | e2e_total_ms | 964.184 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:837] +--------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:878] [OmniTiming] req=chatcmpl-298a1e96b0c44416-a113efb5 total=0.96s engine=0.96s stages=[0:0.96s]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] [StageRequestStats [request_id=chatcmpl-298a1e96b0c44416-a113efb5]]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] +---------------------------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | Field                           |       0 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] +---------------------------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | batch_id                        |       1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | batch_size                      |       1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | denoise_step_latency_ms         | 240.872 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | diffusion_engine_exec_time_s    |   0.957 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | height                          | 512.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | image_num                       |   1.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | image_pixels                    | 262,144 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | image_time_to_first_output_ms   | 963.466 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | inter_output_latencies_ms       |         |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | num_inference_steps             |   4.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | output_unit_count               |       1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | output_unit_type                |   image |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | postprocess_time_s              |   0.006 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | preprocess_time_s               |   0.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | resolution                      | 640.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | scheduler_queue_wait_s          |   0.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | serving_time_to_first_output_ms | 963.828 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | stage_gen_time_ms               | 963.487 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | vllm_itls_ms                    |         |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] | width                           | 512.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [stats.py:920] +---------------------------------+---------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [serving_chat.py:3943] Diffusion chat completed for request chatcmpl-298a1e96b0c44416: output_type=image, image_count=1
+(DiffusionWorker pid=2864057) INFO:     127.0.0.1:48938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+OmniServer started successfully
+--- Running test: test_text_to_image[step]
+[diffusion] request#1 success in 1.110s
+PASSED
+tests/e2e/online_serving/test_qwen_image_21.py::test_image_edit[step] (DiffusionWorker pid=2864057) INFO 09-21 08:46:13 [serving_chat.py:3651] Diffusion chat request chatcmpl-6b7e1eb7fb774af6: ref_images=1, params={'negative_prompt': 'blurry, low quality', 'height': 512, 'seed': 42, 'true_cfg_scale': 4.0, 'width': 512, 'num_inference_steps': 4}
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 4115, ((1, 64, 64), (1, 32, 32)), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('uncond', 1, 4115, ((1, 64, 64), (1, 32, 32)), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] [Overall Summary]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] | Field                       |     Value |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] | e2e_requests                |         1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] | e2e_wall_time_ms            | 1,422.300 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] | e2e_avg_time_per_request_ms | 1,422.300 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] | e2e_stage_0_wall_time_ms    | 1,422.209 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:811] +-----------------------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] [RequestE2EStats [request_id=chatcmpl-6b7e1eb7fb774af6-9450ba42]]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] | Field        |     Value |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] | e2e_total_ms | 1,422.209 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:837] +--------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:878] [OmniTiming] req=chatcmpl-6b7e1eb7fb774af6-9450ba42 total=1.42s engine=1.42s stages=[0:1.42s]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] 
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] [StageRequestStats [request_id=chatcmpl-6b7e1eb7fb774af6-9450ba42]]
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | Field                           |         0 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | batch_id                        |         2 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | batch_size                      |         1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | denoise_step_latency_ms         |   355.232 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | diffusion_engine_exec_time_s    |     1.370 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | height                          |   512.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | image_num                       |     1.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | image_pixels                    |   262,144 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | image_time_to_first_output_ms   | 1,420.895 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | inter_output_latencies_ms       |           |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | num_inference_steps             |     4.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | output_unit_count               |         1 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | output_unit_type                |     image |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | postprocess_time_s              |     0.012 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | preprocess_time_s               |     0.039 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | resolution                      |   640.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | scheduler_queue_wait_s          |     0.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | serving_time_to_first_output_ms | 1,421.129 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | stage_gen_time_ms               | 1,420.927 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | vllm_itls_ms                    |           |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] | width                           |   512.000 |
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [stats.py:920] +---------------------------------+-----------+
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [serving_chat.py:3943] Diffusion chat completed for request chatcmpl-6b7e1eb7fb774af6: output_type=image, image_count=1
+(DiffusionWorker pid=2864057) INFO:     127.0.0.1:48944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+--- Running test: test_image_edit[step]
+[diffusion] request#1 success in 1.479s
+PASSED(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [launcher.py:124] [shutdown] API server: shutdown triggered
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [launcher.py:140] [shutdown] API server: stopping engine client mode=abort timeout=0.0s
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [omni_base.py:771] [AsyncOmni] Shutting down
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [omni_engine_base.py:1144] [OmniEngine] Shutting down Orchestrator
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [orchestrator.py:512] [Orchestrator] Received shutdown signal
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:14 [orchestrator.py:2720] [Orchestrator] Shutting down all 1 client(s)
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:15 [stage_pool.py:1362] [StagePool] Stage 0 replica 0 shut down
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:15 [launcher.py:149] [shutdown] API server: engine client stopped
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:15 [launcher.py:152] [shutdown] API server: signalling HTTP server shutdown
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:15 [launcher.py:173] [shutdown] API server: shutting down FastAPI HTTP server
+(DiffusionWorker pid=2864057) INFO:     Shutting down
+(DiffusionWorker pid=2864057) INFO:     Waiting for application shutdown.
+(DiffusionWorker pid=2864057) INFO:     Application shutdown complete.
+(DiffusionWorker pid=2864057) INFO 09-21 08:46:15 [omni_base.py:771] [AsyncOmni] Shutting down
+OmniServer stopping...
+Pre-test device status:
+[Device Memory Monitor] Waiting for device(s) 0, 1, 2, 3 to free memory, Condition: Memory usage ratio ≤ 5.0%
+[Device Memory Status] Current usage:
+  Device 0: 0.7GiB/184.3GiB (0.4%)
+  Device 1: 0.7GiB/184.3GiB (0.4%)
+  Device 2: 0.7GiB/184.3GiB (0.4%)
+  Device 3: 0.7GiB/184.3GiB (0.4%)
+[Device Memory Freed] Device(s) 0, 1, 2, 3 meet memory condition
+   Condition: Memory usage ratio ≤ 5.0%
+   Wait time: 0.0 seconds (0.0 minutes)
+Post-test device status:
+
+================================================================================
+NVIDIA GPU Information (nvidia-smi)
+================================================================================
+Mon Sep 21 08:46:17 2026       
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 580.126.09             Driver Version: 580.126.09     CUDA Version: 13.0     |
++-----------------------------------------+------------------------+----------------------+
+| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
+|                                         |                        |               MIG M. |
+|=========================================+========================+======================|
+|   0  NVIDIA GB200                   Off |   00000000:89:00.0 Off |                    0 |
+| N/A   27C    P0            219W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   1  NVIDIA GB200                   Off |   00000000:8A:00.0 Off |                    0 |
+| N/A   26C    P0            229W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+|   2  NVIDIA GB200                   Off |   00000000:C1:00.0 Off |                    0 |
+| N/A   26C    P0            201W / 1200W |     695MiB / 189471MiB |      0%      Default |
+|                                         |                        |             Disabled |
++-----------------------------------------+------------------------+----------------------+
+... (showing first 20 of 35 lines)
+
+================================================================================
+Detailed GPU Processes (nvidia-smi pmon)
+================================================================================
+# gpu         pid   type     sm    mem    enc    dec    jpg    ofa    command 
+# Idx           #    C/G      %      %      %      %      %      %    name 
+    0    2853490     C      -      -      -      -      -      -    python         
+    1    2853490     C      0      0      -      -      -      -    python         
+    2    2853490     C      0      0      -      -      -      -    python         
+    3    2853490     C      0      0      -      -      -      -    python
+OmniServer stopped
+
+
+=============================== warnings summary ===============================
+.venv/lib/python3.12/site-packages/torch/jit/_script.py:365: 14 warnings
+  /home/mozf/vllm-omni/.venv/lib/python3.12/site-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`.
+    warnings.warn(
+
+vllm_omni/version.py:55
+  /home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+   --> vLLM-Omni version 0.26.1.dev1+g76de642c5
+   --> vLLM version 0.29.0
+  This will likely cause compatibility issues.
+    warn_if_misaligned_vllm_version()
+
+-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
+--- Running Summary
+================== 4 passed, 15 warnings in 173.25s (0:02:53) ==================

artifacts/pr7759-review/gen_images_30094.out

@@ -0,0 +1,204 @@
+node: gb200-rack1-01
+=== T2I ===
+/home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.26.1.dev1+g76de642c5
+ --> vLLM version 0.29.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+INFO 09-21 08:49:45 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-21 08:49:46 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL).
+INFO 09-21 08:49:46 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM.
+INFO 09-21 08:49:46 [patch.py:503] inductor factorable-divisibility patch: installed.
+INFO 09-21 08:49:46 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-21 08:49:47 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell
+INFO 09-21 08:49:49 [weight_utils.py:58] Using model weights format ['*']
+INFO 09-21 08:49:49 [omni_base.py:208] [Omni] Initializing with model Qwen/Qwen-Image-2.1
+INFO 09-21 08:49:49 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1
+INFO 09-21 08:49:51 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'.
+INFO 09-21 08:49:51 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages
+INFO 09-21 08:49:51 [stage_init_utils.py:775] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+INFO 09-21 08:49:51 [weight_utils.py:58] Using model weights format ['*']
+INFO 09-21 08:49:51 [weight_utils.py:58] Using model weights format ['*']
+INFO 09-21 08:49:51 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
+INFO 09-21 08:49:51 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+INFO 09-21 08:49:51 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
+INFO 09-21 08:49:51 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment.
+INFO 09-21 08:49:51 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True).
+INFO 09-21 08:49:51 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0]
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:52 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor']
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:52 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock
+(DiffusionWorker pid=2877460) ␍Fetching 22 files:   0%|          | 0/22 [00:00<?, ?it/s]␍Fetching 22 files: 100%|██████████| 22/22 [00:00<00:00, 1979.08it/s]
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:52 [hub_prefetch.py:344] Prefetch complete for Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2877460) /home/mozf/vllm-omni/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py:205: UserWarning: The `local_dir_use_symlinks` argument is deprecated and ignored in `hf_hub_download`. Downloading to a local directory does not use symlinks anymore.
+(DiffusionWorker pid=2877460)   warnings.warn(
+(DiffusionWorker pid=2877460) ␍Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]␍Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 1477.00it/s]
+(DiffusionWorker pid=2877460) ␍Loading weights:   0%|          | 0/750 [00:00<?, ?it/s]␍Loading weights:   3%|▎         | 23/750 [00:00<00:03, 225.47it/s]␍Loading weights:   6%|▋         | 48/750 [00:00<00:02, 237.05it/s]␍Loading weights:  10%|▉         | 72/750 [00:00<00:03, 225.49it/s]␍Loading weights:  13%|█▎        | 101/750 [00:00<00:02, 249.63it/s]␍Loading weights:  17%|█▋        | 127/750 [00:00<00:02, 232.96it/s]␍Loading weights:  21%|██        | 158/750 [00:00<00:02, 248.63it/s]␍Loading weights:  25%|██▍       | 184/750 [00:00<00:02, 243.05it/s]␍Loading weights:  28%|██▊       | 213/750 [00:00<00:02, 250.02it/s]␍Loading weights:  32%|███▏      | 239/750 [00:00<00:02, 245.90it/s]␍Loading weights:  36%|███▌      | 268/750 [00:01<00:01, 255.40it/s]␍Loading weights:  39%|███▉      | 294/750 [00:01<00:01, 252.61it/s]␍Loading weights:  43%|████▎     | 323/750 [00:01<00:01, 261.60it/s]␍Loading weights:  47%|████▋     | 350/750 [00:01<00:01, 260.07it/s]␍Loading weights:  50%|█████     | 378/750 [00:01<00:01, 261.11it/s]␍Loading weights:  62%|██████▏   | 467/750 [00:01<00:00, 442.11it/s]␍Loading weights:  95%|█████████▌| 716/750 [00:01<00:00, 1039.01it/s]␍Loading weights: 100%|██████████| 750/750 [00:01<00:00, 428.30it/s] 
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:55 [platform.py:338] Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_100, cuDNN 92000, head_dim 128)
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:55 [selector.py:93] Resolved diffusion attention backend 'CUDNN_ATTN' for role='self' via platform default
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:57 [weight_utils.py:58] Using model weights format ['transformer/diffusion_pytorch_model-00001-of-00002.safetensors', 'transformer/diffusion_pytorch_model-00002-of-00002.safetensors']
+(DiffusionWorker pid=2877460) ␍Multi-thread loading shards:   0% Completed | 0/2 [00:00<?, ?it/s]
+(DiffusionWorker pid=2877460) ␍Multi-thread loading shards:  50% Completed | 1/2 [00:00<00:00,  1.88it/s]
+(DiffusionWorker pid=2877460) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:01<00:00,  1.05it/s]
+(DiffusionWorker pid=2877460) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:01<00:00,  1.12it/s]
+(DiffusionWorker pid=2877460) 
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:59 [diffusers_loader.py:1098] Loading weights took 1.86 seconds
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:59 [diffusion_model_runner.py:351] Model loading took 30.2190 GiB and 8.080112 seconds
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:59 [diffusion_model_runner.py:357] Model runner: Model loaded successfully.
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:59 [diffusion_model_runner.py:243] Model runner: transformer uses CUDA graph decode instead of torch.compile.
+(DiffusionWorker pid=2877460) INFO 09-21 08:49:59 [diffusion_model_runner.py:448] Model runner: Initialization complete.
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:00 [diffusion_worker.py:425] Worker 0: Process-scoped GPU memory after model loading: 31.01 GiB.
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:00 [manager.py:112] Initializing DiffusionLoRAManager: device=cuda:0, dtype=torch.bfloat16, max_cached_adapters=1, static_lora_path=None
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:00 [diffusion_worker.py:302] Worker 0: Initialization complete.
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:00 [uniproc_executor.py:74] Diffusion worker initialized in-process (uniproc executor)
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:00 [diffusion_engine.py:402] [RequestBatch] engine init max_num_seqs=1 max_wait_ms=0.0
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:00 [diffusion_engine.py:1289] dummy run to warm up the model
+(DiffusionWorker pid=2877460) ␍  0%|          | 0/2 [00:00<?, ?it/s]␍ 50%|█████     | 1/2 [00:00<00:00,  4.72it/s](DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 10, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+␍100%|██████████| 2/2 [00:00<00:00,  4.59it/s]␍100%|██████████| 2/2 [00:00<00:00,  4.60it/s]
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [inline_stage_diffusion_client.py:82] [InlineStageDiffusionClient] stage-0 [rep-0] initialized inline (max_num_seqs=1)
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [stage_runtime.py:1226] [StageRuntime] Stage 0 replica 0 initialized (diffusion)
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [orchestrator.py:420] [Orchestrator] Starting event loop
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [omni_engine_base.py:344] [OmniEngine] Orchestrator ready with 1 stages
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [omni_base.py:227] [Omni] AsyncOmniEngine initialized in 11.73 seconds
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [omni_base.py:246] [Omni] Initialized with 1 stages for model Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2877460) 
+(DiffusionWorker pid=2877460) ============================================================
+(DiffusionWorker pid=2877460) Generation Configuration:
+(DiffusionWorker pid=2877460)   Model: Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2877460)   Inference steps: 20
+(DiffusionWorker pid=2877460)   Cache backend: None (no acceleration)
+(DiffusionWorker pid=2877460)   Quantization: None (BF16)
+(DiffusionWorker pid=2877460)   Parallel configuration: tensor_parallel_size=deploy/default, ulysses_degree=1, ulysses_mode=strict, ring_degree=1, cfg_parallel_size=1, vae_patch_parallel_size=1, enable_expert_parallel=False.
+(DiffusionWorker pid=2877460)   CPU offload: False; CPU Layerwise Offload: False
+(DiffusionWorker pid=2877460)   Image size: 1024x1024
+(DiffusionWorker pid=2877460) ============================================================
+(DiffusionWorker pid=2877460) 
+(DiffusionWorker pid=2877460) ␍Processed prompts:   0%|          | 0/1 [00:00<?, ?it/s]
+(DiffusionWorker pid=2877460) ␍  0%|          | 0/20 [00:00<?, ?it/s]
+(DiffusionWorker pid=2877460) ␍  5%|▌         | 1/20 [00:00<00:05,  3.22it/s](DiffusionWorker pid=2877460) INFO 09-21 08:50:01 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 18, ((1, 64, 64),), torch.bfloat16, 0, 'CUDNN_ATTN')
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:02 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('uncond', 1, 13, ((1, 64, 64),), torch.bfloat16, 0, 'CUDNN_ATTN')
+
+(DiffusionWorker pid=2877460) ␍ 10%|█         | 2/20 [00:00<00:08,  2.03it/s]
+(DiffusionWorker pid=2877460) ␍ 15%|█▌        | 3/20 [00:01<00:05,  2.96it/s]
+(DiffusionWorker pid=2877460) ␍ 20%|██        | 4/20 [00:01<00:04,  3.77it/s]
+(DiffusionWorker pid=2877460) ␍ 25%|██▌       | 5/20 [00:01<00:03,  4.44it/s]
+(DiffusionWorker pid=2877460) ␍ 30%|███       | 6/20 [00:01<00:02,  5.00it/s]
+(DiffusionWorker pid=2877460) ␍ 35%|███▌      | 7/20 [00:01<00:02,  5.43it/s]
+(DiffusionWorker pid=2877460) ␍ 40%|████      | 8/20 [00:01<00:02,  5.77it/s]
+(DiffusionWorker pid=2877460) ␍ 45%|████▌     | 9/20 [00:01<00:01,  6.01it/s]
+(DiffusionWorker pid=2877460) ␍ 50%|█████     | 10/20 [00:02<00:01,  6.21it/s]
+(DiffusionWorker pid=2877460) ␍ 55%|█████▌    | 11/20 [00:02<00:01,  6.35it/s]
+(DiffusionWorker pid=2877460) ␍ 60%|██████    | 12/20 [00:02<00:01,  6.40it/s]
+(DiffusionWorker pid=2877460) ␍ 65%|██████▌   | 13/20 [00:02<00:01,  6.46it/s]
+(DiffusionWorker pid=2877460) ␍ 70%|███████   | 14/20 [00:02<00:00,  6.53it/s]
+(DiffusionWorker pid=2877460) ␍ 75%|███████▌  | 15/20 [00:02<00:00,  6.57it/s]
+(DiffusionWorker pid=2877460) ␍ 80%|████████  | 16/20 [00:03<00:00,  6.57it/s]
+(DiffusionWorker pid=2877460) ␍ 85%|████████▌ | 17/20 [00:03<00:00,  6.57it/s]
+(DiffusionWorker pid=2877460) ␍ 90%|█████████ | 18/20 [00:03<00:00,  6.60it/s]
+(DiffusionWorker pid=2877460) ␍ 95%|█████████▌| 19/20 [00:03<00:00,  6.61it/s]
+(DiffusionWorker pid=2877460) ␍100%|██████████| 20/20 [00:03<00:00,  6.63it/s]␍100%|██████████| 20/20 [00:03<00:00,  5.47it/s]
+(DiffusionWorker pid=2877460) ␍Processed prompts: 100%|██████████| 1/1 [00:03<00:00,  3.94s/it]␍Processed prompts: 100%|██████████| 1/1 [00:03<00:00,  3.94s/it]
+(DiffusionWorker pid=2877460) Total generation time: 3.9453 seconds (3945.34 ms)
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:05 [text_to_image.py:794] Outputs: [OmniRequestOutput(request_id='0_770e0883-477e-4c55-8e47-3328e985b949', finished=True, stage_id=0, final_output_type='image', prompt_token_ids=None, outputs=[], images=[1 PIL Images], prompt={'prompt': 'A red ceramic teapot on a wooden table.', 'modalities': ['image'], 'negative_prompt': 'blurry, low quality', 'additional_information': {}}, latents=None, metrics={'stage_metrics': {'0': {'stage_id': 0, 'final_output_type': 'image', 'finish_reason': None, 'num_tokens_in': 0, 'num_tokens_out': 0, 'stage_gen_time_ms': 3943.441867828369, 'postprocess_time_ms': 0.0, 'audio_frames': 0, 'audio_sample_rate': 0, 'audio_duration_s': 0.0, 'image_pixels': 1048576, 'denoise_step_latency_ms': 197.17209339141846, 'output_unit_type': 'image', 'output_unit_count': 1, 'serving_time_to_first_output_ms': 3943.847417831421, 'image_time_to_first_output_ms': 3943.420171737671, 'time_per_output_unit_ms': 0.0, 'inter_output_latency_ms': 0.0, 'inter_output_latencies_ms': [], 'vllm_ttft_ms': 0.0, 'vllm_tpot_ms': 0.0, 'vllm_itl_ms': 0.0, 'vllm_itls_ms': [], 'stage_name': 'diffusion'}}, 'stage_id': 0, 'final_output_type': 'image', 'num_tokens_in': 0, 'num_tokens_out': 0}, multimodal_output={}, custom_output={}, stage_durations={'queue_wait_ms': 0.4212159983580932, 'stage_0_gen_ms': 3943.441867828369}, peak_memory_mb=40320.0)]
+(DiffusionWorker pid=2877460) Saved generated image to /home/mozf/vllm-omni/artifacts/pr7759-review/e2e_images/t2i_teapot.png
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:05 [omni_engine_base.py:1144] [OmniEngine] Shutting down Orchestrator
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:05 [orchestrator.py:512] [Orchestrator] Received shutdown signal
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:05 [orchestrator.py:2720] [Orchestrator] Shutting down all 1 client(s)
+(DiffusionWorker pid=2877460) INFO 09-21 08:50:06 [stage_pool.py:1362] [StagePool] Stage 0 replica 0 shut down
+=== I2I (edit the T2I output) ===
+/home/mozf/vllm-omni/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.26.1.dev1+g76de642c5
+ --> vLLM version 0.29.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+INFO 09-21 08:50:24 [patch.py:252] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-21 08:50:26 [quack_fp8.py:76] Quack FP8 fused-bias GEMM enabled (CuteDSL).
+INFO 09-21 08:50:26 [quack_fp8.py:155] Patched FlashInfer FP8 ScaledMM to use quack fused-bias GEMM.
+INFO 09-21 08:50:26 [patch.py:503] inductor factorable-divisibility patch: installed.
+INFO 09-21 08:50:26 [patch.py:568] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-21 08:50:29 [fa.py:105] Using CuTe FlashAttention-4 on Blackwell
+INFO 09-21 08:50:31 [weight_utils.py:58] Using model weights format ['*']
+INFO 09-21 08:50:31 [omni_base.py:208] [Omni] Initializing with model Qwen/Qwen-Image-2.1
+INFO 09-21 08:50:31 [omni_engine_base.py:163] [OmniEngine] Initializing with model Qwen/Qwen-Image-2.1
+INFO 09-21 08:50:33 [config_factory.py:697] Resolved generic diffusion final_output_type='image' for model_class_name='QwenImage21Pipeline'.
+INFO 09-21 08:50:33 [omni_engine_base.py:312] [OmniEngine] Launching Orchestrator thread with 1 stages
+INFO 09-21 08:50:33 [stage_init_utils.py:775] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+INFO 09-21 08:50:33 [weight_utils.py:58] Using model weights format ['*']
+INFO 09-21 08:50:33 [weight_utils.py:58] Using model weights format ['*']
+INFO 09-21 08:50:33 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
+INFO 09-21 08:50:33 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
+INFO 09-21 08:50:33 [diffusion_worker.py:332] Final IR op priority after setting vLLM-Omni overrides: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
+INFO 09-21 08:50:33 [diffusion_worker.py:344] Worker 0: Initialized device and distributed environment.
+INFO 09-21 08:50:33 [parallel_state.py:610] Building SP subgroups from explicit sp_group_ranks (sp_size=1, ulysses=1, ring=1, use_ulysses_low=True).
+INFO 09-21 08:50:33 [parallel_state.py:652] SP group details for rank 0: sp_group=[0], ulysses_group=[0], ring_group=[0]
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:34 [hub_prefetch.py:299] Prefetching Qwen/Qwen-Image-2.1 subfolders: ['scheduler', 'text_encoder', 'vae', 'processor']
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:34 [hub_prefetch.py:224] Acquired flock prefetch lock for Qwen/Qwen-Image-2.1 at /mnt/lustre/hf-models/locks/vllm-omni-prefetch/Qwen__Qwen-Image-2.1.lock
+(DiffusionWorker pid=2879936) ␍Fetching 22 files:   0%|          | 0/22 [00:00<?, ?it/s]␍Fetching 22 files: 100%|██████████| 22/22 [00:00<00:00, 1980.48it/s]
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:34 [hub_prefetch.py:344] Prefetch complete for Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2879936) /home/mozf/vllm-omni/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py:205: UserWarning: The `local_dir_use_symlinks` argument is deprecated and ignored in `hf_hub_download`. Downloading to a local directory does not use symlinks anymore.
+(DiffusionWorker pid=2879936)   warnings.warn(
+(DiffusionWorker pid=2879936) ␍Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]␍Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 1553.88it/s]
+(DiffusionWorker pid=2879936) ␍Loading weights:   0%|          | 0/750 [00:00<?, ?it/s]␍Loading weights:   3%|▎         | 22/750 [00:00<00:03, 218.48it/s]␍Loading weights:   6%|▋         | 48/750 [00:00<00:03, 223.49it/s]␍Loading weights:  10%|▉         | 72/750 [00:00<00:02, 229.47it/s]␍Loading weights:  13%|█▎        | 99/750 [00:00<00:02, 244.50it/s]␍Loading weights:  17%|█▋        | 124/750 [00:00<00:02, 243.42it/s]␍Loading weights:  20%|█▉        | 149/750 [00:00<00:02, 234.90it/s]␍Loading weights:  24%|██▍       | 180/750 [00:00<00:02, 249.70it/s]␍Loading weights:  27%|██▋       | 205/750 [00:00<00:02, 240.97it/s]␍Loading weights:  31%|███▏      | 235/750 [00:00<00:02, 252.01it/s]␍Loading weights:  35%|███▍      | 261/750 [00:01<00:01, 247.02it/s]␍Loading weights:  39%|███▊      | 290/750 [00:01<00:01, 253.05it/s]␍Loading weights:  42%|████▏     | 316/750 [00:01<00:01, 249.97it/s]␍Loading weights:  46%|████▌     | 345/750 [00:01<00:01, 250.71it/s]␍Loading weights:  50%|████▉     | 373/750 [00:01<00:01, 257.52it/s]␍Loading weights:  57%|█████▋    | 424/750 [00:01<00:00, 328.28it/s]␍Loading weights:  85%|████████▍ | 635/750 [00:01<00:00, 842.34it/s]␍Loading weights: 100%|██████████| 750/750 [00:01<00:00, 417.44it/s]
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:37 [platform.py:338] Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_100, cuDNN 92000, head_dim 128)
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:37 [selector.py:93] Resolved diffusion attention backend 'CUDNN_ATTN' for role='self' via platform default
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:39 [weight_utils.py:58] Using model weights format ['transformer/diffusion_pytorch_model-00001-of-00002.safetensors', 'transformer/diffusion_pytorch_model-00002-of-00002.safetensors']
+(DiffusionWorker pid=2879936) ␍Multi-thread loading shards:   0% Completed | 0/2 [00:00<?, ?it/s]
+(DiffusionWorker pid=2879936) ␍Multi-thread loading shards:  50% Completed | 1/2 [00:00<00:00,  1.87it/s]
+(DiffusionWorker pid=2879936) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:01<00:00,  1.05it/s]
+(DiffusionWorker pid=2879936) ␍Multi-thread loading shards: 100% Completed | 2/2 [00:01<00:00,  1.12it/s]
+(DiffusionWorker pid=2879936) 
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:41 [diffusers_loader.py:1098] Loading weights took 1.86 seconds
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_model_runner.py:351] Model loading took 30.2190 GiB and 8.371849 seconds
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_model_runner.py:357] Model runner: Model loaded successfully.
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_model_runner.py:243] Model runner: transformer uses CUDA graph decode instead of torch.compile.
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_model_runner.py:448] Model runner: Initialization complete.
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_worker.py:425] Worker 0: Process-scoped GPU memory after model loading: 31.01 GiB.
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [manager.py:112] Initializing DiffusionLoRAManager: device=cuda:0, dtype=torch.bfloat16, max_cached_adapters=1, static_lora_path=None
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_worker.py:302] Worker 0: Initialization complete.
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [uniproc_executor.py:74] Diffusion worker initialized in-process (uniproc executor)
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_engine.py:402] [RequestBatch] engine init max_num_seqs=1 max_wait_ms=0.0
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:42 [diffusion_engine.py:1289] dummy run to warm up the model
+(DiffusionWorker pid=2879936) ␍  0%|          | 0/2 [00:00<?, ?it/s]␍ 50%|█████     | 1/2 [00:00<00:00,  4.82it/s](DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 10, ((1, 32, 32),), torch.bfloat16, 0, 'CUDNN_ATTN')
+␍100%|██████████| 2/2 [00:00<00:00,  4.93it/s]␍100%|██████████| 2/2 [00:00<00:00,  4.91it/s]
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [inline_stage_diffusion_client.py:82] [InlineStageDiffusionClient] stage-0 [rep-0] initialized inline (max_num_seqs=1)
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [stage_runtime.py:1226] [StageRuntime] Stage 0 replica 0 initialized (diffusion)
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [orchestrator.py:420] [Orchestrator] Starting event loop
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [omni_engine_base.py:344] [OmniEngine] Orchestrator ready with 1 stages
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [omni_base.py:227] [Omni] AsyncOmniEngine initialized in 12.25 seconds
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:43 [omni_base.py:246] [Omni] Initialized with 1 stages for model Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2879936) Pipeline loaded
+(DiffusionWorker pid=2879936) 
+(DiffusionWorker pid=2879936) ============================================================
+(DiffusionWorker pid=2879936) Generation Configuration:
+(DiffusionWorker pid=2879936)   Model: Qwen/Qwen-Image-2.1
+(DiffusionWorker pid=2879936)   Inference steps: 20
+(DiffusionWorker pid=2879936)   Cache backend: None (no acceleration)
+(DiffusionWorker pid=2879936)   Input image size: (1024, 1024)
+(DiffusionWorker pid=2879936)   Parallel configuration: ulysses_degree=1, ring_degree=1, cfg_parallel_size=1, tensor_parallel_size=1, enable_expert_parallel: False
+(DiffusionWorker pid=2879936) ============================================================
+(DiffusionWorker pid=2879936) 
+(DiffusionWorker pid=2879936) ␍Processed prompts:   0%|          | 0/1 [00:00<?, ?it/s]
+(DiffusionWorker pid=2879936) ␍  0%|          | 0/20 [00:00<?, ?it/s]
+(DiffusionWorker pid=2879936) ␍  5%|▌         | 1/20 [00:00<00:12,  1.52it/s](DiffusionWorker pid=2879936) INFO 09-21 08:50:44 [decode_graph.py:446] Captured Qwen-Image-2.1 decode graph for key=('cond', 1, 4116, ((1, 64, 64), (1, 64, 64)), torch.bfloat16, 0, 'CUDNN_ATTN')
+
+(DiffusionWorker pid=2879936) ␍ 10%|█         | 2/20 [00:00<00:08,  2.24it/s]
+(DiffusionWorker pid=2879936) ␍ 20%|██        | 4/20 [00:01<00:03,  4.57it/s]
+(DiffusionWorker pid=2879936) ␍ 30%|███       | 6/20 [00:01<00:02,  6.40it/s]
+(DiffusionWorker pid=2879936) ␍ 40%|████      | 8/20 [00:01<00:01,  7.85it/s]
+(DiffusionWorker pid=2879936) ␍ 50%|█████     | 10/20 [00:01<00:01,  8.95it/s]
+(DiffusionWorker pid=2879936) ␍ 60%|██████    | 12/20 [00:01<00:00,  9.76it/s]
+(DiffusionWorker pid=2879936) ␍ 70%|███████   | 14/20 [00:01<00:00, 10.40it/s]
+(DiffusionWorker pid=2879936) ␍ 80%|████████  | 16/20 [00:02<00:00, 10.87it/s]
+(DiffusionWorker pid=2879936) ␍ 90%|█████████ | 18/20 [00:02<00:00, 11.19it/s]
+(DiffusionWorker pid=2879936) ␍100%|██████████| 20/20 [00:02<00:00, 11.42it/s]␍100%|██████████| 20/20 [00:02<00:00,  8.09it/s]
+(DiffusionWorker pid=2879936) ␍Processed prompts: 100%|██████████| 1/1 [00:02<00:00,  3.00s/it]␍Processed prompts: 100%|██████████| 1/1 [00:02<00:00,  3.00s/it]
+(DiffusionWorker pid=2879936) Total generation time: 2.9985 seconds (2998.46 ms)
+(DiffusionWorker pid=2879936) Saved edited image to /home/mozf/vllm-omni/artifacts/pr7759-review/e2e_images/i2i_teapot_blue.png
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:46 [omni_engine_base.py:1144] [OmniEngine] Shutting down Orchestrator
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:46 [orchestrator.py:512] [Orchestrator] Received shutdown signal
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:46 [orchestrator.py:2720] [Orchestrator] Shutting down all 1 client(s)
+(DiffusionWorker pid=2879936) INFO 09-21 08:50:47 [stage_pool.py:1362] [StagePool] Stage 0 replica 0 shut down
+total 1589
+drwxrwxr-x 2 mozf mozf    4096 Sep 21 08:50 .
+drwxrwxr-x 3 mozf mozf    4096 Sep 21 08:49 ..
+-rw-rw-r-- 1 mozf mozf 2279235 Sep 21 08:50 i2i_teapot_blue.png
+-rw-rw-r-- 1 mozf mozf 1615157 Sep 21 08:50 t2i_teapot.png

artifacts/pr7759-review/gen_images_slurm.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#SBATCH --partition=batch
+#SBATCH --nodes=1
+#SBATCH --time=60
+#SBATCH --job-name=qi21-imgout
+#SBATCH --output=/home/mozf/vllm-omni/artifacts/pr7759-review/gen_images_%j.out
+
+set -euo pipefail
+cd /home/mozf/vllm-omni
+
+export HF_HOME=/mnt/lustre/hf-models
+export VLLM_LOGGING_LEVEL=INFO
+OUT=/home/mozf/vllm-omni/artifacts/pr7759-review/e2e_images
+mkdir -p "$OUT"
+
+echo "node: $(hostname)"
+
+PY=.venv/bin/python
+
+echo "=== T2I ==="
+$PY examples/offline_inference/text_to_image/text_to_image.py \
+  --model Qwen/Qwen-Image-2.1 \
+  --prompt "A red ceramic teapot on a wooden table." \
+  --negative-prompt "blurry, low quality" \
+  --seed 42 \
+  --height 1024 --width 1024 \
+  --num-inference-steps 20 \
+  --output "$OUT/t2i_teapot.png"
+
+echo "=== I2I (edit the T2I output) ==="
+$PY examples/offline_inference/image_to_image/image_edit.py \
+  --model Qwen/Qwen-Image-2.1 \
+  --image "$OUT/t2i_teapot.png" \
+  --prompt "Make the teapot blue." \
+  --seed 42 \
+  --num-inference-steps 20 \
+  --cfg-scale 1.0 \
+  --output "$OUT/i2i_teapot_blue.png"
+
+ls -la "$OUT"

benchmarks/README.md

@@ -41,6 +41,17 @@ LingBot-Video MoE transformer.
 - **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings
 - **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity
 
+### [LingBot-World](lingbot_world/README.md) — Realtime Streaming Cadence
+
+Client-side benchmark for a `WS /v1/realtime/video` world-model session, where one
+request is a whole rollout and one AR block streams out as one video chunk. It
+measures a cadence rather than a throughput: there is exactly one session, and what
+matters is whether the next chunk arrives before the viewer finishes the last one.
+
+- **Layout**: `lingbot_world/benchmark_lingbot_world_realtime.py` (WS client), `lingbot_world/workload.py` (rollout and metric math), `lingbot_world/configs/` (single-GPU eager and Ulysses-4 compiled deploy configs)
+- **Workload**: built-in image-conditioned rollout with a generated camera script, or a JSON spec with per-chunk actions and mid-rollout prompt updates
+- **Key metrics**: TTFC (time to first chunk), steady-state chunk inter-arrival percentiles, VIDEO_RTF (wall seconds per video second; lower is better), chunk-deadline attainment, and simulated playback underruns
+
 ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing
 
 RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks.

benchmarks/lingbot_world/README.md

@@ -0,0 +1,224 @@
+# LingBot-World Benchmarks — Realtime Streaming Cadence
+
+`benchmark_lingbot_world_realtime.py` measures LingBot-World 2.0 as it is
+actually deployed: one `WS /v1/realtime/video` session that streams one video
+chunk per AR block, for as long as the world keeps running.
+
+## Why this is not the diffusion serving benchmark
+
+[`benchmarks/diffusion`](../diffusion/README.md) measures "one request, one
+finished video" and reports request throughput. LingBot-World has neither shape:
+
+- **One request is the whole rollout.** `prepare_encode` runs once, then every AR
+  block is four DMD steps plus one clean-KV commit forward, and each block
+  streams out as a chunk. Throughput in requests per second is meaningless.
+- **There is exactly one session.** The stepwise deploy config pins
+  `max_num_seqs: 1`, and the AR-Diffusion consumer rejects a stage with more than
+  one replica, because session-affine routing does not exist yet. There is no
+  concurrency to sweep.
+- **What matters is cadence.** A deployment is healthy when chunk *N + 1* arrives
+  before a viewer finishes watching chunk *N*. That is a latency distribution
+  against a wall clock, not a throughput number.
+
+## Quick start
+
+Serve the model, then point the benchmark at it. The benchmark never starts a
+server.
+
+```bash
+# Terminal 1 — one GPU, eager (portable smoke configuration)
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+  --port 8000
+
+# Terminal 2
+pip install websockets
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --host 127.0.0.1 --port 8000 \
+  --num-chunks 16 \
+  --output-json /tmp/lingbot_world_realtime.json
+```
+
+With no `--image`, the first frame comes from the shared vLLM image asset, so the
+default workload needs no setup and this repository carries no benchmark binary.
+Pass `--image /path/to/frame.png` to condition the rollout on your own frame.
+
+### Realtime topology
+
+The single-GPU config is a smoke configuration, not a realtime one: one card runs
+all five forwards of every block serially and nothing is compiled. Use the
+four-GPU config for numbers that mean anything about realtime behavior.
+
+```bash
+vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+  --omni \
+  --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+  --port 8000
+
+python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \
+  --port 8000 --num-chunks 40 --warmup-sessions 1 --sessions 3 \
+  --target-fps 12 --output-json /tmp/lingbot_world_repeated.json
+```
+
+`usp4_compiled.yaml` sets pure Ulysses sequence parallelism over four ranks and
+`enforce_eager: false`. The pipeline validates the parallel shape itself:
+`sequence_parallel_size` must equal `ulysses_degree`, `ring_degree` and
+`allgather_degree` must be 1, `ulysses_mode` must be `strict`, and pipeline,
+CFG, and VAE parallel sizes above 1 are rejected. This benchmark config sets
+`tensor_parallel_size: 1`; the pipeline validator does not reject larger TP sizes.
+
+Compiled mode can pay compilation and capture costs on the first rollout.
+Use `--warmup-sessions 1` before measured repetitions. Excluding the first six
+chunks addresses the attention-window ramp; it does not guarantee that
+compilation and capture have finished.
+
+## Metrics
+
+| Metric | Meaning |
+| --- | --- |
+| `ttfc_ms` | **Time to first chunk**: `session.start` sent until the first chunk's bytes arrive. Pays for prompt encode, first-frame VAE encode, and any compile or capture. |
+| `interval_all` | Inter-arrival latency across every chunk: mean, median, std, min, p90, p95, p99, max. |
+| `interval_steady` | The same summary over steady-state chunks only. |
+| `video_rtf` | Wall seconds spent per second of video, matching vLLM-Omni's own RTF (`generation_s / video_duration`). **Lower is better; below 1.0 is real time.** |
+| `video_rtfx` | The reciprocal, under the name this repo uses for it: video seconds per wall second, where higher is better. |
+| `steady_video_rtf` | `video_rtf` restricted to steady-state chunks. |
+| `chunk_deadline_ms` | Wall-clock budget one steady chunk must meet to hold real time: 12 frames ÷ `--target-fps`. Override directly with `--chunk-slo-ms`. |
+| `slo_attainment` | Fraction of steady chunks that met the deadline. |
+| `playback` | Underrun count, total stall, and stall ratio from a simulated viewer. |
+
+### Which frame rate counts as real time
+
+**The checkpoint declares no frame rate.** There is none in `model_index.json`, the
+scheduler config, or the transformer config, and the pipeline never reads one. A
+chunk is twelve pixel frames; how long those frames *last* is a property of how the
+output is played, not of the model.
+
+That choice moves the verdict without moving anything measured:
+
+| basis | video per chunk | deadline | RTF at a measured 1357 ms/chunk |
+| --- | --- | --- | --- |
+| 16 fps | 0.750 s | 750 ms | 1.809 |
+| 12 fps | 1.000 s | 1000 ms | 1.357 |
+
+So `--fps` (sent to the server, which uses it to label the muxed MP4) and
+`--target-fps` (the real-time basis for RTF, the deadline, and the playback
+simulation) are separate flags. `--target-fps` defaults to `--fps`; set it when the
+two differ, and the report prints both. State the basis whenever you quote a number
+— an RTF without its fps is not a measurement. (RTF is a cost here, so a bigger number is a
+worse result; a steady chunk's RTF is simply its interval divided by its video duration.)
+An RTF above 1 means the measured deployment is slower than real time at the
+stated target frame rate. The benchmark measures this failure as well as success.
+
+### Why warmup chunks are not optional
+
+The DiT attends over a sliding window of `sliding_window_num_frames = 18` latent
+frames with `sink_size = 9`, so the key count a block attends over keeps growing
+until eighteen latent frames of history exist — six chunks at three frames per
+block. **Chunk cost is not stationary before chunk six.** A mean over a short
+rollout is therefore lower than anything a running session will sustain, which is
+why `--warmup-chunks` defaults to 6 and steady-state metrics are reported
+separately. Keep `--num-chunks` comfortably above the warmup, or the benchmark
+tells you no chunk reached steady state. The terminal chunk skips next-chunk
+preparation, so it is excluded from steady metrics too; it remains in overall
+RTF, all-chunk intervals, and playback simulation.
+
+The default 16-chunk rollout provides only nine steady intervals. Percentiles
+are interpolated sample summaries; the report warns when fewer than 100
+intervals contribute, and even 100 samples do not guarantee a stable p99.
+Use longer rollouts for tail analysis and at least `--sessions 3` after
+`--warmup-sessions 1` for reported comparisons. Session warmup pays compilation
+cost; chunk warmup separately excludes the attention-window ramp. The command
+above yields 33 steady intervals per session (indices 6 through 38), or 99 across
+three sessions. Report each session's mean and the spread across sessions; this
+sample still does not establish a reliable p99.
+
+### Why playback is simulated
+
+A p99 interval alone cannot say whether a viewer saw a stall: the same p99 is
+invisible behind a three-chunk buffer and a visible freeze behind none. The
+benchmark replays chunk arrivals against a wall clock — playback starts once
+`--playback-buffer-chunks` chunks have landed and then consumes video at `--target-fps`
+— and reports every moment the player ran dry.
+
+### What the client-side number is worth
+
+On a four-GPU Ulysses run the client's mean inter-arrival over all fifteen
+post-first-chunk intervals was **1326.743 ms**, against the server's own
+`StageRequestStats.inter_output_latency_ms` of **1327.479 ms** for the same
+request — a difference of **0.74 ms**. This cross-check shows agreement in
+average output cadence for that run, including warmup and the terminal chunk.
+It does not measure encoding or transport latency: a fixed delivery delay can
+cancel out of inter-arrival differences. Attributing that latency requires
+per-chunk timestamps at both boundaries or server-side profiling.
+
+Compare matching interval populations when investigating discrepancies; both
+measurement bugs and delivery buffering can change the observed cadence.
+
+### What this benchmark cannot see
+
+Chunk metadata carries no server-side timestamps, so the benchmark cannot split
+DiT time from VAE decode time; use `--enable-diffusion-pipeline-profiler` on the
+server for that. It also reports no GPU memory, which is a server-side quantity.
+
+## Custom rollouts
+
+`--workload rollout.json` replaces the built-in workload, including mid-rollout
+prompt changes, which the CLI flags cannot express:
+
+```json
+{
+  "prompt": "The camera moves slowly forward through the scene.",
+  "image": "/path/to/first_frame.png",
+  "num_chunks": 24,
+  "camera_pattern": "orbit",
+  "fps": 16,
+  "seed": 42,
+  "prompt_updates": [
+    {"after_chunk": 8, "prompt": "Rain begins to fall", "transition_chunks": 2}
+  ]
+}
+```
+
+Supply `camera_action_script` directly instead of `num_chunks` to control every
+frame's action. It must hold exactly one three-entry action list per chunk; one
+AR block is three latent frames, and the server rejects any other shape.
+Explicit `--image`, `--prompt`, `--negative-prompt`, `--width`, `--height`,
+`--fps`, `--seed`, and `--flow-shift` flags override the matching workload-file
+values. `--num-chunks` and `--camera-pattern` apply only to the built-in
+workload; when using `--workload`, set those fields in the file instead.
+An explicit `camera_action_script` takes precedence over the file's
+`num_chunks` and `camera_pattern`. Each prompt update must use a distinct
+`after_chunk` boundary so its interaction event ID is unique.
+
+Camera actions change what the world does, not what it costs, so `--camera-pattern`
+(`forward`, `orbit`, `hold`) keeps a rollout representative rather than sweeping a
+cost dimension.
+
+## Options worth knowing
+
+| Flag | Purpose |
+| --- | --- |
+| `--target-fps` | Playback rate the real-time verdict is measured against, independent of the mux label. |
+| `--sessions N` | N sequential measured rollouts; the report adds spread across them. |
+| `--warmup-sessions N` | Unmeasured rollouts first, to pay compile and capture cost. |
+| `--print-chunks` | One line per chunk as it arrives. |
+| `--save-video PATH` | Remux the last measured session for eyeballing (needs vLLM-Omni). |
+| `--ping-interval` | `session.ping` cadence keeping the server's stall clock fresh during a slow first chunk. `0` disables. |
+| `--first-chunk-timeout` | Raise it when the server is still compiling or capturing graphs. |
+
+## Output
+
+`--output-json` writes the run's configuration, every chunk's arrival time and
+size, per-session metrics, and the aggregate, along with the git revision and
+hostname so a number can be traced back to what produced it.
+
+## Tests
+
+`tests/benchmarks/test_lingbot_world_realtime.py` covers the chunk and frame
+arithmetic against the server's own formula, the metric math, and the protocol
+handling against a scripted WebSocket server. It needs no GPU:
+
+```bash
+pytest tests/benchmarks/test_lingbot_world_realtime.py
+```

benchmarks/lingbot_world/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py

@@ -0,0 +1,642 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Realtime streaming benchmark for LingBot-World 2.0 over ``WS /v1/realtime/video``.
+
+Every other diffusion benchmark in this repository measures "one request, one
+finished video". LingBot-World is not that: one request is a whole rollout, and
+the thing a deployment lives or dies by is whether chunk *N + 1* keeps arriving
+before a viewer has finished watching chunk *N*. This benchmark therefore
+measures a **cadence**, not a throughput:
+
+* ``TTFC`` -- session.start to the first chunk's bytes, which pays for prompt
+  encode, the first-frame VAE encode, and any compile or capture the deploy
+  config asks for;
+* **inter-arrival latency** over all chunks and, separately, over the steady
+  state after the attention window saturates;
+* ``VIDEO_RTF`` -- wall seconds spent per second of video generated, matching how
+  vLLM-Omni computes RTF everywhere else (``generation_s / video_duration``).
+  **Lower is better and below 1.0 is real time**; ``VIDEO_RTFX`` is reported
+  alongside it as the reciprocal, under the repo's own name for that;
+* **playback continuity** -- a simulated viewer with a start buffer, because a
+  p99 that is harmless behind a three-chunk buffer is a visible stall behind
+  none.
+
+The server is not started here. Bring one up first (see the README), then point
+this script at it. Requires ``websockets``; ``--save-video`` additionally needs
+a vLLM-Omni installation for the remux helper.
+
+Example:
+
+    python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \\
+      --host 127.0.0.1 --port 8000 \\
+      --num-chunks 16 \\
+      --output-json /tmp/lingbot_world_realtime.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import mimetypes
+import os
+import platform
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from benchmarks.lingbot_world.workload import (  # noqa: E402
+    CAMERA_PATTERNS,
+    DEFAULT_FLOW_SHIFT,
+    DEFAULT_FPS,
+    DEFAULT_HEIGHT,
+    DEFAULT_MODEL,
+    DEFAULT_PROMPT,
+    DEFAULT_SEED,
+    DEFAULT_WARMUP_CHUNKS,
+    DEFAULT_WIDTH,
+    ChunkRecord,
+    Workload,
+    aggregate_metrics,
+    compute_metrics,
+    default_workload,
+    load_workload,
+    steady_chunk_deadline_ms,
+)
+
+# The shared vLLM asset keeps the default workload runnable with no setup and
+# adds no binary to this repository. Any real photograph conditions a rollout.
+_DEFAULT_IMAGE_ASSET = "2560px-Gfp-wisconsin-madison-the-nature-boardwalk"
+_WS_PATH = "/v1/realtime/video"
+
+
+class BenchmarkError(RuntimeError):
+    """A session failed in a way that invalidates its numbers."""
+
+
+# --------------------------------------------------------------------------- #
+# Input resolution
+# --------------------------------------------------------------------------- #
+
+
+def resolve_image_reference(value: str | None) -> str:
+    """Return an ``http(s)`` or ``data:`` URL for the session's first frame."""
+    if value is None:
+        from vllm.assets.image import ImageAsset
+
+        value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg"))
+    if value.startswith(("http://", "https://", "data:")):
+        return value
+    path = Path(value).expanduser()
+    if not path.is_file():
+        raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}")
+    media_type = mimetypes.guess_type(path.name)[0] or "image/png"
+    return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}"
+
+
+def build_workload(args: argparse.Namespace) -> Workload:
+    if args.workload is not None:
+        # The file names its own first frame, so resolving happens inside
+        # load_workload -- never eagerly, which would fetch the default asset a
+        # --workload run has no use for.
+        return load_workload(
+            args.workload,
+            image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image),
+            overrides={
+                "width": args.width_override,
+                "height": args.height_override,
+                "fps": args.fps_override,
+                "seed": args.seed_override,
+                "prompt": args.prompt_override,
+                "flow_shift": args.flow_shift,
+                "negative_prompt": args.negative_prompt,
+            },
+        )
+    return default_workload(
+        image_reference=resolve_image_reference(args.image),
+        num_chunks=args.num_chunks,
+        camera_pattern=args.camera_pattern,
+        prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT,
+        width=args.width_override if args.width_override is not None else DEFAULT_WIDTH,
+        height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT,
+        fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS,
+        seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED,
+        flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT,
+        negative_prompt=args.negative_prompt,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# One streaming session
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class SessionResult:
+    request_id: str | None
+    records: list[ChunkRecord]
+    media: bytes
+    ttfc_s: float | None
+    video_start_s: float | None
+    wall_s: float
+    done_event: dict[str, Any]
+
+
+async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None:
+    """Refresh the server's stall clock while a slow first chunk is generating.
+
+    A compiled deploy config records CUDA graphs on the first block, which can
+    outlast the server's ~60 s stall timeout. ``session.ping`` is the documented
+    way to say the client is still there.
+    """
+    while True:
+        await asyncio.sleep(interval)
+        async with send_lock:
+            await websocket.send(json.dumps({"type": "session.ping"}))
+
+
+async def run_session(
+    url: str,
+    model: str,
+    workload: Workload,
+    *,
+    first_chunk_timeout: float,
+    chunk_timeout: float,
+    ping_interval: float,
+    collect_media: bool,
+    print_chunks: bool,
+) -> SessionResult:
+    """Drive one rollout and timestamp every chunk as its bytes land."""
+    try:
+        from websockets.asyncio.client import connect
+    except ImportError:  # pragma: no cover - exercised only without websockets
+        try:
+            from websockets import connect  # type: ignore[attr-defined]
+        except ImportError as exc:
+            raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc
+
+    payload = workload.session_start_payload(model)
+    records: list[ChunkRecord] = []
+    media = bytearray()
+    pending: dict[str, Any] | None = None
+    request_id: str | None = None
+    ttfc_s: float | None = None
+    video_start_s: float | None = None
+    last_arrival: float | None = None
+    done_event: dict[str, Any] = {}
+    send_lock = asyncio.Lock()
+    keepalive: asyncio.Task[None] | None = None
+    # Fire remaining prompt updates in chunk order as their boundary passes.
+    updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk)
+    update_index = 0
+
+    async with connect(url, max_size=None, ping_interval=None) as websocket:
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        started = time.perf_counter()
+        if ping_interval > 0:
+            keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval))
+
+        try:
+            while True:
+                timeout = first_chunk_timeout if not records else chunk_timeout
+                try:
+                    deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout
+                    remaining = deadline - time.perf_counter()
+                    if remaining <= 0:
+                        raise asyncio.TimeoutError
+                    message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
+                except asyncio.TimeoutError as exc:
+                    raise BenchmarkError(
+                        f"No media chunk for {timeout:g} s after {len(records)} chunk(s); "
+                        "the server may still be loading, compiling, or capturing graphs."
+                    ) from exc
+                now = time.perf_counter()
+
+                if isinstance(message, (bytes, bytearray)):
+                    if pending is None:
+                        raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.")
+                    if collect_media:
+                        media.extend(message)
+                    if pending.get("kind") == "media":
+                        index = int(pending["generation_chunk_index"])
+                        arrival = now - started
+                        if ttfc_s is None:
+                            ttfc_s = arrival
+                        records.append(
+                            ChunkRecord(
+                                index=index,
+                                arrival_s=arrival,
+                                inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0),
+                                byte_length=len(message),
+                                num_frames=int(pending["num_frames"]),
+                            )
+                        )
+                        last_arrival = arrival
+                        if print_chunks:
+                            print(
+                                f"  chunk {index:3d}  t={arrival:8.3f}s  "
+                                f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms  "
+                                f"frames={records[-1].num_frames:3d}  bytes={len(message)}",
+                                flush=True,
+                            )
+                        while update_index < len(updates) and updates[update_index].after_chunk <= index:
+                            async with send_lock:
+                                await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False))
+                            update_index += 1
+                    pending = None
+                    continue
+
+                event = json.loads(message)
+                event_type = event.get("type")
+                if event_type == "video.chunk_metadata":
+                    pending = event
+                elif event_type == "video.start":
+                    request_id = event.get("request_id")
+                    video_start_s = now - started
+                elif event_type == "error":
+                    # Say what was asked for alongside what the server refused:
+                    # some builds cap a rollout's num_frames, and the operator
+                    # needs to know which --num-chunks would fit.
+                    raise BenchmarkError(
+                        f"Server rejected the session after {len(records)} chunk(s): "
+                        f"{event.get('message', event)} "
+                        f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})"
+                    )
+                elif event_type == "session.done":
+                    done_event = event
+                    break
+        finally:
+            if keepalive is not None:
+                keepalive.cancel()
+                try:
+                    await keepalive
+                except asyncio.CancelledError:
+                    pass
+
+        wall_s = time.perf_counter() - started
+
+    if not records:
+        raise BenchmarkError("The session finished without delivering a single media chunk.")
+    if done_event.get("stopped"):
+        raise BenchmarkError(f"The session was stopped before it finished: {done_event}")
+    observed = [record.index for record in records]
+    if observed != list(range(len(records))):
+        raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}")
+    if len(records) != workload.num_chunks:
+        raise BenchmarkError(
+            f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}."
+        )
+
+    return SessionResult(
+        request_id=request_id,
+        records=records,
+        media=bytes(media),
+        ttfc_s=ttfc_s,
+        video_start_s=video_start_s,
+        wall_s=wall_s,
+        done_event=done_event,
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Reporting
+# --------------------------------------------------------------------------- #
+
+
+def _git_revision() -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return None
+
+
+def _print_row(label: str, value: str) -> None:
+    print(f"{label:<44}{value:>16}")
+
+
+def print_report(
+    metrics: dict[str, Any],
+    *,
+    workload: Workload,
+    endpoint: str,
+    sessions: int,
+    target_fps: float,
+) -> None:
+    header = " LingBot-World realtime streaming benchmark "
+    print("\n{s:{c}^{n}}".format(s=header, n=60, c="="))
+    _print_row("Endpoint:", endpoint)
+    _print_row("Geometry:", f"{workload.width}x{workload.height}")
+    _print_row("Chunks per session:", str(workload.num_chunks))
+    _print_row("Sessions:", str(sessions))
+    frames = workload.num_frames
+    _print_row("Real-time basis:", f"{target_fps:g} fps")
+    _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)")
+    if abs(target_fps - workload.fps) > 1e-9:
+        _print_row("Mux label sent to server:", f"{workload.fps} fps")
+
+    if sessions == 1:
+        print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-"))
+        if metrics.get("ttfc_ms") is not None:
+            _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms")
+        interval_all = metrics.get("interval_all") or {}
+        interval_steady = metrics.get("interval_steady") or {}
+        for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)):
+            if not summary:
+                continue
+            print(f"  {title} (n={int(summary['count'])}):")
+            _print_row("    mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms")
+            _print_row("    std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms")
+            tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms"
+            _print_row("    p90 / p95 / p99:", tail)
+            if summary["count"] < 100:
+                print(
+                    "    NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates."
+                )
+
+        print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-"))
+        _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}")
+        _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}")
+        if metrics.get("steady_video_rtf") is not None:
+            _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}")
+        _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}")
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        if metrics.get("slo_attainment") is not None:
+            _print_row(
+                "Steady chunks within deadline:",
+                f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)",
+            )
+        if not metrics.get("steady_state_reached", False):
+            print(
+                f"  NOTE: no chunk reached the steady state; the attention window needs "
+                f"{metrics['warmup_chunks']} chunks. Raise --num-chunks."
+            )
+
+        playback = metrics.get("playback") or {}
+        print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-"))
+        _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)")
+        _print_row("Underruns:", str(playback.get("underrun_count")))
+        _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms")
+        _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%")
+        if playback.get("first_underrun_chunk") is not None:
+            _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"]))
+    else:
+        print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-"))
+        if metrics.get("ttfc_ms_mean") is not None:
+            _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms")
+        if metrics.get("steady_interval_ms_mean") is not None:
+            _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms")
+            _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms")
+        _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}")
+        _print_row(
+            "VIDEO_RTF best / worst:",
+            f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}",
+        )
+        _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms")
+        _print_row("Underruns (all sessions):", str(metrics["underrun_count"]))
+    print("=" * 60)
+
+
+def save_video(media: bytes, path: Path, *, fps: int) -> None:
+    from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
+
+    playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_bytes(playback_bytes)
+    os.replace(tmp_path, path)
+    print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}")
+
+
+# --------------------------------------------------------------------------- #
+# CLI
+# --------------------------------------------------------------------------- #
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    endpoint = parser.add_argument_group("endpoint")
+    endpoint.add_argument("--host", default="127.0.0.1")
+    endpoint.add_argument("--port", type=int, default=8000)
+    endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.")
+    endpoint.add_argument("--model", default=DEFAULT_MODEL)
+
+    workload = parser.add_argument_group("workload")
+    workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.")
+    workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.")
+    workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}")
+    workload.add_argument("--negative-prompt", default=None)
+    workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.")
+    workload.add_argument(
+        "--camera-pattern",
+        choices=CAMERA_PATTERNS,
+        default="forward",
+        help="Camera pattern for the built-in workload only; ignored with --workload.",
+    )
+    workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}")
+    workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}")
+    workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}")
+    workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}")
+    workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}")
+
+    run = parser.add_argument_group("run")
+    run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.")
+    run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).")
+    run.add_argument(
+        "--warmup-chunks",
+        type=int,
+        default=DEFAULT_WARMUP_CHUNKS,
+        help="Chunks excluded from steady state; the attention window saturates after 6.",
+    )
+    run.add_argument(
+        "--chunk-slo-ms",
+        type=float,
+        default=None,
+        help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.",
+    )
+    run.add_argument(
+        "--target-fps",
+        type=float,
+        default=None,
+        help=(
+            "Playback rate used for real-time factor, the chunk deadline, and the playback "
+            "simulation. The checkpoint declares no frame rate, so this is a consumption "
+            "choice, not a model property. Default: the --fps sent to the server."
+        ),
+    )
+    run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.")
+    run.add_argument("--first-chunk-timeout", type=float, default=900.0)
+    run.add_argument("--chunk-timeout", type=float, default=300.0)
+    run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.")
+
+    output = parser.add_argument_group("output")
+    output.add_argument("--output-json", default=None)
+    output.add_argument("--save-video", default=None, help="Write the last measured session's video here.")
+    output.add_argument("--print-chunks", action="store_true")
+
+    args = parser.parse_args(argv)
+    if args.sessions < 1:
+        parser.error("--sessions must be at least 1.")
+    if args.warmup_sessions < 0:
+        parser.error("--warmup-sessions must be non-negative.")
+    if args.warmup_chunks < 0:
+        parser.error("--warmup-chunks must be non-negative.")
+    if args.playback_buffer_chunks < 1:
+        parser.error("--playback-buffer-chunks must be at least 1.")
+    if args.target_fps is not None and args.target_fps <= 0:
+        parser.error("--target-fps must be positive.")
+    if args.workload is None and args.num_chunks < 1:
+        parser.error("--num-chunks must be at least 1.")
+    return args
+
+
+def endpoint_url(args: argparse.Namespace) -> str:
+    if args.base_url is not None:
+        base = args.base_url.rstrip("/")
+        for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")):
+            if base.startswith(prefix):
+                base = replacement + base[len(prefix) :]
+                break
+        return base if base.endswith(_WS_PATH) else base + _WS_PATH
+    return f"ws://{args.host}:{args.port}{_WS_PATH}"
+
+
+async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]:
+    workload = build_workload(args)
+    url = endpoint_url(args)
+    # The mux label and the real-time basis are different questions; only the
+    # latter decides whether a run passes.
+    target_fps = args.target_fps if args.target_fps is not None else float(workload.fps)
+    if max(1, args.warmup_chunks) >= workload.num_chunks - 1:
+        print(
+            f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a "
+            f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.",
+            file=sys.stderr,
+        )
+
+    print(f"Connecting to {url}")
+    print(
+        f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, "
+        f"{workload.video_seconds:.2f}s of video"
+    )
+    print(
+        f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is "
+        f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video"
+        + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)")
+    )
+
+    for index in range(args.warmup_sessions):
+        print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...")
+        await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=False,
+            print_chunks=False,
+        )
+
+    session_metrics: list[dict[str, Any]] = []
+    session_payloads: list[dict[str, Any]] = []
+    last_result: SessionResult | None = None
+    for index in range(args.sessions):
+        if args.sessions > 1:
+            print(f"Session {index + 1}/{args.sessions}...")
+        result = await run_session(
+            url,
+            args.model,
+            workload,
+            first_chunk_timeout=args.first_chunk_timeout,
+            chunk_timeout=args.chunk_timeout,
+            ping_interval=args.ping_interval,
+            collect_media=args.save_video is not None,
+            print_chunks=args.print_chunks,
+        )
+        last_result = result
+        metrics = compute_metrics(
+            result.records,
+            fps=target_fps,
+            warmup_chunks=args.warmup_chunks,
+            slo_ms=args.chunk_slo_ms,
+            playback_buffer_chunks=args.playback_buffer_chunks,
+            ttfc_s=result.ttfc_s,
+            session_wall_s=result.wall_s,
+        )
+        session_metrics.append(metrics)
+        session_payloads.append(
+            {
+                "session_index": index,
+                "request_id": result.request_id,
+                "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None,
+                "chunks": [record.to_dict() for record in result.records],
+                "metrics": metrics,
+            }
+        )
+
+    aggregate = aggregate_metrics(session_metrics, fps=target_fps)
+    print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps)
+
+    if args.save_video is not None and last_result is not None:
+        save_video(last_result.media, Path(args.save_video), fps=workload.fps)
+
+    document = {
+        "benchmark": "lingbot_world_realtime",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "endpoint": url,
+        "model": args.model,
+        "git_revision": _git_revision(),
+        "hostname": platform.node(),
+        "workload": workload.describe(),
+        "config": {
+            "sessions": args.sessions,
+            "warmup_sessions": args.warmup_sessions,
+            "warmup_chunks": args.warmup_chunks,
+            "camera_pattern": args.camera_pattern,
+            "chunk_slo_ms": args.chunk_slo_ms,
+            "target_fps": target_fps,
+            "mux_fps": workload.fps,
+            "playback_buffer_chunks": args.playback_buffer_chunks,
+            "workload_file": args.workload,
+        },
+        "sessions": session_payloads,
+        "metrics": aggregate,
+    }
+    if args.output_json is not None:
+        path = Path(args.output_json)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
+        print(f"Wrote {path}")
+    return document
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        asyncio.run(run_benchmark(args))
+    except BenchmarkError as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/lingbot_world/configs/single_gpu_eager.yaml

@@ -0,0 +1,45 @@
+# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager.
+#
+# This is the portable smoke configuration -- it matches the shipped
+# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is
+# comparable to the default deployment. It is NOT a realtime configuration: one
+# GPU runs all five forwards of every AR block serially and nothing is compiled,
+# so expect a chunk interval several times the real-time deadline.
+#
+# Use configs/usp4_compiled.yaml for the realtime topology.
+#
+#   vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \
+#     --port 8000
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0"
+    # AR-Diffusion binds one session per runner invocation; the runner rejects
+    # anything larger.
+    max_num_seqs: 1
+    enforce_eager: true
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    default_sampling_params:
+      # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the
+      # step budget before the pipeline is reached.
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      # The AR cache geometry is fixed at load time, so a benchmark request must
+      # ask for exactly this resolution.
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/configs/usp4_compiled.yaml

@@ -0,0 +1,60 @@
+# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled.
+#
+# This is the realtime topology. Two things differ from the single-GPU config,
+# and both matter to the number the benchmark reports:
+#
+#   parallel_config   Pure Ulysses sequence parallelism over four ranks. The
+#                     pipeline validates this: LingBot requires
+#                     sequence_parallel_size == ulysses_degree, ring_degree and
+#                     allgather_degree of 1, ulysses_mode "strict", and
+#                     ulysses_a2a_permute disabled. Pipeline, CFG, and VAE
+#                     parallel sizes above 1 are rejected. This config uses
+#                     TP=1; the validator does not reject larger TP sizes.
+#   enforce_eager     false puts LingBotDMDBlockRunner on the compiled path
+#                     (regional torch.compile with cudagraph step marking). The
+#                     first rollout therefore pays compile and capture cost --
+#                     use --warmup-sessions 1 before measured repetitions.
+#                     Chunk warmup only excludes the attention-window ramp.
+#
+# Serve it under a GPU reservation, since it needs four cards:
+#
+#   gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
+#     --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \
+#     --port 8000
+#
+# `devices` is the rank->card mapping inside the reservation, so it stays
+# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected.
+
+pipeline: lingbot_world
+async_chunk: false
+distributed_executor_backend: mp
+dtype: bfloat16
+
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enforce_eager: false
+    model_class_name: LingBotWorldCausalDMDPipeline
+    engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine
+    step_execution: true
+    streaming_output: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 4
+      ulysses_degree: 4
+      ring_degree: 1
+    default_sampling_params:
+      num_inference_steps: 4
+      output_type: pil
+      extra_args:
+        flow_shift: 5.0
+    model_config:
+      ar_diffusion_height: 480
+      ar_diffusion_width: 832
+      ar_diffusion_kv_config:
+        # Each rank holds its own shard of the paged KV cache, so the per-card
+        # fraction can stay at the single-GPU value; lower it if the four-rank
+        # start OOMs on smaller cards.
+        gpu_memory_fraction: 0.6
+        warmup_cudagraph: true

benchmarks/lingbot_world/workload.py

@@ -0,0 +1,631 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Workload construction and metric math for the LingBot-World realtime benchmark.
+
+Nothing here imports torch, vllm, or websockets, so the arithmetic that decides
+what a benchmark number *means* is unit-testable without a GPU or a server.
+
+Two model properties drive every formula in this module, both read from the
+checkpoint's ``transformer/config.json`` rather than assumed:
+
+``num_frames_per_block = 3``
+    One AR block commits three latent frames, so one streamed chunk is three
+    latent frames and a request's ``camera_action_script`` carries exactly one
+    three-entry action list per chunk.
+
+``sliding_window_num_frames = 18`` with ``sink_size = 9``
+    Self-attention sees nine sink frames plus a nine-frame recent window, so the
+    key count a block attends over keeps growing until eighteen latent frames of
+    history exist -- six chunks. Chunk cost is therefore *not* stationary before
+    chunk six, and a mean taken over a short rollout reports a number no steady
+    session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that
+    reason and is not a round-number guess.
+
+The causal VAE expands the first latent frame to one pixel frame and every later
+latent frame to four, so chunk zero carries nine pixel frames and every chunk
+after it carries twelve. Video-duration math has to respect that asymmetry or
+real-time factor comes out wrong on short rollouts.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# transformer.config.num_frames_per_block: latent frames committed per AR block.
+FRAMES_PER_BLOCK = 3
+# vae_scale_factor_temporal: latent frames -> pixel frames, after the first.
+TEMPORAL_COMPRESSION = 4
+# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention
+# window saturates and per-chunk cost becomes stationary.
+DEFAULT_WARMUP_CHUNKS = 6
+
+DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
+DEFAULT_PROMPT = "The camera moves slowly forward through the scene."
+# The AR KV cache geometry is fixed at load time, so a request must ask for
+# exactly the resolution the deploy config declares.
+DEFAULT_WIDTH = 832
+DEFAULT_HEIGHT = 480
+DEFAULT_FPS = 16
+DEFAULT_FLOW_SHIFT = 5.0
+DEFAULT_SEED = 42
+
+_HOLD = "hold"
+_ACTION_KEYS = ("w", "a", "s", "d")
+
+
+def num_frames_for_chunks(num_chunks: int) -> int:
+    """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks.
+
+    Inverse of :func:`chunks_for_num_frames`, which is the server's own
+    ``((num_frames - 1) // 4 + 1) // 3``.
+    """
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+
+
+def chunks_for_num_frames(num_frames: int) -> int:
+    """Chunks the server generates for ``num_frames``, mirroring its own formula."""
+    if num_frames < 1:
+        raise ValueError("num_frames must be at least 1.")
+    return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK
+
+
+def pixel_frames_in_chunk(chunk_index: int) -> int:
+    """Pixel frames carried by one chunk.
+
+    The causal decoder expands the session's opening latent frame once and every
+    later latent frame four times, so chunk zero is short by three frames.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if chunk_index == 0:
+        return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1
+    return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION
+
+
+def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]:
+    """One three-entry action list for ``chunk_index``.
+
+    Camera actions select camera embeddings; they change what the world does,
+    not how much arithmetic it costs. The pattern therefore exists to keep a
+    benchmark rollout representative rather than to sweep a cost dimension.
+    """
+    if chunk_index < 0:
+        raise ValueError("chunk_index must be non-negative.")
+    if pattern == "forward":
+        return [["w"] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == _HOLD:
+        return [[] for _ in range(FRAMES_PER_BLOCK)]
+    if pattern == "orbit":
+        key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)]
+        return [[key] for _ in range(FRAMES_PER_BLOCK)]
+    raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.")
+
+
+CAMERA_PATTERNS = ("forward", "orbit", _HOLD)
+
+
+def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]:
+    """Full ``camera_action_script``: one action list per generated chunk."""
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1.")
+    return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)]
+
+
+def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]:
+    validated: list[list[list[str]]] = []
+    for chunk_index, actions in enumerate(script):
+        if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)):
+            raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.")
+        if len(actions) != FRAMES_PER_BLOCK:
+            raise ValueError(
+                f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists "
+                f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames."
+            )
+        frames: list[list[str]] = []
+        for frame_index, frame in enumerate(actions):
+            if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.")
+            if any(not isinstance(action, str) for action in frame):
+                raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.")
+            frames.append([str(action) for action in frame])
+        validated.append(frames)
+    if not validated:
+        raise ValueError("camera_action_script must not be empty.")
+    return validated
+
+
+@dataclass(frozen=True)
+class PromptUpdate:
+    """A mid-rollout ``session.interaction`` scheduled on a chunk boundary."""
+
+    after_chunk: int
+    prompt: str
+    transition_chunks: int = 3
+
+    def __post_init__(self) -> None:
+        if self.after_chunk < 0:
+            raise ValueError("PromptUpdate.after_chunk must be non-negative.")
+        if not self.prompt.strip():
+            raise ValueError("PromptUpdate.prompt must contain non-whitespace text.")
+        if self.transition_chunks < 0:
+            raise ValueError("PromptUpdate.transition_chunks must be non-negative.")
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "type": "session.interaction",
+            "interaction": {
+                "event_id": f"chunk-{self.after_chunk}",
+                "event": {"prompt": self.prompt},
+                "transition_chunks": self.transition_chunks,
+            },
+        }
+
+
+@dataclass(frozen=True)
+class Workload:
+    """One rollout: what the benchmark asks the server to generate."""
+
+    prompt: str
+    image_reference: str
+    camera_script: list[list[list[str]]]
+    width: int = DEFAULT_WIDTH
+    height: int = DEFAULT_HEIGHT
+    fps: int = DEFAULT_FPS
+    seed: int = DEFAULT_SEED
+    flow_shift: float = DEFAULT_FLOW_SHIFT
+    negative_prompt: str | None = None
+    prompt_updates: tuple[PromptUpdate, ...] = ()
+    extra_params: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.prompt.strip():
+            raise ValueError("Workload.prompt must contain non-whitespace text.")
+        if not self.image_reference:
+            raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.")
+        if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16:
+            raise ValueError("Workload width/height must be positive multiples of 16.")
+        if self.fps <= 0:
+            raise ValueError("Workload.fps must be positive.")
+        object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script))
+        seen_update_chunks: set[int] = set()
+        for update in self.prompt_updates:
+            if update.after_chunk in seen_update_chunks:
+                raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.")
+            seen_update_chunks.add(update.after_chunk)
+            if update.after_chunk >= self.num_chunks:
+                raise ValueError(
+                    f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout."
+                )
+
+    @property
+    def num_chunks(self) -> int:
+        return len(self.camera_script)
+
+    @property
+    def num_frames(self) -> int:
+        return num_frames_for_chunks(self.num_chunks)
+
+    @property
+    def video_seconds(self) -> float:
+        return self.num_frames / self.fps
+
+    def session_start_payload(self, model: str) -> dict[str, Any]:
+        """The ``session.start`` event for ``WS /v1/realtime/video``."""
+        extra_params: dict[str, Any] = {
+            "flow_shift": self.flow_shift,
+            "camera_action_script": self.camera_script,
+        }
+        extra_params.update(self.extra_params)
+        payload: dict[str, Any] = {
+            "type": "session.start",
+            "model": model,
+            "prompt": self.prompt,
+            "image_reference": {"image_url": self.image_reference},
+            "width": self.width,
+            "height": self.height,
+            "num_frames": self.num_frames,
+            "fps": self.fps,
+            "seed": self.seed,
+            "extra_params": extra_params,
+        }
+        if self.negative_prompt is not None:
+            payload["negative_prompt"] = self.negative_prompt
+        return payload
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "num_chunks": self.num_chunks,
+            "num_frames": self.num_frames,
+            "width": self.width,
+            "height": self.height,
+            "fps": self.fps,
+            "seed": self.seed,
+            "flow_shift": self.flow_shift,
+            "video_seconds": self.video_seconds,
+            "prompt_updates": [
+                {
+                    "after_chunk": update.after_chunk,
+                    "transition_chunks": update.transition_chunks,
+                }
+                for update in self.prompt_updates
+            ],
+        }
+
+
+def default_workload(
+    *,
+    image_reference: str,
+    num_chunks: int,
+    camera_pattern: str = "forward",
+    prompt: str = DEFAULT_PROMPT,
+    width: int = DEFAULT_WIDTH,
+    height: int = DEFAULT_HEIGHT,
+    fps: int = DEFAULT_FPS,
+    seed: int = DEFAULT_SEED,
+    flow_shift: float = DEFAULT_FLOW_SHIFT,
+    negative_prompt: str | None = None,
+) -> Workload:
+    """Built-in rollout: a fixed prompt and a generated camera script."""
+    return Workload(
+        prompt=prompt,
+        image_reference=image_reference,
+        camera_script=build_camera_script(num_chunks, camera_pattern),
+        width=width,
+        height=height,
+        fps=fps,
+        seed=seed,
+        flow_shift=flow_shift,
+        negative_prompt=negative_prompt,
+    )
+
+
+def load_workload(
+    path: str | Path,
+    *,
+    image_resolver: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+) -> Workload:
+    """Read a rollout spec from JSON.
+
+    ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or
+    ``data:`` URL; the caller owns that because inlining a local file is I/O.
+    Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags
+    stay meaningful next to ``--workload``.
+    """
+    document = json.loads(Path(path).read_text())
+    if not isinstance(document, dict):
+        raise ValueError("A workload file must contain a JSON object.")
+
+    merged = dict(document)
+    for key, value in (overrides or {}).items():
+        if value is not None:
+            merged[key] = value
+
+    camera_script = merged.get("camera_action_script")
+    if camera_script is None:
+        num_chunks = merged.get("num_chunks")
+        if num_chunks is None:
+            raise ValueError("A workload file needs either camera_action_script or num_chunks.")
+        camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward")))
+
+    image = merged.get("image") or merged.get("image_reference")
+    if image is None:
+        raise ValueError("A workload file needs an image or image_reference field.")
+    image_reference = image_resolver(str(image)) if image_resolver is not None else str(image)
+
+    updates_field = merged.get("prompt_updates") or ()
+    prompt_updates = tuple(
+        PromptUpdate(
+            after_chunk=int(entry["after_chunk"]),
+            prompt=str(entry["prompt"]),
+            transition_chunks=int(entry.get("transition_chunks", 3)),
+        )
+        for entry in updates_field
+    )
+
+    extra_params = merged.get("extra_params") or {}
+    if not isinstance(extra_params, Mapping):
+        raise ValueError("Workload extra_params must be a JSON object.")
+
+    return Workload(
+        prompt=str(merged.get("prompt", DEFAULT_PROMPT)),
+        image_reference=image_reference,
+        camera_script=camera_script,
+        width=int(merged.get("width", DEFAULT_WIDTH)),
+        height=int(merged.get("height", DEFAULT_HEIGHT)),
+        fps=int(merged.get("fps", DEFAULT_FPS)),
+        seed=int(merged.get("seed", DEFAULT_SEED)),
+        flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)),
+        negative_prompt=merged.get("negative_prompt"),
+        prompt_updates=prompt_updates,
+        extra_params=dict(extra_params),
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Metrics
+# --------------------------------------------------------------------------- #
+
+
+@dataclass
+class ChunkRecord:
+    """One media chunk as the client observed it."""
+
+    index: int
+    arrival_s: float
+    """Seconds from ``session.start`` being sent to this chunk's bytes arriving."""
+    inter_arrival_s: float
+    """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``."""
+    byte_length: int
+    num_frames: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "arrival_s": self.arrival_s,
+            "inter_arrival_ms": self.inter_arrival_s * 1000.0,
+            "byte_length": self.byte_length,
+            "num_frames": self.num_frames,
+        }
+
+
+def percentile(values: Sequence[float], fraction: float) -> float:
+    """Linearly interpolated percentile, so a 10-sample run still reports a p95.
+
+    Implemented here rather than pulled from numpy to keep this module importable
+    in a bare environment; the benchmark's own numbers must not depend on which
+    optional package happens to be installed.
+    """
+    if not values:
+        raise ValueError("percentile() needs at least one value.")
+    if not 0.0 <= fraction <= 1.0:
+        raise ValueError("percentile fraction must be within [0, 1].")
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    position = fraction * (len(ordered) - 1)
+    lower = int(position)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = position - lower
+    return ordered[lower] * (1.0 - weight) + ordered[upper] * weight
+
+
+def _stddev(values: Sequence[float]) -> float:
+    if len(values) < 2:
+        return 0.0
+    mean = sum(values) / len(values)
+    return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5
+
+
+def _latency_summary(values: Sequence[float]) -> dict[str, float]:
+    """Millisecond summary of a set of inter-arrival intervals."""
+    if not values:
+        return {}
+    milliseconds = [value * 1000.0 for value in values]
+    return {
+        "count": float(len(milliseconds)),
+        "mean_ms": sum(milliseconds) / len(milliseconds),
+        "median_ms": percentile(milliseconds, 0.5),
+        "std_ms": _stddev(milliseconds),
+        "min_ms": min(milliseconds),
+        "p90_ms": percentile(milliseconds, 0.90),
+        "p95_ms": percentile(milliseconds, 0.95),
+        "p99_ms": percentile(milliseconds, 0.99),
+        "max_ms": max(milliseconds),
+    }
+
+
+@dataclass(frozen=True)
+class PlaybackResult:
+    """What a viewer would have seen, given a start buffer and a playback rate."""
+
+    underrun_count: int
+    total_stall_s: float
+    playback_span_s: float
+    first_underrun_chunk: int | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "underrun_count": self.underrun_count,
+            "total_stall_ms": self.total_stall_s * 1000.0,
+            "playback_span_s": self.playback_span_s,
+            "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0,
+            "first_underrun_chunk": self.first_underrun_chunk,
+        }
+
+
+def simulate_playback(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    buffer_chunks: int = 1,
+) -> PlaybackResult:
+    """Replay arrivals against a wall clock and count stalls.
+
+    Playback starts when ``buffer_chunks`` chunks have arrived and then consumes
+    video at ``fps``. A chunk that has not arrived by the moment the player needs
+    it is an underrun, and the player resumes only when the bytes land -- which
+    is what a viewer experiences, and what a single p99 number cannot tell you:
+    the same p99 is harmless behind a three-chunk buffer and fatal behind none.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    if buffer_chunks < 1:
+        raise ValueError("buffer_chunks must be at least 1.")
+    if not records:
+        return PlaybackResult(0, 0.0, 0.0, None)
+    if buffer_chunks > len(records):
+        raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    start = ordered[buffer_chunks - 1].arrival_s
+    cursor = start
+    underruns = 0
+    total_stall = 0.0
+    first_underrun: int | None = None
+
+    for record in ordered:
+        if record.arrival_s > cursor:
+            stall = record.arrival_s - cursor
+            underruns += 1
+            total_stall += stall
+            if first_underrun is None:
+                first_underrun = record.index
+            cursor = record.arrival_s
+        cursor += record.num_frames / fps
+
+    return PlaybackResult(
+        underrun_count=underruns,
+        total_stall_s=total_stall,
+        playback_span_s=cursor - start,
+        first_underrun_chunk=first_underrun,
+    )
+
+
+def steady_chunk_deadline_ms(fps: float) -> float:
+    """Wall-clock budget a steady chunk must meet to hold real time.
+
+    ``fps`` here is the rate the frames are *intended to be played at*, which the
+    checkpoint does not declare: nothing in ``model_index.json``, the scheduler
+    config, or the transformer config carries a frame rate, and the pipeline
+    never reads one. It is a property of how the output is consumed, so both
+    this deadline and real-time factor scale linearly with the choice and the
+    caller has to state it rather than inherit it from a muxing default.
+    """
+    if fps <= 0:
+        raise ValueError("fps must be positive.")
+    return pixel_frames_in_chunk(1) / fps * 1000.0
+
+
+def compute_metrics(
+    records: Sequence[ChunkRecord],
+    *,
+    fps: float,
+    warmup_chunks: int = DEFAULT_WARMUP_CHUNKS,
+    slo_ms: float | None = None,
+    playback_buffer_chunks: int = 1,
+    ttfc_s: float | None = None,
+    session_wall_s: float | None = None,
+) -> dict[str, Any]:
+    """Summarize one session.
+
+    ``warmup_chunks`` are excluded from the steady-state block because the
+    attention window has not saturated before then (see the module docstring).
+    The terminal chunk also stays out of steady metrics because it skips
+    next-chunk preparation. All chunks stay in the playback simulation, which is
+    what a session really costs end to end.
+    """
+    if not records:
+        raise ValueError("compute_metrics() needs at least one chunk record.")
+    if warmup_chunks < 0:
+        raise ValueError("warmup_chunks must be non-negative.")
+
+    ordered = sorted(records, key=lambda record: record.index)
+    deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps)
+
+    all_intervals = [record.inter_arrival_s for record in ordered[1:]]
+    # The terminal chunk skips next-chunk preparation and is not representative
+    # of sustained generation. Keep it in end-to-end and playback metrics.
+    steady = [record for record in ordered[:-1] if record.index >= warmup_chunks]
+    steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0]
+
+    total_frames = sum(record.num_frames for record in ordered)
+    video_seconds = total_frames / fps
+    wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s
+
+    metrics: dict[str, Any] = {
+        "chunks_received": len(ordered),
+        "warmup_chunks": warmup_chunks,
+        "steady_chunks": len(steady_intervals),
+        "steady_state_reached": bool(steady_intervals),
+        "total_frames": total_frames,
+        "total_bytes": sum(record.byte_length for record in ordered),
+        "video_seconds": video_seconds,
+        "wall_seconds": wall_seconds,
+        # RTF is processing time over content duration, as vLLM-Omni computes it
+        # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency
+        # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is
+        # real time. RTFX is the reciprocal the repo reports under its own name,
+        # kept here because "fraction of real time achieved" reads better on a
+        # dashboard -- but the two must never be confused for one another.
+        "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0,
+        "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0,
+        "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0,
+        "chunk_deadline_ms": deadline_ms,
+        "interval_all": _latency_summary(all_intervals),
+        "interval_steady": _latency_summary(steady_intervals),
+    }
+    if ttfc_s is not None:
+        metrics["ttfc_ms"] = ttfc_s * 1000.0
+
+    if steady_intervals:
+        steady_frames = sum(record.num_frames for record in steady if record.index > 0)
+        steady_wall = sum(steady_intervals)
+        steady_video = steady_frames / fps
+        metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0
+        metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0
+        within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms)
+        metrics["slo_attainment"] = within / len(steady_intervals)
+        metrics["slo_violations"] = len(steady_intervals) - within
+
+    metrics["playback"] = simulate_playback(
+        ordered,
+        fps=fps,
+        buffer_chunks=playback_buffer_chunks,
+    ).to_dict()
+    metrics["playback"]["buffer_chunks"] = playback_buffer_chunks
+    return metrics
+
+
+def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]:
+    """Combine per-session metrics across sequential repeats.
+
+    Steady intervals are pooled rather than averaging per-session means, so a
+    session that happened to produce fewer steady chunks does not carry the same
+    weight as a full one.
+    """
+    entries = list(sessions)
+    if not entries:
+        raise ValueError("aggregate_metrics() needs at least one session.")
+    if len(entries) == 1:
+        single = dict(entries[0])
+        single["sessions"] = 1
+        return single
+
+    deadline_ms = entries[0]["chunk_deadline_ms"]
+    if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries):
+        raise ValueError("Cannot aggregate sessions with different chunk deadlines.")
+    ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None]
+    # Lower is better, so "max" is the worst session, not the best.
+    rtfs = [float(entry["video_rtf"]) for entry in entries]
+    steady_means = [
+        float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count")
+    ]
+    steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries)
+    steady_total_ms = sum(
+        entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"]
+        for entry in entries
+        if entry.get("interval_steady", {}).get("count")
+    )
+    return {
+        "sessions": len(entries),
+        "chunks_received": sum(int(entry["chunks_received"]) for entry in entries),
+        "total_frames": sum(int(entry["total_frames"]) for entry in entries),
+        "video_seconds": sum(float(entry["video_seconds"]) for entry in entries),
+        "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries),
+        "video_rtf_mean": sum(rtfs) / len(rtfs),
+        "video_rtf_best": min(rtfs),
+        "video_rtf_worst": max(rtfs),
+        "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None,
+        "ttfc_ms_max": max(ttfcs) if ttfcs else None,
+        "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None,
+        "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None,
+        "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries),
+        "chunk_deadline_ms": deadline_ms,
+    }

docs/configuration/environment_variables.md

@@ -85,7 +85,7 @@ depends on the installed kernels and model path.
 | `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
 | `VLLM_OMNI_ABORT_TIMEOUT` | Float seconds; default `2` | Engine abort wait for Videos DELETE and `generate()` cancel/error cleanup; read when `async_omni` imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
 | `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; defaults to on for Qwen3-TTS and off for other pipelines | Pipeline default computed from `pipeline_config.model_type` at engine initialization; env override resolved at `Orchestrator` construction and separately when the serving-side final-output drain starts | An explicit env value wins; otherwise both consumers use the engine's pipeline default. Values are stripped and case-normalized; any unrecognized value selects the legacy poll loop. Set before server startup. | Experimental |
 | `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
 | `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
 | `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |

docs/configuration/stage_configs.md

@@ -293,19 +293,25 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/depl
 
 ## Qwen3-TTS with Model Runner V2
 
-Qwen3-TTS can opt into the native CUDA Model Runner V2 pipeline on vLLM
-0.29.0. Select one of these deployment profiles:
+Qwen3-TTS runs the native CUDA Model Runner V2 pipeline by default on vLLM
+0.29.0. MRV2 is an experimental feature for this model: the bundled default
+profile selects it on CUDA only, and its scheduler and delivery paths are
+still being qualified. Set `model_runner: v1` in a copy of the deploy config
+to opt out; the `platforms:` sections of `qwen3_tts.yaml` keep V1 on NPU, XPU,
+ROCm and MUSA. Select one of these deployment profiles:
 
 | Profile | Runner | Code2Wav graph batches | Intended use |
 | --- | --- | --- | --- |
-| `qwen3_tts.yaml` | V1 | Existing defaults | Existing deployment / regression control |
-| `qwen3_tts_mrv2.yaml` | V2 | B1 | Native runner validation |
+| `qwen3_tts.yaml` | V2 (default) | Existing defaults | Shipped default; experimental |
+| `qwen3_tts_mrv2.yaml` | V2 | B1 | Explicit MRV2 profile (same runner selection as the default) |
 | `qwen3_tts_high_concurrency_mrv2.yaml` | V2 | B1, B2 | Opt-in throughput tuning |
 | `qwen3_tts_high_concurrency_mrv2_b4.yaml` | V2 | B1, B2, B3, B4 | Experimental throughput / buffered playback |
+| `qwen3_tts_high_concurrency.yaml` | V1 | Existing defaults | V1 high-concurrency control |
 
 ```bash
+# MRV2 is the default; pass a copy with `model_runner: v1` to force V1.
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni \
-  --deploy-config vllm_omni/deploy/qwen3_tts_mrv2.yaml
+  --deploy-config /path/to/qwen3_tts_v1.yaml
 ```
 
 The V2 profiles bound Talker prefill to 512 tokens per step and select the

docs/contributing/ci/test_execution_guide.md

@@ -177,10 +177,10 @@ Failed jobs: 1/2
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+    pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow
 
     # Reliability: Qwen3-Omni (H100 × 2)
     pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow

docs/contributing/ci/test_writing_guide.md

@@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
 - ***Run Command***:
-    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
+    - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_<model>.py` alongside a matching JSON config)
     - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`).
 - ***Script Example***:
 
@@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
 
 ##### Stability
 
-When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
+When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_<model>.py`. The following illustrates the Qwen3-Omni shape:
 
 ```json
 {

docs/contributing/profiling.md

@@ -22,7 +22,7 @@ profiler_config:
 Supported fields:
 
 | Field | Description |
-|---|---|
+| --- | --- |
 | `profiler` | Profiler backend. Supported values: `torch`, `cuda`. Use `torch` for `trace.json`, Excel operator tables, and optional memory snapshots. Use `cuda` for Nsight Systems only. |
 | `torch_profiler_dir` | Output directory for torch-profiler artifacts. Required when `profiler: torch`. |
 | `torch_profiler_use_gzip` | Compress `trace_rank*.json` into `trace_rank*.json.gz`. |
@@ -287,6 +287,52 @@ Recommended viewers:
 
 For upstream background on the underlying vLLM profiling infrastructure, see the [vLLM profiling guide](https://docs.vllm.ai/en/stable/contributing/profiling/).
 
+## 6. Profiling Omni AR/TTS stages
+
+For Omni AR and TTS stages, put `profiler_config` under each stage that should
+be captured. The serving API then controls a narrow window with
+`POST /start_profile` and `POST /stop_profile`; model initialization and
+warmup stay outside the trace.
+
+Example stage override for a host-stack torch trace:
+
+```bash
+PROFILE='{"profiler":"torch","torch_profiler_dir":"/tmp/qwen3_tts_profile",'\
+'"torch_profiler_with_stack":true,"torch_profiler_record_shapes":false,'\
+'"torch_profiler_use_gzip":true,"torch_profiler_dump_cuda_time_total":true}'
+
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
+  --deploy-config vllm_omni/deploy/qwen3_tts_high_concurrency.yaml \
+  --stage-overrides "{\"0\":{\"profiler_config\":$PROFILE},\"1\":{\"profiler_config\":$PROFILE}}"
+```
+
+After one or more warmup requests, capture one short measured request:
+
+```bash
+curl -sS -X POST http://127.0.0.1:8000/start_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+# send one fixed /v1/audio/speech or /v1/chat/completions request here
+curl -sS -X POST http://127.0.0.1:8000/stop_profile \
+  -H 'Content-Type: application/json' -d '{"stages":[0,1]}'
+```
+
+For an Nsight Systems CUDA timeline, use the same stage override with
+`{"profiler":"cuda"}` and wrap the server:
+
+```bash
+nsys profile --trace=cuda,nvtx,osrt --sample=process-tree \
+  --cpuctxsw=process-tree --trace-fork-before-exec=true \
+  --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown \
+  --cuda-graph-trace=node -o /tmp/qwen3_tts_nsys \
+  vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni ...
+```
+
+For Omni AR workers, `profiler="cuda"` requires the native CUDA profiler
+wrapper and uses `cudaProfilerStart/Stop`; the report is only valid if the
+`/start_profile` and `/stop_profile` calls both succeed. Use `nsys stats` for
+CUDA API, GPU kernel, OS runtime, and NVTX summaries. Treat profiler latency as
+diagnostic only; use a separate no-profiler run for performance numbers.
+
 ## 7. Orchestrator Monitor
 
 > **Warning:** Diagnostic only. Adds lightweight counters on the orchestrator poll loop and per-replica queue depth. Disable in production unless you are actively debugging orchestrator saturation.
@@ -314,9 +360,11 @@ If unset, the monitor writes `./vllm_omni_orch_monitor_<MMDDHHMM>.json` at shutd
 Each 1-second window records:
 
 | Series | Description |
-|---|---|
+| --- | --- |
 | `windows.duration_s` | Wall time covered by the window |
 | `windows.loop_idle` / `windows.loop_active` | Orchestrator poll-loop iterations with no work vs. active forwarding |
+| `windows.dispatch_queue_size` | Ready-queue size sampled at each monitor window boundary |
+| `windows.dispatch_queue_high_water` | Maximum Event Driven ready-queue size observed in each window |
 | `replicas.<stage,replica>.outputs_queue_size` | MP client `outputs_queue` backlog for that replica |
 | `replicas.<stage,replica>.inflight` | Requests currently bound/routed to the replica |
 
@@ -331,12 +379,16 @@ orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still
 records one iteration per routed output. Window counts and the `loop_active_pct`
 summary are therefore not comparable across the two modes.
 
+For Event Driven runs, use `dispatch_queue_high_water` rather than only the
+window-boundary `dispatch_queue_size`: a serial dispatcher can drain a burst
+before the window closes, hiding a transient ready-queue backlog in the latter.
+
 ### Relationship to other diagnostics
 
 This monitor is intentionally separate from the existing profiling tools:
 
 | Tool | Scope | Output |
-|---|---|---|
+| --- | --- | --- |
 | `--enable-diffusion-pipeline-profiler` | Diffusion stage functions (`vae.decode`, `diffuse`, …) | Per-request `stage_durations` / logs |
 | `--enable-ar-profiler` | AR stage generation time | Per-request `stage_durations` |
 | `profiler_config` (`torch` / `cuda`) | GPU/CPU kernels inside a stage worker | `trace.json`, Nsight, operator tables |

docs/design/module/engine_orchestration.md

@@ -79,7 +79,7 @@ the boundary affected by the in-flight stage client/process refactor in
 responsibilities proposed only by that PR are not current contracts.
 
 The orchestration loop also has an opt-in event-driven mode
-(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off) proposed in
+(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off except for Qwen3-TTS) proposed in
 [#5221](https://github.com/vllm-project/vllm-omni/pull/5221). It changes poll
 cadence only: the routing, ordering, and terminal-state contracts below hold
 identically on both loops.

docs/features/sleep_mode.md

@@ -9,9 +9,11 @@ This feature is inherited from [vLLM’s Sleep Mode](https://blog.vllm.ai/2025/1
 ## 1. Feature Documentation
 
 ### Overview
+
 Omni Sleep Mode provides a mechanism to "sleep" specific model stages. When a stage enters sleep, its physical VRAM is reclaimed by the system, while the process state is preserved for rapid "wake-up" without full re-initialization.
 
 ### Sleep Levels
+
 We support two levels of hibernation to balance recovery speed and memory efficiency:
 
 | Level | Name | Mechanism | Recovery Speed | Memory Freed |
@@ -29,15 +31,67 @@ Omni Sleep Mode is optimized for high-performance computing backends:
 * **Huawei NPU**: Supported via Ascend memory scavenging.
 
 ### Hardware Requirements
+
 * **Memory Considerations**: System RAM must be sufficient to hold offloaded weights during sleep.
 * **TP Support**: Tensor Parallel groups synchronize sleep/wake transitions across all workers.
 
----
+### Quiescing a busy diffusion stage: `pause_generation(mode="keep")`
 
+`sleep()` on a diffusion stage is a worker-level RPC: it offloads memory but does not stop the
+stage's scheduler, so a stage that still has queued requests must be made quiet first. Aborting
+discards the batch that is running. `pause_generation(mode="keep")` keeps it instead:
+
+* The batch that is already running finishes on the weights it started with and its outputs are
+  delivered normally.
+* Requests that have not started stay queued and run after `resume_generation()`, on whatever
+  weights are loaded by then. A request never continues denoising across a weight change.
+* The call returns only after every replica has acknowledged: the batch that was running has
+  finished executing, its asynchronous output copies are done, and the device is synchronized; its
+  outputs are then delivered on the normal path. On a non-preemptible backend that acknowledgement
+  arrives at the next batch boundary, so expect it to take as long as the remaining batch.
+* Control RPCs (`list_loras`, `sleep`, `wake_up`, `resume_generation`) keep working while paused.
+
+Recommended order for a weight update on a diffusion stage:
+
+```python
+await engine.pause_generation(mode="keep", clear_cache=False)
+await engine.sleep(level=1)          # optional: free memory for the trainer
+await engine.wake_up()               # admission stays closed until resume
+# install new weights here
+await engine.resume_generation()
+```
+
+Support matrix and limits:
+
+| Configuration | `mode="keep"` |
+| :--- | :--- |
+| Request-level execution (`step_execution=False`) with the in-process (`uni`) executor | Supported |
+| Request-level execution with the multi-process (`mp`) executor, including asynchronous output | Supported; the acknowledgement waits for the output copies |
+| Request-level execution with asynchronous KV prefetch | Supported; the acknowledgement joins any prefetch still in flight |
+| Step-level execution (`step_execution=True` / streaming output) | Not supported: the call raises `NotImplementedError` |
+| `mode="abort"` / `mode="wait"` / `wait_for_inflight_requests=True` on a diffusion stage | Unchanged: frontend admission is paused only |
+
+Behavior notes:
+
+* Callers that already passed `mode="keep"` for a diffusion stage used to get an immediate return
+  that only closed frontend admission. They now also stop the backend and wait for the ACK.
+* A timeout or error from `pause_generation` means the pause did **not** complete; the scheduler
+  stays closed and the caller must retry the pause or call `resume_generation()` explicitly. Do not
+  sleep or mutate weights after a failed pause.
+* `resume_generation(stage_ids=...)` that leaves another paused stage closed (an AR stage, or a
+  keep-paused diffusion stage) keeps frontend admission closed as well, so new requests cannot
+  queue on a stage that will not schedule them. Admission reopens once every stage paused by
+  `pause_generation` has been resumed.
+* `clear_cache` has no diffusion-specific effect.
+* Without a pause call nothing changes: no barrier runs and batching, abort and output ordering are
+  as before.
+
+---
 
 ## 2. Usage Examples
 
 ### Python API Example
+
 You can programmatically control the lifecycle of stages using the `AsyncOmni` engine.
 
 ```python
@@ -65,6 +119,7 @@ if __name__ == "__main__":
 ```
 
 ### server command Example
+
 Start the server with sleep mode enabled:
 
 The first method
@@ -92,9 +147,6 @@ python3 -m vllm_omni.entrypoints.openai.api_server \
 
 ```
 
-
-
-
 ### Test Scenarios & Commands
 
 #### Scenario 1: LLM Engine Sleep
@@ -113,9 +165,8 @@ curl -X POST http://localhost:8000/v1/omni/sleep \
 
 Tip: Open a new terminal and run rocm-smi or nvidia-smi or to observe the immediate drop in VRAM usage.
 
-
-
 #### Scenario 2: Diffusion Sleep
+
 Objective: Verify VRAM reclamation for Stage 1 (Diffusion).
 
 Trigger sleep (Level 1 or Level 2) via client:
@@ -128,9 +179,8 @@ curl -X POST http://localhost:8000/v1/omni/sleep \
 
 ```
 
-
-
 #### Scenario 3: Multi-Stage Coordinated Stress Test
+
 Objective: Test concurrent sleep and rapid wake-up across multiple stages.
 
 Concurrent Sleep (Stage 0 & 1):
@@ -143,7 +193,6 @@ curl -X POST http://localhost:8000/v1/omni/sleep \
 
 ```
 
-
 Rapid Wake-up:
 
 ```
@@ -154,8 +203,8 @@ curl -X POST http://localhost:8000/v1/omni/wakeup \
 
 ```
 
-
 #### Scenario 4: Full Lifecycle Memory Audit & Functional Integrity
+
 Objective: Audit the complete flow from Sleep to Wake-up followed by an Inference validation.
 
 Check Initial State: Observe baseline VRAM usage.
@@ -196,20 +245,16 @@ curl -X POST http://localhost:8000/v1/images/generations \
 
 ```
 
-
-
-
 ## 3. API Reference
 
-
 ### Methods
 
 | Method | Arguments | Return Type | Description |
 | :--- | :--- | :--- | :--- |
 | **sleep** | `stage_ids: List[int], level: int` | `List[OmniACK]` | Triggers hibernation for specified stages. |
 | **wake_up** | `stage_ids: List[int]` | `List[OmniACK]` | Reloads weights and re-maps memory. |
-
-
+| **pause_generation** | `mode: str, stage_ids: List[int]` | `None` | Stops admission; with `mode="keep"` also stops diffusion schedulers and returns after their ACK. |
+| **resume_generation** | `stage_ids: List[int]` | `None` | Reopens paused schedulers, then admission. |
 
 ### OmniACK Dataclass Fields
 
@@ -233,7 +278,8 @@ The metadata field is a dynamic dictionary containing hardware-specific telemetr
 }
 ```
 
-#### Core Utility:
+#### Core Utility
+
 **VRAM Reclamation Audit (total_freed_gib)**: Converts raw freed_bytes into human-readable GiB. It serves as the primary metric to verify that Level 2 sleep has successfully purged model weights from VRAM.
 
 **Residual & Fragmentation Monitoring (rank_residual_gib)**: Reports the remaining VRAM footprint after memory de-mapping. A low residual value (e.g., 2.07 GiB) confirms a successful "clean" state, ensuring the device is ready for high-memory co-located tasks like training or diffusion pipelines.

docs/serving/speech_api.md

@@ -957,22 +957,24 @@ Use `/v1/audio/voices` to list available voices for the loaded model.
 ## Orchestration Loop (experimental)
 
 Multi-stage omni deployments route stage outputs through a single orchestrator
-loop. By default that loop polls every stage replica on a 1 ms cadence. An
-opt-in event-driven mode replaces the poll with one reader task per live stage
+loop. The legacy loop polls every stage replica on a 1 ms cadence. The
+event-driven mode replaces the poll with one reader task per live stage
 replica awaiting its client directly, and switches the serving-side
-final-output drain to a condition-variable wakeup at the same time.
+final-output drain to a condition-variable wakeup at the same time. Qwen3-TTS
+uses this mode by default; other pipelines keep the legacy poll unless enabled.
 
 **Configuration (environment variables):**
 
 | Variable | Default | Description |
 | --- | --- | --- |
-| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. |
+| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | On for Qwen3-TTS; off for other pipelines | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. An explicit value wins; otherwise the pipeline default computed at engine initialization is used. The override is resolved at orchestrator construction and separately when the final-output drain starts. `1`, `true`, `yes`, or `on` enables it, ignoring case and surrounding whitespace; other values select the legacy poll loop. |
 
-Set it on the process that runs the orchestrator (stage 0 of an omni
-deployment) before starting the server:
+Set it on the process that runs the orchestrator (stage 0 of an omni deployment)
+before starting the server when overriding the pipeline default. For example,
+explicitly disable event-driven orchestration for Qwen3-TTS:
 
 ```bash
-export VLLM_OMNI_EVENT_DRIVEN_ORCH=1
+VLLM_OMNI_EVENT_DRIVEN_ORCH=0 \
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
     --omni \
     --port 8091
@@ -982,8 +984,9 @@ The server logs the selected loop mode and its reader/poller counts once at
 startup, so you can confirm which loop is live.
 
 Routing, output ordering, and terminal-state behavior are identical on both
-loops; only the poll cadence changes. Leaving the variable unset keeps the
-legacy poll loop, which is the supported default.
+loops; only the poll cadence changes. Leaving the variable unset selects the
+pipeline default: event-driven for Qwen3-TTS, and legacy polling for other
+pipelines, including Qwen3-Omni.
 
 **Known limitations:**
 

docs/user_guide/examples/online_serving/aura_omni.md

@@ -6,29 +6,29 @@
 ASR -> AURA -> Qwen3-TTS Talker -> Code2Wav
 ```
 
-Qwen3-TTS remains two engine stages so the pipeline reuses the existing native
-Talker and Code2Wav implementation.
+**Supported online path:** Realtime duplex at `/v1/realtime?duplex=1`.
+
+The default deploy profile (`vllm_omni/deploy/aura_omni.yaml`) sets
+`session_mode: duplex`. The `aura_omni` pipeline declares `duplex_plugin`, and
+`DuplexOmniEngine` requires that mode (same pattern as MiniCPM-o duplex
+deploys). Turn-based `chat/completions` is not a supported AURA online mode
+against this profile.
+
+`async_chunk` defaults to `false` in that file (safer if a turn-based
+`OmniOrchestrator` ever loads it). Duplex Realtime Stage2→3 still needs
+async chunks — use the smoke deploy below, or set `async_chunk: true`
+yourself.
 
 ```bash
-vllm serve aurateam/AURA \
-  --omni \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
-  --served-model-name aurateam/AURA \
-  --trust-remote-code
+bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh
+python examples/online_serving/aura_omni/smoke_duplex_realtime_client.py
 ```
 
-Configure local checkpoints by editing per-stage `model` values in
-`vllm_omni/deploy/aura_omni.yaml`. The deploy file sets
-`pipeline: aura_omni`, so the four-stage topology is used even if the
-command-line `--model` points at one of the component checkpoints.
+Configure local checkpoints by editing per-stage `model` values in the deploy
+or smoke YAML. The file sets `pipeline: aura_omni`, so the four-stage topology
+is used even if the command-line `--model` points at one component checkpoint.
 
-Send requests with `"model": "aurateam/AURA"`. The ASR, AURA, and Qwen3-TTS
-checkpoint paths are internal stage models from the deploy YAML, not the
-OpenAI-facing served model name.
-
-The AURA stage can emit `<|silent|>`. Silent outputs are treated as a gate:
-they produce no Qwen3-TTS Talker input, so no audio is synthesized for that
-turn.
+Silent Stage1 outputs (`<|silent|>` / id `151669`) skip TTS for that turn.
 
 ## GPU Utilization Recommendation
 
@@ -40,26 +40,21 @@ VRAM each stage can reserve. Start with this split for a single GPU:
 - Stage 2 (Qwen3-TTS Talker): `0.20`
 - Stage 3 (Qwen3-TTS Code2Wav): `0.20`
 
-## TTS Modes
+## TTS modes (stage extras)
 
-`aura_omni` can pass AURA text to Qwen3-TTS in two task modes:
+When the duplex session supplies TTS extras, AURA text can feed Qwen3-TTS as:
 
-- `Base`: voice clone from `tts_ref_audio` with ICL enabled in the AURA
-  pipeline. Provide both `tts_ref_audio` and `tts_ref_text`. Set
-  `tts_x_vector_only_mode=true` to disable ICL and use speaker embedding only.
-- `CustomVoice`: predefined speaker mode. Use a Qwen3-TTS CustomVoice
-  checkpoint for stages 2 and 3 in `aura_omni.yaml`, then pass
-  `tts_task_type=CustomVoice` and `tts_speaker`.
+- `Base`: voice clone from `tts_ref_audio` (optional x-vector-only mode)
+- `CustomVoice`: predefined speaker (`tts_speaker`) with a CustomVoice checkpoint
+  on stages 2 and 3
 
-By default, AURA responses are passed to Qwen3-TTS as text. Set
-`additional_information.tts_pass_token_ids=true` to pass AURA-generated
-assistant token ids directly instead. Even when token passthrough is disabled,
-the stage processor uses AURA token ids when available to estimate the Talker
-prefill length, so scheduling does not rely on raw character length.
+Optional `tts_pass_token_ids` passes AURA assistant token ids into Talker
+instead of detokenized text.
 
-The example client exposes this as:
+## Old chat / Gradio / curl scripts (unsupported)
 
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-pass-token-ids
-```
+`examples/online_serving/aura_omni/` still contains chat-completions, curl, and
+Gradio helpers from an older turn-based path. They do **not** work against the
+shipped duplex profile. That path was an incomplete stand-in before streaming
+I/O and session history were available; it is not a supported AURA mode. Use
+the Realtime duplex smoke client for online verification.

docs/user_guide/examples/online_serving/bagel.md

@@ -247,12 +247,18 @@ curl http://localhost:8091/v1/chat/completions \
   -d @payload.json
 ```
 
-BAGEL img2img derives its generated image size from the input image. It
-preserves the input aspect ratio, aligns dimensions to the latent stride, and
-applies the checkpoint size limit; explicit `height` and `width` values do not
-override this model-specific resize policy. In step mode, the effective size is
-resolved before scheduler admission so requests with different resulting
-shapes are not combined.
+When no output size is requested (`size=auto` on `/v1/images/edits`, or no
+`height`/`width` in a chat request), BAGEL img2img derives the canvas from the
+input image: it preserves the input aspect ratio, aligns dimensions to the
+latent stride (16 px), and applies the checkpoint size limit. An explicit
+`size`, `height`, or `width` is honored instead, in both the single-stage and
+the two-stage deployment; the input image is still resized to a stride-aligned
+size for the VAE/ViT prefill. The latent grid needs multiples of 16, so a
+requested side that is not one is floored to the next multiple (1000x700
+generates 992x688) and the server logs a warning; the response reports the
+generated size. A side above the checkpoint limit is rejected with HTTP 400.
+In step mode, the effective size is resolved before scheduler admission so
+requests with different resulting shapes are not combined.
 
 ### Image to Text (img2text)
 
@@ -318,8 +324,8 @@ curl http://localhost:8091/v1/chat/completions \
 | `--server` / `-s` | `http://localhost:8091` | Server URL |
 | `--image-url` / `-i` | `None` | Input image URL or local path (img2img/img2text) |
 | `--modality` / `-m` | `text2img` | `text2img`, `img2img`, `img2text`, `text2text` |
-| `--height` | `512` | Text-to-image output height; BAGEL img2img derives its size from the input image |
-| `--width` | `512` | Text-to-image output width; BAGEL img2img derives its size from the input image |
+| `--height` | `512` | Output height, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
+| `--width` | `512` | Output width, a multiple of 16; omit both `--height` and `--width` to let img2img derive the size from the input image |
 | `--steps` | `25` | Number of inference steps |
 | `--seed` | `42` | Random seed |
 | `--negative` | `None` | Negative prompt for CFG |

examples/online_serving/aura_omni/README.md

@@ -6,139 +6,89 @@
 Qwen3-ASR -> AURA/Qwen3-VL -> Qwen3-TTS Talker -> Qwen3-TTS Code2Wav
 ```
 
-The pipeline has three semantic modules, but four engine stages because the
-existing Qwen3-TTS implementation is natively split into Talker and Code2Wav.
+**AURA's supported online path is Realtime duplex** (`/v1/realtime?duplex=1`).
+The default deploy profile sets `session_mode: duplex` because the pipeline
+declares `duplex_plugin` and `DuplexOmniEngine` requires that mode.
 
-Start the server with the deploy profile:
+This is AURA v1 / Qwen3-VL (silent / ChatML ids `151669` / `151645`), not
+AURA v2 / Qwen3.5-VL.
+
+## Duplex Realtime (only supported online path)
+
+Stage2→3 codec handoff needs `async_chunk: true`. The default
+`vllm_omni/deploy/aura_omni.yaml` keeps `async_chunk: false` as a safer
+baseline (turn-based `OmniOrchestrator` never sees the duplex async-chunk
+gate). For Realtime, use the smoke deploy (already `async_chunk: true`) or
+set that flag yourself:
 
 ```bash
+bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh
+python examples/online_serving/aura_omni/smoke_duplex_realtime_client.py
+```
+
+Smoke deploy file: `examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml`.
+
+Equivalent serve with the default topology and async chunks enabled:
+
+```bash
+# Copy or edit so async_chunk: true, then:
 vllm serve aurateam/AURA \
   --omni \
   --port 8091 \
-  --deploy-config vllm_omni/deploy/aura_omni.yaml \
+  --deploy-config examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml \
   --served-model-name aurateam/AURA \
   --trust-remote-code
 ```
 
-The deploy file sets per-stage model repos:
+Connect clients to `/v1/realtime?duplex=1`. Silent Stage1 outputs gate TTS (no
+audio for that turn). Overlapped input and vision-follow are AURA duplex
+capabilities.
+
+### Browser UI
+
+AURA has no client VAD: control is **push-to-talk** on the shared shell from
+[#7585](https://github.com/vllm-project/vllm-omni/pull/7585)
+([`examples/online_serving/realtime_web/`](../realtime_web/README.md)), profile
+`aura-ptt` (hold = `is_speech=true` PCM + sticky frames; release = `commit`;
+silent+frames for vision-follow). Thin wrapper:
+
+```bash
+python -m examples.online_serving.aura_omni.realtime_web \
+    --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862
+```
+
+Do **not** fork a separate `aura_omni/realtime_web` asset tree; MiniCPM / Qwen
+profiles are unchanged and hide the PTT control.
+
+### Per-stage models
+
+Edit `model` on each stage in `vllm_omni/deploy/aura_omni.yaml` (or the smoke
+YAML) for local checkpoints:
 
 - Stage 0 ASR: `Qwen/Qwen3-ASR-1.7B`
 - Stage 1 AURA: `aurateam/AURA`
-- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base`
+- Stage 2/3 TTS: `Qwen/Qwen3-TTS-12Hz-1.7B-Base` (or CustomVoice)
 
-For local weights, edit the `model` value on each stage in
-`vllm_omni/deploy/aura_omni.yaml`. The deploy profile includes
-`pipeline: aura_omni`, so the server uses this four-stage topology even when
-the command-line model path points at one component checkpoint.
+### GPU utilization
 
-Expected request shape:
-
-- Send microphone audio as the Stage 0 multimodal audio input.
-- Include video frames in the original request `multi_modal_data`; the
-  `asr2aura` processor carries them forward to AURA.
-- Optional `additional_information` keys:
-  - `aura_system_prompt`
-  - `tts_task_type`
-  - `tts_language`
-  - `tts_speaker`
-  - `tts_instruct`
-  - `tts_ref_audio`
-  - `tts_ref_text`
-  - `tts_x_vector_only_mode`
-  - `tts_pass_token_ids`
-
-If AURA emits `<|silent|>`, the `aura2tts` processor returns no TTS request, so
-the TTS stages are skipped for that turn.
-
-## GPU Utilization Recommendation
-
-Tune `gpu_memory_utilization` per stage in `vllm_omni/deploy/aura_omni.yaml`.
-Recommended baseline on one GPU for H200
+Tune `gpu_memory_utilization` per stage. Baseline on one large GPU:
 
 - Stage 0 (ASR): `0.10`
-- Stage 1 (AURA): `0.4`
-- Stage 2 (Qwen3-TTS Talker): `0.20`
-- Stage 3 (Qwen3-TTS Code2Wav): `0.20`
+- Stage 1 (AURA): `0.40`
+- Stage 2 (Talker): `0.20`
+- Stage 3 (Code2Wav): `0.20`
 
-## Python Client
+## Old chat / Gradio / curl scripts (unsupported)
 
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --host localhost \
-  --port 8091 \
-  --model aurateam/AURA \
-  --modalities text,audio
-```
+`openai_chat_completion_client.py`, `run_curl_multimodal_generation.sh`, and
+`run_gradio_demo.sh` target the older turn-based Omni path. They do **not**
+work against the shipped `session_mode: duplex` profile.
 
-Use local media:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --audio-path /path/to/input.wav \
-  --video-path /path/to/video.mp4 \
-  --output-dir output_aura_omni_online
-```
-
-Base voice clone mode (default, recommended as x-vector while debugging ICL):
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-task-type Base \
-  --tts-ref-audio vllm-omni/tests/assets/qwen3_tts/clone_2.wav \
-  --tts-ref-text "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-```
-
-Enable AURA token-id passthrough explicitly:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-pass-token-ids
-```
-
-CustomVoice mode requires stages 2 and 3 in `aura_omni.yaml` to point at a
-Qwen3-TTS CustomVoice checkpoint:
-
-```bash
-python examples/online_serving/aura_omni/openai_chat_completion_client.py \
-  --tts-task-type CustomVoice \
-  --tts-speaker Vivian
-```
-
-By default, AURA responses are passed to Qwen3-TTS as text. Set
-`tts_pass_token_ids=true` to pass AURA-generated assistant token ids directly
-to Qwen3-TTS instead. The processor still uses AURA token ids, when available,
-to estimate the Talker prompt length in the default text path.
-
-## Curl
-
-```bash
-cd examples/online_serving/aura_omni
-bash run_curl_multimodal_generation.sh
-```
-
-Set `PORT`, `MODEL`, or `OUTPUT_DIR` to override defaults:
-
-```bash
-PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh
-TTS_PASS_TOKEN_IDS=true PORT=8666 MODEL=aurateam/AURA bash run_curl_multimodal_generation.sh
-```
-
-## Gradio
-
-Launch the server and Gradio UI together:
-
-```bash
-cd examples/online_serving/aura_omni
-bash run_gradio_demo.sh
-```
-
-If the server is already running:
-
-```bash
-python examples/online_serving/aura_omni/gradio_demo.py \
-  --model aurateam/AURA \
-  --api-base http://localhost:8091/v1
-```
+That turn-based path was an incomplete June-era stand-in: vLLM-Omni did not
+yet support streaming I/O for this stack, and adding session history then would
+have required a much larger orchestrator change. It is not a supported AURA
+mode. Do not run those scripts against the default deploy. Prefer the duplex
+smoke client above.
 
 ## Offline
 

examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml

@@ -0,0 +1,119 @@
+# AURA duplex Realtime smoke (local weights, session_mode: duplex).
+# Stage1 = /workspace/models/AURA + AuraQwen3VLForConditionalGeneration (AURA v1 / Qwen3-VL).
+pipeline: aura_omni
+# Qwen3-TTS Stage2→3 codec handoff requires async_chunk=true (same as native /
+# duplex explore). async_chunk=false reaches talker/code_predictor then never
+# delivers audio to Stage3/client.
+async_chunk: true
+session_mode: duplex
+duplex_session:
+  max_sessions: 1
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      codec_streaming: true
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 3000
+      connector_get_max_wait: 300
+      codec_chunk_frames: 25
+      codec_left_context_frames: 72
+      # Match native/duplex explore. initial=1 can stall Code2Wav on the first
+      # undersized chunk (observed hang after "Code2Wav codec: frames=1").
+      initial_codec_chunk_frames: 8
+      decode_cudagraph_capture_sizes: [25, 73, 97, 169, 325]
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.08
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: true
+    async_scheduling: true
+    max_model_len: 4096
+    max_num_batched_tokens: 2048
+    devices: "0"
+    model: /workspace/models/Qwen3-ASR-1.7B
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 256
+
+  - stage_id: 1
+    max_num_seqs: 1
+    gpu_memory_utilization: 0.55
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: true
+    async_scheduling: true
+    max_model_len: 32768
+    max_num_batched_tokens: 8192
+    devices: "0"
+    model: /workspace/models/AURA
+    model_arch: AuraQwen3VLForConditionalGeneration
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 256
+      repetition_penalty: 1.0
+      stop_token_ids: [151669, 151645]
+
+  - stage_id: 2
+    # Allow prior TTS to drain while the next Talker starts (not assumed 1).
+    max_num_seqs: 2
+    gpu_memory_utilization: 0.12
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    async_scheduling: true
+    max_num_batched_tokens: 4096
+    max_model_len: 4096
+    devices: "0"
+    # Stage1→2 is orchestrator-fed via aura2tts (custom_process_input_func).
+    # A from_stage_1 SHM edge parks Talker in WAITING_FOR_CHUNK and clears
+    # additional_information.text → Missing Qwen3-TTS text conditioning.
+    # Only Stage2→3 codec handoff needs a connector.
+    output_connectors:
+      to_stage_3: connector_of_shared_memory
+    model: /workspace/models/Qwen3-TTS-12Hz-1.7B-CustomVoice
+    default_sampling_params:
+      temperature: 0.5
+      top_k: 50
+      # 12Hz × 240 ≈ 20s. Stay under the smoke 25s audio gate so a missed
+      # 2150 (English/quotes humming) cannot overlap the next vision-follow.
+      max_tokens: 240
+      min_tokens: 8
+      repetition_penalty: 1.05
+      # Qwen3-TTS Talker codec EOS — without this Talker can run to max_tokens
+      # (~26s garbage audio while Stage1 text was correct).
+      stop_token_ids: [2150]
+    subtalker_sampling_params:
+      do_sample: true
+      temperature: 0.5
+      top_k: 50
+      top_p: 1.0
+
+  - stage_id: 3
+    max_num_seqs: 2
+    gpu_memory_utilization: 0.12
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    async_scheduling: true
+    max_num_batched_tokens: 16384
+    max_model_len: 16384
+    devices: "0"
+    input_connectors:
+      from_stage_2: connector_of_shared_memory
+    model: /workspace/models/Qwen3-TTS-12Hz-1.7B-CustomVoice
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 16384
+      seed: 42
+      repetition_penalty: 1.0

examples/online_serving/aura_omni/realtime_web/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""AURA push-to-talk browser call UI."""

examples/online_serving/aura_omni/realtime_web/__main__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from examples.online_serving.realtime_web.server import main
+
+if __name__ == "__main__":
+    main(default_profile="aura-ptt")

examples/online_serving/aura_omni/run_duplex_smoke_serve.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Minimal AURA duplex Realtime server for smoke.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+cd "$REPO_ROOT"
+
+PORT="${PORT:-8099}"
+MODEL="${MODEL:-/workspace/models/AURA}"
+DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/examples/online_serving/aura_omni/aura_omni_duplex_smoke.yaml}"
+
+# AURA v1 / Qwen3-VL ChatML ids (do not use AURA v2 248070/248046 here).
+export VLLM_AURA_SILENT_TOKEN_ID="${VLLM_AURA_SILENT_TOKEN_ID:-151669}"
+export VLLM_AURA_IM_END_TOKEN_ID="${VLLM_AURA_IM_END_TOKEN_ID:-151645}"
+export VLLM_AURA_IM_START_TOKEN_ID="${VLLM_AURA_IM_START_TOKEN_ID:-151644}"
+export VLLM_AURA_ASSISTANT_TOKEN_ID="${VLLM_AURA_ASSISTANT_TOKEN_ID:-77091}"
+# Multi-stage colocated init can make earlier stages' Process.sentinel spuriously
+# readable; prefer spawn for stage engine subprocesses (override if needed).
+export VLLM_WORKER_MULTIPROC_METHOD="${VLLM_WORKER_MULTIPROC_METHOD:-spawn}"
+# Do NOT set TORCHDYNAMO_DISABLE here: it breaks Stage0/1 AOT when those
+# stages are not enforce_eager. Stage2/3 use enforce_eager; code_predictor
+# honors that and skips its own torch.compile.
+
+exec "$REPO_ROOT/.venv/bin/vllm-omni" serve "$MODEL" \
+  --omni \
+  --deploy-config "$DEPLOY_CONFIG" \
+  --trust-remote-code \
+  --host 0.0.0.0 \
+  --port "$PORT"

examples/online_serving/aura_omni/smoke_duplex_realtime_client.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Minimal AURA duplex Realtime smoke via DuplexClient."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import sys
+import time
+from io import BytesIO
+from pathlib import Path
+
+from PIL import Image
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from vllm_omni.clients.duplex import (  # noqa: E402
+    DuplexClient,
+    EventCollector,
+    SessionConfig,
+    read_pcm16_wav,
+)
+
+PROGRESS_TYPES = {
+    "response.created",
+    "response.done",
+    "response.audio.delta",
+    "response.output_audio.delta",
+    "response.audio_transcript.delta",
+    "response.output_text.delta",
+    "response.text.delta",
+    "response.listen",
+    "response.model_listen",
+    "error",
+}
+
+
+def _tiny_jpeg_b64() -> str:
+    img = Image.new("RGB", (64, 64), color=(200, 40, 40))
+    buf = BytesIO()
+    img.save(buf, format="JPEG", quality=85)
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+async def run(url: str, model: str, wav: Path, timeout_s: float) -> int:
+    pcm16 = read_pcm16_wav(wav)
+    if not pcm16:
+        print("empty wav", file=sys.stderr)
+        return 2
+
+    frame = _tiny_jpeg_b64()
+    collector = EventCollector()
+    config = SessionConfig(
+        modalities=("text", "audio"),
+        instructions="You are AURA. Reply briefly in Chinese.",
+        extra_body={
+            "aura_system_prompt": "You are AURA. Reply briefly in Chinese.",
+            "tts_task_type": "CustomVoice",
+            "tts_language": "Chinese",
+            "tts_speaker": "Vivian",
+        },
+    )
+
+    started = time.time()
+    client = DuplexClient(
+        url,
+        model=model,
+        config=config,
+        reconnect=None,
+        heartbeat_interval_s=None,
+        handshake_timeout_s=min(60.0, timeout_s),
+    )
+
+    async with client:
+        consume_task = asyncio.create_task(collector.consume(client))
+        collector.add({"type": "session.created", "session": getattr(client, "session_info", {})})
+
+        await client.append_audio(pcm16, is_speech=True, video_frames=[frame])
+        await client.commit(create_response=True)
+
+        progress = False
+        got_done = False
+        while time.time() - started < timeout_s:
+            types = [e.get("type") for e in collector.events]
+            if any(t in PROGRESS_TYPES for t in types):
+                progress = True
+            if "response.done" in types or "response.completed" in types:
+                got_done = True
+                progress = True
+                break
+            for e in collector.events:
+                meta = e.get("metadata") if isinstance(e.get("metadata"), dict) else {}
+                if meta.get("model_listen") or meta.get("listen_source") == "aura_silent":
+                    progress = True
+                    break
+            if collector.count("error") > 0 or collector.count("session.closed") > 0:
+                break
+            # Keep draining until response.done so TTS is not truncated.
+            await asyncio.sleep(0.2)
+
+        try:
+            await client.close(timeout_s=30.0)
+        except Exception as exc:  # noqa: BLE001
+            print(f"close_warn={exc}", file=sys.stderr)
+        try:
+            await asyncio.wait_for(consume_task, timeout=5.0)
+        except (TimeoutError, asyncio.CancelledError):
+            consume_task.cancel()
+
+    event_types = [e.get("type") for e in collector.events]
+    errors = [json.dumps(e, ensure_ascii=False)[:500] for e in collector.errors()]
+    result = {
+        "ok": progress and (got_done or collector.count("response.done") > 0),
+        "got_response_progress": progress,
+        "got_response_done": got_done or collector.count("response.done") > 0,
+        "errors": errors,
+        "event_types": event_types,
+        "n_events": len(event_types),
+        "audio_bytes": len(collector.audio_bytes()),
+        "llm_text": collector.response_text(collector.response_ids[0]) if collector.response_ids else "",
+        "elapsed_s": round(time.time() - started, 2),
+    }
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    log_path = Path("/tmp/aura_duplex_smoke/client_events.jsonl")
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    log_path.write_text(
+        "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in collector.events),
+        encoding="utf-8",
+    )
+    print(f"events_log={log_path}", file=sys.stderr)
+    return 0 if progress else 1
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime")
+    p.add_argument("--model", default="/workspace/models/AURA")
+    p.add_argument(
+        "--wav",
+        default=str(REPO_ROOT / "tests/assets/minicpmo_4_5/response_required_16k.wav"),
+    )
+    p.add_argument("--timeout-s", type=float, default=300.0)
+    args = p.parse_args()
+    return asyncio.run(run(args.url, args.model, Path(args.wav), args.timeout_s))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/online_serving/realtime_web/README.md

@@ -9,6 +9,7 @@ This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/722
 | `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
 | `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
 | `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
+| `aura-ptt` | Hold **Hold to talk**; release commits. Hold also stops local playback | Sticky frames with speech; vision-follow every 2 frames only while unlocked (locked from PTT release until text-final / listen) | Yes |
 
 Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
 while replies stream, and speech can interrupt generation and playback. The
@@ -29,6 +30,38 @@ The compatibility wrapper serves the shared assets with `minicpm-native`.
 The current native query, `extra_body`, reference voice, continuous microphone
 upload, camera frames and playback acknowledgements are retained.
 
+## AURA: push-to-talk duplex
+
+AURA has no client VAD. Start the duplex serve, then the thin wrapper:
+
+```bash
+bash examples/online_serving/aura_omni/run_duplex_smoke_serve.sh
+
+python -m examples.online_serving.aura_omni.realtime_web \
+    --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862
+```
+
+Or select the profile on the shared host:
+
+```bash
+python -m examples.online_serving.realtime_web --profile aura-ptt \
+    --ws-backend ws://127.0.0.1:8099 --model aurateam/AURA --port 7862
+```
+
+Open the UI, start a session, optionally enable **Camera**, then **hold**
+**Hold to talk** to stream `is_speech=true` PCM (with sticky camera frames).
+**Release** to `input_audio_buffer.commit`. Holding the button stops playback
+in the browser only; the in-flight response is not cancelled on the server.
+With the camera on, frames are still captured at 2 fps. While the button is
+up they are committed once per two frames (`is_speech=false` + silent PCM).
+Both frames of that commit are sent and packed as one video (`<|video_pad|>`).
+Turns still open from this client commit, not from a server-side auto trigger.
+but only after the prior spoken turn unlocks (text-final or `response.listen`).
+Release itself locks vision immediately so a follow-up frame cannot abort the
+just-committed speech turn on Stage0/1 (`max_num_seqs=1`). Audio still playing
+does not hold the next vision turn once text is done.
+MiniCPM / Qwen profiles do not set `pushToTalk`, so the PTT control stays hidden.
+
 ## Qwen3: explicit-turn STT adapter
 
 Start the backend, then the UI in a second terminal:
@@ -49,6 +82,7 @@ accepts only model selection, so system-prompt controls are hidden in this mode.
 The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
 `vllm_omni/entrypoints/openai/realtime_connection.py`:
 
+
 1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
    It does not enable STT on a duplex deployment.
 2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
@@ -181,7 +215,7 @@ conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
 
 ## Shared host options
 
-The shared entry point can select either profile explicitly:
+The shared entry point can select a profile explicitly:
 
 ```bash
 python -m examples.online_serving.realtime_web --profile qwen3-turn \
@@ -193,7 +227,7 @@ python -m examples.online_serving.realtime_web --profile qwen3-turn \
 - `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
   static host proxies `/v1/realtime` on the same origin.
 - `--host`, `--port`: UI bind address and port (default port 7862).
-- `--ref-audio`: required for MiniCPM; rejected for Qwen3.
+- `--ref-audio`: required for MiniCPM; rejected for Qwen3 and AURA.
 
 Microphone and camera access require `localhost` or HTTPS. For a remote backend,
 use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support.

examples/online_serving/realtime_web/app/index.html

@@ -22,6 +22,7 @@
         <div class="call-actions">
           <button id="callButton" class="primary" type="button">Start session</button>
           <button id="sendTurnButton" class="secondary" type="button" hidden disabled>Send turn</button>
+          <button id="pttButton" class="secondary" type="button" hidden disabled>Hold to talk</button>
           <button id="muteButton" class="secondary" type="button" disabled>Mute</button>
           <button id="cameraButton" class="secondary" type="button" disabled>Camera</button>
           <video id="cameraPreview" muted playsinline style="display:none;width:96px;height:72px;border-radius:8px;object-fit:cover;background:#000"></video>
@@ -75,6 +76,7 @@
     <script src="static/profiles/common.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
     <script src="static/profiles/minicpm_native.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
     <script src="static/profiles/qwen3_turn.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
+    <script src="static/profiles/aura_ptt.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
     <script src="static/app.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
   </body>
 </html>

examples/online_serving/realtime_web/app/static/app.js

@@ -5,6 +5,7 @@
   const profile = window.OmniRealtimeProfiles[config.profile || 'minicpm-native'](config);
   const callButton = document.getElementById('callButton');
   const sendTurnButton = document.getElementById('sendTurnButton');
+  const pttButton = document.getElementById('pttButton');
   const muteButton = document.getElementById('muteButton');
   const cameraButton = document.getElementById('cameraButton');
   const cameraPreview = document.getElementById('cameraPreview');
@@ -28,6 +29,7 @@
   const ECHO_GUARD_MS = 300;
   const INITIAL_PLAYBACK_BUFFER_MS = 400;
   const SESSION_CLOSE_TIMEOUT_MS = 1000;
+  const SILENT_PCM_SAMPLES = 160;
 
   const PROMPT_PRESETS = profile.presets;
   document.title = profile.title;
@@ -36,7 +38,21 @@
   document.getElementById('profileDescription').textContent = profile.description;
   document.getElementById('policyLabel').textContent = profile.policy;
   cameraButton.hidden = !profile.camera;
+  if (profile.cameraPreviewLarge) {
+    cameraPreview.classList.add('camera-preview-large');
+    // Inline, so a cached stylesheet cannot leave the 96×72 HTML default.
+    cameraPreview.style.width = '100%';
+    cameraPreview.style.height = 'auto';
+    cameraPreview.style.maxHeight = '480px';
+    cameraPreview.style.flex = '0 0 100%';
+    const cameraActions = cameraPreview.closest('.call-actions');
+    if (cameraActions) {
+      cameraActions.style.flexWrap = 'wrap';
+      cameraActions.style.gridColumn = '1 / -1';
+    }
+  }
   sendTurnButton.hidden = !profile.clientCommit;
+  pttButton.hidden = !profile.pushToTalk;
   promptPreset.replaceChildren();
   for (const name of [...Object.keys(PROMPT_PRESETS), 'custom']) {
     const option = document.createElement('option');
@@ -64,6 +80,12 @@
   let cameraStream = null;
   let cameraTimer = null;
   let cameraPendingFrame = null;
+  let cameraLastFrame = null;
+  // AURA vision clock only. Other profiles leave this empty and never read it.
+  let visionFollowQueue = [];
+  let visionTurnLocked = false;
+  let suppressedPlaybackId = null;
+  let pttHeld = false;
   const cameraCanvas = document.createElement('canvas');
   let playbackRate = OUTPUT_RATE;
   let pendingCapture = [];
@@ -87,6 +109,8 @@
   const interruptedResponses = new Set();
   const pendingEvents = new Set();
   let assistantTextChannel = null;
+  const assistantRowByResponse = new Map();
+  let lastClosedAssistantText = '';
   let connectionReady = false;
   let turnCounter = 0;
   let stopping = null;
@@ -134,7 +158,7 @@
       assistantTextChannel = null;
       if (profile.halfDuplex) pendingCapture = [];
       sendTurnButton.disabled = !profile.clientCommit;
-      setModel(profile.waiting);
+      if (!pttHeld) setModel(profile.waiting);
     }, ECHO_GUARD_MS);
   }
 
@@ -240,6 +264,7 @@
       current.value = finalText;
       current.text.textContent = finalText;
     }
+    if (role === 'assistant' && current.value) lastClosedAssistantText = current.value;
     current.row.classList.remove('turn-live');
     if (role === 'user') liveUserTurn = null;
     else liveAssistantTurn = null;
@@ -317,11 +342,67 @@
   }
 
   function microphoneUploadEnabled() {
-    return running && connectionReady && !muted && (!profile.halfDuplex || !assistantActive);
+    if (!running || !connectionReady || muted) return false;
+    if (profile.pushToTalk) return pttHeld;
+    return !profile.halfDuplex || !assistantActive;
+  }
+
+  function queueVisionFrame(frame) {
+    visionFollowQueue.push(frame);
+    if (visionFollowQueue.length > 2) visionFollowQueue.shift();
+  }
+
+  function commitVisionFollow() {
+    // Two frames still gate the turn (client commit). Both frames are one clip;
+    // the server packs them as a single <|video_pad|>, same as Native.
+    const frames = visionFollowQueue.slice();
+    visionFollowQueue = [];
+    const silent = new Int16Array(SILENT_PCM_SAMPLES);
+    const event = profile.append(int16ToBase64(silent), frames, { isSpeech: false });
+    socket.send(JSON.stringify(event));
+    for (const message of profile.commitMessages()) socket.send(JSON.stringify(message));
+  }
+
+  function releaseVisionTurn(responseId) {
+    if (!profile.visionFollowWhileSpeaking || !visionTurnLocked) return;
+    if (responseId && currentResponseId && responseId !== currentResponseId) return;
+    visionTurnLocked = false;
+  }
+
+  function suppressLocalPlayback() {
+    // Hold-to-talk cuts the speaker only. The in-flight response stays on the server.
+    if (currentResponseId) suppressedPlaybackId = currentResponseId;
+    if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
+    playbackComplete = true;
+    responseHasAudio = false;
+    setPlayback('Idle');
+  }
+
+  function playbackSuppressed(responseId) {
+    return Boolean(suppressedPlaybackId) && responseId === suppressedPlaybackId;
   }
 
   function flushCapture() {
-    if (!socket || socket.readyState !== WebSocket.OPEN || pendingCapture.length === 0) return;
+    if (!socket || socket.readyState !== WebSocket.OPEN) return;
+
+    // AURA: capture stays 2 fps, but a turn opens only every 2 frames, and
+    // not until the previous response's text is done (or it listened).
+    // Playback still running is not the lock. PTT speech is not batched here.
+    if (profile.pushToTalk && profile.visionFollowWhileSpeaking && !pttHeld) {
+      if (cameraPendingFrame) {
+        queueVisionFrame(cameraPendingFrame);
+        cameraPendingFrame = null;
+      }
+      if (!visionTurnLocked && visionFollowQueue.length >= 2) commitVisionFollow();
+      return;
+    }
+
+    if (profile.pushToTalk && !pttHeld) {
+      pendingCapture = [];
+      return;
+    }
+
+    if (pendingCapture.length === 0) return;
     if (!microphoneUploadEnabled()) {
       pendingCapture = [];
       return;
@@ -335,11 +416,44 @@
     }
     pendingCapture = [];
     const pcm = resampleInt16(merged, captureRate, profile.inputSampleRate || INPUT_RATE);
-    const appendEvent = profile.append(int16ToBase64(pcm), cameraPendingFrame);
+    const frame = cameraPendingFrame || (profile.stickyCamera ? cameraLastFrame : null);
     cameraPendingFrame = null;
+    const appendEvent = profile.pushToTalk
+      ? profile.append(int16ToBase64(pcm), frame, { isSpeech: true })
+      : profile.append(int16ToBase64(pcm), frame);
     socket.send(JSON.stringify(appendEvent));
   }
 
+  function setPttHeld(held) {
+    if (!profile.pushToTalk || !running || pttHeld === held) return;
+    if (held) {
+      pttHeld = true;
+      suppressLocalPlayback();
+      pttButton.classList.toggle('is-active', true);
+      pttButton.textContent = 'Release to send';
+      setModel('Talking');
+      appendLog('PTT down');
+      return;
+    }
+    // Flush remaining speech while still held, then commit.
+    flushCapture();
+    if (socket && socket.readyState === WebSocket.OPEN) {
+      for (const event of profile.commitMessages()) socket.send(JSON.stringify(event));
+    }
+    // Lock vision before clearing pttHeld. Stage0/1 are max_num_seqs=1: a
+    // vision-follow commit in the gap before response.begin aborts the speech
+    // turn (server sees a new ephemeral and kills the prior ASR request).
+    if (profile.visionFollowWhileSpeaking) {
+      visionTurnLocked = true;
+      visionFollowQueue = [];
+    }
+    pttHeld = false;
+    pttButton.classList.toggle('is-active', false);
+    pttButton.textContent = 'Hold to talk';
+    setModel(profile.waiting);
+    appendLog('PTT up · commit');
+  }
+
   function beginAssistant(responseId) {
     currentResponseId = responseId || currentResponseId;
     responseHasAudio = false;
@@ -351,6 +465,7 @@
   }
 
   function feedPlayback(decoded, responseId) {
+    if (playbackSuppressed(responseId || currentResponseId)) return;
     if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
     const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
     responseHasAudio = true;
@@ -392,13 +507,18 @@
   }
 
   async function handleAudioEvent(action) {
+    const responseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
+    if (playbackSuppressed(responseId)) return;
     markBusy();
-    currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
+    currentResponseId = responseId;
     setModel('Speaking');
     const generation = sessionGeneration;
-    const responseId = currentResponseId;
     const decoded = await decodeAudioDelta(action.event);
-    if (generation === sessionGeneration && !interruptedResponses.has(responseId)) feedPlayback(decoded, responseId);
+    if (
+      generation === sessionGeneration
+      && !interruptedResponses.has(responseId)
+      && !playbackSuppressed(responseId)
+    ) feedPlayback(decoded, responseId);
   }
 
   function handleTranscriptEvent(action) {
@@ -408,12 +528,43 @@
       if (profile.deduplicateTranscript && assistantTextChannel && action.channel !== assistantTextChannel) return;
       if (action.text) assistantTextChannel = action.channel || assistantTextChannel;
     }
-    if (action.kind === 'text') addTranscript(action.role, action.text);
-    else if (profile.deduplicateTranscript && action.role === 'assistant') {
+    const responseId = action.responseId || null;
+    const existing = responseId && action.role === 'assistant' ? assistantRowByResponse.get(responseId) : null;
+    if (existing) {
+      if (action.kind === 'text') {
+        if (!action.text) return;
+        existing.value += action.text;
+      } else if (action.text) {
+        existing.value = action.text;
+      }
+      existing.text.textContent = existing.value;
+      liveAssistantTurn = existing;
+      return;
+    }
+    if (
+      action.role === 'assistant'
+      && !liveAssistantTurn
+      && action.text
+      && action.text === lastClosedAssistantText
+    ) {
+      // response.done already closed this sentence. A later full transcript,
+      // including one stamped with a newer vision-follow response id, must
+      // not open a second bubble with the same words.
+      return;
+    }
+    if (action.kind === 'text') {
+      addTranscript(action.role, action.text);
+      if (action.role === 'assistant' && responseId && liveAssistantTurn) {
+        assistantRowByResponse.set(responseId, liveAssistantTurn);
+      }
+      return;
+    }
+    if (profile.deduplicateTranscript && action.role === 'assistant') {
       if (action.text) {
         const turn = ensureTurn('assistant');
         turn.value = action.text;
         turn.text.textContent = action.text;
+        if (responseId) assistantRowByResponse.set(responseId, turn);
       }
     } else finishTranscript(action.role, action.text);
   }
@@ -459,12 +610,14 @@
       }
       case 'listen':
         assistantActive = false;
+        releaseVisionTurn(responseId);
         setModel(profile.waiting);
         break;
       case 'begin':
         if (profile.halfDuplex) armTurnTimeout();
         if (echoTimer !== null) { clearTimeout(echoTimer); echoTimer = null; }
         beginAssistant(responseId);
+        if (profile.visionFollowWhileSpeaking) visionTurnLocked = true;
         break;
       case 'audio':
         await handleAudioEvent(action);
@@ -474,11 +627,13 @@
         break;
       case 'text': case 'text-final':
         handleTranscriptEvent(action);
+        if (action.kind === 'text-final' && action.role === 'assistant') releaseVisionTurn(responseId);
         break;
       case 'done':
         clearTimeout(turnTimeout);
         responseComplete = true;
         finishTranscript('assistant');
+        releaseVisionTurn(responseId);
         requestPlaybackDrain(responseId);
         finishResponseIfReady();
         break;
@@ -671,7 +826,11 @@
       await openSocket();
       running = true;
       muted = false;
+      pttHeld = false;
       assistantActive = false;
+      visionFollowQueue = [];
+      visionTurnLocked = false;
+      suppressedPlaybackId = null;
       sendTimer = window.setInterval(flushCapture, profile.sendIntervalMs || SEND_INTERVAL_MS);
       startClock();
       callButton.textContent = 'End session';
@@ -679,6 +838,7 @@
       muteButton.disabled = false;
       cameraButton.disabled = !profile.camera;
       sendTurnButton.disabled = !profile.clientCommit;
+      pttButton.disabled = !profile.pushToTalk;
       setConnection('Connected', 'online');
       setModel(profile.waiting);
       appendLog('session started');
@@ -698,8 +858,9 @@
     cameraPreview.srcObject = cameraStream;
     cameraPreview.style.display = '';
     await cameraPreview.play().catch(() => {});
-    // Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
-    // no client-side resize (the server normalizes at scale_resolution=448).
+    // JPEG quality 0.7; no client-side resize (server scale_resolution=448).
+    // Interval is per-profile: AURA 2 fps, MiniCPM/Qwen stay at 1 s.
+    const cameraIntervalMs = profile.cameraIntervalMs || 1000;
     const captureCameraFrame = () => {
       if (!cameraStream || cameraPreview.videoWidth === 0) return;
       const scale = profile.cameraMaxDimension
@@ -712,14 +873,17 @@
         if (connectionReady && socket?.readyState === WebSocket.OPEN) {
           for (const event of profile.imageMessages(frame)) socket.send(JSON.stringify(event));
         }
-      } else cameraPendingFrame = frame;
+      } else {
+        cameraPendingFrame = frame;
+        if (profile.stickyCamera) cameraLastFrame = frame;
+      }
     };
     // Do not make the first spoken turn race a one-second timer.
     captureCameraFrame();
-    cameraTimer = window.setInterval(captureCameraFrame, 1000);
+    cameraTimer = window.setInterval(captureCameraFrame, cameraIntervalMs);
     cameraButton.textContent = 'Camera off';
     cameraButton.classList.add('is-active');
-    appendLog('camera on (1 fps omni frames)');
+    appendLog(`camera on (${(1000 / cameraIntervalMs).toFixed(0)} fps omni frames)`);
   }
 
   function stopCamera() {
@@ -730,6 +894,7 @@
     }
     cameraStream = null;
     cameraPendingFrame = null;
+    cameraLastFrame = null;
     cameraPreview.srcObject = null;
     cameraPreview.style.display = 'none';
     cameraButton.textContent = 'Camera';
@@ -778,9 +943,18 @@
     playbackComplete = true;
     turnSubmitted = false;
     assistantTextChannel = null;
+    assistantRowByResponse.clear();
+    lastClosedAssistantText = '';
     sendTurnButton.disabled = true;
+    pttHeld = false;
+    pttButton.disabled = true;
+    pttButton.classList.remove('is-active');
+    pttButton.textContent = 'Hold to talk';
     running = false;
     assistantActive = false;
+    visionFollowQueue = [];
+    visionTurnLocked = false;
+    suppressedPlaybackId = null;
     pendingCapture = [];
     if (sendTimer !== null) clearInterval(sendTimer);
     if (clockTimer !== null) clearInterval(clockTimer);
@@ -854,6 +1028,23 @@
     setModel('Thinking');
     appendLog('turn submitted');
   });
+  const pttDown = (event) => {
+    event.preventDefault();
+    setPttHeld(true);
+  };
+  const pttUp = (event) => {
+    event.preventDefault();
+    setPttHeld(false);
+  };
+  pttButton.addEventListener('pointerdown', pttDown);
+  pttButton.addEventListener('pointerup', pttUp);
+  pttButton.addEventListener('pointercancel', pttUp);
+  pttButton.addEventListener('pointerleave', () => {
+    if (pttHeld) setPttHeld(false);
+  });
+  window.addEventListener('pointerup', () => {
+    if (pttHeld) setPttHeld(false);
+  });
   muteButton.addEventListener('click', toggleMute);
   clearLogButton.addEventListener('click', () => {
     eventLog.textContent = '';

examples/online_serving/realtime_web/app/static/profiles/aura_ptt.js

@@ -0,0 +1,79 @@
+(function (root) {
+  'use strict';
+  const profiles = root.OmniRealtimeProfiles;
+  profiles['aura-ptt'] = () => ({
+    title: 'AURA Realtime',
+    eyebrow: 'Push-to-talk duplex',
+    policy: 'Hold to talk • release to commit',
+    description:
+      'Hold to talk, release to send. Camera stays on whether you speak or not. '
+    + 'Holding the button stops local playback.',
+    waiting: 'Ready',
+    camera: true,
+    // Display only. Shared CSS puts .camera-preview-large on its own row.
+    // Other profiles keep the 96×72 preview in index.html.
+    cameraPreviewLarge: true,
+    playbackAck: true,
+    clientCommit: false,
+    pushToTalk: true,
+    stickyCamera: true,
+    // OmniInteract / Native AURA clock: 2 fps. Other profiles omit this and stay at 1 s.
+    cameraIntervalMs: 500,
+    visionFollowWhileSpeaking: true,
+    deduplicateTranscript: true,
+    halfDuplex: false,
+    closeSession: true,
+    reconnectEachTurn: false,
+    readyEvent: 'session.updated',
+    instructions: true,
+    sendIntervalMs: 200,
+    presets: {
+      omni: 'You are receiving a live video stream where the final frame is the present moment. Respond only when a response is needed based on the user\'s message or the visual context. Otherwise, output `<|silent|>` to signify silence.',
+    },
+    url(config, location) {
+      const url = profiles.url(config, location);
+      url.searchParams.set('duplex', '1');
+      url.searchParams.set('model', config.model || 'aurateam/AURA');
+      return url.toString();
+    },
+    initialMessages(config, instructions) {
+      const session = {
+        modalities: ['audio', 'text'],
+        extra_body: { auto_response: true },
+      };
+      if (instructions) session.instructions = instructions;
+      return [{ type: 'session.update', session }];
+    },
+    append(audio, frame, opts) {
+      const isSpeech = !opts || opts.isSpeech !== false;
+      const event = {
+        type: 'input_audio_buffer.append',
+        audio,
+        format: 'pcm16',
+        sample_rate_hz: 16000,
+        is_speech: isSpeech,
+      };
+      const frames = Array.isArray(frame) ? frame.filter(Boolean) : (frame ? [frame] : []);
+      if (frames.length) event.video_frames = frames;
+      return event;
+    },
+    commitMessages: () => [{ type: 'input_audio_buffer.commit', create_response: true }],
+    ack(responseId, playedMs) {
+      return {
+        type: 'playback.ack',
+        response_id: responseId,
+        item_id: `item_${responseId}`,
+        played_ms: playedMs,
+        committed_ms: playedMs,
+      };
+    },
+    mapEvent(event) {
+      if (event.type === 'response.listen') return { kind: 'listen' };
+      if (event.type === 'playback.acknowledged') {
+        return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
+      }
+      return profiles.event(event);
+    },
+    connectionHint: 'Check the AURA duplex serve (run_duplex_smoke_serve.sh) and --ws-backend port.',
+  });
+})(globalThis);

examples/online_serving/realtime_web/app/static/styles.css

@@ -71,6 +71,18 @@ main { width: min(1040px, calc(100% - 40px)); margin: 28px auto 48px; }
 }
 
 .call-actions { display: flex; gap: 8px; }
+/* AURA only: class is added solely by the aura-ptt profile. */
+.call-bar:has(.camera-preview-large) .call-actions { grid-column: 1 / -1; }
+.call-actions:has(.camera-preview-large) { flex-wrap: wrap; align-items: center; }
+.call-actions:has(.camera-preview-large) .camera-preview-large {
+  order: -1;
+  flex: 0 0 100%;
+  width: 100% !important;
+  height: auto !important;
+  max-height: 480px;
+  aspect-ratio: 4 / 3;
+}
+.call-actions:has(.camera-preview-large) button { flex: 0 0 auto; }
 button { border-radius: 5px; border: 1px solid transparent; cursor: pointer; }
 button:disabled { opacity: .46; cursor: default; }
 .primary { padding: 10px 15px; color: #ffffff; background: #16834b; font-weight: 700; }

examples/online_serving/realtime_web/server.py

@@ -76,15 +76,20 @@ def build_app(
     profile: str = "minicpm-native",
     adapter: str = "stt",
 ) -> FastAPI:
-    if profile not in {"minicpm-native", "qwen3-turn"}:
+    if profile not in {"minicpm-native", "qwen3-turn", "aura-ptt"}:
         raise ValueError(f"Unknown profile: {profile}")
     if adapter not in {"stt", "vad"}:
         raise ValueError(f"Unknown adapter: {adapter}")
     if profile == "qwen3-turn" and ref_audio:
         raise ValueError("Qwen3 profile does not accept --ref-audio")
+    if profile == "aura-ptt" and ref_audio:
+        raise ValueError("AURA profile does not accept --ref-audio")
     native = profile == "minicpm-native"
-    ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
-    model = model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
+    aura = profile == "aura-ptt"
+    ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native or aura else "ws://127.0.0.1:8091")
+    model = model or (
+        "openbmb/MiniCPM-o-4_5" if native else "aurateam/AURA" if aura else "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+    )
     app = FastAPI(title="Omni Realtime Web UI")
     index_path = APP_DIR / "index.html"
     app_version_hash = hashlib.sha256()
@@ -182,7 +187,7 @@ def main(default_profile: str = "minicpm-native") -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--host", default="0.0.0.0")
     parser.add_argument("--port", type=int, default=7862)
-    parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn"), default=default_profile)
+    parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn", "aura-ptt"), default=default_profile)
     parser.add_argument("--ws-backend", "--backend", dest="ws_backend")
     mode = parser.add_mutually_exclusive_group()
     mode.add_argument("--stt", dest="adapter", action="store_const", const="stt")
@@ -202,12 +207,15 @@ def main(default_profile: str = "minicpm-native") -> None:
     )
     args = parser.parse_args()
     native = args.profile == "minicpm-native"
+    aura = args.profile == "aura-ptt"
     if native and not args.ref_audio:
         parser.error("--ref-audio is required for the minicpm-native profile")
     if not native and args.ref_audio:
         parser.error("--ref-audio is only supported by minicpm-native")
-    args.model = args.model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
-    args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
+    args.model = args.model or (
+        "openbmb/MiniCPM-o-4_5" if native else "aurateam/AURA" if aura else "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+    )
+    args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native or aura else "ws://127.0.0.1:8091")
 
     logging.basicConfig(level=logging.INFO)
     uvicorn.run(

recipes/Bagel/BAGEL-7B-MoT.md

@@ -120,12 +120,19 @@ python examples/offline_inference/image_to_image/image_edit.py \
 The `--extra-args` JSON forwards BAGEL-specific parameters (e.g. `cfg_text_scale`,
 `cfg_img_scale`, `cfg_interval`, `cfg_renorm_type`) into
 `OmniDiffusionSamplingParams.extra_args` via the model-extras registry.
-For BAGEL img2img, the pipeline derives the generated image size from the input
-image: it preserves the input aspect ratio, aligns dimensions to the latent
-stride, and applies the checkpoint size limit. Explicit `--height` and
-`--width` values do not override that BAGEL-specific resize policy. Step-mode
-admission resolves this effective size before batching so differently shaped
-requests are scheduled separately.
+For BAGEL img2img, an explicit `--height`/`--width` (or `size` on
+`/v1/images/edits`) sets the output canvas, in both the single-stage and the
+two-stage deployment; the input image is still resized to a stride-aligned size
+for the VAE/ViT prefill. The latent grid needs multiples of 16, so a requested
+side that is not one is floored to the next multiple (1000x700 generates
+992x688) with a server-side warning, and the response reports the generated
+size. A side above the checkpoint limit is rejected with HTTP 400. When no size
+is requested
+(`size=auto`, or no `height`/`width`), the pipeline derives the canvas from the
+input image: it preserves the input aspect ratio, aligns dimensions to the
+latent stride, and applies the checkpoint size limit. Step-mode admission
+resolves this effective size before batching so differently shaped requests are
+scheduled separately.
 
 Run text-to-text with the shared understanding example. BAGEL's default
 `bagel.yaml` deploy config is discovered from the checkpoint, so no model-specific

recipes/Qwen/Qwen3-TTS.md

@@ -71,8 +71,10 @@ Alternatively, use the convenience script:
 ```
 
 The bundled deploy config (`vllm_omni/deploy/qwen3_tts.yaml`) enables async
-chunking for low first-audio latency. For advanced deployment tuning, pass a
-custom deploy config:
+chunking for low first-audio latency, and selects the experimental Model
+Runner V2 on CUDA. Copy it and set `model_runner: v1` to run V1; NPU, XPU,
+ROCm and MUSA keep V1 through the `platforms:` sections. For advanced
+deployment tuning, pass a custom deploy config:
 
 ```bash
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \

recipes/Robbyant/LingBot-World-2.0.md

@@ -53,6 +53,26 @@ source frames, at least as many as requested and at most 4096. The runtime
 consumes only the prefix needed by the request. Longer rollouts also require
 sufficient device memory.
 
+## Experimental KV writeback reuse
+
+To reuse KV from the final denoising step instead of running the additional
+clean-latent writeback, pass this option in the `model_config` supplied to
+`Omni` or `AsyncOmni`:
+
+```python
+model_config = {"lingbot_reuse_last_step_kv": True}
+```
+
+The default is `False`. The value must be a boolean, not a string or integer.
+It is validated and fixed when the pipeline instance is constructed; changing
+it requires a new instance. It is not a per-request `extra_args` option.
+The previous `VLLM_OMNI_LINGBOT_REUSE_LAST_STEP_KV` environment variable is no
+longer read.
+
+This experimental mode stores the final noisy probe's KV rather than clean
+`x0` KV, so generated outputs can differ. The four denoising steps, chunk
+boundaries, and paged-context finalization remain the same.
+
 ## Realtime in-process generation (deprecated)
 
 Prefer [Streaming video serving](#streaming-video-serving) for mid-session
@@ -193,6 +213,47 @@ accepts:
   `s`, `d`, `i`, `j`, `k`, and `l`;
 - `lingbot.camera_trajectory.v1` for explicit pose/intrinsics trajectories.
 
+## Online FP8 linear layers (experimental)
+
+Pass `quantization_config={"method": "fp8"}` to the existing `AsyncOmni`
+construction to use vLLM's online FP8 linear implementation. On the tested
+Hopper-class GPU this dispatches to CUTLASS FP8 GEMM with online per-tensor
+activation scaling; no LingBot-specific quantization kernel is introduced.
+
+Only the transformer's existing vLLM parallel linear layers are eligible:
+self-attention, cross-attention, FFN, camera injectors, and C2WS projections.
+The ordinary PyTorch linear layers (including the output head and
+time/text embeddings), normalization, convolutions, VAE, text encoder, and
+AR KV cache are not converted to FP8 by this option.
+
+Quality-sensitive projections can remain BF16 through fully qualified
+`ignored_layers` names. For example, the following configuration tests
+retaining the camera/C2WS path in BF16:
+
+```python
+quantization_config = {
+    "method": "fp8",
+    "ignored_layers": [
+        f"transformer.blocks.{i}.cam_injector_layer{j}"
+        for i in range(40)
+        for j in (1, 2)
+    ]
+    + [
+        "transformer.c2ws_hidden_states_layer1",
+        "transformer.c2ws_hidden_states_layer2",
+    ],
+}
+```
+
+Use `transformer.blocks.<i>.self_attn.qkv` to exclude the fused Q/K/V
+projection as a unit. The example exclusion list is an ablation, not a
+universal quality guarantee or a default. Validate the intended scene, seed,
+camera trajectory, and session length against BF16 before selecting a policy.
+An FP8 kernel speedup alone is not an end-to-end performance claim.
+The long-session E8 quality gate in
+[#7074](https://github.com/vllm-project/vllm-omni/issues/7074) remains required
+for Tier-2 acceptance.
+
 ## Validation
 
 Real-checkpoint validation uses 480x832 output, four DMD steps, and seed 42.
@@ -226,7 +287,7 @@ tested commit.
   AR blocks in one request. `max_num_seqs` must be one in both cases.
 - Stateful streaming VAE decode is not implemented; the realtime example emits
   latent chunks.
-- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization,
+- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization methods other than online FP8,
   Cache-DiT, TeaCache, causal-pretrain, and the 1.3B checkpoint are not claimed.
 - No AMD GPU, Ascend NPU, or Intel GPU support is claimed.
 

tests/benchmarks/test_aura_omniinteract_clock.py

@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+import pytest
+
+from vllm_omni.benchmarks.omniinteract import _session_capabilities, _wants_annotation_video_clock
+from vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock import (
+    stream_annotation_clock,
+    wants_annotation_video_clock,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class _Events:
+    def __init__(self, events):
+        self.events = events
+
+
+class _Client:
+    def __init__(self, capabilities):
+        self.events = _Events([{"type": "session.created", "session": {"capabilities": capabilities}}])
+        self._client = None
+
+
+def test_annotation_clock_follows_capabilities_not_model_name():
+    aura = {"required_input_modalities": ["video"], "supports_turn_commit_only": True}
+    minicpm = {"required_input_modalities": ["audio"], "supports_turn_commit_only": False}
+    audio_commit = {"required_input_modalities": ["audio"], "supports_turn_commit_only": True}
+    assert wants_annotation_video_clock(aura)
+    assert not wants_annotation_video_clock(minicpm)
+    assert not wants_annotation_video_clock(audio_commit)
+    assert not wants_annotation_video_clock(None)
+    assert _wants_annotation_video_clock(_Client(minicpm)) is False
+    assert _wants_annotation_video_clock(_Client(aura)) is True
+    assert _session_capabilities(_Client(aura))["required_input_modalities"] == ["video"]
+
+
+def test_annotation_clock_commits_only_the_due_wav(monkeypatch, tmp_path: Path):
+    video = tmp_path / "videos" / "0002.mp4"
+    video.parent.mkdir()
+    video.write_bytes(b"")
+    sent = []
+
+    class Client:
+        async def send(self, event):
+            sent.append(event)
+
+        async def commit(self):
+            sent.append({"type": "commit"})
+
+    class Playback:
+        async def acknowledge(self, client):
+            del client
+
+    monkeypatch.setattr(
+        "vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock.load_annotation_utterances",
+        lambda path: [{"at_sec": 0.0, "pcm": b"\x01\x00" * 160, "sent": False, "qa_index": 0}],
+    )
+    chunks, frames, _, _ = asyncio.run(
+        stream_annotation_clock(
+            Client(),
+            ["frame-a", "frame-b"],
+            Playback(),
+            video_path=video,
+            fps=2.0,
+        )
+    )
+    assert (chunks, frames) == (2, 2)
+    assert sent[0]["is_speech"] is True
+    assert sent[0]["video_frames"] == ["frame-a"]
+    assert sent[1] == {"type": "commit"}
+    # The trailing silent frame is one vision turn, committed after the clip.
+    assert sent[2]["is_speech"] is False
+    assert sent[2]["video_frames"] == ["frame-b"]
+    assert sent[3] == {"type": "commit"}
+    assert sent.count({"type": "commit"}) == 2
+
+
+def test_annotation_clock_commits_silent_frames_in_pairs(monkeypatch, tmp_path: Path):
+    video = tmp_path / "videos" / "0002.mp4"
+    video.parent.mkdir()
+    video.write_bytes(b"")
+    sent = []
+
+    class Client:
+        async def send(self, event):
+            sent.append(event)
+
+        async def commit(self):
+            sent.append({"type": "commit"})
+
+    class Playback:
+        async def acknowledge(self, client):
+            del client
+
+    monkeypatch.setattr(
+        "vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock.load_annotation_utterances",
+        lambda path: [{"at_sec": 100.0, "pcm": b"\x01\x00" * 160, "sent": False, "qa_index": 0}],
+    )
+    asyncio.run(
+        stream_annotation_clock(
+            Client(),
+            ["f1", "f2", "f3", "f4"],
+            Playback(),
+            video_path=video,
+            fps=2.0,
+        )
+    )
+    vision = [event for event in sent if event.get("is_speech") is False]
+    assert [event["video_frames"] for event in vision] == [["f1", "f2"], ["f3", "f4"]]
+    assert sent.count({"type": "commit"}) == 2
+
+
+def test_annotation_clock_holds_vision_while_a_response_is_open(monkeypatch, tmp_path: Path):
+    video = tmp_path / "videos" / "0002.mp4"
+    video.parent.mkdir()
+    video.write_bytes(b"")
+    sent = []
+
+    class Events:
+        def __init__(self):
+            self.events = [{"type": "response.created", "response": {"id": "resp-open"}}]
+
+    class Client:
+        def __init__(self):
+            self.events = Events()
+
+        async def send(self, event):
+            sent.append(event)
+
+        async def commit(self):
+            sent.append({"type": "commit"})
+
+    class Playback:
+        async def acknowledge(self, client):
+            del client
+
+    monkeypatch.setattr(
+        "vllm_omni.model_executor.models.aura_omni.benchmarks.omniinteract_clock.load_annotation_utterances",
+        lambda path: [{"at_sec": 100.0, "pcm": b"\x01\x00" * 160, "sent": False, "qa_index": 0}],
+    )
+    asyncio.run(
+        stream_annotation_clock(
+            Client(),
+            ["f1", "f2", "f3", "f4"],
+            Playback(),
+            video_path=video,
+            fps=2.0,
+        )
+    )
+    assert sent == []

tests/benchmarks/test_lingbot_world_realtime.py

@@ -0,0 +1,798 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Unit tests for the LingBot-World realtime streaming benchmark.
+
+The arithmetic is tested against the server's own formulas rather than against
+hand-copied expectations, and the protocol handling runs against a real
+WebSocket server that replays ``/v1/realtime/video`` with scripted delays -- so
+a benchmark number can be wrong only if the server itself is, not because the
+client parsed incorrectly an event or timed the wrong edge.
+"""
+
+from __future__ import annotations
+
+# ruff: noqa: E402, I001
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from benchmarks.lingbot_world.benchmark_lingbot_world_realtime import (
+    BenchmarkError,
+    build_workload,
+    print_report,
+    endpoint_url,
+    parse_args,
+    resolve_image_reference,
+    run_session,
+)
+from benchmarks.lingbot_world.workload import (
+    DEFAULT_WARMUP_CHUNKS,
+    FRAMES_PER_BLOCK,
+    ChunkRecord,
+    PromptUpdate,
+    Workload,
+    aggregate_metrics,
+    build_camera_script,
+    chunks_for_num_frames,
+    compute_metrics,
+    default_workload,
+    load_workload,
+    num_frames_for_chunks,
+    percentile,
+    pixel_frames_in_chunk,
+    simulate_playback,
+    steady_chunk_deadline_ms,
+)
+
+_IMAGE = "data:image/png;base64,Zm9v"
+
+
+def _workload(num_chunks: int = 8, fps: int = 16) -> Workload:
+    return default_workload(image_reference=_IMAGE, num_chunks=num_chunks, fps=fps)
+
+
+# --------------------------------------------------------------------------- #
+# Chunk and frame arithmetic
+# --------------------------------------------------------------------------- #
+
+
+@pytest.mark.parametrize("num_chunks", list(range(1, 33)))
+def test_frame_count_round_trips_through_the_server_formula(num_chunks: int) -> None:
+    """``num_frames`` must land on a rollout of exactly the requested chunks.
+
+    The server derives chunk count as ``((num_frames - 1) // 4 + 1) // 3``; if
+    the benchmark's inverse is off by one frame the run silently generates a
+    different number of blocks than it reports.
+    """
+    assert chunks_for_num_frames(num_frames_for_chunks(num_chunks)) == num_chunks
+
+
+def test_recipe_frame_counts_match_the_documented_chunk_counts() -> None:
+    # The recipe states three chunks for num_frames 33 and seven for 81.
+    assert chunks_for_num_frames(33) == 3
+    assert chunks_for_num_frames(81) == 7
+    assert num_frames_for_chunks(3) == 33
+    assert num_frames_for_chunks(7) == 81
+
+
+def test_first_chunk_is_short_because_the_decoder_expands_it_once() -> None:
+    assert pixel_frames_in_chunk(0) == 9
+    assert pixel_frames_in_chunk(1) == 12
+    assert pixel_frames_in_chunk(7) == 12
+    # A whole rollout's frames are the sum of its chunks' frames.
+    assert sum(pixel_frames_in_chunk(index) for index in range(5)) == num_frames_for_chunks(5)
+
+
+def test_camera_script_holds_one_three_entry_action_list_per_chunk() -> None:
+    script = build_camera_script(4, "orbit")
+    assert len(script) == 4
+    assert all(len(actions) == FRAMES_PER_BLOCK for actions in script)
+    # orbit advances through the action keys so successive chunks differ.
+    assert script[0] != script[1]
+
+
+def test_unknown_camera_pattern_is_rejected() -> None:
+    with pytest.raises(ValueError, match="Unknown camera pattern"):
+        build_camera_script(2, "sideways")
+
+
+# --------------------------------------------------------------------------- #
+# Workload
+# --------------------------------------------------------------------------- #
+
+
+def test_session_start_payload_matches_the_documented_protocol() -> None:
+    workload = _workload(num_chunks=3)
+    payload = workload.session_start_payload("robbyant/lingbot-world-v2-14b-causal-fast-diffusers")
+    assert payload["type"] == "session.start"
+    assert payload["num_frames"] == 33
+    assert payload["image_reference"] == {"image_url": _IMAGE}
+    assert payload["extra_params"]["flow_shift"] == 5.0
+    assert len(payload["extra_params"]["camera_action_script"]) == 3
+
+
+def test_workload_rejects_a_camera_script_with_the_wrong_block_width() -> None:
+    with pytest.raises(ValueError, match="exactly 3 action lists"):
+        Workload(prompt="p", image_reference=_IMAGE, camera_script=[[["w"], ["w"]]])
+
+
+def test_workload_rejects_geometry_the_cache_cannot_serve() -> None:
+    with pytest.raises(ValueError, match="multiples of 16"):
+        Workload(prompt="p", image_reference=_IMAGE, camera_script=build_camera_script(1, "forward"), width=831)
+
+
+def test_prompt_update_beyond_the_rollout_is_rejected() -> None:
+    with pytest.raises(ValueError, match="never fires"):
+        Workload(
+            prompt="p",
+            image_reference=_IMAGE,
+            camera_script=build_camera_script(2, "forward"),
+            prompt_updates=(PromptUpdate(after_chunk=5, prompt="later"),),
+        )
+
+
+@pytest.mark.parametrize("from_file", [False, True])
+def test_duplicate_prompt_update_boundaries_are_rejected(tmp_path: Path, from_file: bool) -> None:
+    updates = [
+        {"after_chunk": 1, "prompt": "Rain begins"},
+        {"after_chunk": 0, "prompt": "Clouds gather"},
+        {"after_chunk": 1, "prompt": "Rain stops"},
+    ]
+    with pytest.raises(ValueError, match="Duplicate prompt update after chunk 1"):
+        if from_file:
+            path = tmp_path / "rollout.json"
+            path.write_text(json.dumps({"image": _IMAGE, "num_chunks": 3, "prompt_updates": updates}))
+            build_workload(parse_args(["--workload", str(path)]))
+        else:
+            Workload(
+                prompt="p",
+                image_reference=_IMAGE,
+                camera_script=build_camera_script(3, "forward"),
+                prompt_updates=(
+                    PromptUpdate(after_chunk=1, prompt="Rain begins"),
+                    PromptUpdate(after_chunk=0, prompt="Clouds gather"),
+                    PromptUpdate(after_chunk=1, prompt="Rain stops"),
+                ),
+            )
+
+
+def test_workload_file_is_loaded_and_cli_overrides_win(tmp_path: Path) -> None:
+    path = tmp_path / "rollout.json"
+    path.write_text(
+        json.dumps(
+            {
+                "prompt": "A quiet street",
+                "image": "https://example.invalid/frame.png",
+                "num_chunks": 4,
+                "camera_pattern": "hold",
+                "fps": 12,
+                "seed": 7,
+                "prompt_updates": [{"after_chunk": 1, "prompt": "Rain begins", "transition_chunks": 2}],
+            }
+        )
+    )
+    workload = load_workload(path, overrides={"fps": 24, "seed": None})
+    assert workload.prompt == "A quiet street"
+    assert workload.num_chunks == 4
+    assert workload.fps == 24, "an explicit CLI value must beat the file"
+    assert workload.seed == 7, "a None override must leave the file value alone"
+    assert workload.prompt_updates[0].transition_chunks == 2
+
+
+def test_workload_file_without_a_rollout_length_is_rejected(tmp_path: Path) -> None:
+    path = tmp_path / "rollout.json"
+    path.write_text(json.dumps({"image": "https://example.invalid/frame.png"}))
+    with pytest.raises(ValueError, match="camera_action_script or num_chunks"):
+        load_workload(path)
+
+
+def test_local_image_is_inlined_as_a_data_url(tmp_path: Path) -> None:
+    image = tmp_path / "frame.png"
+    image.write_bytes(b"\x89PNG\r\n\x1a\n")
+    reference = resolve_image_reference(str(image))
+    assert reference.startswith("data:image/png;base64,")
+    assert resolve_image_reference("https://example.invalid/a.png") == "https://example.invalid/a.png"
+    with pytest.raises(BenchmarkError, match="neither a URL nor a readable file"):
+        resolve_image_reference(str(tmp_path / "missing.png"))
+
+
+# --------------------------------------------------------------------------- #
+# Metric math
+# --------------------------------------------------------------------------- #
+
+
+def test_rtf_is_a_cost_not_a_speed() -> None:
+    """RTF must match vLLM-Omni's own formula, where lower is better.
+
+    ``vllm_omni/metrics/definitions.py::compute_audio_rtf`` is
+    ``latency / duration`` with the docstring "SLO red line < 1", and
+    ``benchmarks/patch/patch.py`` sets ``video_rtf = generation_s /
+    video_duration``. Emitting the reciprocal under the same field name would
+    silently invert every dashboard that aggregates it, so the direction is
+    pinned here rather than left to a comment.
+    """
+    # Two chunks, 9 + 12 = 21 frames at 12 fps = 1.75 s of video, generated in 3.5 s.
+    slow = compute_metrics(_records([1.0, 2.5]), fps=12, warmup_chunks=0, session_wall_s=3.5)
+    assert slow["video_seconds"] == pytest.approx(1.75)
+    assert slow["video_rtf"] == pytest.approx(2.0), "twice as slow as real time"
+    assert slow["video_rtfx"] == pytest.approx(0.5)
+
+    fast = compute_metrics(_records([0.4, 0.475]), fps=12, warmup_chunks=0, session_wall_s=0.875)
+    assert fast["video_rtf"] == pytest.approx(0.5), "twice as fast as real time"
+    assert fast["video_rtf"] < 1.0 < slow["video_rtf"]
+    assert fast["video_rtf"] * fast["video_rtfx"] == pytest.approx(1.0)
+
+
+def test_steady_rtf_is_the_chunk_interval_over_its_deadline() -> None:
+    """A steady chunk's RTF is just its interval divided by its video duration."""
+    metrics = compute_metrics(_records([1.0] + [1.357] * 15), fps=12, warmup_chunks=6)
+    # 12 frames at 12 fps = 1000 ms of video per steady chunk.
+    assert metrics["chunk_deadline_ms"] == pytest.approx(1000.0)
+    assert metrics["steady_video_rtf"] == pytest.approx(1.357, abs=1e-3)
+
+
+def test_percentile_interpolates_so_short_runs_still_report_tails() -> None:
+    values = [10.0, 20.0, 30.0, 40.0]
+    assert percentile(values, 0.0) == 10.0
+    assert percentile(values, 1.0) == 40.0
+    assert percentile(values, 0.5) == 25.0
+    assert percentile([5.0], 0.99) == 5.0
+
+
+def _records(intervals: list[float]) -> list[ChunkRecord]:
+    """One record per entry; ``intervals[0]`` is the TTFC, the rest are gaps."""
+    records: list[ChunkRecord] = []
+    arrival = 0.0
+    for index, interval in enumerate(intervals):
+        arrival += interval
+        records.append(
+            ChunkRecord(
+                index=index,
+                arrival_s=arrival,
+                inter_arrival_s=interval,
+                byte_length=1000,
+                num_frames=pixel_frames_in_chunk(index),
+            )
+        )
+    return records
+
+
+def test_steady_state_excludes_the_unsaturated_attention_window() -> None:
+    """Warmup chunks must not drag the steady-state mean down.
+
+    The first six chunks attend over a growing window, so they are genuinely
+    cheaper. A benchmark that averages them in reports a cadence the session
+    will never sustain.
+    """
+    intervals = [1.0] + [0.2] * (DEFAULT_WARMUP_CHUNKS - 1) + [0.8] * 4
+    metrics = compute_metrics(_records(intervals), fps=16, warmup_chunks=DEFAULT_WARMUP_CHUNKS)
+    assert metrics["interval_steady"]["count"] == 3
+    assert metrics["interval_steady"]["mean_ms"] == pytest.approx(800.0)
+    assert metrics["interval_all"]["mean_ms"] < metrics["interval_steady"]["mean_ms"]
+
+
+def test_short_rollouts_report_that_steady_state_was_never_reached() -> None:
+    metrics = compute_metrics(_records([1.0, 0.5, 0.5]), fps=16, warmup_chunks=DEFAULT_WARMUP_CHUNKS)
+    assert metrics["steady_state_reached"] is False
+    assert metrics["interval_steady"] == {}
+    assert "slo_attainment" not in metrics
+
+
+def test_real_time_factor_and_deadline_use_the_true_frame_counts() -> None:
+    # Eight chunks: 9 + 7*12 = 93 frames at 16 fps = 5.8125 s of video.
+    metrics = compute_metrics(_records([1.0] + [0.5] * 7), fps=16, warmup_chunks=0, session_wall_s=5.8125)
+    assert metrics["total_frames"] == 93
+    assert metrics["video_seconds"] == pytest.approx(5.8125)
+    assert metrics["video_rtf"] == pytest.approx(1.0)
+    assert metrics["video_rtfx"] == pytest.approx(1.0)
+    # A steady chunk carries 12 frames, so real time allows 750 ms at 16 fps.
+    assert metrics["chunk_deadline_ms"] == pytest.approx(750.0)
+    assert steady_chunk_deadline_ms(12) == pytest.approx(1000.0)
+
+
+def test_slo_attainment_counts_only_steady_chunks() -> None:
+    intervals = [1.0] + [0.1] * 5 + [0.7, 0.9, 0.7, 0.9]
+    metrics = compute_metrics(_records(intervals), fps=16, warmup_chunks=6, slo_ms=750.0)
+    assert metrics["slo_attainment"] == pytest.approx(2 / 3)
+    assert metrics["slo_violations"] == 1
+
+
+def test_playback_simulation_finds_stalls_a_percentile_would_hide() -> None:
+    """One late chunk behind a one-chunk buffer is a visible stall."""
+    # 12 frames at 16 fps = 750 ms of playback per chunk; chunk 3 takes 2 s.
+    intervals = [1.0, 0.5, 0.5, 2.0, 0.5]
+    result = simulate_playback(_records(intervals), fps=16, buffer_chunks=1)
+    assert result.underrun_count >= 1
+    assert result.first_underrun_chunk == 3
+    assert result.total_stall_s > 0.0
+
+
+def test_a_deeper_buffer_absorbs_the_same_jitter() -> None:
+    intervals = [1.0, 0.5, 0.5, 1.4, 0.2, 0.2]
+    fast = simulate_playback(_records(intervals), fps=16, buffer_chunks=1)
+    buffered = simulate_playback(_records(intervals), fps=16, buffer_chunks=3)
+    assert buffered.underrun_count < fast.underrun_count
+
+
+def test_playback_rejects_a_buffer_deeper_than_the_rollout() -> None:
+    with pytest.raises(ValueError, match="exceeds"):
+        simulate_playback(_records([1.0, 0.5]), fps=16, buffer_chunks=9)
+
+
+def test_aggregate_pools_sessions_and_reports_spread() -> None:
+    fast = compute_metrics(_records([1.0] + [0.1] * 5 + [0.6] * 4), fps=16, warmup_chunks=6)
+    slow = compute_metrics(_records([1.0] + [0.1] * 5 + [0.9] * 4), fps=16, warmup_chunks=6)
+    aggregate = aggregate_metrics([fast, slow], fps=16)
+    assert aggregate["sessions"] == 2
+    assert aggregate["steady_interval_ms_mean"] == pytest.approx(750.0)
+    assert aggregate["steady_interval_ms_spread"] == pytest.approx(300.0)
+    assert aggregate["video_rtf_best"] <= aggregate["video_rtf_mean"] <= aggregate["video_rtf_worst"]
+
+
+def test_single_session_aggregate_is_the_session_itself() -> None:
+    metrics = compute_metrics(_records([1.0] + [0.5] * 3), fps=16, warmup_chunks=0)
+    aggregate = aggregate_metrics([metrics], fps=16)
+    assert aggregate["sessions"] == 1
+    assert aggregate["video_rtf"] == metrics["video_rtf"]
+
+
+# --------------------------------------------------------------------------- #
+# CLI
+# --------------------------------------------------------------------------- #
+
+
+def test_endpoint_url_normalizes_http_and_appends_the_ws_path() -> None:
+    assert endpoint_url(parse_args([])) == "ws://127.0.0.1:8000/v1/realtime/video"
+    assert endpoint_url(parse_args(["--host", "h", "--port", "9"])) == "ws://h:9/v1/realtime/video"
+    args = parse_args(["--base-url", "http://example.invalid:8099"])
+    assert endpoint_url(args) == "ws://example.invalid:8099/v1/realtime/video"
+    args = parse_args(["--base-url", "ws://example.invalid:8099/v1/realtime/video"])
+    assert endpoint_url(args) == "ws://example.invalid:8099/v1/realtime/video"
+
+
+def test_cli_rejects_a_zero_chunk_buffer() -> None:
+    with pytest.raises(SystemExit):
+        parse_args(["--playback-buffer-chunks", "0"])
+    with pytest.raises(SystemExit):
+        parse_args(["--target-fps", "0"])
+
+
+def test_the_real_time_basis_is_independent_of_the_mux_label() -> None:
+    """The checkpoint declares no frame rate, so the basis must be statable.
+
+    The same measured cadence has RTF 1.809 at 16 fps and 1.357 at 12 fps;
+    tying that verdict to the fps sent to the muxer would let a playback label
+    silently decide whether a run passes.
+    """
+    records = _records([1.0] + [1.357] * 15)
+    at_16 = compute_metrics(records, fps=16, warmup_chunks=6)
+    at_12 = compute_metrics(records, fps=12, warmup_chunks=6)
+    assert at_16["chunk_deadline_ms"] == pytest.approx(750.0)
+    assert at_12["chunk_deadline_ms"] == pytest.approx(1000.0)
+    # RTF is cost, so the harsher 16 fps basis scores WORSE (higher).
+    assert at_16["video_rtf"] > at_12["video_rtf"]
+    # Same wall clock, same bytes: only the basis moved.
+    assert at_16["video_rtf"] / at_12["video_rtf"] == pytest.approx(16 / 12)
+    assert parse_args([]).target_fps is None, "default defers to the workload fps"
+
+
+# --------------------------------------------------------------------------- #
+# Protocol handling against a scripted server
+# --------------------------------------------------------------------------- #
+
+
+def _serve(script):
+    """Return an async context manager yielding a ws:// URL for ``script``."""
+    from websockets.asyncio.server import Server, serve
+
+    class _Server:
+        def __init__(self) -> None:
+            self.server: Server | None = None
+            self.start_payloads: list[dict] = []
+
+        async def __aenter__(self) -> str:
+            async def handler(websocket) -> None:
+                self.start_payloads.append(json.loads(await websocket.recv()))
+                await script(websocket, self.start_payloads[-1])
+
+            self.server = await serve(handler, "127.0.0.1", 0)
+            port = next(iter(self.server.sockets)).getsockname()[1]
+            return f"ws://127.0.0.1:{port}/v1/realtime/video"
+
+        async def __aexit__(self, *exc) -> None:
+            assert self.server is not None
+            self.server.close()
+            await self.server.wait_closed()
+
+    return _Server()
+
+
+async def _emit_rollout(websocket, payload, *, intervals=None, chunks=None, trailer=True) -> None:
+    """Replay the documented event order: start, metadata+binary per chunk, done."""
+    num_chunks = chunks if chunks is not None else len(payload["extra_params"]["camera_action_script"])
+    await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"}))
+    for index in range(num_chunks):
+        if intervals is not None:
+            await asyncio.sleep(intervals[index])
+        frames = pixel_frames_in_chunk(index)
+        body = bytes([index % 256]) * (100 + index)
+        await websocket.send(
+            json.dumps(
+                {
+                    "type": "video.chunk_metadata",
+                    "request_id": "req-1",
+                    "kind": "media",
+                    "num_frames": frames,
+                    "byte_length": len(body),
+                    "generation_chunk_index": index,
+                }
+            )
+        )
+        await websocket.send(body)
+    if trailer:
+        tail = b"\x00" * 8
+        await websocket.send(
+            json.dumps(
+                {
+                    "type": "video.chunk_metadata",
+                    "request_id": "req-1",
+                    "kind": "trailer",
+                    "num_frames": 0,
+                    "byte_length": len(tail),
+                    "generation_chunk_index": None,
+                }
+            )
+        )
+        await websocket.send(tail)
+    await websocket.send(json.dumps({"type": "session.done", "stopped": False, "chunks": num_chunks}))
+
+
+def _run(coro):
+    return asyncio.run(asyncio.wait_for(coro, timeout=30))
+
+
+def test_session_records_one_chunk_per_block_and_ignores_the_trailer() -> None:
+    workload = _workload(num_chunks=4)
+
+    async def scenario() -> None:
+        server = _serve(_emit_rollout)
+        url = await server.__aenter__()
+        try:
+            result = await run_session(
+                url,
+                "model",
+                workload,
+                first_chunk_timeout=10,
+                chunk_timeout=10,
+                ping_interval=0,
+                collect_media=True,
+                print_chunks=False,
+            )
+        finally:
+            await server.__aexit__(None, None, None)
+
+        assert [record.index for record in result.records] == [0, 1, 2, 3]
+        assert [record.num_frames for record in result.records] == [9, 12, 12, 12]
+        assert result.request_id == "req-1"
+        assert result.ttfc_s is not None and result.ttfc_s > 0
+        # The trailer's bytes are kept for remuxing but carry no chunk record.
+        assert len(result.media) == sum(record.byte_length for record in result.records) + 8
+        assert server.start_payloads[0]["num_frames"] == workload.num_frames
+
+    _run(scenario())
+
+
+def test_measured_intervals_track_the_server_cadence() -> None:
+    workload = _workload(num_chunks=3)
+
+    async def script(websocket, payload) -> None:
+        await _emit_rollout(websocket, payload, intervals=[0.05, 0.30, 0.05])
+
+    async def scenario() -> None:
+        server = _serve(script)
+        url = await server.__aenter__()
+        try:
+            result = await run_session(
+                url,
+                "model",
+                workload,
+                first_chunk_timeout=10,
+                chunk_timeout=10,
+                ping_interval=0,
+                collect_media=False,
+                print_chunks=False,
+            )
+        finally:
+            await server.__aexit__(None, None, None)
+
+        intervals = [record.inter_arrival_s for record in result.records]
+        assert intervals[1] == pytest.approx(0.30, abs=0.15)
+        assert intervals[2] < intervals[1], "a fast chunk must not inherit the slow one's interval"
+        assert result.records[0].arrival_s == pytest.approx(result.ttfc_s)
+
+    _run(scenario())
+
+
+def test_a_short_rollout_fails_loudly_instead_of_reporting_a_partial_cadence() -> None:
+    workload = _workload(num_chunks=4)
+
+    async def script(websocket, payload) -> None:
+        await _emit_rollout(websocket, payload, chunks=2)
+
+    async def scenario() -> None:
+        server = _serve(script)
+        url = await server.__aenter__()
+        try:
+            with pytest.raises(BenchmarkError, match="Expected 4 chunks"):
+                await run_session(
+                    url,
+                    "model",
+                    workload,
+                    first_chunk_timeout=10,
+                    chunk_timeout=10,
+                    ping_interval=0,
+                    collect_media=False,
+                    print_chunks=False,
+                )
+        finally:
+            await server.__aexit__(None, None, None)
+
+    _run(scenario())
+
+
+def test_a_server_error_event_aborts_the_session() -> None:
+    workload = _workload(num_chunks=2)
+
+    async def script(websocket, payload) -> None:
+        await websocket.send(json.dumps({"type": "error", "message": "resolution mismatch"}))
+
+    async def scenario() -> None:
+        server = _serve(script)
+        url = await server.__aenter__()
+        try:
+            with pytest.raises(BenchmarkError, match="resolution mismatch.*requested 2 chunks"):
+                await run_session(
+                    url,
+                    "model",
+                    workload,
+                    first_chunk_timeout=10,
+                    chunk_timeout=10,
+                    ping_interval=0,
+                    collect_media=False,
+                    print_chunks=False,
+                )
+        finally:
+            await server.__aexit__(None, None, None)
+
+    _run(scenario())
+
+
+def test_a_stalled_server_times_out_rather_than_hanging() -> None:
+    workload = _workload(num_chunks=2)
+
+    async def script(websocket, payload) -> None:
+        await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"}))
+        await asyncio.sleep(5)
+
+    async def scenario() -> None:
+        server = _serve(script)
+        url = await server.__aenter__()
+        try:
+            with pytest.raises(BenchmarkError, match="No media chunk for"):
+                await run_session(
+                    url,
+                    "model",
+                    workload,
+                    first_chunk_timeout=0.4,
+                    chunk_timeout=0.4,
+                    ping_interval=0,
+                    collect_media=False,
+                    print_chunks=False,
+                )
+        finally:
+            await server.__aexit__(None, None, None)
+
+    _run(scenario())
+
+
+def test_keepalive_pings_reach_the_server_during_a_slow_first_chunk() -> None:
+    """A compiled first block can outlast the server's stall timeout."""
+    workload = _workload(num_chunks=1)
+    seen: list[str] = []
+
+    async def script(websocket, payload) -> None:
+        await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"}))
+
+        async def drain() -> None:
+            async for message in websocket:
+                event = json.loads(message)
+                seen.append(event["type"])
+                if event["type"] == "session.ping":
+                    await websocket.send(json.dumps({"type": "session.pong"}))
+
+        drainer = asyncio.create_task(drain())
+        await asyncio.sleep(0.35)
+        await _emit_rollout(websocket, payload, trailer=False)
+        drainer.cancel()
+
+    async def scenario() -> None:
+        server = _serve(script)
+        url = await server.__aenter__()
+        try:
+            result = await run_session(
+                url,
+                "model",
+                workload,
+                first_chunk_timeout=10,
+                chunk_timeout=10,
+                ping_interval=0.1,
+                collect_media=False,
+                print_chunks=False,
+            )
+        finally:
+            await server.__aexit__(None, None, None)
+        assert len(result.records) == 1
+        assert "session.ping" in seen
+
+    _run(scenario())
+
+
+def test_prompt_updates_are_sent_on_their_chunk_boundary() -> None:
+    workload = Workload(
+        prompt="start",
+        image_reference=_IMAGE,
+        camera_script=build_camera_script(3, "forward"),
+        prompt_updates=(PromptUpdate(after_chunk=0, prompt="Rain begins", transition_chunks=1),),
+    )
+    received: list[dict] = []
+
+    async def script(websocket, payload) -> None:
+        async def drain() -> None:
+            async for message in websocket:
+                received.append(json.loads(message))
+
+        drainer = asyncio.create_task(drain())
+        await _emit_rollout(websocket, payload, intervals=[0.02, 0.15, 0.02], trailer=False)
+        await asyncio.sleep(0.05)
+        drainer.cancel()
+
+    async def scenario() -> None:
+        server = _serve(script)
+        url = await server.__aenter__()
+        try:
+            await run_session(
+                url,
+                "model",
+                workload,
+                first_chunk_timeout=10,
+                chunk_timeout=10,
+                ping_interval=0,
+                collect_media=False,
+                print_chunks=False,
+            )
+        finally:
+            await server.__aexit__(None, None, None)
+        interactions = [event for event in received if event.get("type") == "session.interaction"]
+        assert len(interactions) == 1
+        assert interactions[0]["interaction"]["event"]["prompt"] == "Rain begins"
+        assert interactions[0]["interaction"]["transition_chunks"] == 1
+
+    _run(scenario())
+
+
+@pytest.mark.parametrize("delivered", [0, 1])
+def test_pongs_do_not_extend_media_deadlines(delivered) -> None:
+    seen = []
+
+    async def script(websocket, payload) -> None:
+        if delivered:
+            await websocket.send(
+                json.dumps(
+                    {
+                        "type": "video.chunk_metadata",
+                        "kind": "media",
+                        "generation_chunk_index": 0,
+                        "num_frames": 9,
+                    }
+                )
+            )
+            await websocket.send(b"media")
+        async for message in websocket:
+            seen.append(json.loads(message)["type"])
+            await websocket.send(json.dumps({"type": "session.pong"}))
+
+    async def scenario() -> None:
+        async with _serve(script) as url:
+            with pytest.raises(BenchmarkError, match=f"after {delivered} chunk"):
+                await asyncio.wait_for(
+                    run_session(
+                        url,
+                        "model",
+                        _workload(num_chunks=2),
+                        first_chunk_timeout=0.25,
+                        chunk_timeout=0.25,
+                        ping_interval=0.03,
+                        collect_media=False,
+                        print_chunks=False,
+                    ),
+                    timeout=2,
+                )
+        assert "session.ping" in seen
+
+    _run(scenario())
+
+
+def test_terminal_chunk_is_only_excluded_from_steady_metrics() -> None:
+    records = _records([1.0] + [0.1] * 5 + [1.2, 0.8, 0.1])
+    metrics = compute_metrics(records, fps=12, warmup_chunks=6)
+    assert metrics["steady_chunks"] == 2
+    assert metrics["interval_steady"]["mean_ms"] == pytest.approx(1000)
+    assert metrics["steady_video_rtf"] == pytest.approx(1)
+    assert metrics["slo_attainment"] == pytest.approx(0.5)
+    assert metrics["interval_all"]["count"] == 8
+    assert metrics["total_frames"] == 105
+    assert metrics["wall_seconds"] == pytest.approx(records[-1].arrival_s)
+    assert metrics["playback"]["total_stall_ms"] == pytest.approx(
+        simulate_playback(records, fps=12).total_stall_s * 1000
+    )
+
+
+def test_terminal_chunk_alone_does_not_establish_steady_state() -> None:
+    metrics = compute_metrics(_records([1.0] * 7), fps=12, warmup_chunks=6)
+    assert not metrics["steady_state_reached"]
+    assert "steady_video_rtf" not in metrics
+
+
+def test_aggregate_preserves_custom_deadline_and_weights_intervals() -> None:
+    fast = compute_metrics(_records([1.0, 0.5, 0.1]), fps=12, warmup_chunks=0, slo_ms=700)
+    slow = compute_metrics(_records([1.0, 0.9, 0.9, 0.9, 0.1]), fps=12, warmup_chunks=0, slo_ms=700)
+    aggregate = aggregate_metrics([fast, slow], fps=12)
+    assert aggregate["chunk_deadline_ms"] == 700
+    assert aggregate["steady_interval_ms_mean"] == pytest.approx(800)
+    slow["chunk_deadline_ms"] = 800
+    with pytest.raises(ValueError, match="different chunk deadlines"):
+        aggregate_metrics([fast, slow], fps=12)
+
+
+@pytest.mark.parametrize(
+    "flags, expected_shift, expected_negative",
+    [
+        ([], 7.0, "file negative"),
+        (["--flow-shift", "5", "--negative-prompt", "CLI negative"], 5.0, "CLI negative"),
+        (["--negative-prompt", ""], 7.0, ""),
+    ],
+)
+def test_cli_workload_generation_overrides(tmp_path, flags, expected_shift, expected_negative) -> None:
+    path = tmp_path / "workload.json"
+    path.write_text(
+        json.dumps(
+            {
+                "image": _IMAGE,
+                "num_chunks": 3,
+                "flow_shift": 7.0,
+                "negative_prompt": "file negative",
+            }
+        )
+    )
+    workload = build_workload(parse_args(["--workload", str(path), *flags]))
+    assert workload.flow_shift == expected_shift
+    assert workload.negative_prompt == expected_negative
+
+
+def test_builtin_workload_keeps_default_flow_shift() -> None:
+    assert build_workload(parse_args(["--image", _IMAGE])).flow_shift == 5.0
+
+
+def test_short_sample_report_qualifies_tail_percentiles(capsys) -> None:
+    metrics = compute_metrics(_records([1.0] * 16), fps=12)
+    print_report(metrics, workload=_workload(num_chunks=16), endpoint="ws://test", sessions=1, target_fps=12)
+    assert "not reliable tail estimates" in capsys.readouterr().out

tests/clients/test_duplex_client.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import asyncio
 import base64
 import json
+import struct
 import wave
 from collections.abc import Callable
 from typing import Any
@@ -33,6 +34,7 @@ from vllm_omni.clients.duplex import (
     ReconnectPolicy,
     SessionConfig,
     SessionResumed,
+    SessionUpdated,
     build_realtime_url,
     chunk_period_ms,
     duplex_unit_boundary_ms,
@@ -797,6 +799,52 @@ async def test_inline_client_submits_typed_commands_in_order():
         assert ack.played_ms == 120 and ack.response_id == "resp-1"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("accepted", [True, False])
+@pytest.mark.parametrize(
+    "patch",
+    [
+        {"input_audio_format": "pcm_f32le", "sample_rate_hz": 24_000},
+        {"audio": {"input": {"format": "pcm_f32le", "sample_rate_hz": 24_000}}},
+    ],
+)
+async def test_inline_input_defaults_change_only_after_session_updated(accepted, patch):
+    client, omni = make_inline_client()
+    patch = {**patch, "overlap_silence_rms": 0.1}
+    async with client:
+        handle = omni.handles[0]
+        previous_session = dict(client.session_info)
+        await client.send({"type": "session.update", "session": patch})
+        # Sending a patch is not an acknowledgement of its acceptance.
+        assert client.session_info == previous_session
+        assert client._input_defaults().input_sample_rate_hz == 16_000
+        updated_session = {**previous_session, **patch}
+        event: dict[str, object] = (
+            {"type": "session.updated", "session": updated_session}
+            if accepted
+            else {"type": "error", "error": {"code": "invalid_session_update", "message": "rejected"}}
+        )
+        handle.feed(event)
+        received = await client.wait_for("session.updated" if accepted else "error", timeout_s=2.0)
+        assert isinstance(received, SessionUpdated if accepted else ErrorEvent)
+
+        pcm = struct.pack("<f", 0.05) * 16
+        await client.send({"type": "input_audio_buffer.append", "audio": _b64(pcm)})
+        command = handle.commands[-1]
+        assert isinstance(command, duplex_commands.AppendAudio)
+        assert command.format == "pcm_f32le"
+        if accepted:
+            assert client.session_info == updated_session
+            assert command.audio == pcm
+            assert command.sample_rate_hz == 24_000
+            assert command.is_speech is False
+        else:
+            assert client.session_info == previous_session
+            assert len(command.audio) == 2 * len(pcm)  # Still decoded as pcm16.
+            assert command.sample_rate_hz == 16_000
+            assert command.is_speech is True
+
+
 @pytest.mark.asyncio
 async def test_inline_session_close_maps_to_handle_close():
     client, omni = make_inline_client()

tests/config/test_environment_variables.py

@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
         EnvironmentVariableCategory.PUBLIC_OMNI: 29,
         EnvironmentVariableCategory.INHERITED_VLLM: 20,
         EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
-        EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
+        EnvironmentVariableCategory.MODEL_SPECIFIC: 59,
         EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 21,
         EnvironmentVariableCategory.INTERNAL: 2,
     }
@@ -181,7 +181,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
         ModelEnvironmentVariableDisposition.PROMOTE: 31,
         ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 5,
         ModelEnvironmentVariableDisposition.EXTERNAL: 0,
-        ModelEnvironmentVariableDisposition.INTERNALIZE: 14,
+        ModelEnvironmentVariableDisposition.INTERNALIZE: 16,
         ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 7,
     }
 

tests/config/test_omni_config.py

@@ -816,19 +816,25 @@ def test_mrv2_fails_fast_on_platforms_without_native_workers(platform: str):
         _apply_platform_overrides(DeployConfig(model_runner="v2"), platform=platform)
 
 
-@pytest.mark.parametrize(
-    ("default_name", "mrv2_name"),
-    [
-        ("qwen3_tts.yaml", "qwen3_tts_mrv2.yaml"),
-        (
-            "qwen3_tts_high_concurrency.yaml",
-            "qwen3_tts_high_concurrency_mrv2.yaml",
-        ),
-    ],
-)
-def test_qwen3_mrv2_profiles_are_explicit_opt_in(default_name: str, mrv2_name: str):
-    assert load_deploy_config(_DEPLOY_DIR / default_name).model_runner == "v1"
-    assert load_deploy_config(_DEPLOY_DIR / mrv2_name).model_runner == "v2"
+def test_qwen3_tts_high_concurrency_mrv2_profile_is_explicit_opt_in():
+    assert load_deploy_config(_DEPLOY_DIR / "qwen3_tts_high_concurrency.yaml").model_runner == "v1"
+    assert load_deploy_config(_DEPLOY_DIR / "qwen3_tts_high_concurrency_mrv2.yaml").model_runner == "v2"
+
+
+def test_qwen3_tts_default_profile_is_experimental_mrv2_with_v1_platform_fallback():
+    deploy_path = _DEPLOY_DIR / "qwen3_tts.yaml"
+    assert load_deploy_config(deploy_path).model_runner == "v2"
+
+    pipeline = _resolve_pipeline_or_skip("qwen3_tts")
+    stages = merge_pipeline_deploy(pipeline, load_deploy_config(deploy_path))
+    assert all(stage.yaml_engine_args["use_v2_model_runner"] is True for stage in stages)
+
+    assert _apply_platform_overrides(load_deploy_config(deploy_path), platform="cuda").model_runner == "v2"
+    for platform in ("npu", "xpu", "rocm", "musa"):
+        assert _apply_platform_overrides(load_deploy_config(deploy_path), platform=platform).model_runner == "v1"
+
+    # The explicit MRV2 profile resolves to the same runner selection.
+    assert load_deploy_config(_DEPLOY_DIR / "qwen3_tts_mrv2.yaml").model_runner == "v2"
 
 
 def test_qwen3_tts_mrv2_retunes_do_not_change_default_mrv1_profile():

tests/dfx/stability/scripts/run_stability_hunyuan_image.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 Hunyuan-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``.
 
@@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
-@pytest.mark.slow
-@pytest.mark.diffusion
+# Pytest marks (hardware, local_model, slow, diffusion) come from test_hunyuan_image.json.
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
 def test_stability_hunyuan_image(omni_server, stability_benchmark_params):

tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""
+HunyuanVideo-1.5 I2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``.
+
+Configuration: ``tests/dfx/stability/tests/test_hunyuanvideo15.json``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.dfx.conftest import (
+    create_benchmark_indices,
+    create_test_parameter_mapping,
+    create_unique_server_pytest_params,
+    load_configs,
+)
+from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop
+
+STABILITY_DIR = Path(__file__).resolve().parent.parent
+DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
+CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_hunyuanvideo15.json")
+DEFAULT_NUM_PROMPTS_PER_BATCH = 50
+STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600", "--init-timeout", "900"]
+
+try:
+    BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
+except FileNotFoundError:
+    BENCHMARK_CONFIGS = []
+
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
+benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
+
+
+# Pytest marks (hardware, local_model, slow, diffusion) come from test_hunyuanvideo15.json.
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
+def test_stability_hunyuanvideo15(omni_server, stability_benchmark_params):
+    test_name = stability_benchmark_params["test_name"]
+    params = stability_benchmark_params["params"]
+    duration_sec = params.get("duration_sec", 300)
+    num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH)
+    request_rate = params.get("request_rate")
+    max_concurrency = params.get("max_concurrency")
+
+    bench_params = {
+        k: v
+        for k, v in params.items()
+        if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch")
+    }
+
+    result = run_stability_benchmark_loop(
+        host=omni_server.host,
+        port=omni_server.port,
+        model=omni_server.model,
+        duration_sec=duration_sec,
+        params=bench_params,
+        request_rate=request_rate,
+        max_concurrency=max_concurrency,
+        result_dir=str(STABILITY_DIR),
+        num_prompts_per_batch=num_prompts_per_batch,
+        run_one_batch=_run_one_diffusion_batch,
+    )
+
+    assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}"
+    assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed"

tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""
+MiniCPM-o-4.5 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration.
+
+Configuration: ``tests/dfx/stability/tests/test_minicpmo_4_5.json``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.dfx.conftest import (
+    create_benchmark_indices,
+    create_test_parameter_mapping,
+    create_unique_server_pytest_params,
+    load_configs,
+)
+from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop
+
+STABILITY_DIR = Path(__file__).resolve().parent.parent
+DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
+CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minicpmo_4_5.json")
+DEFAULT_NUM_PROMPTS_PER_BATCH = 20
+STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"]
+
+try:
+    BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
+except FileNotFoundError:
+    BENCHMARK_CONFIGS = []
+
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
+benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
+
+
+# Pytest marks (hardware, local_model, slow, omni) come from test_minicpmo_4_5.json.
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
+def test_stability_minicpmo_4_5(omni_server, stability_benchmark_params):
+    test_name = stability_benchmark_params["test_name"]
+    params = stability_benchmark_params["params"]
+    duration_sec = params.get("duration_sec", 300)
+    num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH)
+    request_rate = params.get("request_rate")
+    max_concurrency = params.get("max_concurrency")
+
+    bench_params = {
+        k: v
+        for k, v in params.items()
+        if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch")
+    }
+
+    result = run_stability_benchmark_loop(
+        host=omni_server.host,
+        port=omni_server.port,
+        model=omni_server.model,
+        duration_sec=duration_sec,
+        params=bench_params,
+        request_rate=request_rate,
+        max_concurrency=max_concurrency,
+        result_dir=str(STABILITY_DIR),
+        num_prompts_per_batch=num_prompts_per_batch,
+        run_one_batch=_run_one_vllm_bench_batch,
+    )
+
+    assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}"
+    assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed"

tests/dfx/stability/scripts/run_stability_minimax_h3.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""
+MiniMax-H3 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``.
+
+Configuration: ``tests/dfx/stability/tests/test_minimax_h3.json``.
+
+``--lora-path`` is resolved like e2e FastH3 / Buildkite ``test-merge.yml``:
+set ``VLLM_TEST_MINIMAX_H3_FASTH3_LORA`` to the adapter file, or populate the local
+HF cache for ``FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.dfx.conftest import (
+    create_benchmark_indices,
+    create_test_parameter_mapping,
+    create_unique_server_pytest_params,
+    load_configs,
+)
+from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop
+from tests.e2e.online_serving.minimax_h3._common import FASTH3_LORA
+
+STABILITY_DIR = Path(__file__).resolve().parent.parent
+DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
+CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minimax_h3.json")
+DEFAULT_NUM_PROMPTS_PER_BATCH = 20
+STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "1800", "--init-timeout", "1800"]
+
+
+def _inject_fasth3_lora_path(configs: list[dict]) -> list[dict]:
+    """Fill serve_args.lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA / HF cache."""
+    if not FASTH3_LORA:
+        return configs
+    for config in configs:
+        serve_args = config.setdefault("server_params", {}).setdefault("serve_args", {})
+        serve_args["lora-path"] = FASTH3_LORA
+    return configs
+
+
+try:
+    BENCHMARK_CONFIGS = _inject_fasth3_lora_path(load_configs(CONFIG_FILE_PATH))
+except FileNotFoundError:
+    BENCHMARK_CONFIGS = []
+
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
+benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
+
+
+# Pytest marks (hardware, local_model, slow, diffusion) come from test_minimax_h3.json.
+@pytest.mark.skipif(
+    FASTH3_LORA is None,
+    reason="set VLLM_TEST_MINIMAX_H3_FASTH3_LORA or populate the local HF cache",
+)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
+def test_stability_minimax_h3(omni_server, stability_benchmark_params):
+    test_name = stability_benchmark_params["test_name"]
+    params = stability_benchmark_params["params"]
+    duration_sec = params.get("duration_sec", 300)
+    num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH)
+    request_rate = params.get("request_rate")
+    max_concurrency = params.get("max_concurrency")
+
+    bench_params = {
+        k: v
+        for k, v in params.items()
+        if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch")
+    }
+
+    result = run_stability_benchmark_loop(
+        host=omni_server.host,
+        port=omni_server.port,
+        model=omni_server.model,
+        duration_sec=duration_sec,
+        params=bench_params,
+        request_rate=request_rate,
+        max_concurrency=max_concurrency,
+        result_dir=str(STABILITY_DIR),
+        num_prompts_per_batch=num_prompts_per_batch,
+        run_one_batch=_run_one_diffusion_batch,
+    )
+
+    assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}"
+    assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed"

tests/dfx/stability/scripts/run_stability_qwen3_omni.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 Qwen3-Omni stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration.
 
@@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
-@pytest.mark.slow
-@pytest.mark.omni
+# Pytest marks (hardware, local_model, slow, omni) come from test_qwen3_omni.json.
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
 def test_stability_qwen3_omni(omni_server, stability_benchmark_params):

tests/dfx/stability/scripts/run_stability_qwen3_tts.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 Qwen3-TTS stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration.
 
@@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
-@pytest.mark.slow
-@pytest.mark.tts
+# Pytest marks (hardware, local_model, slow, tts) come from test_qwen3_tts.json.
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
 def test_stability_qwen3_tts(omni_server, stability_benchmark_params):

tests/dfx/stability/scripts/run_stability_qwen_image.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 Qwen-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``.
 
@@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
-@pytest.mark.slow
-@pytest.mark.diffusion
+# Pytest marks (hardware, local_model, slow, diffusion) come from test_qwen_image.json.
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
 def test_stability_qwen_image(omni_server, stability_benchmark_params):

tests/dfx/stability/scripts/run_stability_wan22.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 """
 Wan2.2 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``.
 
@@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
-@pytest.mark.slow
-@pytest.mark.diffusion
+# Pytest marks (hardware, local_model, slow, diffusion) come from test_wan22.json.
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
 def test_stability_wan22(omni_server, stability_benchmark_params):

tests/dfx/stability/tests/test_hunyuan_image.json

@@ -7,10 +7,11 @@
                     "res": {
                         "cuda": "H800"
                     },
-                    "num_cards": 2
+                    "num_cards": 4
                 }
             },
             "local_model",
+            "slow",
             "diffusion"
         ],
         "server_params": {
@@ -44,5 +45,53 @@
                 "enable_negative_prompt": true
             }
         ]
+    },
+    {
+        "test_name": "test_hunyuan_image_stability_tp2_sp2_a3",
+        "description": "A3: same as perf test_hunyuan_image_tp2_sp2 — DiT-only, TP=2 USP=2, 4 dies, no fp8",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "npu": "A3"
+                    },
+                    "num_cards": 4
+                }
+            },
+            "local_model",
+            "slow",
+            "diffusion"
+        ],
+        "server_params": {
+            "model": "tencent/HunyuanImage-3.0-Instruct",
+            "serve_args": {
+                "deploy-config": "vllm_omni/deploy/hunyuan_image3_dit.yaml",
+                "tensor-parallel-size": 2,
+                "ulysses-degree": 2,
+                "vae-patch-parallel-size": 4,
+                "vae-use-tiling": true,
+                "enable-diffusion-pipeline-profiler": true
+            }
+        },
+        "benchmark_params": [
+            {
+                "dataset": "random",
+                "task": "t2i",
+                "backend": "vllm-omni",
+                "duration_sec": 86400,
+                "max_concurrency": 1,
+                "num_prompts_per_batch": 50,
+                "width": {
+                    "min": 512,
+                    "max": 1024
+                },
+                "height": {
+                    "min": 512,
+                    "max": 1024
+                },
+                "num_inference_steps": 50,
+                "enable_negative_prompt": true
+            }
+        ]
     }
 ]

tests/dfx/stability/tests/test_hunyuanvideo15.json

@@ -0,0 +1,57 @@
+[
+  {
+    "test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2_stability",
+    "description": "HunyuanVideo-1.5 I2V 24h: CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": ["H800"]
+          },
+          "num_cards": 2
+        }
+      },
+      "local_model",
+      "slow",
+      "diffusion"
+    ],
+    "server_params": {
+      "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
+      "serve_args": {
+        "flow-shift": 5.0,
+        "cache-backend": "cache_dit",
+        "tensor-parallel-size": 2,
+        "vae-patch-parallel-size": 2,
+        "vae-use-tiling": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "dataset": "random",
+        "task": "i2v",
+        "backend": "v1/videos",
+        "duration_sec": 86400,
+        "max_concurrency": 1,
+        "num_prompts_per_batch": 50,
+        "num_input_images": 1,
+        "seed": 42,
+        "enable_negative_prompt": true,
+        "random_request_config": [
+          {
+            "width": 832,
+            "height": 480,
+            "num_inference_steps": 4,
+            "num_frames": 33,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra_body": {
+          "guidance_scale": 6.0,
+          "flow_shift": 5.0
+        }
+      }
+    ]
+  }
+]

tests/dfx/stability/tests/test_minicpmo_4_5.json

@@ -0,0 +1,53 @@
+[
+    {
+        "test_name": "test_minicpmo_4_5_stability_ready",
+        "description": "MiniCPM-o-4.5 ready deploy + seed-tts chat/audio stability (from perf test_minicpmo_4_5_ready)",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800"
+                    },
+                    "num_cards": 1
+                }
+            },
+            "local_model",
+            "slow",
+            "omni"
+        ],
+        "server_params": {
+            "model": "openbmb/MiniCPM-o-4_5",
+            "extra_cli_args": [
+                "--deploy-config",
+                "vllm_omni/deploy/minicpmo_4_5.yaml",
+                "--trust-remote-code"
+            ]
+        },
+        "benchmark_params": [
+            {
+                "dataset_name": "seed-tts",
+                "dataset_path": "zhaochenyang20/seed-tts-eval",
+                "backend": "openai-chat-omni",
+                "endpoint": "/v1/chat/completions",
+                "duration_sec": 86400,
+                "max_concurrency": 1,
+                "num_prompts_per_batch": 50,
+                "no_oversample": true,
+                "disable_shuffle": true,
+                "trust_remote_code": true,
+                "seed_tts_locale": "en",
+                "extra_body": {
+                    "modalities": [
+                        "text",
+                        "audio"
+                    ],
+                    "chat_template_kwargs": {
+                        "enable_thinking": false,
+                        "use_tts_template": true
+                    }
+                },
+                "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
+            }
+        ]
+    }
+]

tests/dfx/stability/tests/test_minimax_h3.json

@@ -0,0 +1,62 @@
+[
+    {
+        "test_name": "test_minimax_h3_t2v_stability_4gpu",
+        "description": "MiniMax-H3 FastH3 T2VA on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile); fused lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA; requests must use num_inference_steps=4",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800"
+                    },
+                    "num_cards": 4
+                }
+            },
+            "local_model",
+            "slow",
+            "diffusion"
+        ],
+        "server_params": {
+            "model": "MiniMaxAI/MiniMax-H3",
+            "serve_args": {
+                "trust-remote-code": true,
+                "task-type": "fl2va",
+                "num-gpus": 4,
+                "usp": 4,
+                "ring": 1,
+                "use-hsdp": true,
+                "hsdp-shard-size": 4,
+                "text-encoder-tp-size": 4,
+                "vae-patch-parallel-size": 4,
+                "vae-parallel-mode": "tile",
+                "vae-use-tiling": true,
+                "stage-init-timeout": 1800,
+                "init-timeout": 1800
+            }
+        },
+        "benchmark_params": [
+            {
+                "dataset": "random",
+                "task": "t2v",
+                "backend": "v1/videos",
+                "duration_sec": 86400,
+                "max_concurrency": 1,
+                "num_prompts_per_batch": 50,
+                "seed": 42,
+                "random_request_config": [
+                    {
+                        "width": 1344,
+                        "height": 768,
+                        "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
+                        "num_inference_steps": 4,
+                        "num_frames": 209,
+                        "fps": 24,
+                        "weight": 1
+                    }
+                ],
+                "extra_body": {
+                    "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\"}"
+                }
+            }
+        ]
+    }
+]

tests/dfx/stability/tests/test_qwen3_omni.json

@@ -5,12 +5,14 @@
             {
                 "hardware_marks": {
                     "res": {
-                        "cuda": "H800"
+                        "cuda": "H800",
+                        "npu": "A3"
                     },
                     "num_cards": 2
                 }
             },
             "local_model",
+            "slow",
             "omni"
         ],
         "server_params": {

tests/dfx/stability/tests/test_qwen3_tts.json

@@ -12,6 +12,7 @@
                 }
             },
             "local_model",
+            "slow",
             "tts"
         ],
         "server_params": {

tests/dfx/stability/tests/test_qwen_image.json

@@ -11,6 +11,7 @@
                 }
             },
             "local_model",
+            "slow",
             "diffusion"
         ],
         "server_params": {
@@ -36,5 +37,44 @@
                 "enable_negative_prompt": true
             }
         ]
+    },
+    {
+        "test_name": "test_qwen_image_stability_a3",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "npu": "A3"
+                    },
+                    "num_cards": 1
+                }
+            },
+            "local_model",
+            "slow",
+            "diffusion"
+        ],
+        "server_params": {
+            "model": "Qwen/Qwen-Image"
+        },
+        "benchmark_params": [
+            {
+                "dataset": "random",
+                "task": "t2i",
+                "backend": "vllm-omni",
+                "duration_sec": 86400,
+                "max_concurrency": 1,
+                "num_prompts_per_batch": 50,
+                "width": {
+                    "min": 512,
+                    "max": 1000
+                },
+                "height": {
+                    "min": 512,
+                    "max": 1000
+                },
+                "num_inference_steps": 50,
+                "enable_negative_prompt": true
+            }
+        ]
     }
 ]

tests/dfx/stability/tests/test_wan22.json

@@ -12,6 +12,7 @@
                 }
             },
             "local_model",
+            "slow",
             "diffusion"
         ],
         "server_params": {
@@ -54,6 +55,7 @@
                 }
             },
             "local_model",
+            "slow",
             "diffusion"
         ],
         "server_params": {

tests/diffusion/attention/test_attention_config.py

@@ -42,6 +42,14 @@ from vllm_omni.diffusion.data import (
 pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
 
 
+@pytest.fixture(autouse=True)
+def _clear_inherited_attention_backend(monkeypatch):
+    # Weekly CPU exports DIFFUSION_ATTENTION_BACKEND; OmniDiffusionConfig
+    # treats that as an explicit default so auto-backend cases break.
+    # Tests that setenv the same variable still work: this fixture runs first.
+    monkeypatch.delenv("DIFFUSION_ATTENTION_BACKEND", raising=False)
+
+
 class TestAttentionSpec:
     def test_construct_no_skip_softmax(self):
         spec = AttentionSpec(backend="FLASH_ATTN")
@@ -416,13 +424,6 @@ class TestBuildAttentionConfig:
 class TestOmniDiffusionConfigAttentionParsing:
     """Test OmniDiffusionConfig attention shorthand and structured config."""
 
-    @pytest.fixture(autouse=True)
-    def _clear_diffusion_attention_backend_env(self, monkeypatch):
-        # OmniDiffusionConfig.__post_init__ applies DIFFUSION_ATTENTION_BACKEND via
-        # build_attention_config(); clear it so these tests assert config defaults,
-        # not whatever the process inherited from CI / sibling tests.
-        monkeypatch.delenv("DIFFUSION_ATTENTION_BACKEND", raising=False)
-
     def test_diffusion_attention_backend_sets_default(self):
         config = OmniDiffusionConfig.from_kwargs(diffusion_attention_backend="SAGE_ATTN")
         assert isinstance(config.diffusion_attention_config, AttentionConfig)

tests/diffusion/distributed/test_pause_keep_two_ranks.py

@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""
+Two-rank ``pause_generation(mode="keep")``: every rank runs the pause barrier
+before the ACK, and a barrier failure on a non-zero rank fails the pause
+without aborting or dropping the running batch.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections import defaultdict
+from contextlib import ExitStack
+
+import pytest
+
+from tests.e2e.features.custom_pipeline.test_diffusion_pause_keep import (
+    CUSTOM_PIPELINE_CLASS,
+    MODEL,
+    WORKER_EXTENSION_CLASS,
+    _generate,
+    _pause_round,
+    _wait_for,
+)
+from tests.helpers.mark import hardware_test
+from vllm_omni.entrypoints.async_omni import AsyncOmni
+
+
+def _all_true(results) -> bool:
+    flat = []
+    stack = list(results)
+    while stack:
+        item = stack.pop()
+        if isinstance(item, list):
+            stack.extend(item)
+        else:
+            flat.append(item)
+    return bool(flat) and all(item is True for item in flat)
+
+
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "L4"}, num_cards=2)
+@pytest.mark.asyncio
+async def test_keep_pause_two_ranks_ack_and_rank_failure():
+    with ExitStack() as after:
+        engine = AsyncOmni(
+            model=MODEL,
+            custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS},
+            worker_extension_cls=WORKER_EXTENSION_CLASS,
+            enforce_eager=True,
+            enable_sleep_mode=True,
+            num_gpus=2,
+            parallel_config={"ulysses_degree": 2, "ulysses_mode": "advanced_uaa"},
+        )
+        after.callback(engine.shutdown)
+        events: dict[str, dict[str, float]] = defaultdict(dict)
+        assert _all_true(await engine.collective_rpc(method="start_counting_synchronize_device"))
+
+        pause_latencies = [await _pause_round(engine, events, round_index) for round_index in range(3)]
+        # The result is the all-rank AND, so True means both ranks ran the barrier.
+        assert _all_true(await engine.collective_rpc(method="synchronize_device_seen"))
+
+        assert _all_true(await engine.collective_rpc(method="set_synchronize_device_failure", args=(1, True)))
+        a = asyncio.create_task(_generate(engine, "a-fail", events))
+        await _wait_for(lambda: "started" in events["a-fail"])
+        with pytest.raises(RuntimeError, match="pause_scheduler failed"):
+            await engine.pause_generation(mode="keep", clear_cache=False)
+        assert await engine.is_paused()
+        a_output = await asyncio.wait_for(a, 60.0)
+        assert a_output.images, "a failed barrier must not abort the running batch"
+
+        assert _all_true(await engine.collective_rpc(method="set_synchronize_device_failure", args=(1, False)))
+        await engine.resume_generation()
+        assert not await engine.is_paused()
+        recovered = await asyncio.wait_for(_generate(engine, "after-recovery", events), 60.0)
+        assert recovered.images
+
+        assert not engine.request_states
+        print(f"[sp2] rounds=3 pause_latency_s min/max={min(pause_latencies):.3f}/{max(pause_latencies):.3f}")

tests/diffusion/distributed/test_wan_spatial_shard.py

@@ -493,6 +493,84 @@ def test_dist_zero_pad_only_applies_global_width_edges(monkeypatch: pytest.Monke
     assert last.shape == (1, 1, 2, 3)
 
 
+@pytest.mark.core_model
+@pytest.mark.cpu
+class _PlainDecoder(torch.nn.Module):
+    """A decoder with nothing to patch, so the install only wraps ``forward``."""
+
+    def __init__(self):
+        super().__init__()
+        self.calls: list[dict] = []
+
+    def forward(self, x, feat_cache=None, feat_idx=None, first_chunk=False):
+        self.calls.append({"feat_cache": feat_cache, "feat_idx": feat_idx, "first_chunk": first_chunk})
+        return x
+
+
+@pytest.mark.core_model
+@pytest.mark.cpu
+@pytest.mark.parametrize("world_size, dst", [(4, -1), (4, 4), (4, 5), (1, 1), (4, 1.5)])
+def test_install_rejects_invalid_destination_before_mutating_decoder(monkeypatch, world_size, dst):
+    monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, world_size))
+    vae = SimpleNamespace(decoder=torch.nn.Identity())
+    original_forward = vae.decoder.forward
+
+    def unexpected_patch(*args, **kwargs):
+        pytest.fail("invalid destination must fail before patching modules")
+
+    monkeypatch.setattr(wan_spatial_shard, "_patch_decoder_modules", unexpected_patch)
+    with pytest.raises(ValueError, match="dst must be None or an integer"):
+        wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), dst=dst)
+    assert vae.decoder.forward == original_forward
+    assert not getattr(vae, "_vllm_omni_wan_spatial_shard_installed", False)
+
+
+def _install_on_plain_decoder(monkeypatch: pytest.MonkeyPatch, *, dst):
+    monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (1, 2))
+    monkeypatch.setattr(
+        wan_spatial_shard,
+        "split_for_parallel_decode",
+        lambda x, *, upsample_count, split_dim, group: (x, x.shape[wan_spatial_shard._spatial_dim(split_dim)]),
+    )
+    gathers: list[dict] = []
+
+    def gather(x, *, expected_extent, split_dim, group, dst=None):
+        gathers.append({"expected_extent": expected_extent, "split_dim": split_dim, "dst": dst})
+        return x
+
+    monkeypatch.setattr(wan_spatial_shard, "gather_and_trim_extent", gather)
+    vae = SimpleNamespace(decoder=_PlainDecoder())
+    wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=dst)
+    return vae, gathers
+
+
+@pytest.mark.core_model
+@pytest.mark.cpu
+@pytest.mark.parametrize("dst", [0, 1, None])
+def test_install_passes_the_assembling_rank_through_to_the_gather(monkeypatch: pytest.MonkeyPatch, dst):
+    vae, gathers = _install_on_plain_decoder(monkeypatch, dst=dst)
+    cache = [None]
+
+    out = vae.decoder(torch.zeros(1, 3, 1, 4, 6), feat_cache=cache, feat_idx=[0], first_chunk=True)
+
+    assert out.shape == (1, 3, 1, 4, 6)
+    assert gathers == [{"expected_extent": 6, "split_dim": "width", "dst": dst}]
+    # The streaming arguments reach the wrapped decoder untouched.
+    assert vae.decoder.calls == [{"feat_cache": cache, "feat_idx": [0], "first_chunk": True}]
+    assert vae._vllm_omni_wan_spatial_shard_dst == dst
+
+
+@pytest.mark.core_model
+@pytest.mark.cpu
+def test_install_refuses_to_change_the_assembling_rank(monkeypatch: pytest.MonkeyPatch):
+    vae, _ = _install_on_plain_decoder(monkeypatch, dst=None)
+
+    # Same settings again: a no-op, as before.
+    wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=None)
+    with pytest.raises(ValueError, match="already patched to assemble on None"):
+        wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=0)
+
+
 @pytest.mark.core_model
 @pytest.mark.cpu
 def test_spatial_shard_height_gate_falls_back_for_partial_group(monkeypatch: pytest.MonkeyPatch):
@@ -626,6 +704,105 @@ def _spatial_shard_decode_worker(rank: int, split_dim: str, return_dict, master_
             dist.destroy_process_group()
 
 
+_STREAMING_SHARD_CHUNKS = 3
+
+
+def _streaming_shard_worker(rank: int, return_dict, master_port: str) -> None:
+    """Sharded streaming decode with every rank keeping the frame, against an unsharded streaming reference.
+
+    Two sessions are interleaved chunk by chunk, so the per-session temporal cache
+    and the halo exchange are exercised across successive calls, then one session
+    is reset and must reproduce its opening chunk. Every rank checks every chunk.
+    """
+    from vllm_omni.diffusion.distributed.parallel_state import (
+        destroy_model_parallel,
+        get_sp_group,
+        init_distributed_environment,
+        initialize_model_parallel,
+    )
+    from vllm_omni.experimental.ar_diffusion.streaming_decode import WanStreamingDecoder
+
+    os.environ["MASTER_ADDR"] = "127.0.0.1"
+    os.environ["MASTER_PORT"] = master_port
+    device = current_omni_platform.get_torch_device(rank)
+    current_omni_platform.set_device(device)
+    dtype = torch.float32
+    backend = current_omni_platform.dist_backend
+    init_distributed_environment(world_size=_SPATIAL_SHARD_WORLD_SIZE, rank=rank, local_rank=rank, backend=backend)
+    initialize_model_parallel(
+        sequence_parallel_size=_SPATIAL_SHARD_WORLD_SIZE, ulysses_degree=_SPATIAL_SHARD_WORLD_SIZE, backend=backend
+    )
+    # VLLM_OMNI_TEST_WAN_VAE may point at a local diffusers checkpoint (with or
+    # without a ``vae`` subfolder) for machines without hub access.
+    model = os.environ.get("VLLM_OMNI_TEST_WAN_VAE", _SPATIAL_SHARD_MODEL)
+    local_vae_only = os.path.isdir(model) and not os.path.isdir(os.path.join(model, "vae"))
+    load = {"torch_dtype": dtype} if local_vae_only else {"subfolder": _SPATIAL_SHARD_SUBFOLDER, "torch_dtype": dtype}
+    try:
+        reference_vae = DistributedAutoencoderKLWan.from_pretrained(model, **load).to(device=device, dtype=dtype).eval()
+        sharded_vae = DistributedAutoencoderKLWan.from_pretrained(model, **load).to(device=device, dtype=dtype).eval()
+        wan_spatial_shard.install_wan_spatial_shard_decode(
+            sharded_vae, get_sp_group().device_group, split_dim="width", dst=None
+        )
+        reference = WanStreamingDecoder(reference_vae, bytes_per_pixel_fp32=1.0)
+        sharded = WanStreamingDecoder(sharded_vae, bytes_per_pixel_fp32=1.0)
+
+        generator = torch.Generator(device=device).manual_seed(0)
+        # Width 104 latents = 832 px: not divisible by two ranks' halo-padded shards without padding.
+        shape = (1, reference_vae.config.z_dim, 1, _SPATIAL_SHARD_LATENT_HEIGHT, _SPATIAL_SHARD_LATENT_WIDTH)
+        chunks = {
+            name: [
+                torch.randn(shape, generator=generator, device=device, dtype=dtype)
+                for _ in range(_STREAMING_SHARD_CHUNKS)
+            ]
+            for name in ("a", "b")
+        }
+        max_diff = 0.0
+        with torch.inference_mode():
+            ref_states = {name: reference.new_decode_state(name) for name in chunks}
+            shard_states = {name: sharded.new_decode_state(name) for name in chunks}
+            for index in range(_STREAMING_SHARD_CHUNKS):
+                for name in ("a", "b"):
+                    expected = reference.decode_chunk(chunks[name][index], ref_states[name]).float()
+                    got = sharded.decode_chunk(chunks[name][index], shard_states[name]).float()
+                    if got.shape != expected.shape:
+                        raise AssertionError(
+                            f"rank {rank} {name}[{index}]: {tuple(got.shape)} != {tuple(expected.shape)}"
+                        )
+                    max_diff = max(max_diff, (got - expected).abs().max().item())
+            # Reset session a: a fresh state must reproduce its opening chunk, on this rank too.
+            shard_states["a"].release()
+            ref_states["a"].release()
+            shard_states["a"] = sharded.new_decode_state("a")
+            ref_states["a"] = reference.new_decode_state("a")
+            expected = reference.decode_chunk(chunks["a"][0], ref_states["a"]).float()
+            got = sharded.decode_chunk(chunks["a"][0], shard_states["a"]).float()
+            max_diff = max(max_diff, (got - expected).abs().max().item())
+        return_dict[f"max_abs_diff_rank{rank}"] = max_diff
+        return_dict[f"shape_rank{rank}"] = tuple(got.shape)
+    finally:
+        destroy_model_parallel()
+        if dist.is_initialized():
+            dist.destroy_process_group()
+
+
+@pytest.mark.full_model
+@pytest.mark.diffusion
+@pytest.mark.parallel
+@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=_SPATIAL_SHARD_WORLD_SIZE)
+def test_streaming_shard_decode_matches_unsharded_streaming_on_every_rank():
+    manager = mp.get_context("spawn").Manager()
+    return_dict = manager.dict()
+    mp.spawn(_streaming_shard_worker, args=(return_dict, "29510"), nprocs=_SPATIAL_SHARD_WORLD_SIZE, join=True)
+
+    for rank in range(_SPATIAL_SHARD_WORLD_SIZE):
+        assert f"max_abs_diff_rank{rank}" in return_dict, f"rank {rank} did not report"
+        diff = return_dict[f"max_abs_diff_rank{rank}"]
+        shape = return_dict[f"shape_rank{rank}"]
+        print(f"streaming shard rank {rank}: max_abs_diff={diff:.6e} shape={shape}")
+        assert shape[-1] == _SPATIAL_SHARD_LATENT_WIDTH * 8 and shape[-2] == _SPATIAL_SHARD_LATENT_HEIGHT * 8
+        assert diff <= _SPATIAL_SHARD_TOLERANCE, f"rank {rank}: {diff} exceeds {_SPATIAL_SHARD_TOLERANCE}"
+
+
 @pytest.mark.full_model
 @pytest.mark.diffusion
 @pytest.mark.parallel

tests/diffusion/model_loader/test_hub_prefetch.py

@@ -3,21 +3,21 @@
 
 import contextlib
 
-import huggingface_hub
 import pytest
 
+from tests.helpers.mock import patch_hf_snapshot_download
 from vllm_omni.diffusion.model_loader import hub_prefetch
 
 pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
 
 
-def test_prefetch_subfolders_propagates_revision(monkeypatch):
+def test_prefetch_subfolders_propagates_revision(monkeypatch, tmp_path):
     calls = []
 
-    def fake_snapshot_download(self, **kwargs):
+    def fake_snapshot_download(**kwargs):
         calls.append(kwargs)
 
-    monkeypatch.setattr(huggingface_hub.HfApi, "snapshot_download", fake_snapshot_download)
+    patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path)
     monkeypatch.setattr(hub_prefetch, "_repo_prefetch_lock", lambda _model: contextlib.nullcontext())
 
     hub_prefetch.prefetch_subfolders(

tests/diffusion/models/bagel/test_step_execution.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Focused CPU tests for BAGEL step execution and packed batching."""
 
 from __future__ import annotations
@@ -18,12 +18,14 @@ from vllm_omni.diffusion.models.bagel.bagel_transformer import Bagel, NaiveCache
 from vllm_omni.diffusion.models.bagel.pipeline_bagel import (
     BagelGenParams,
     BagelPipeline,
+    _bagel_canvas_requested,
     get_bagel_pre_process_func,
 )
 from vllm_omni.diffusion.request import OmniDiffusionRequest
 from vllm_omni.diffusion.sched import StepScheduler
 from vllm_omni.diffusion.worker.input_batch import InputBatch
 from vllm_omni.diffusion.worker.utils import StepRequestState
+from vllm_omni.errors import OmniClientError
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -118,7 +120,7 @@ def test_bagel_step_preprocessor_separates_incompatible_cfg_settings(first_extra
     assert first.batch_compatibility_key != second.batch_compatibility_key
 
 
-def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
+def _pre_process_with_stub_checkpoint(tmp_path):
     (tmp_path / "config.json").write_text(
         '{"vae_config":{"downsample":8},"latent_patch_size":2,"max_latent_size":32}',
         encoding="utf-8",
@@ -131,9 +133,22 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
         '{"weight_map":{"latent_pos_embed.pos_embed":"ema.safetensors"}}',
         encoding="utf-8",
     )
-    pre_process = get_bagel_pre_process_func(
-        types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True)
+    return get_bagel_pre_process_func(types.SimpleNamespace(model=str(tmp_path), revision=None, step_execution=True))
+
+
+def _edit_request(source_size, sampling_params, request_id="req"):
+    return OmniDiffusionRequest(
+        prompt={
+            "prompt": "edit this image",
+            "multi_modal_data": {"image": [Image.new("RGB", source_size)]},
+        },
+        sampling_params=sampling_params,
+        request_id=request_id,
     )
+
+
+def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
     requests = [
         OmniDiffusionRequest(
             prompt={
@@ -163,6 +178,172 @@ def test_bagel_step_preprocessor_buckets_effective_img2img_sizes(tmp_path):
     assert schedule.num_waiting_reqs == 1
 
 
+def test_bagel_pre_process_keeps_explicitly_requested_canvas(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request((512, 1024), OmniDiffusionSamplingParams(num_inference_steps=2, height=1024, width=1024))
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
+    assert _bagel_canvas_requested(request.sampling_params)
+
+
+def test_bagel_pre_process_aligns_source_derived_canvas_for_size_auto(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request(
+        (500, 700),
+        OmniDiffusionSamplingParams(
+            num_inference_steps=2,
+            height=700,
+            width=500,
+            height_not_provided=True,
+            width_not_provided=True,
+        ),
+    )
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (704, 496)
+    assert not _bagel_canvas_requested(request.sampling_params)
+
+
+def test_bagel_pre_process_derives_canvas_when_caller_asked_for_none(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request((800, 400), OmniDiffusionSamplingParams(num_inference_steps=2))
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (400, 800)
+    assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (True, True)
+    assert not _bagel_canvas_requested(request.sampling_params)
+
+
+def test_bagel_pre_process_marks_kv_metadata_canvas_as_requested(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request(
+        (500, 700),
+        OmniDiffusionSamplingParams(
+            num_inference_steps=2,
+            height=700,
+            width=500,
+            height_not_provided=True,
+            width_not_provided=True,
+            kv_metadata={"image_shape": (1024, 1024)},
+        ),
+    )
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
+    assert (request.sampling_params.height_not_provided, request.sampling_params.width_not_provided) == (False, False)
+    assert _bagel_canvas_requested(request.sampling_params)
+
+
+def test_bagel_pre_process_prefers_explicit_canvas_over_kv_metadata(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request(
+        (1280, 720),
+        OmniDiffusionSamplingParams(
+            num_inference_steps=2,
+            height=1024,
+            width=1024,
+            kv_metadata={"image_shape": (576, 1024)},
+        ),
+    )
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (1024, 1024)
+    assert _bagel_canvas_requested(request.sampling_params)
+
+
+def test_bagel_pre_process_floors_unaligned_explicit_canvas_with_warning(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request(
+        (1280, 720),
+        OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
+    )
+
+    with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning:
+        pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (688, 992)
+    assert _bagel_canvas_requested(request.sampling_params)
+    warning.assert_called_once()
+    assert "width 1000->992, height 700->688" in warning.call_args.args[-1]
+
+
+def test_bagel_pre_process_keeps_aligned_explicit_canvas_silent(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request(
+        (1280, 720),
+        OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=1024),
+    )
+
+    with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.logger.warning") as warning:
+        pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (512, 1024)
+    warning.assert_not_called()
+
+
+def test_bagel_pre_process_rejects_explicit_canvas_over_checkpoint_limit(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = _edit_request(
+        (1280, 720),
+        OmniDiffusionSamplingParams(num_inference_steps=2, height=512, width=2048),
+    )
+
+    with pytest.raises(OmniClientError, match="width=2048 exceeds the BAGEL checkpoint limit of 1024") as excinfo:
+        pre_process(request)
+
+    assert excinfo.value.status_code == 400
+
+
+def test_bagel_pre_process_aligns_text2img_canvas_too(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = OmniDiffusionRequest(
+        prompt="a cat",
+        sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
+        request_id="req",
+    )
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (688, 992)
+
+
+def test_bagel_pre_process_skips_canvas_alignment_for_text_output(tmp_path):
+    pre_process = _pre_process_with_stub_checkpoint(tmp_path)
+    request = OmniDiffusionRequest(
+        prompt={
+            "prompt": "describe this image",
+            "modalities": ["text"],
+            "multi_modal_data": {"image": [Image.new("RGB", (1280, 720))]},
+        },
+        sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, height=700, width=1000),
+        request_id="req",
+    )
+
+    pre_process(request)
+
+    assert (request.sampling_params.height, request.sampling_params.width) == (700, 1000)
+    assert request.use_step_execution is False
+
+
+@pytest.mark.parametrize(
+    ("sampling_kwargs", "requested"),
+    [
+        ({"height": 1024, "width": 1024}, True),
+        ({"height": 1024, "width": 512, "height_not_provided": True, "width_not_provided": True}, False),
+        ({}, False),
+    ],
+    ids=["explicit_size", "size_auto", "unset"],
+)
+def test_bagel_canvas_requested_separates_asked_for_from_derived(sampling_kwargs, requested):
+    assert _bagel_canvas_requested(OmniDiffusionSamplingParams(**sampling_kwargs)) is requested
+
+
 def _cache(length: int, value: float, num_layers: int = 1) -> NaiveCache:
     cache = NaiveCache(num_layers)
     if length:

tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_it2i_multi_image.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Multi-image input regression for HunyuanImage3 IT2I prompt construction.
 
 The official HunyuanImage-3.0-Instruct supports up to 3 reference images
@@ -41,7 +41,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
 
 class FakeTokenizer:
-    """Recording fake tokenizer mirroring the one in test_prompt_utils.
+    """Recording fake tokenizer mirroring the one in test_hunyuan_image3_prompt_utils.
 
     Special token ids: `<|startoftext|>`=1, `<img>`=2, `<think>`=3,
     `<recaption>`=4. encode() returns one id per character starting at

tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_prompt_utils.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 from __future__ import annotations
 
 import ast

tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py

@@ -134,6 +134,8 @@ def _install_vllm_stubs() -> None:
             **kwargs,
         ) -> None:
             super().__init__()
+            self.quant_config = kwargs.pop("quant_config", None)
+            self.prefix = kwargs.pop("prefix", "")
             del kwargs
             self.return_bias = return_bias
             self.weight = nn.Parameter(torch.empty(output_size, input_size))
@@ -164,6 +166,8 @@ def _install_vllm_stubs() -> None:
             **kwargs,
         ) -> None:
             super().__init__()
+            self.quant_config = kwargs.pop("quant_config", None)
+            self.prefix = kwargs.pop("prefix", "")
             del kwargs
             self.num_heads = total_num_heads
             self.num_kv_heads = total_num_kv_heads or total_num_heads

tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py

@@ -35,6 +35,8 @@ def _tiny_model(
     num_layers: int = 2,
     num_frames_per_block: int = 1,
     sliding_window_num_frames: int = 3,
+    quant_config=None,
+    prefix: str = "",
     attention_head_dim: int = 2,
     rope_max_seq_len: int = 16,
 ):
@@ -55,6 +57,8 @@ def _tiny_model(
         num_frames_per_block=num_frames_per_block,
         sliding_window_num_frames=sliding_window_num_frames,
         local_attn_size=-1,
+        quant_config=quant_config,
+        prefix=prefix,
     )
 
 
@@ -485,11 +489,52 @@ def test_constructor_rejects_non_null_image_embedding_fields(field: str) -> None
 
 
 @pytest.mark.cpu
-def test_constructor_rejects_unsupported_quantization_with_runtime_error() -> None:
+@pytest.mark.parametrize("prefix", ["", "transformer"])
+@pytest.mark.parametrize("quantized", [False, True])
+def test_constructor_propagates_quantization_to_parallel_linears(prefix: str, quantized: bool) -> None:
     module = attention_tests._load_module()
+    quant_config = object() if quantized else None
+    model = _tiny_model(module, num_layers=1, quant_config=quant_config, prefix=prefix)
+    block = model.blocks[0]
+    quantized_linears = [
+        model.c2ws_hidden_states_layer1,
+        model.c2ws_hidden_states_layer2,
+        block.self_attn.qkv,
+        block.self_attn.o,
+        block.cross_attn.q,
+        block.cross_attn.k,
+        block.cross_attn.v,
+        block.cross_attn.o,
+        block.ffn[0],
+        block.ffn[2],
+        block.cam_injector_layer1,
+        block.cam_injector_layer2,
+    ]
 
-    with pytest.raises(RuntimeError, match="quant_config.*not supported"):
-        module.CausalLingBotWorldTransformer3DModel(quant_config=object())
+    assert len(quantized_linears) == 12
+    assert all(layer.quant_config is quant_config for layer in quantized_linears)
+    expected_names = {
+        "c2ws_hidden_states_layer1",
+        "c2ws_hidden_states_layer2",
+        *(
+            f"blocks.0.{name}"
+            for name in (
+                "self_attn.qkv",
+                "self_attn.o",
+                "cross_attn.q",
+                "cross_attn.k",
+                "cross_attn.v",
+                "cross_attn.o",
+                "ffn.0",
+                "ffn.2",
+                "cam_injector_layer1",
+                "cam_injector_layer2",
+            )
+        ),
+    }
+    assert {layer.prefix for layer in quantized_linears} == {
+        f"{prefix}.{name}" if prefix else name for name in expected_names
+    }
 
 
 @pytest.mark.cpu

tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py

@@ -737,6 +737,99 @@ def test_component_discovery_uses_official_checkpoint_contract() -> None:
     assert module._loader_state.prefetch_calls == [("checkpoint", ("tokenizer", "text_encoder", "vae"), False)]
 
 
+def _ulysses_config(size: int):
+    parallel_config = _od_config().parallel_config
+    parallel_config.sequence_parallel_size = size
+    parallel_config.ulysses_degree = size
+    return _od_config(parallel_config=parallel_config)
+
+
+def _record_shard_install(monkeypatch, module, *, world_size: int):
+    """Stand in for the Ulysses group and the decoder patch; return what the install was asked for."""
+    installs: list[dict] = []
+    group = object()
+
+    def install(vae, group_arg, split_dim, *, dst):
+        installs.append({"vae": vae, "group": group_arg, "split_dim": split_dim, "dst": dst})
+
+    monkeypatch.setattr(module, "install_wan_spatial_shard_decode", install)
+    monkeypatch.setattr(module.LingBotWorldCausalDMDPipeline, "_vae_shard_group", lambda self: (group, world_size))
+    return installs, group
+
+
+def test_a_multi_rank_deployment_shards_the_decoder_across_the_ulysses_ranks(monkeypatch) -> None:
+    """Multi-rank deployments shard decode by default."""
+    module = _load_pipeline_module()
+    installs, group = _record_shard_install(monkeypatch, module, world_size=2)
+
+    pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(2))
+
+    # Along the width, and assembled on every rank: each rank's post_decode consumes the frame.
+    assert installs == [{"vae": pipeline.vae, "group": group, "split_dim": "width", "dst": None}]
+    assert pipeline._vae_shard_split_dim == "width"
+
+
+def test_a_single_rank_deployment_leaves_the_decoder_alone(monkeypatch) -> None:
+    module = _load_pipeline_module()
+    installs, _ = _record_shard_install(monkeypatch, module, world_size=1)
+
+    pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(1))
+
+    assert installs == [] and pipeline._vae_shard_split_dim is None
+
+
+@pytest.mark.parametrize("enabled", [True, False])
+@pytest.mark.parametrize("width, local_width", [(832, 208), (840, 216)])
+def test_streaming_decode_reservation_uses_padded_shards(monkeypatch, enabled, width, local_width):
+    module = _load_pipeline_module()
+    installs, _ = _record_shard_install(monkeypatch, module, world_size=4)
+    config = _ulysses_config(4)
+    config.model_config.update(lingbot_vae_spatial_sharding=enabled, ar_diffusion_height=480, ar_diffusion_width=width)
+    pipeline = module.LingBotWorldCausalDMDPipeline(od_config=config)
+    from vllm_omni.experimental.ar_diffusion.streaming_decode import WanStreamingDecoder
+
+    # Exercise the real byte estimator without constructing a checkpoint VAE.
+    decoder = object.__new__(WanStreamingDecoder)
+    decoder._bytes_per_pixel_fp32 = 16.0
+    monkeypatch.setattr(pipeline, "_streaming_decoder", lambda: decoder)
+    expected_width = local_width if enabled else width
+    assert pipeline._streaming_decode_bytes_per_session() == 16 * 480 * expected_width
+    assert len(installs) == int(enabled)
+    assert pipeline._vae_shard_world_size == (4 if enabled else 1)
+    assert config.parallel_config.sequence_parallel_size == config.parallel_config.ulysses_degree == 4
+
+
+def test_disabling_vae_sharding_does_not_require_a_shard_group(monkeypatch):
+    module = _load_pipeline_module()
+    config = _ulysses_config(4)
+    config.model_config["lingbot_vae_spatial_sharding"] = False
+
+    def unexpected_group(self):
+        pytest.fail("disabled VAE sharding must not access its process group")
+
+    monkeypatch.setattr(module.LingBotWorldCausalDMDPipeline, "_vae_shard_group", unexpected_group)
+    pipeline = module.LingBotWorldCausalDMDPipeline(od_config=config)
+    assert pipeline._vae_shard_split_dim is None
+
+
+@pytest.mark.parametrize("value", ["false", 0, None])
+def test_vae_sharding_switch_requires_a_boolean(value):
+    module = _load_pipeline_module()
+    config = _ulysses_config(4)
+    config.model_config["lingbot_vae_spatial_sharding"] = value
+    with pytest.raises(ValueError, match="lingbot_vae_spatial_sharding must be a boolean"):
+        module.LingBotWorldCausalDMDPipeline(od_config=config)
+
+
+def test_vae_shard_refuses_a_group_of_the_wrong_size(monkeypatch) -> None:
+    module = _load_pipeline_module()
+    installs, _ = _record_shard_install(monkeypatch, module, world_size=4)
+
+    with pytest.raises(RuntimeError, match="sequence_parallel_size=2 but the Ulysses group has 4 ranks"):
+        module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(2))
+    assert installs == []
+
+
 @pytest.mark.parametrize(
     ("field", "value", "feature"),
     [
@@ -758,8 +851,9 @@ def test_unsupported_parallel_modes_fail_before_component_loading(field: str, va
     assert module._loader_state.prefetch_calls == []
 
 
-def test_pure_ulysses_parallel_config_is_supported() -> None:
+def test_pure_ulysses_parallel_config_is_supported(monkeypatch) -> None:
     module = _load_pipeline_module()
+    _record_shard_install(monkeypatch, module, world_size=2)
     parallel_config = _od_config().parallel_config
     parallel_config.sequence_parallel_size = 2
     parallel_config.ulysses_degree = 2
@@ -795,13 +889,14 @@ def test_unsupported_sp_config_fails_before_component_loading(overrides):
     assert module._loader_state.prefetch_calls == []
 
 
-def test_unsupported_quantization_fails_before_component_loading() -> None:
+def test_quantization_reaches_transformer_factory() -> None:
     module = _load_pipeline_module()
+    quant_config = object()
 
-    with pytest.raises(NotImplementedError, match="quantization"):
-        module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=object()))
+    pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=quant_config))
 
-    assert module._loader_state.prefetch_calls == []
+    assert pipeline.transformer is not None
+    assert _FakeTransformerFactory.last_call[1:] == (quant_config, "transformer")
 
 
 def test_official_scheduler_config_matches_fixed_dmd_contract() -> None:
@@ -1358,10 +1453,34 @@ def test_first_frame_condition_and_camera_fold_match_transformer_contract() -> N
     assert expected_camera.shape == (1, 384, 3, 2, 2)
 
 
-def test_fixed_dmd_transition_and_cache_commit_trace() -> None:
+@pytest.mark.parametrize("value", ["true", "false", 1, 0, None])
+def test_reuse_last_step_kv_rejects_non_boolean_config(value: object) -> None:
+    config = _od_config()
+    config.model_config["lingbot_reuse_last_step_kv"] = value
+    with pytest.raises(ValueError, match="lingbot_reuse_last_step_kv must be a bool"):
+        _pipeline(_load_pipeline_module(), od_config=config)
+
+
+def test_reuse_last_step_kv_is_fixed_per_instance(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("VLLM_OMNI_LINGBOT_REUSE_LAST_STEP_KV", "1")
+    module = _load_pipeline_module()
+    config = _od_config()
+    config.model_config["lingbot_reuse_last_step_kv"] = True
+    reuse = _pipeline(module, od_config=config)
+    default = _pipeline(module)
+    config.model_config["lingbot_reuse_last_step_kv"] = False
+    assert reuse._dmd_blocks.reuse_last_step_kv is True
+    assert default._dmd_blocks.reuse_last_step_kv is False
+    assert reuse._dmd_blocks is reuse._dmd_blocks
+
+
+@pytest.mark.parametrize("reuse_last_step_kv", [False, True])
+def test_fixed_dmd_transition_and_cache_commit_trace(reuse_last_step_kv: bool) -> None:
     module = _load_pipeline_module()
     transformer = _RecordingTransformer()
-    pipeline = _pipeline(module, transformer=transformer)
+    config = _od_config()
+    config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv
+    pipeline = _pipeline(module, transformer=transformer, od_config=config)
 
     result = pipeline(_request())
 
@@ -1383,14 +1502,18 @@ def test_fixed_dmd_transition_and_cache_commit_trace() -> None:
             current = x0
     torch.testing.assert_close(result.output, current)
 
+    expected_timesteps = [timestep for timestep, _ in warped_schedule]
+    if not reuse_last_step_kv:
+        expected_timesteps.append(0.0)
     torch.testing.assert_close(
-        torch.cat([call["timestep"] for call in transformer.calls]),
-        torch.tensor([*(timestep for timestep, _ in warped_schedule), 0.0]),
+        torch.cat([call["timestep"] for call in transformer.calls]), torch.tensor(expected_timesteps)
     )
-    assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True]
-    assert [call["start_frame"] for call in transformer.calls] == [0, 0, 0, 0, 0]
+    commit_flags = [False] * (len(expected_timesteps) - 1) + [True]
+    assert [call["update_cache"] for call in transformer.calls] == commit_flags
+    assert [call["start_frame"] for call in transformer.calls] == [0] * len(expected_timesteps)
     assert len({call["cache_id"] for call in transformer.calls}) == 1
-    torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], result.output)
+    committed_latent = result.output + warped_schedule[-1][1] if reuse_last_step_kv else result.output
+    torch.testing.assert_close(transformer.calls[-1]["hidden_states"][:, :16], committed_latent)
 
 
 @pytest.mark.parametrize(
@@ -2389,10 +2512,13 @@ def test_forward_rejects_a_stepwise_camera_action_script(extra_args) -> None:
         pipeline(_request(sampling=sampling))
 
 
-def test_stepwise_progress_metadata_and_commit_trace() -> None:
+@pytest.mark.parametrize("reuse_last_step_kv", [False, True])
+def test_stepwise_progress_metadata_and_commit_trace(reuse_last_step_kv: bool) -> None:
     module = _load_pipeline_module()
     transformer = _RecordingTransformer()
-    pipeline = _pipeline(module, transformer=transformer)
+    config = _od_config()
+    config.model_config["lingbot_reuse_last_step_kv"] = reuse_last_step_kv
+    pipeline = _pipeline(module, transformer=transformer, od_config=config)
     pipeline._ar_height = 16
     pipeline._ar_width = 16
     state = _stepwise_state(num_frames=21)
@@ -2406,8 +2532,9 @@ def test_stepwise_progress_metadata_and_commit_trace() -> None:
     assert [output.chunk_index for output in outputs] == [0, 1]
     assert all(output.total_chunks == 2 for output in outputs)
     assert outputs[-1].finished is True
-    assert [call["update_cache"] for call in transformer.calls] == [False, False, False, False, True] * 2
-    assert [call["start_frame"] for call in transformer.calls] == [0] * 5 + [3] * 5
+    calls_per_chunk = 4 if reuse_last_step_kv else 5
+    assert [call["update_cache"] for call in transformer.calls] == ([False] * (calls_per_chunk - 1) + [True]) * 2
+    assert [call["start_frame"] for call in transformer.calls] == [0] * calls_per_chunk + [3] * calls_per_chunk
     assert fake.commits == ["main", "main"]
     for chunk_index, output in enumerate(outputs):
         metadata = output.output["metadata"]["ar_diffusion"]

tests/diffusion/models/magi2/test_pipeline_magi2.py

@@ -171,7 +171,7 @@ def test_huggingface_url_resolves_pinned_snapshot(tmp_path, monkeypatch):
         else pytest.fail("unexpected snapshot request")
     )
     monkeypatch.setattr(
-        "vllm.transformers_utils.repo_utils.hf_api",
+        "vllm_omni.transformers_utils.repo_utils.hf_api",
         lambda: api,
     )
     assert _resolve_checkpoint_root(

tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py

@@ -3,7 +3,7 @@
 
 import json
 import sys
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from multiprocessing.reduction import ForkingPickler
 from types import SimpleNamespace
 from typing import Any, TypeVar
@@ -2837,41 +2837,63 @@ def test_dit_encoder_selection_keeps_vae_resident():
     component.offload_to_cpu.assert_not_called()
 
 
-def test_keyframe_encode_pins_and_restores_cudnn_settings():
+@pytest.mark.parametrize(
+    ("capability", "allow_tf32"),
+    [((8, 6), True), ((9, 0), False), ((10, 0), True), ((10, 3), True), (None, True)],
+)
+@pytest.mark.parametrize("initial_tf32", [False, True])
+@pytest.mark.parametrize("fail_encode", [False, True])
+def test_keyframe_encode_pins_and_restores_cudnn_settings(
+    monkeypatch, capability, allow_tf32, initial_tf32, fail_encode
+):
+    from vllm.platforms.interface import DeviceCapability
+
     from vllm_omni.diffusion.models.minimax_h3.vae import (
         _minimax_h3_keyframe_encode_context,
     )
+    from vllm_omni.platforms import current_omni_platform
+
+    def get_capability(device_id):
+        assert device_id == 1
+        return DeviceCapability(*capability) if capability is not None else None
+
+    monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: True)
+    monkeypatch.setattr(current_omni_platform, "get_device_capability", get_capability)
 
     cudnn = torch.backends.cudnn
-    original = (
-        cudnn.enabled,
-        cudnn.benchmark,
-        cudnn.deterministic,
-        cudnn.allow_tf32,
-    )
-    try:
-        cudnn.enabled = False
-        cudnn.benchmark = True
-        cudnn.deterministic = False
-        cudnn.allow_tf32 = False
-
-        with _minimax_h3_keyframe_encode_context(torch.device("cuda")):
+    with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=initial_tf32):
+        error_context = pytest.raises(RuntimeError, match="encode failed") if fail_encode else nullcontext()
+        with error_context, _minimax_h3_keyframe_encode_context(torch.device("cuda:1")):
             assert cudnn.enabled
             assert not cudnn.benchmark
             assert cudnn.deterministic
-            assert cudnn.allow_tf32
+            assert cudnn.allow_tf32 is allow_tf32
+            if fail_encode:
+                raise RuntimeError("encode failed")
 
         assert not cudnn.enabled
         assert cudnn.benchmark
         assert not cudnn.deterministic
-        assert not cudnn.allow_tf32
-    finally:
-        (
-            cudnn.enabled,
-            cudnn.benchmark,
-            cudnn.deterministic,
-            cudnn.allow_tf32,
-        ) = original
+        assert cudnn.allow_tf32 is initial_tf32
+
+
+def test_keyframe_encode_cpu_keeps_cudnn_settings(monkeypatch):
+    from vllm_omni.diffusion.models.minimax_h3.vae import (
+        _minimax_h3_keyframe_encode_context,
+    )
+    from vllm_omni.platforms import current_omni_platform
+
+    def unexpected_capability_query(device_id):
+        pytest.fail("CPU keyframe encode must not query CUDA capability")
+
+    monkeypatch.setattr(current_omni_platform, "get_device_capability", unexpected_capability_query)
+    cudnn = torch.backends.cudnn
+    with cudnn.flags(enabled=False, benchmark=True, deterministic=False, allow_tf32=False):
+        with _minimax_h3_keyframe_encode_context(torch.device("cpu")):
+            assert not cudnn.enabled
+            assert cudnn.benchmark
+            assert not cudnn.deterministic
+            assert not cudnn.allow_tf32
 
 
 @pytest.mark.parametrize("fail_encode", [False, True])

tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py

@@ -10,7 +10,6 @@ from vllm_omni.diffusion.layers.indexed_modulation import (
     _MAX_1D_GRID_SIZE,
     _iter_row_chunks,
     _launch_row_chunks,
-    _use_hopper_bf16_affine_semantics,
     indexed_gate,
     indexed_gate_rms_norm_scale_shift,
     indexed_scale_shift_,
@@ -194,52 +193,28 @@ def test_fused_modulation_preserves_bf16_residual_boundary() -> None:
 
 
 @pytest.mark.cuda
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
-@pytest.mark.skipif(
-    torch.cuda.is_available() and torch.cuda.get_device_capability()[0] != 9,
-    reason="SM90-specific precision path",
-)
+@pytest.mark.skipif(not current_omni_platform.is_cuda(), reason="CUDA required")
 @pytest.mark.skipif(not HAS_TRITON, reason="Triton required")
-def test_hopper_fused_modulation_matches_pytorch_reference() -> None:
-    torch.manual_seed(42)
-    rows, hidden_size, conditions = 128, 3072, 4
-    residual = torch.randn(rows, hidden_size, device="cuda", dtype=torch.bfloat16)
-    gate = torch.randn(conditions, hidden_size, device="cuda", dtype=torch.bfloat16)
-    branch = torch.randn(rows, hidden_size, device="cuda", dtype=torch.bfloat16)
-    weight = torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16)
-    shift = torch.randn(conditions, hidden_size, device="cuda", dtype=torch.bfloat16)
-    scale = torch.randn(conditions, hidden_size, device="cuda", dtype=torch.bfloat16)
-    indices = torch.arange(rows, device="cuda") % conditions
+@pytest.mark.parametrize("fused_gate", [False, True])
+def test_fused_modulation_keeps_fp32_affine_intermediates(fused_gate: bool) -> None:
+    device = current_omni_platform.get_torch_device()
+    rows, hidden_size = 2, 5376
+    x = torch.ones(rows, hidden_size, device=device, dtype=torch.bfloat16)
+    weight = torch.full((hidden_size,), 1.125, device=device, dtype=x.dtype)
+    shift = torch.zeros(1, hidden_size, device=device, dtype=x.dtype)
+    scale = torch.full_like(shift, 0.09375)
+    indices = torch.zeros(rows, device=device, dtype=torch.int64)
     eps = 1e-6
+    if fused_gate:
+        residual, actual = indexed_gate_rms_norm_scale_shift(
+            x, torch.zeros_like(shift), x, weight, shift, scale, indices, eps
+        )
+        assert torch.equal(residual, x)
+    else:
+        actual = rms_norm_indexed_scale_shift(x, weight, shift, scale, indices, eps)
 
-    reference_residual = (residual + gate.index_select(0, indices) * branch).to(torch.bfloat16)
-    normalized = reference_residual.float()
-    variance = normalized.pow(2).mean(-1, keepdim=True)
-    normalized = (weight.float() * (normalized * torch.rsqrt(variance + eps))).to(torch.bfloat16)
-    reference_modulated = (normalized * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to(
-        torch.bfloat16
-    )
-
-    actual_residual, actual_modulated = indexed_gate_rms_norm_scale_shift(
-        residual,
-        gate,
-        branch,
-        weight,
-        shift,
-        scale,
-        indices,
-        eps,
-    )
-
-    torch.testing.assert_close(actual_residual, reference_residual, atol=5e-2, rtol=5e-2)
-    torch.testing.assert_close(actual_modulated, reference_modulated, atol=5e-2, rtol=5e-2)
-
-
-@pytest.mark.cuda
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
-def test_hopper_precision_dispatch_is_architecture_scoped(monkeypatch) -> None:
-    tensor = torch.empty(0, device="cuda")
-    monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (9, 0))
-    assert _use_hopper_bf16_affine_semantics(tensor)
-    monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (10, 3))
-    assert not _use_hopper_bf16_affine_semantics(tensor)
+    # Without the RMS epsilon, 1.125 * 1.09375 = 1.23046875 is exactly
+    # halfway between two BF16 values. FP32 normalization keeps the epsilon
+    # and rounds down to 1.2265625. Premature BF16 rounding drops the epsilon
+    # and instead rounds the tie up to 1.234375, changing every output row.
+    assert torch.equal(actual, torch.full_like(x, 1.2265625))

tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py

@@ -16,6 +16,7 @@ from tests.diffusion.quantization.test_quantization_quality import (
     _maybe_save_output,
 )
 from tests.helpers.mark import hardware_marks
+from tests.helpers.mock import patch_hf_snapshot_download
 
 _MINIMAX_H3_REPO = "MiniMaxAI/MiniMax-H3"
 _MINIMAX_H3_REVISION = "48d93ede732756e404a3b1b2f3b3a9b5a22f6cfc"
@@ -258,13 +259,13 @@ def test_resolve_fl2va_model_ref(tmp_path, monkeypatch):
     fl2va_root.mkdir()
     (fl2va_root / "model_index.json").write_text("{}", encoding="utf-8")
 
-    def fake_snapshot_download(self, *, repo_id, revision, allow_patterns):
+    def fake_snapshot_download(*, repo_id, revision, allow_patterns):
         assert repo_id == _MINIMAX_H3_REPO
         assert revision == _MINIMAX_H3_REVISION
         assert allow_patterns == ["FL2VA/**"]
         return str(tmp_path)
 
-    monkeypatch.setattr("huggingface_hub.HfApi.snapshot_download", fake_snapshot_download)
+    patch_hf_snapshot_download(monkeypatch, fake_snapshot_download, hf_home=tmp_path)
     assert _resolve_fl2va_model_ref() == str(fl2va_root)
 
 

tests/diffusion/models/nextstep_1_1/test_nextstep_rectangular.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from types import MethodType, SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+import torch
+
+from vllm_omni.diffusion.models.nextstep_1_1 import pipeline_nextstep_1_1 as pipeline_module
+from vllm_omni.diffusion.models.nextstep_1_1.modeling_nextstep import NextStepModel
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
+
+
+@pytest.mark.parametrize("height,width", [(32, 32), (32, 48), (48, 32)])
+@pytest.mark.parametrize("batch", [1, 2])
+def test_forward_preserves_latent_grid(monkeypatch, height, width, batch):
+    pipeline = object.__new__(pipeline_module.NextStep11Pipeline)
+    torch.nn.Module.__init__(pipeline)
+    pipeline._NextStep11Pipeline__device = torch.device("cpu")
+    pipeline.config = SimpleNamespace(use_gen_pos_embed=False)
+    pipeline.down_factor = 16
+    pipeline.scaling_factor = 2.0
+    pipeline.shift_factor = 0.25
+    grid = SimpleNamespace(config=SimpleNamespace(latent_channels=2, latent_patch_size=2))
+    expected = torch.arange(batch * 2 * (height // 8) * (width // 8), dtype=torch.float32)
+    expected = expected.reshape(batch, 2, height // 8, width // 8)
+    tokens = NextStepModel.patchify(grid, expected)
+    pipeline.model = Mock(
+        return_value=SimpleNamespace(last_hidden_state=torch.zeros(batch, 1, 4), past_key_values=None)
+    )
+    pipeline.model.forward_model.return_value = pipeline.model.return_value
+    pipeline.model.unpatchify = MethodType(NextStepModel.unpatchify, grid)
+    pipeline.tokenizer = Mock(
+        bos_token=None,
+        return_value=SimpleNamespace(
+            input_ids=torch.ones(batch, 2, dtype=torch.long), attention_mask=torch.ones(batch, 2, dtype=torch.long)
+        ),
+    )
+    pipeline._check_input = Mock(return_value=(["teapot"], None))
+    pipeline._build_captions = Mock(return_value=(["teapot"] * batch, None, 1, 1.0))
+    pipeline._add_prefix_ids = lambda _hw, ids, mask: (ids, mask)
+    pipeline.decoding = Mock(return_value=tokens)
+    pipeline.vae = SimpleNamespace(dtype=torch.float32, decode=lambda latent: SimpleNamespace(sample=latent))
+    monkeypatch.setattr(pipeline_module, "StaticCache", lambda **_kwargs: None)
+    params = OmniDiffusionSamplingParams(
+        height=height, width=width, guidance_scale=1.0, num_outputs_per_prompt=batch, seed=None
+    )
+    output = pipeline.forward(SimpleNamespace(prompts=["teapot"], sampling_params=params))
+    torch.testing.assert_close(output.output, expected / 2 + 0.25, rtol=0, atol=0)

tests/diffusion/test_async_output_worker.py

@@ -497,3 +497,99 @@ class TestWorkerProcShutdown:
         result_mq.shutdown.assert_called_once_with()
         assert proc.mq is None
         assert proc.result_mq is None
+
+
+class TestSynchronizeDeviceBarrier:
+    """The pause barrier must not report a quiet device while outputs still pack."""
+
+    @staticmethod
+    def _rpc(timeout):
+        return {
+            "method": "synchronize_device",
+            "args": (),
+            "kwargs": {"timeout": timeout},
+            "output_rank": 0,
+            "exec_all_ranks": False,
+            "collect_rank_status": False,
+        }
+
+    def test_rejects_undrained_output(self, mocker):
+        proc = _make_worker_proc(step_execution=False)
+        proc.worker = MagicMock()
+        mocker.patch.object(proc, "drain_async_outputs", return_value=False)
+
+        with pytest.raises(TimeoutError, match="did not drain"):
+            proc._execute_rpc(self._rpc(0.01))
+
+        proc.worker.execute_method.assert_not_called()
+
+    @pytest.mark.parametrize("timeout", [0.5, None])
+    def test_drains_with_the_call_timeout_then_synchronizes(self, mocker, timeout):
+        proc = _make_worker_proc(step_execution=False)
+        proc.worker = MagicMock()
+        call_order: list = []
+        mocker.patch.object(
+            proc,
+            "drain_async_outputs",
+            side_effect=lambda **kw: call_order.append(("drain", kw)) or True,
+        )
+        proc.worker.execute_method.side_effect = lambda *a, **kw: call_order.append(("sync", a, kw))
+
+        _, should_reply = proc._execute_rpc(self._rpc(timeout))
+
+        assert call_order == [
+            ("drain", {"timeout": timeout}),
+            ("sync", ("synchronize_device",), {"timeout": timeout}),
+        ]
+        assert should_reply is True
+
+    def test_completes_only_after_output_ready_is_published(self, mocker):
+        proc = _make_worker_proc(step_execution=False)
+        proc.worker = MagicMock()
+
+        mock_platform = mocker.MagicMock()
+        mock_platform.record_device_event.return_value = MagicMock()
+        mocker.patch(
+            "vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform",
+            mock_platform,
+        )
+        mock_torch = mocker.MagicMock()
+        mock_torch.accelerator.current_accelerator.return_value.type = "cpu"
+        mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.torch", mock_torch)
+        release = threading.Event()
+        mocker.patch(
+            "vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm",
+            side_effect=lambda *a, **kw: release.wait(timeout=5.0),
+        )
+
+        proc._return_result(DiffusionOutput(output="data"), rpc_id="1")
+        loop_thread = threading.Thread(target=proc._async_output_loop, daemon=True)
+        loop_thread.start()
+
+        barrier_done = threading.Event()
+
+        def run_barrier():
+            proc._execute_rpc(self._rpc(5.0))
+            barrier_done.set()
+
+        barrier_thread = threading.Thread(target=run_barrier, daemon=True)
+        barrier_thread.start()
+        try:
+            assert not barrier_done.wait(timeout=0.2)
+            proc.worker.execute_method.assert_not_called()
+
+            release.set()
+            assert barrier_done.wait(timeout=5.0)
+
+            kinds = [
+                call.args[0].kind
+                for call in proc.result_mq.enqueue.call_args_list
+                if isinstance(call.args[0], AsyncDiffusionOutput)
+            ]
+            assert kinds == [AsyncOutputKind.COMPUTE_DONE, AsyncOutputKind.OUTPUT_READY]
+            proc.worker.execute_method.assert_called_once_with("synchronize_device", timeout=5.0)
+        finally:
+            release.set()
+            proc._async_output_queue.put(None)
+            loop_thread.join(timeout=2.0)
+            barrier_thread.join(timeout=2.0)

tests/diffusion/test_diffusion_engine_rpc_routing.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Regression tests for the busy-loop-based RPC routing in DiffusionEngine.
 
 These tests guard the invariants established when ``collective_rpc`` was
@@ -60,6 +60,12 @@ class _ConcurrencyTrackingExecutor:
         self.is_failed = False
         self._closed = False
         self.od_config = SimpleNamespace()
+        # Pause tests: a call whose gate is registered (by request id, or by
+        # method name for ``synchronize_device``) blocks inside the executor
+        # until the gate is set; ``version`` tags each call so a test can tell
+        # which "weights" a request ran on.
+        self.gates: dict[str, threading.Event] = {}
+        self.version = "v1"
 
     def collective_rpc(
         self,
@@ -81,16 +87,25 @@ class _ConcurrencyTrackingExecutor:
                     "kwargs": kwargs,
                     "unique_reply_rank": unique_reply_rank,
                     "thread": threading.get_ident(),
+                    "version": self.version,
                 }
             )
         try:
             if self.rpc_delay:
                 time.sleep(self.rpc_delay)
+            if method == "synchronize_device":
+                gate = self.gates.get(method)
+                if gate is not None:
+                    gate.wait(timeout=5.0)
+                return [None]
             # Distinguish per-request execution from raw RPC by method name.
             if method in {"execute_model", "execute_stepwise", "generate"}:
                 # args[0] is the request-like object for execute_model.
                 req = args[0] if args else None
                 tag = req.request_id if req is not None and hasattr(req, "request_id") else "unknown"
+                gate = self.gates.get(tag)
+                if gate is not None:
+                    gate.wait(timeout=5.0)
                 return DiffusionOutput(error=f"result_for_{tag}", finished=True)
             tag = args[0] if args else method
             return DiffusionOutput(error=f"rpc_result_for_{tag}")
@@ -133,21 +148,27 @@ def _make_request(tag: str):
 
 
 def _make_engine_with_loop(
-    loop: asyncio.AbstractEventLoop,
+    loop: asyncio.AbstractEventLoop | None,
     rpc_delay: float = 0.0,
+    *,
+    scheduler: RequestScheduler | None = None,
+    start_loop: bool = True,
 ):
     """Construct a ``DiffusionEngine`` skeleton with a real busy loop.
 
     The engine is wired with a fake executor that asserts no concurrent
-    calls and a real ``RequestScheduler``.
+    calls and a real ``RequestScheduler``. ``start_loop=False`` leaves the
+    engine in its bootstrap state (no busy-loop thread).
     """
     engine = DiffusionEngine.__new__(DiffusionEngine)
     engine._closed = False
     engine.od_config = SimpleNamespace(streaming_output=False)
     engine.executor = _ConcurrencyTrackingExecutor(rpc_delay=rpc_delay)
 
-    sched = RequestScheduler()
-    sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0))
+    sched = scheduler
+    if sched is None:
+        sched = RequestScheduler()
+        sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0))
     engine.scheduler = sched
     engine.step_execution = False
     engine.supports_request_batch = False
@@ -160,9 +181,14 @@ def _make_engine_with_loop(
     engine._closed = False
     engine.abort_queue = queue.Queue()
     engine._rpc_queue = queue.Queue()
+    engine._scheduling_paused = False
 
     engine.main_loop = loop
     engine.stop_event = threading.Event()
+    if not start_loop:
+        engine._loop_started = False
+        engine.worker_thread = None
+        return engine
     engine._loop_started = True
     engine.worker_thread = threading.Thread(target=engine._busy_loop, daemon=True)
     engine.worker_thread.start()
@@ -170,6 +196,10 @@ def _make_engine_with_loop(
 
 
 def _stop_engine(engine: DiffusionEngine) -> None:
+    if engine.worker_thread is None:
+        return
+    for gate in engine.executor.gates.values():
+        gate.set()
     with engine._cv:
         engine.stop_event.set()
         engine._cv.notify_all()
@@ -565,3 +595,319 @@ async def test_rpc_and_request_results_do_not_swap():
     assert req_res.error == "result_for_req1"
     # Belt and braces: never overlapped on the executor.
     assert engine.executor.max_concurrent == 1
+
+
+# ───────────────────────── scheduler pause (keep) ─────────────────────────
+
+
+class _GatedAdmissionScheduler(RequestScheduler):
+    """Batching wait that only ends when the test opens admission."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.admission_open = threading.Event()
+
+    def get_admission_wait_decision(self, *, now: float, dp_concurrent: bool = False):
+        decision = super().get_admission_wait_decision(now=now, dp_concurrent=dp_concurrent)
+        return decision.__class__(should_wait=True, deadline=None, max_batch=self.max_num_running_reqs)
+
+    def should_end_admission_wait(self, decision, *, now: float, stable_since: float) -> bool:
+        return self.admission_open.is_set()
+
+
+def _pause(engine: DiffusionEngine, timeout: float | None = None):
+    return engine.async_collective_rpc("pause_scheduler", timeout=timeout, kwargs={"mode": "keep"})
+
+
+def _resume(engine: DiffusionEngine):
+    return engine.async_collective_rpc("resume_scheduler")
+
+
+def _methods(engine: DiffusionEngine) -> list[str]:
+    return [call["method"] for call in engine.executor.calls]
+
+
+def _executed(engine: DiffusionEngine) -> list[tuple[str, str]]:
+    return [
+        (call["args"][0].request_id, call["version"])
+        for call in engine.executor.calls
+        if call["method"] == "execute_model"
+    ]
+
+
+def _submit(engine: DiffusionEngine, tag: str) -> asyncio.Task:
+    return asyncio.create_task(_consume_final_output(engine.async_add_req_and_stream_response(_make_request(tag))))
+
+
+async def _wait_until(predicate, timeout: float = 3.0) -> None:
+    deadline = time.monotonic() + timeout
+    while not predicate():
+        if time.monotonic() > deadline:
+            raise AssertionError("condition not reached in time")
+        await asyncio.sleep(0.005)
+
+
+@pytest.mark.asyncio
+async def test_keep_pause_retains_waiting_request_and_serves_control_rpcs():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    try:
+        assert await _pause(engine) is None
+        queued = _submit(engine, "queued")
+        result = await engine.async_collective_rpc("list_loras", unique_reply_rank=0)
+        assert result.error == "rpc_result_for_list_loras"
+        await asyncio.sleep(0.05)
+        assert engine.scheduler.num_waiting_requests() == 1
+        assert _executed(engine) == []
+        assert not queued.done()
+
+        await _resume(engine)
+        assert (await asyncio.wait_for(queued, 3.0)).error == "result_for_queued"
+        assert _methods(engine) == ["synchronize_device", "list_loras", "execute_model"]
+    finally:
+        _stop_engine(engine)
+
+
+@pytest.mark.asyncio
+async def test_keep_pause_acks_after_running_batch_and_freezes_queue():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    gate = engine.executor.gates["A"] = threading.Event()
+    try:
+        a = _submit(engine, "A")
+        await _wait_until(lambda: _executed(engine) == [("A", "v1")])
+        b = _submit(engine, "B")
+        await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
+
+        pause = asyncio.create_task(_pause(engine))
+        await asyncio.sleep(0.1)
+        assert not pause.done()
+
+        gate.set()
+        await asyncio.wait_for(pause, 3.0)
+        assert _methods(engine) == ["execute_model", "synchronize_device"]
+        assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
+        assert engine.scheduler.num_waiting_requests() == 1
+        assert not b.done()
+
+        engine.executor.version = "v2"
+        await _resume(engine)
+        assert (await asyncio.wait_for(b, 3.0)).error == "result_for_B"
+        assert _executed(engine) == [("A", "v1"), ("B", "v2")]
+    finally:
+        _stop_engine(engine)
+
+
+@pytest.mark.asyncio
+async def test_keep_pause_ack_does_not_release_later_rpcs_before_the_batch_is_emitted():
+    """A pause ACK must not open the queue to RPCs that touch device memory
+    (sleep) before the batch that just ran has been delivered.
+    """
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    barrier_gate = engine.executor.gates["synchronize_device"] = threading.Event()
+    methods_at_emit: list[list[str]] = []
+    emit_outputs = engine._emit_outputs
+
+    def _recording_emit(*args, **kwargs):
+        methods_at_emit.append(_methods(engine))
+        return emit_outputs(*args, **kwargs)
+
+    engine._emit_outputs = _recording_emit
+    try:
+        gate = engine.executor.gates["A"] = threading.Event()
+        a = _submit(engine, "A")
+        await _wait_until(lambda: _executed(engine) == [("A", "v1")])
+        pause = asyncio.create_task(_pause(engine))
+        await _wait_until(lambda: engine._scheduling_paused)
+        gate.set()
+        # The barrier holds the busy loop inside the post-execute RPC drain,
+        # so the sleep is queued before the pause ACK is published.
+        await _wait_until(lambda: "synchronize_device" in _methods(engine))
+        sleep_rpc = asyncio.create_task(engine.async_collective_rpc("handle_sleep_task", args=("t",)))
+        await _wait_until(lambda: not engine._rpc_queue.empty())
+        barrier_gate.set()
+
+        await asyncio.wait_for(pause, 3.0)
+        assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
+        assert (await asyncio.wait_for(sleep_rpc, 3.0)).error == "rpc_result_for_t"
+    finally:
+        _stop_engine(engine)
+
+    assert _methods(engine) == ["execute_model", "synchronize_device", "handle_sleep_task"]
+    assert methods_at_emit and "handle_sleep_task" not in methods_at_emit[0]
+
+
+@pytest.mark.asyncio
+async def test_timed_out_pause_still_ends_the_drain_before_the_batch_is_emitted():
+    """A pause whose caller already gave up is still a drain barrier: its
+    cancelled task must not let a later RPC run before the batch is emitted.
+    """
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    methods_at_emit: list[list[str]] = []
+    emit_outputs = engine._emit_outputs
+
+    def _recording_emit(*args, **kwargs):
+        methods_at_emit.append(_methods(engine))
+        return emit_outputs(*args, **kwargs)
+
+    engine._emit_outputs = _recording_emit
+    try:
+        gate = engine.executor.gates["A"] = threading.Event()
+        a = _submit(engine, "A")
+        await _wait_until(lambda: _executed(engine) == [("A", "v1")])
+
+        # The caller times out while A is still executing: the pause future is
+        # cancelled, but its task stays queued ahead of the sleep.
+        with pytest.raises(TimeoutError):
+            await _pause(engine, timeout=0.1)
+        sleep_rpc = asyncio.create_task(engine.async_collective_rpc("handle_sleep_task", args=("t",)))
+        await _wait_until(lambda: engine._rpc_queue.qsize() == 2)
+        gate.set()
+
+        assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
+        assert (await asyncio.wait_for(sleep_rpc, 3.0)).error == "rpc_result_for_t"
+    finally:
+        _stop_engine(engine)
+
+    assert _methods(engine) == ["execute_model", "handle_sleep_task"]
+    assert methods_at_emit and "handle_sleep_task" not in methods_at_emit[0]
+
+
+@pytest.mark.asyncio
+async def test_pause_rejects_unsupported_modes_without_closing_the_gate():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    try:
+        for kwargs in ({"mode": "abort"}, {"mode": "wait"}, {}):
+            with pytest.raises(ValueError):
+                await engine.async_collective_rpc("pause_scheduler", kwargs=kwargs)
+        engine.execution_mode = DiffusionExecutionMode.STEP_BATCH
+        with pytest.raises(NotImplementedError):
+            await _pause(engine)
+        engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH
+
+        assert (await asyncio.wait_for(_submit(engine, "still-runs"), 3.0)).error == "result_for_still-runs"
+        assert _methods(engine) == ["execute_model"]
+    finally:
+        _stop_engine(engine)
+
+
+@pytest.mark.asyncio
+async def test_keep_pause_during_batching_wait_holds_the_batch():
+    loop = asyncio.get_running_loop()
+    sched = _GatedAdmissionScheduler()
+    sched.initialize(SimpleNamespace(max_num_seqs=4, request_batch_max_wait_ms=1000.0))
+    engine = _make_engine_with_loop(loop, scheduler=sched)
+    try:
+        waiting = _submit(engine, "W")
+        await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
+        await asyncio.sleep(0.05)
+
+        await asyncio.wait_for(_pause(engine), 3.0)
+        assert _methods(engine) == ["synchronize_device"]
+        assert engine.scheduler.num_waiting_requests() == 1
+
+        await _resume(engine)
+        await asyncio.sleep(0.05)
+        assert _executed(engine) == []
+        sched.admission_open.set()
+        assert (await asyncio.wait_for(waiting, 3.0)).error == "result_for_W"
+    finally:
+        sched.admission_open.set()
+        _stop_engine(engine)
+
+
+@pytest.mark.asyncio
+async def test_pause_timeout_leaves_gate_closed_until_explicit_resume():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    gate = engine.executor.gates["A"] = threading.Event()
+    try:
+        a = _submit(engine, "A")
+        await _wait_until(lambda: _executed(engine) == [("A", "v1")])
+        b = _submit(engine, "B")
+        await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
+
+        with pytest.raises(TimeoutError):
+            await _pause(engine, timeout=0.1)
+        gate.set()
+        assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
+        await asyncio.sleep(0.1)
+        assert _executed(engine) == [("A", "v1")]
+        assert not b.done()
+
+        await _resume(engine)
+        assert (await asyncio.wait_for(b, 3.0)).error == "result_for_B"
+        assert "synchronize_device" not in _methods(engine)
+    finally:
+        _stop_engine(engine)
+
+
+@pytest.mark.asyncio
+async def test_repeated_pause_and_resume_are_idempotent():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    try:
+        await _resume(engine)
+        await _pause(engine)
+        await _pause(engine)
+        queued = _submit(engine, "Q")
+        await asyncio.sleep(0.05)
+        assert _executed(engine) == []
+        await _resume(engine)
+        await _resume(engine)
+        assert (await asyncio.wait_for(queued, 3.0)).error == "result_for_Q"
+        assert _methods(engine) == ["synchronize_device", "synchronize_device", "execute_model"]
+    finally:
+        _stop_engine(engine)
+
+
+@pytest.mark.asyncio
+async def test_abort_of_waiting_request_while_paused_delivers_terminal():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    try:
+        await _pause(engine)
+        b = _submit(engine, "B")
+        await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
+        engine.abort("B")
+
+        result = await asyncio.wait_for(b, 3.0)
+        assert result.aborted is True
+        assert engine.scheduler.get_request_state("B") is None
+        assert _executed(engine) == []
+
+        await _resume(engine)
+        assert (await asyncio.wait_for(_submit(engine, "C"), 3.0)).error == "result_for_C"
+        assert _executed(engine) == [("C", "v1")]
+        assert engine.worker_thread.is_alive()
+    finally:
+        _stop_engine(engine)
+
+
+def test_pause_and_resume_before_loop_starts_use_engine_local_path():
+    engine = _make_engine_with_loop(None, start_loop=False)
+
+    assert engine.collective_rpc("pause_scheduler", kwargs={"mode": "keep"}) is None
+    assert _methods(engine) == ["synchronize_device"]
+
+    assert engine.collective_rpc("resume_scheduler") is None
+    assert engine.add_req_and_wait_for_response(_make_request("X")).error == "result_for_X"
+    assert _executed(engine) == [("X", "v1")]
+
+
+@pytest.mark.asyncio
+async def test_requests_without_pause_never_trigger_the_barrier():
+    loop = asyncio.get_running_loop()
+    engine = _make_engine_with_loop(loop)
+    try:
+        first = await asyncio.wait_for(_submit(engine, "r1"), 3.0)
+        await engine.async_collective_rpc("ping", args=("p",), unique_reply_rank=0)
+        second = await asyncio.wait_for(_submit(engine, "r2"), 3.0)
+    finally:
+        _stop_engine(engine)
+
+    assert (first.error, second.error) == ("result_for_r1", "result_for_r2")
+    assert _methods(engine) == ["execute_model", "ping", "execute_model"]

tests/diffusion/test_diffusion_scheduler.py

@@ -2092,3 +2092,19 @@ class TestStepScheduler:
 
         with pytest.raises(ValueError):
             self.scheduler.add_request(request)
+
+
+class TestPendingFinishedRequestIds:
+    def test_reports_finished_ids_that_still_hold_state_without_clearing_them(self):
+        sched = RequestScheduler()
+        sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0))
+        sched.add_request(_make_request("a"))
+        sched.add_request(_make_request("b"))
+
+        sched.finish_requests("a", DiffusionRequestStatus.FINISHED_ABORTED)
+        assert sched.pending_finished_request_ids() == {"a"}
+
+        sched.pop_request_state("a")
+        assert sched.pending_finished_request_ids() == set()
+        # The next wave still ships the id to the worker for its own cleanup.
+        assert sched.schedule().finished_req_ids == {"a"}

tests/diffusion/test_diffusion_worker_synchronize_device.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Unit tests for the device step of the diffusion pause barrier."""
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def _make_worker(runner: object) -> DiffusionWorker:
+    worker = object.__new__(DiffusionWorker)
+    worker.model_runner = runner
+    worker.rank = 0
+    return worker
+
+
+def test_synchronize_device_waits_for_prefetch_before_device_sync(mocker):
+    platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
+    order: list = []
+    manager = MagicMock()
+    manager.wait_prefetch.side_effect = lambda timeout: order.append(("prefetch", timeout)) or True
+    platform.synchronize.side_effect = lambda: order.append("sync")
+    worker = _make_worker(SimpleNamespace(kv_transfer_manager=manager))
+
+    worker.synchronize_device(timeout=2.5)
+
+    assert order == [("prefetch", 2.5), "sync"]
+
+
+def test_synchronize_device_fails_when_prefetch_is_still_outstanding(mocker):
+    platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
+    manager = MagicMock()
+    manager.wait_prefetch.return_value = False
+    worker = _make_worker(SimpleNamespace(kv_transfer_manager=manager))
+
+    with pytest.raises(TimeoutError, match="prefetch"):
+        worker.synchronize_device(timeout=0.01)
+
+    platform.synchronize.assert_not_called()
+
+
+def test_synchronize_device_without_transfer_manager_only_syncs(mocker):
+    platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
+    worker = _make_worker(SimpleNamespace(kv_transfer_manager=None))
+
+    worker.synchronize_device(timeout=None)
+
+    platform.synchronize.assert_called_once_with()

tests/distributed/omni_connectors/test_kv_async_prefetch.py

@@ -5,6 +5,7 @@ mock connector, dedup, per-call sender_info isolation, role classification, and
 miss/abort paths.  GPU H2D and D2D paths need CUDA (integration tests).
 """
 
+import threading
 from types import SimpleNamespace
 
 import pytest
@@ -354,3 +355,41 @@ def test_consume_then_apply_attaches_payload():
     receiver._record_stream_for_prefetched(data)
     receiver.apply_kv_cache_to_request(req, data)
     assert req.past_key_values is not None
+
+
+# --------------------------------------------------------------------------- #
+#  wait_prefetch: the pause barrier's view of outstanding background loads
+# --------------------------------------------------------------------------- #
+
+
+def test_wait_prefetch_is_true_when_nothing_outstanding():
+    _, receiver, _ = _make_sender_receiver()
+    assert receiver.wait_prefetch(timeout=0.01) is True
+
+
+def test_wait_prefetch_is_true_when_prefetch_disabled():
+    _, receiver, _ = _make_sender_receiver(async_prefetch=False)
+    assert receiver.wait_prefetch(timeout=0.01) is True
+
+
+def test_wait_prefetch_reports_a_running_prefetch_and_keeps_its_payload():
+    sender, receiver, connector = _make_sender_receiver()
+    _seed_payload(sender, "rid-w")
+    gate = threading.Event()
+    real_get = connector.get
+
+    def slow_get(*args, **kwargs):
+        gate.wait(timeout=5.0)
+        return real_get(*args, **kwargs)
+
+    connector.get = slow_get
+    receiver.start_prefetch({"request_id": "rid-w", "kv_sender_info": _SENDER_INFO})
+    try:
+        assert receiver.wait_prefetch(timeout=0.05) is False
+        gate.set()
+        assert receiver.wait_prefetch(timeout=5.0) is True
+        data, _ = receiver.consume_prefetched_kv(_req("rid-w"))
+        assert data is not None
+    finally:
+        gate.set()
+        receiver.shutdown_prefetch()

tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py

@@ -32,7 +32,6 @@ from huggingface_hub import hf_hub_download, snapshot_download
 from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure
 
 from tests.e2e.accuracy.helpers import reset_artifact_dir
-from tests.helpers import skip_if_gated_repo_inaccessible
 from tests.helpers.mark import hardware_test
 
 OFFICIAL_REPOSITORY = "https://github.com/Lightricks/LTX-2.git"
@@ -303,32 +302,6 @@ def _resolve_ltx25_image() -> Path:
     )
 
 
-def _require_ltx25_model_access() -> None:
-    configured_omni_model = os.environ.get("VLLM_TEST_LTX25_MODEL")
-    if not (configured_omni_model and Path(configured_omni_model).exists()):
-        omni_model = configured_omni_model or LTX25_OMNI_MODEL_ID
-        omni_revision = os.environ.get("VLLM_TEST_LTX25_MODEL_REVISION")
-        if omni_revision is None and omni_model == LTX25_OMNI_MODEL_ID:
-            omni_revision = LTX25_OMNI_MODEL_REVISION
-        skip_if_gated_repo_inaccessible(
-            omni_model,
-            revision=omni_revision,
-            filename="model_index.json",
-        )
-
-    configured_official_model = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL")
-    if not (configured_official_model and Path(configured_official_model).exists()):
-        official_model = configured_official_model or LTX25_OFFICIAL_MODEL_ID
-        official_revision = os.environ.get("VLLM_TEST_LTX25_OFFICIAL_MODEL_REVISION")
-        if official_revision is None and official_model == LTX25_OFFICIAL_MODEL_ID:
-            official_revision = LTX25_OFFICIAL_MODEL_REVISION
-        skip_if_gated_repo_inaccessible(
-            official_model,
-            revision=official_revision,
-            filename=LTX25_OFFICIAL_COMMON_FILES["text_encoder"],
-        )
-
-
 def _ltx25_request(image: Path | None = None) -> dict[str, object]:
     request: dict[str, object] = {
         "prompt": LTX25_PROMPT,
@@ -475,7 +448,6 @@ def _assert_strict_similarity(
 @pytest.mark.parametrize("task", ["t2v", "i2v"])
 def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path, task: str) -> None:
     """Compare Omni with official canonical DiffVAE and matching connector weights."""
-    _require_ltx25_model_access()
     artifact_parent = accuracy_artifact_root / "ltx_official"
     output_root = reset_artifact_dir(artifact_parent / f"ltx2_5_distilled_{task}")
     official_root, official_revision = _ltx25_official_source(artifact_parent)
@@ -600,7 +572,6 @@ def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path
 @pytest.mark.parametrize("pipeline_mode", ["full_one_stage", "full_two_stage"])
 def test_ltx25_full_matches_official(accuracy_artifact_root: Path, task: str, pipeline_mode: str) -> None:
     """Compare raw official Full/SFT and DiffVAE weights with Omni."""
-    _require_ltx25_model_access()
     artifact_parent = accuracy_artifact_root / "ltx_official"
     output_root = reset_artifact_dir(artifact_parent / f"ltx2_5_{pipeline_mode}_{task}")
     model_class_name = {

tests/e2e/features/comfyui/mock_videos_server.py

@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests.
+
+Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test
+can assert exactly what the client serialized, and returns a synthetic MP4.
+"""
+
+import asyncio
+import json
+import os
+import subprocess
+import uuid
+
+from fastapi import FastAPI, Request
+from fastapi.responses import Response
+from starlette.datastructures import UploadFile
+
+app = FastAPI()
+_JOBS: dict[str, dict] = {}
+_STATE_FILE = os.environ.get("MOCK_STATE_FILE")
+
+
+def _record(fields: dict) -> None:
+    if _STATE_FILE:
+        with open(_STATE_FILE, "w") as f:
+            json.dump(fields, f)
+
+
+def _synthetic_mp4() -> bytes:
+    return subprocess.run(
+        [
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel",
+            "error",
+            "-y",
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=size=320x240:rate=24:duration=1",
+            "-f",
+            "lavfi",
+            "-i",
+            "sine=frequency=440:sample_rate=32000:duration=1",
+            "-map",
+            "0:v",
+            "-map",
+            "1:a",
+            "-shortest",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "aac",
+            "-ar",
+            "32000",
+            "-ac",
+            "2",
+            "-movflags",
+            "frag_keyframe+empty_moov",
+            "-f",
+            "mp4",
+            "pipe:1",
+        ],
+        check=True,
+        capture_output=True,
+    ).stdout
+
+
+@app.post("/v1/videos")
+async def create_video(request: Request):
+    form = await request.form()
+    fields = {}
+    for key, value in form.items():
+        if isinstance(value, UploadFile):
+            data = await value.read()
+            fields[key] = {
+                "file": True,
+                "size": len(data),
+                "filename": value.filename,
+                "content_type": value.content_type,
+            }
+            if value.content_type == "application/json":
+                fields[key]["json"] = json.loads(data)
+        else:
+            fields[key] = str(value)
+    _record(fields)
+
+    job_id = str(uuid.uuid4())
+    _JOBS[job_id] = {"status": "queued"}
+    return {"id": job_id, "status": "queued"}
+
+
+@app.get("/v1/videos/{job_id}")
+async def get_video(job_id: str):
+    if job_id not in _JOBS:
+        return {"status": "failed", "error": "unknown job"}
+    return {"id": job_id, "status": "completed"}
+
+
+@app.get("/v1/videos/{job_id}/content")
+async def get_video_content(job_id: str):
+    if job_id not in _JOBS:
+        return Response(status_code=404)
+    mp4 = await asyncio.to_thread(_synthetic_mp4)
+    return Response(content=mp4, media_type="video/mp4")
+
+
+@app.delete("/v1/videos/{job_id}")
+async def delete_video(job_id: str):
+    _JOBS.pop(job_id, None)
+    return {"deleted": True}

tests/e2e/features/comfyui/test_latent_mask.py

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Unit tests for latent-mask serialization helpers."""
+
+import json
+
+import pytest
+import torch
+from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_scalar_mask_to_json():
+    assert scalar_mask_to_json(0.5) == "0.5"
+    assert scalar_mask_to_json(1.0) == "1.0"
+    with pytest.raises(ValueError):
+        scalar_mask_to_json(1.5)
+    with pytest.raises(ValueError):
+        scalar_mask_to_json(-0.1)
+
+
+def test_video_mask_grid_shape():
+    mask = torch.zeros(256, 448)
+    grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107))
+    assert len(grid) == 2 + 5 * ((107 - 5) // 17)
+    assert len(grid[0]) == 256 // 16
+    assert len(grid[0][0]) == 448 // 16
+
+
+def test_video_mask_grid_shape_non_aligned_frames():
+    # 96 snaps up to 107 (17n+5), so Tv == 32, matching the server.
+    grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96))
+    assert len(grid) == 32
+
+
+def test_video_mask_grid_shape_short_clip():
+    # num_frames <= 5 -> Tv == 2.
+    grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1))
+    assert len(grid) == 2
+
+
+def test_video_mask_grid_shape_unaligned_width():
+    # width=500 floors to 480 (multiple of 32) -> gw == 30, not 31.
+    grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22))
+    assert len(grid[0][0]) == 30
+
+
+def test_video_mask_3d_input():
+    grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107))
+    assert grid[0][0][0] == 1.0
+
+
+def test_video_mask_temporal():
+    # 3D mask: first half 0 (preserve), second half 1 (regenerate).
+    mask = torch.zeros(7, 64, 64)
+    mask[4:] = 1.0
+    grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22))
+    assert len(grid) == 7
+    assert all(v == 0.0 for row in grid[0] for v in row)
+    assert all(v == 1.0 for row in grid[-1] for v in row)
+
+
+def test_video_mask_temporal_resize():
+    # 2 temporal slices resize to tv == 7.
+    mask = torch.zeros(2, 64, 64)
+    mask[1] = 1.0
+    grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22))
+    assert len(grid) == 7
+
+
+def test_video_mask_uniform_preserved():
+    grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107))
+    assert all(v == 0.5 for row in grid for cell in row for v in cell)
+
+
+def test_video_mask_values_in_range():
+    grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22))
+    assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell)